From de7531ba7b89c5b1fb37afcf1d81fd336e5e04ce Mon Sep 17 00:00:00 2001 From: Jonah Larson Date: Wed, 27 May 2026 15:56:05 -0500 Subject: [PATCH 01/87] Initial Driver System (#1335) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Refactored `transformers` so that bridge configuration code is source-agnostic * Initial breakup of Bridge into core and bridge only components. Setup remote driver in prep * Initial setup of the vllm source Cherry-picked from 98bbb835 on feature/vllm-integration. Resolved against dev-4.x's Phase A type split: - bridge.py: kept the shim (TransformerBridge content lives in transformer_bridge.py now). - sources/transformers.py: accepted the scaffold's package restructure (transformers.py → transformers/{__init__,helpers,source}.py + _hf_format.py). - sources/transformers/source.py: re-applied Phase A's explicit TransformersDriver construction in boot(). sources/vllm/source.py's boot_vllm currently sets bridge._forward_impl, which Phase A removed. The function is dead until the next chunk lifts it onto VLLMDriver + RemoteBridge — landing as-is so the scaffold tree is on dev-4.x for incremental Phase B work. Co-Authored-By: Claude Opus 4.7 (1M context) * Revision round 1 of the vLLM system * Full vLLM integration initial completion * jupiter patch * dynamo patching * pass through batch size * add max model length to remote bridge boot * Updating config construction to properly pass through to vLLM * Wiring TLWorkerExtension * Additional testing + diagnostic * Another spot check * Remove logging * Fix issue with logit generation on hook_out * Adjust to use token_ids directly instead of logprobs * Updating Step 5 and driver bridge cleanup * Add decoder layer for materializing the residual stream * Updating tests due to changes on `dev` * Fix pip install path * Testing ln_final divergence * documenting differences in ln_final * Format cleanup * Fix bug with beartype --------- Co-authored-by: Claude Opus 4.7 (1M context) --- demos/vLLM_Bridge_Integration_Test.ipynb | 420 ++ .../test_qwen3_5_adapter.py | 28 +- .../unit/model_bridge/test_driver_protocol.py | 733 ++++ tests/unit/model_bridge/test_remote_bridge.py | 312 ++ tests/unit/model_bridge/test_vllm_boot.py | 183 + tests/unit/model_bridge/test_vllm_driver.py | 258 ++ transformer_lens/model_bridge/__init__.py | 4 +- transformer_lens/model_bridge/bridge.py | 3759 +---------------- transformer_lens/model_bridge/bridge_core.py | 865 ++++ .../model_bridge/driver_protocol.py | 192 + .../model_bridge/remote_bridge.py | 100 + .../model_bridge/sources/__init__.py | 2 + .../model_bridge/sources/_bridge_builder.py | 193 + .../model_bridge/sources/_driver_base.py | 67 + .../model_bridge/sources/_hf_format.py | 261 ++ .../model_bridge/sources/transformers.py | 936 ---- .../sources/transformers/__init__.py | 47 + .../sources/transformers/helpers.py | 157 + .../sources/transformers/source.py | 435 ++ .../sources/transformers_driver.py | 75 + .../model_bridge/sources/vllm/__init__.py | 12 + .../model_bridge/sources/vllm/driver.py | 207 + .../model_bridge/sources/vllm/internals.py | 24 + .../sources/vllm/intervention_specs.py | 9 + .../sources/vllm/overlays/__init__.py | 21 + .../sources/vllm/overlays/base.py | 25 + .../sources/vllm/overlays/decoder_only.py | 65 + .../model_bridge/sources/vllm/plugin.py | 170 + .../model_bridge/sources/vllm/source.py | 207 + .../sources/vllm/worker_extension.py | 101 + .../model_bridge/transformer_bridge.py | 2991 +++++++++++++ 31 files changed, 8167 insertions(+), 4692 deletions(-) create mode 100644 demos/vLLM_Bridge_Integration_Test.ipynb create mode 100644 tests/unit/model_bridge/test_driver_protocol.py create mode 100644 tests/unit/model_bridge/test_remote_bridge.py create mode 100644 tests/unit/model_bridge/test_vllm_boot.py create mode 100644 tests/unit/model_bridge/test_vllm_driver.py create mode 100644 transformer_lens/model_bridge/bridge_core.py create mode 100644 transformer_lens/model_bridge/driver_protocol.py create mode 100644 transformer_lens/model_bridge/remote_bridge.py create mode 100644 transformer_lens/model_bridge/sources/_bridge_builder.py create mode 100644 transformer_lens/model_bridge/sources/_driver_base.py create mode 100644 transformer_lens/model_bridge/sources/_hf_format.py delete mode 100644 transformer_lens/model_bridge/sources/transformers.py create mode 100644 transformer_lens/model_bridge/sources/transformers/__init__.py create mode 100644 transformer_lens/model_bridge/sources/transformers/helpers.py create mode 100644 transformer_lens/model_bridge/sources/transformers/source.py create mode 100644 transformer_lens/model_bridge/sources/transformers_driver.py create mode 100644 transformer_lens/model_bridge/sources/vllm/__init__.py create mode 100644 transformer_lens/model_bridge/sources/vllm/driver.py create mode 100644 transformer_lens/model_bridge/sources/vllm/internals.py create mode 100644 transformer_lens/model_bridge/sources/vllm/intervention_specs.py create mode 100644 transformer_lens/model_bridge/sources/vllm/overlays/__init__.py create mode 100644 transformer_lens/model_bridge/sources/vllm/overlays/base.py create mode 100644 transformer_lens/model_bridge/sources/vllm/overlays/decoder_only.py create mode 100644 transformer_lens/model_bridge/sources/vllm/plugin.py create mode 100644 transformer_lens/model_bridge/sources/vllm/source.py create mode 100644 transformer_lens/model_bridge/sources/vllm/worker_extension.py create mode 100644 transformer_lens/model_bridge/transformer_bridge.py diff --git a/demos/vLLM_Bridge_Integration_Test.ipynb b/demos/vLLM_Bridge_Integration_Test.ipynb new file mode 100644 index 0000000000..e712db1c41 --- /dev/null +++ b/demos/vLLM_Bridge_Integration_Test.ipynb @@ -0,0 +1,420 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# vLLM Bridge \u2014 Integration Test\n", + "\n", + "End-to-end validation of `boot_vllm` on a real GPU. Runs the v4 Driver protocol's vLLM backend against `meta-llama/Llama-3.2-1B`, compares per-hook activations and next-token argmax against the HF transformers backend, and exercises the affine intervention path.\n", + "\n", + "**Branch:** `dev-4.x`. **Hardware:** any CUDA GPU with \u226510 GB VRAM (free Colab T4 is sufficient).\n", + "\n", + "## Scope: observation + mutation\n", + "\n", + "This source extends past vllm-lens's observation-only scope. Each capture hook applies an affine transform (`output = output * scale + bias`, default identity) and returns the modified tensor, so interventions propagate to downstream layers. Step 6 below is the load-bearing verification that this works end-to-end under `torch.compile` + CUDA graphs \u2014 unit tests can't reach the compiled-graph path.\n", + "\n", + "## What this validates\n", + "1. `boot_vllm` returns a `RemoteBridge` end-to-end.\n", + "2. `run_with_cache` populates an `ActivationCache` from the vLLM worker via `collective_rpc`.\n", + "3. Greedy next-token argmax matches `boot_transformers` (greedy parity).\n", + "4. Per-fireable-hook relative L2 < 5e-3 vs HF (hook-fidelity gate; failures abort).\n", + "5. **Mutation smoke** (load-bearing): suppressing `embed.hook_out` zeros the cache there, shifts the argmax, and reverts cleanly on the next forward.\n", + "6. GPU release after `del bridge`." + ], + "id": "b86ce379b49c" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "1. **Runtime \u2192 Change runtime type \u2192 GPU** (T4 / L4 / A100 all work).\n", + "2. **Secrets \u2192 add `HF_TOKEN`** with a token that has access to `meta-llama/Llama-3.2-1B` (gated).\n", + "\n", + "The environment cell below patches `sys.stdout.fileno` because ipykernel's captured stdout doesn't expose a real file descriptor and vLLM's worker init calls `fileno()`. Without the patch, Step 2 fails with `UnsupportedOperation: fileno`." + ], + "id": "3623b9f2ca96" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Install vllm and TransformerLens @ feature/driver-system. ~3-5 minutes.\n", + "# vllm pinned to 0.20.2 \u2014 the version the internal-API walks in\n", + "# transformer_lens/model_bridge/sources/vllm/{plugin,internals}.py were validated against.\n", + "# vLLM rearranges its internal class paths every 4-6 weeks; re-validate before bumping.\n", + "%pip install -q \"vllm==0.20.2\"\n", + "%pip install -q git+https://github.com/TransformerLensOrg/TransformerLens.git@feature/driver-system" + ], + "id": "4ab1eb60e6b1" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import gc\n", + "import os\n", + "import sys\n", + "\n", + "import torch\n", + "\n", + "# HF_TOKEN comes from Colab secrets. Falls back to env var for non-Colab runs.\n", + "try:\n", + " from google.colab import userdata\n", + " os.environ.setdefault(\"HF_TOKEN\", userdata.get(\"HF_TOKEN\"))\n", + "except (ImportError, Exception):\n", + " pass\n", + "assert os.environ.get(\"HF_TOKEN\"), \"Set HF_TOKEN in Colab Secrets (gear icon, left sidebar).\"\n", + "\n", + "# Colab/Jupyter compatibility: ipykernel's stdout doesn't expose a fileno();\n", + "# vLLM's worker init calls sys.stdout.fileno() during parallel-state setup\n", + "# and crashes with UnsupportedOperation: fileno. Patch fileno to return the\n", + "# underlying process FDs (1, 2) \u2014 Colab writes back to those anyway.\n", + "if \"ipykernel\" in sys.modules:\n", + " sys.stdout.fileno = lambda: 1 # type: ignore[method-assign]\n", + " sys.stderr.fileno = lambda: 2 # type: ignore[method-assign]\n", + "\n", + "MODEL = \"meta-llama/Llama-3.2-1B\"\n", + "PROMPT = \"The quick brown fox jumps over the\"\n", + "DTYPE = torch.float16\n", + "torch.manual_seed(0)\n", + "\n", + "print(f\"CUDA available: {torch.cuda.is_available()}\")\n", + "print(f\"Device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU only \u2014 abort'}\")\n", + "assert torch.cuda.is_available(), \"GPU runtime required.\"" + ], + "id": "1748b73ca8b0" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1 \u2014 HF reference\n", + "\n", + "Boot the transformers backend first, capture activations and argmax, then drop it so vLLM has the GPU to itself." + ], + "id": "f6afd736f3bd" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoConfig\n", + "\n", + "from transformer_lens.model_bridge.sources.transformers import boot as boot_transformers\n", + "from transformer_lens.model_bridge.sources.vllm.overlays import get_overlay\n", + "\n", + "# Compute the expected vLLM-fireable hook set from the overlay so HF captures\n", + "# only what we'll compare. Avoids hundreds of MB of CPU clones for hooks vLLM\n", + "# doesn't expose.\n", + "_hf_preview = AutoConfig.from_pretrained(MODEL, token=os.environ[\"HF_TOKEN\"])\n", + "_overlay = get_overlay(_hf_preview.architectures[0])\n", + "EXPECTED_HOOKS = set(_overlay.capture_specs(_hf_preview).keys())\n", + "print(f\"Expecting {len(EXPECTED_HOOKS)} fireable hook(s) per vLLM overlay.\")\n", + "\n", + "bridge_hf = boot_transformers(MODEL, dtype=DTYPE).to(\"cuda\")\n", + "tokens = bridge_hf.to_tokens(PROMPT)\n", + "# no_grad drops the autograd graph; without this, the forward-pass intermediates\n", + "# stay alive (~8+ GiB for a 1B model) and starve vLLM's KV cache allocation later.\n", + "with torch.no_grad():\n", + " logits_hf, cache_hf = bridge_hf.run_with_cache(\n", + " tokens, names_filter=lambda name: name in EXPECTED_HOOKS\n", + " )\n", + "argmax_hf = int(logits_hf[0, -1].argmax().item())\n", + "\n", + "cache_hf_cpu = {name: t.detach().cpu().clone() for name, t in cache_hf.cache_dict.items()}\n", + "next_token_hf = bridge_hf.tokenizer.decode([argmax_hf])\n", + "print(f\"HF argmax token id: {argmax_hf} \u2192 {next_token_hf!r}\")\n", + "print(f\"HF cache: {len(cache_hf_cpu)} entries (filtered to overlay's fireable set)\")\n", + "\n", + "# Move parameters to CPU before deletion to force release even if a reference\n", + "# cycle persists. del + gc.collect alone has been observed to leak ~6 GiB here.\n", + "bridge_hf.to(\"cpu\")\n", + "del bridge_hf, logits_hf, cache_hf, tokens\n", + "gc.collect(); torch.cuda.empty_cache()\n", + "print(f\"GPU memory after HF release: {torch.cuda.memory_allocated() / 1e9:.2f} GB\")" + ], + "id": "fe23ad48fbb1" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2 \u2014 Boot vLLM bridge\n", + "\n", + "`boot_vllm` constructs the LLM, monkey-patches `Worker.load_model` pre-compile to install capture hooks, then wraps it in a `RemoteBridge`." + ], + "id": "f1b2bd162cd9" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from transformer_lens.model_bridge.remote_bridge import RemoteBridge\n", + "\n", + "# max_model_len=2048 caps the KV cache reservation. Llama-3.2-1B's native\n", + "# context is 131072 (128k) \u2014 the default reservation is ~4 GiB and overshoots\n", + "# the free T4 budget. The test prompt is ~10 tokens, so 2048 is plenty.\n", + "bridge = RemoteBridge.boot_vllm(\n", + " MODEL,\n", + " dtype=DTYPE,\n", + " gpu_memory_utilization=0.5,\n", + " max_model_len=2048,\n", + ")\n", + "assert isinstance(bridge, RemoteBridge), f\"Expected RemoteBridge, got {type(bridge).__name__}\"\n", + "print(f\"Architecture: {bridge.cfg.architecture}\")\n", + "print(f\"Fireable hooks: {len(bridge._driver.supported_hook_points)}\")\n", + "print(f\"Non-fireable hooks (fused kernels): {len(bridge._driver.non_fireable_hook_points)}\")" + ], + "id": "49d50957a4c7" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3 \u2014 Capture pipeline\n", + "\n", + "Run a single forward, populate the cache via `collective_rpc \u2192 tl_read_captures`, and sanity-check shapes." + ], + "id": "284538a2c556" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tokens = bridge.to_tokens(PROMPT)\n", + "logits_vllm, cache_vllm = bridge.run_with_cache(tokens)\n", + "argmax_vllm = int(logits_vllm[0, -1].argmax().item())\n", + "next_token_vllm = bridge.tokenizer.decode([argmax_vllm])\n", + "\n", + "print(f\"vLLM argmax token id: {argmax_vllm} \u2192 {next_token_vllm!r}\")\n", + "print(f\"vLLM cache entries: {len(cache_vllm.cache_dict)}\")\n", + "\n", + "for name in sorted(bridge._driver.supported_hook_points)[:5]:\n", + " t = cache_vllm[name]\n", + " finite_pct = 100 * torch.isfinite(t).float().mean().item()\n", + " print(f\" {name}: shape={tuple(t.shape)}, dtype={t.dtype}, finite={finite_pct:.1f}%\")" + ], + "id": "68b83f7ac2db" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4 \u2014 Greedy parity \n", + "\n", + "vLLM and HF must produce the same next-token argmax on the same prompt." + ], + "id": "12a09c8312e5" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "parity = argmax_vllm == argmax_hf\n", + "status = \"\u2705 PASS\" if parity else \"\u274c FAIL\"\n", + "print(f\"{status}: HF\u2192{argmax_hf} ({next_token_hf!r}) vs vLLM\u2192{argmax_vllm} ({next_token_vllm!r})\")\n", + "assert parity, \"Greedy parity failed \u2014 kernel divergence or overlay misconfiguration.\"" + ], + "id": "a59824b0e3be" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5 \u2014 Per-hook L2 (acceptance gate)\n", + "\n", + "Target is relative L2 < 5e-3 in fp16 for every fireable hook. One hook remains exempted from the strict gate:\n", + "\n", + "- **`ln_final.hook_normalized`** \u2014 vLLM's `model.norm` is invoked as part of the fused-residual norm kernel and the captured value scales ~2\u00d7 HF's. Open investigation (likely a residual-fusion semantic discrepancy at the model boundary rather than a real divergence; argmax + downstream parity work correctly).\n", + "\n", + "All other fireable hooks (including `blocks.{i}.hook_out`, which is now materialized from vLLM's `(mlp_delta, residual)` tuple) must be within target." + ], + "id": "3e66a40c8c0a" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "TARGET_REL_L2 = 5e-3\n", + "# Hooks where vLLM's architecture diverges from HF at the capture point in\n", + "# ways we haven't yet reconciled. Only ln_final remains after the layer-hook\n", + "# materialization landed.\n", + "_RESIDUAL_FUSION_DIVERGENT = {\"ln_final.hook_normalized\"}\n", + "\n", + "rows = []\n", + "for name in sorted(bridge._driver.supported_hook_points):\n", + " if name not in cache_hf_cpu:\n", + " rows.append((name, None, None, \"missing in HF cache\"))\n", + " continue\n", + " t_vllm = cache_vllm[name].detach().cpu().float()\n", + " t_hf = cache_hf_cpu[name].float()\n", + " if t_vllm.shape != t_hf.shape:\n", + " rows.append((name, None, None, f\"shape {tuple(t_vllm.shape)} vs HF {tuple(t_hf.shape)}\"))\n", + " continue\n", + " diff = (t_vllm - t_hf).norm().item()\n", + " base = t_hf.norm().item() or 1.0\n", + " rel = diff / base\n", + " if rel < TARGET_REL_L2:\n", + " mark = \"\u2705\"\n", + " note = \"\"\n", + " elif name in _RESIDUAL_FUSION_DIVERGENT:\n", + " mark = \"\u26a0\"\n", + " note = \"residual-fusion (expected)\"\n", + " else:\n", + " mark = \"\u274c\"\n", + " note = \"\"\n", + " rows.append((name, rel, mark, note))\n", + "\n", + "print(f\"{'hook':<40} {'rel L2':>10} status note\")\n", + "print(\"-\" * 80)\n", + "for name, rel, mark, note in rows:\n", + " rel_str = f\"{rel:.3e}\" if isinstance(rel, float) else \"\u2014\"\n", + " print(f\"{name:<40} {rel_str:>10} {mark or '\u2014':<6} {note}\")\n", + "\n", + "# Strict gate: only the non-residual-fusion hooks need to be within target.\n", + "failed = [(name, rel) for name, rel, _, _ in rows\n", + " if rel is not None and rel >= TARGET_REL_L2 and name not in _RESIDUAL_FUSION_DIVERGENT]\n", + "assert not failed, f\"Hooks exceeded fp16 drift (target {TARGET_REL_L2}): {failed}\"\n", + "missing = [(name, note) for name, rel, _, note in rows if rel is None]\n", + "assert not missing, f\"Hooks missing or shape-mismatched vs HF: {missing}\"" + ], + "id": "7abbca1e04e9" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6 \u2014 Intervention smoke (load-bearing)\n", + "\n", + "**This cell is the load-bearing verification for the vLLM source's mutation claim.** Unit tests only exercise the dispatch protocol (mocked LLM); they cannot prove that the affine math (`output = output * scale + bias`) traces correctly through `torch.compile` and propagates to downstream layers under CUDA-graph replay. That guarantee depends on this cell passing.\n", + "\n", + "The test: zero out `embed.hook_out` via `{\"op\": \"suppress\"}`. If the mutation path works, (a) the cache shows zeros there, (b) the next-token argmax shifts vs the clean run, and (c) the immediately-following clean forward reverts (no sticky state)." + ], + "id": "ed7dfca52cbb" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Suppress (zero) the embedding output. Forward should behave very differently.\n", + "logits_suppressed, cache_suppressed = bridge.run_with_cache(\n", + " tokens,\n", + " intervene={\"embed.hook_out\": {\"op\": \"suppress\"}},\n", + ")\n", + "argmax_suppressed = int(logits_suppressed[0, -1].argmax().item())\n", + "\n", + "# Embed cache should be all zeros after suppress.\n", + "embed_norm = cache_suppressed[\"embed.hook_out\"].abs().max().item()\n", + "print(f\"Embed |max| after suppress: {embed_norm:.6f} (should be 0.0)\")\n", + "assert embed_norm == 0.0, \"Suppress did not zero embed.hook_out \u2014 intervention path broken.\"\n", + "\n", + "# Argmax should differ from the clean run.\n", + "argmax_shifted = argmax_suppressed != argmax_vllm\n", + "print(f\"Clean argmax: {argmax_vllm} Suppressed argmax: {argmax_suppressed} Shifted: {argmax_shifted}\")\n", + "\n", + "# Verify the next forward (no intervene) reverts \u2014 interventions are not sticky.\n", + "logits_revert, _ = bridge.run_with_cache(tokens)\n", + "argmax_revert = int(logits_revert[0, -1].argmax().item())\n", + "print(f\"Revert argmax: {argmax_revert} matches clean: {argmax_revert == argmax_vllm}\")\n", + "assert argmax_revert == argmax_vllm, \"Intervention persisted across calls \u2014 reset path broken.\"" + ], + "id": "8c94e6f538f4" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7 \u2014 Lifetime\n", + "\n", + "`bridge.close()` is responsible for releasing **our** resources:\n", + "- detaches the per-Worker capture hooks via `tl_remove_hooks`\n", + "- tears down vLLM's distributed environment (`destroy_distributed_environment`)\n", + "\n", + "It cannot release **vLLM's** internal state in 0.20.2 \u2014 there's no `LLM.shutdown()` API. Model weights, KV cache pool, and Inductor compile cache stay resident in PyTorch's caching allocator until process exit. A clean Colab `Runtime \u2192 Restart session` is the only way to fully reclaim GPU memory.\n", + "\n", + "What we *can* verify: `bridge.close()` ran without error and didn't leak more memory. The residual is informational, not a gate." + ], + "id": "fb8ab542fd3d" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "before = torch.cuda.memory_allocated() / 1e9\n", + "bridge.close() # detaches hooks + tears down vLLM distributed state\n", + "del bridge, logits_vllm, cache_vllm, logits_suppressed, cache_suppressed, logits_revert\n", + "gc.collect(); torch.cuda.empty_cache()\n", + "after = torch.cuda.memory_allocated() / 1e9\n", + "released = before - after\n", + "print(f\"GPU memory before close: {before:.2f} GB after close+del: {after:.2f} GB\")\n", + "print(f\"GPU memory released: {released:.2f} GB\")\n", + "print()\n", + "if released > 0.05:\n", + " print(f\"\u2705 close() released ~{released:.2f} GB (hooks + capture buffers + distributed state).\")\n", + "else:\n", + " print(\"\u26a0 close() released < 50 MB \u2014 likely vLLM's model weights and KV cache pool\")\n", + " print(\" staying resident. This is expected on vLLM 0.20.2; restart the runtime\")\n", + " print(\" for a clean GPU.\")" + ], + "id": "51dbe16a57b2" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "If all asserts above passed, the v4 Driver-protocol vLLM backend is sound on this architecture:\n", + "\n", + "- `boot_vllm` returns `RemoteBridge` end-to-end.\n", + "- `collective_rpc \u2192 tl_read_captures` populates the cache.\n", + "- Greedy argmax matches HF (greedy parity).\n", + "- Per-hook L2 < 5e-3 vs HF across every fireable hook (hook-fidelity gate).\n", + "- Affine interventions (suppress / scale / add / set) apply per-forward and reset cleanly.\n", + "- GPU lifetime is well-behaved.\n", + "\n", + "Next: extend to other architectures (Qwen / Mistral / Gemma) via the `DecoderOnlyOverlay`, or add `clamp` to the intervention vocabulary." + ], + "id": "fedb17c9a539" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + }, + "accelerator": "GPU", + "colab": { + "provenance": [], + "gpuType": "T4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/unit/model_bridge/supported_architectures/test_qwen3_5_adapter.py b/tests/unit/model_bridge/supported_architectures/test_qwen3_5_adapter.py index 393e0620dd..15c671c9a2 100644 --- a/tests/unit/model_bridge/supported_architectures/test_qwen3_5_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_qwen3_5_adapter.py @@ -6,6 +6,7 @@ from types import SimpleNamespace import pytest +import torch.nn as nn from transformer_lens.factories.architecture_adapter_factory import ( SUPPORTED_ARCHITECTURES, @@ -184,6 +185,13 @@ def test_load_weights_false_uses_prepared_text_config( from transformer_lens.model_bridge.bridge import TransformerBridge from transformer_lens.model_bridge.sources import transformers as source + # boot() lives in the submodule after the package split; module-level + # name lookups for TransformerBridge / setup_tokenizer happen there, not + # in the package __init__. + from transformer_lens.model_bridge.sources.transformers import ( + source as boot_module, + ) + text_config = SimpleNamespace( model_type="qwen3_5_text", architectures=["Qwen3_5ForCausalLM"], @@ -208,13 +216,14 @@ def test_load_weights_false_uses_prepared_text_config( eos_token_id=1, ) - class DummyModel: + class DummyModel(nn.Module): + # nn.Module subclass: TransformersDriver's beartype-validated init + # requires the underlying model to be an nn.Module after the type + # split. Plain objects no longer suffice. def __init__(self, config): + super().__init__() self.config = config - def parameters(self): - return iter(()) - class DummyModelClass: seen_config = None @@ -224,7 +233,12 @@ def from_config(cls, config, **kwargs): return DummyModel(config) class DummyBridge(TransformerBridge): - def __init__(self, hf_model, adapter, tokenizer): + # **kwargs absorbs the ``driver=`` kwarg boot() now passes after the + # type-split refactor (TransformersDriver gets constructed in boot() + # and threaded through to the bridge). nn.Module.__init__() must run + # before any nn.Module-valued attribute assignment (e.g. hf_model). + def __init__(self, hf_model, adapter, tokenizer, **kwargs): + nn.Module.__init__(self) self.hf_model = hf_model self.adapter = adapter self.tokenizer = tokenizer @@ -246,8 +260,8 @@ def encode(self, text): "from_pretrained", staticmethod(lambda *args, **kwargs: DummyTokenizer()), ) - monkeypatch.setattr(source, "TransformerBridge", DummyBridge) - monkeypatch.setattr(source, "setup_tokenizer", lambda tokenizer, **kwargs: tokenizer) + monkeypatch.setattr(boot_module, "TransformerBridge", DummyBridge) + monkeypatch.setattr(boot_module, "setup_tokenizer", lambda tokenizer, **kwargs: tokenizer) bridge = source.boot( "Qwen/Qwen3.5-0.8B", diff --git a/tests/unit/model_bridge/test_driver_protocol.py b/tests/unit/model_bridge/test_driver_protocol.py new file mode 100644 index 0000000000..e0b652b989 --- /dev/null +++ b/tests/unit/model_bridge/test_driver_protocol.py @@ -0,0 +1,733 @@ +"""Tests for the Driver protocol and the first implementation (TransformersDriver).""" +from __future__ import annotations + +import numpy as np +import pytest +import torch +from torch import nn + +from transformer_lens.model_bridge.driver_protocol import ( + Driver, + ForwardResult, + TensorLike, + to_torch, + validate_driver, +) + + +def _stub_adapter(architecture: str = "Mock"): + """Minimal adapter satisfying DriverBase's beartype-checked cfg arg.""" + from transformer_lens.config import TransformerBridgeConfig + + cfg = TransformerBridgeConfig( + d_model=4, + d_head=2, + n_layers=1, + n_ctx=8, + n_heads=2, + d_vocab=16, + d_mlp=8, + architecture=architecture, + ) + + class _StubAdapter: + pass + + a = _StubAdapter() + a.cfg = cfg # type: ignore[attr-defined] + return a + + +class TestTensorLikeProtocol: + """TensorLike matches what every tensor library exposes.""" + + def test_torch_tensor_matches(self): + assert isinstance(torch.randn(2, 3), TensorLike) + + def test_numpy_array_matches(self): + assert isinstance(np.zeros((2, 3)), TensorLike) + + def test_bare_object_rejected(self): + class NotATensor: + pass + + assert not isinstance(NotATensor(), TensorLike) + + +class TestToTorch: + """Boundary conversion: TensorLike → torch.Tensor.""" + + def test_torch_passthrough(self): + t = torch.randn(2, 3) + # Identity-equal: drivers returning torch tensors shouldn't pay a clone tax. + assert to_torch(t) is t + + def test_numpy_converts_with_correct_shape(self): + arr = np.arange(6, dtype=np.float32).reshape(2, 3) + out = to_torch(arr) + assert isinstance(out, torch.Tensor) + assert tuple(out.shape) == (2, 3) + assert out.dtype == torch.float32 + + def test_dtype_override(self): + out = to_torch(torch.randn(2, 3, dtype=torch.float32), dtype=torch.float16) + assert out.dtype == torch.float16 + + def test_dlpack_path_used_when_available(self): + """numpy ≥ 1.22 has __dlpack__ — stand-in for JAX/MLX/CuPy device arrays.""" + arr = np.arange(6, dtype=np.float32).reshape(2, 3) + if not hasattr(arr, "__dlpack__"): + pytest.skip("numpy < 1.22 has no __dlpack__; nothing to verify") + out = to_torch(arr) + assert isinstance(out, torch.Tensor) + assert tuple(out.shape) == (2, 3) + assert out.dtype == torch.float32 + + def test_dlpack_failure_falls_back_to_numpy(self): + """Stream-sync / version-skew failures fall through to the __array__ path.""" + + class FailingDLPack: + shape = (2, 3) + dtype = np.float32 + + def __dlpack__(self, *a, **kw): + raise BufferError("simulated stream-sync failure") + + def __array__(self, dtype=None): + return np.arange(6, dtype=np.float32).reshape(2, 3) + + out = to_torch(FailingDLPack()) + assert isinstance(out, torch.Tensor) + assert tuple(out.shape) == (2, 3) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") + def test_cuda_torch_tensor_passthrough(self): + """The passthrough branch must work for non-CPU tensors. np.asarray would crash.""" + t = torch.randn(2, 3, device="cuda") + out = to_torch(t) + assert out is t + assert out.device.type == "cuda" + + +class TestForwardResult: + """ForwardResult is the data envelope across the driver boundary.""" + + def test_default_construction(self): + fr = ForwardResult() + assert fr.logits is None + assert fr.captured == {} + assert fr.new_tokens is None + assert fr.raw_output is None + + def test_frozen(self): + import dataclasses + + fr = ForwardResult(logits=torch.zeros(2)) + # setattr bypasses mypy's static frozen-instance check. + with pytest.raises(dataclasses.FrozenInstanceError): + setattr(fr, "logits", torch.ones(2)) + + +class TestDriverProtocol: + """The Driver protocol is structurally checkable at runtime.""" + + def test_bare_class_does_not_satisfy(self): + class NotADriver: + pass + + assert not isinstance(NotADriver(), Driver) + + def test_capability_flags_route_torch_specific_access(self): + """parameters/state_dict are torch-specific implementation details, not protocol + members. Callers route via supports("...") + hasattr; non-torch drivers + simply don't define them.""" + from transformer_lens.model_bridge.sources._driver_base import DriverBase + from transformer_lens.model_bridge.sources.transformers_driver import ( + TransformersDriver, + ) + + adapter = _stub_adapter() + model = nn.Linear(2, 2) + hf_driver = TransformersDriver(model, adapter, tokenizer=None) + assert hf_driver.supports("parameters") is True + assert hf_driver.supports("gradients") is True + assert hf_driver.supports("intervention_callbacks") is True + assert hf_driver.supports("never-heard-of-it") is False + assert hasattr(hf_driver, "parameters") + assert list(hf_driver.parameters()) == list(model.parameters()) + + class RemoteLike(DriverBase): + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult() + + remote = RemoteLike(_stub_adapter().cfg, tokenizer=None) + assert remote.supports("parameters") is False + assert not hasattr(remote, "parameters") + + def test_transformers_driver_passes_strict_validation(self): + """validate_driver enforces types/signatures, not just hasattr.""" + from transformer_lens.model_bridge.sources.transformers_driver import ( + TransformersDriver, + ) + + driver = TransformersDriver(nn.Linear(2, 2), _stub_adapter(), tokenizer=None) + validate_driver(driver) + + +class TestValidateDriverCatchesBrokenDrivers: + """validate_driver must reject drivers that runtime_checkable Protocol would accept.""" + + def _cfg(self): + return _stub_adapter().cfg + + def test_wrong_forward_signature_rejected(self): + """Driver with forward(only_input_ids) silently swallows the bridge's keyword args.""" + from transformer_lens.model_bridge.sources._driver_base import DriverBase + + class WrongSignature(DriverBase): + def __init__(self, cfg): + super().__init__(cfg, tokenizer=None) + + # Missing capture/intervene/max_new_tokens/return_logits, no **kwargs. + # runtime_checkable accepts this; validate_driver must not. + def forward(self, input_ids): # type: ignore[override] + return ForwardResult() + + driver = WrongSignature(self._cfg()) + assert isinstance(driver, Driver) # Protocol is too weak to catch this + with pytest.raises(TypeError, match="must accept parameters"): + validate_driver(driver) + + def test_overlapping_hook_sets_rejected(self): + """A hook can't be both fireable and not — overlap is a contract bug.""" + from transformer_lens.model_bridge.sources._driver_base import DriverBase + + class OverlappingSets(DriverBase): + supported_hook_points = frozenset({"a", "b"}) + non_fireable_hook_points = frozenset({"b", "c"}) + + def __init__(self, cfg): + super().__init__(cfg, tokenizer=None) + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult() + + driver = OverlappingSets(self._cfg()) + with pytest.raises(TypeError, match="overlap"): + validate_driver(driver) + + def test_wrong_attribute_type_rejected(self): + """architecture: str — a driver that sets it to None or an int fails.""" + from transformer_lens.model_bridge.sources._driver_base import DriverBase + + class WrongTypes(DriverBase): + def __init__(self, cfg): + super().__init__(cfg, tokenizer=None) + self.architecture = 42 # type: ignore[assignment] + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult() + + driver = WrongTypes(self._cfg()) + with pytest.raises(TypeError, match="architecture must be str"): + validate_driver(driver) + + def test_non_string_hookpoint_rejected(self): + from transformer_lens.model_bridge.sources._driver_base import DriverBase + + class BadHookNames(DriverBase): + supported_hook_points = frozenset({"valid", 123}) # type: ignore[arg-type] + + def __init__(self, cfg): + super().__init__(cfg, tokenizer=None) + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult() + + driver = BadHookNames(self._cfg()) + with pytest.raises(TypeError, match="must be str"): + validate_driver(driver) + + def test_empty_hookpoints_post_construction_rejected(self): + """Driver that never declared anything after the bridge backfilled is silently broken.""" + from transformer_lens.model_bridge.sources._driver_base import DriverBase + + class SilentlyEmpty(DriverBase): + def __init__(self, cfg): + super().__init__(cfg, tokenizer=None) + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult() + + driver = SilentlyEmpty(self._cfg()) + validate_driver(driver, after_bridge_construction=False) + with pytest.raises(TypeError, match="empty supported_hook_points AND"): + validate_driver(driver, after_bridge_construction=True) + + def test_bridge_accepts_kw_only_input_ids_driver(self): + """Bridge passes input_ids by keyword so kw-only drivers don't TypeError.""" + from types import SimpleNamespace + from unittest.mock import MagicMock + + from tests.mocks.architecture_adapter import MockArchitectureAdapter + from transformer_lens.model_bridge.bridge import TransformerBridge + from transformer_lens.model_bridge.generalized_components import ( + AttentionBridge, + BlockBridge, + NormalizationBridge, + ) + from transformer_lens.model_bridge.sources._driver_base import DriverBase + + received: dict = {} + + class KwOnlyDriver(DriverBase): + supported_hook_points = frozenset({"sentinel"}) + + def forward( + self, + *, # everything past here is keyword-only — including input_ids + input_ids=None, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + received["input_ids"] = input_ids + return ForwardResult(logits=torch.zeros(1, 1, 16), raw_output=None) + + model = nn.Module() + model.final_norm = nn.LayerNorm(10) + model.encoder = nn.Module() + model.encoder.layers = nn.ModuleList([nn.Module() for _ in range(1)]) + model.encoder.layers[0].norm1 = nn.LayerNorm(10) + model.encoder.layers[0].self_attn = nn.Module() + + adapter = MockArchitectureAdapter() + adapter.component_mapping = { + "ln_final": NormalizationBridge(name="final_norm", config={}), + "blocks": BlockBridge( + name="encoder.layers", + submodules={ + "ln1": NormalizationBridge(name="norm1", config={}), + "attn": AttentionBridge(name="self_attn", config=SimpleNamespace(n_heads=1)), + }, + ), + } + driver = KwOnlyDriver(_stub_adapter().cfg, tokenizer=None) + bridge = TransformerBridge(model, adapter, tokenizer=MagicMock(), driver=driver) + + # Mirrors the bridge's keyword-form call. Positional would TypeError. + input_ids = torch.tensor([[1, 2, 3]]) + result = bridge._driver.forward(input_ids=input_ids) + assert received["input_ids"] is input_ids + assert isinstance(result, ForwardResult) + + def test_bridge_init_rejects_misshapen_driver(self): + """validate_driver runs at end of __init__ — fail fast at construction.""" + from types import SimpleNamespace + from unittest.mock import MagicMock + + from tests.mocks.architecture_adapter import MockArchitectureAdapter + from transformer_lens.model_bridge.bridge import TransformerBridge + from transformer_lens.model_bridge.generalized_components import ( + AttentionBridge, + BlockBridge, + NormalizationBridge, + ) + from transformer_lens.model_bridge.sources._driver_base import DriverBase + + # non_fireable_hook_points as list (not frozenset) passes Protocol + # presence-check but should fail validate_driver at __init__. + class MisshapenDriver(DriverBase): + supported_hook_points = frozenset({"x"}) + non_fireable_hook_points = ["wrong_type"] # type: ignore[assignment] + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult() + + model = nn.Module() + model.final_norm = nn.LayerNorm(10) + model.encoder = nn.Module() + model.encoder.layers = nn.ModuleList([nn.Module() for _ in range(1)]) + model.encoder.layers[0].norm1 = nn.LayerNorm(10) + model.encoder.layers[0].self_attn = nn.Module() + + adapter = MockArchitectureAdapter() + adapter.component_mapping = { + "ln_final": NormalizationBridge(name="final_norm", config={}), + "blocks": BlockBridge( + name="encoder.layers", + submodules={ + "ln1": NormalizationBridge(name="norm1", config={}), + "attn": AttentionBridge(name="self_attn", config=SimpleNamespace(n_heads=1)), + }, + ), + } + bad_driver = MisshapenDriver(_stub_adapter().cfg, tokenizer=None) + + with pytest.raises(TypeError, match="non_fireable_hook_points must be frozenset"): + TransformerBridge(model, adapter, tokenizer=MagicMock(), driver=bad_driver) + + def test_hf_driver_passes_post_construction_check(self): + """Full HF stack populates supported_hook_points; strict gate accepts it.""" + from types import SimpleNamespace + from unittest.mock import MagicMock + + from tests.mocks.architecture_adapter import MockArchitectureAdapter + from transformer_lens.model_bridge.bridge import TransformerBridge + from transformer_lens.model_bridge.generalized_components import ( + AttentionBridge, + BlockBridge, + NormalizationBridge, + ) + + model = nn.Module() + model.final_norm = nn.LayerNorm(10) + model.encoder = nn.Module() + model.encoder.layers = nn.ModuleList([nn.Module() for _ in range(1)]) + model.encoder.layers[0].norm1 = nn.LayerNorm(10) + model.encoder.layers[0].self_attn = nn.Module() + + adapter = MockArchitectureAdapter() + adapter.component_mapping = { + "ln_final": NormalizationBridge(name="final_norm", config={}), + "blocks": BlockBridge( + name="encoder.layers", + submodules={ + "ln1": NormalizationBridge(name="norm1", config={}), + "attn": AttentionBridge(name="self_attn", config=SimpleNamespace(n_heads=1)), + }, + ), + } + bridge = TransformerBridge(model, adapter, tokenizer=MagicMock()) + validate_driver(bridge._driver, after_bridge_construction=True) + + +class TestBridgeConsumesCaptures: + """Bridge replays ForwardResult.captured through its HookPoint tree.""" + + def _build_minimal_bridge(self): + """Tiny bridge over a hand-rolled adapter — avoids HF Hub.""" + from types import SimpleNamespace + from unittest.mock import MagicMock + + from tests.mocks.architecture_adapter import MockArchitectureAdapter + from transformer_lens.model_bridge.bridge import TransformerBridge + from transformer_lens.model_bridge.generalized_components import ( + AttentionBridge, + BlockBridge, + NormalizationBridge, + ) + + model = nn.Module() + model.final_norm = nn.LayerNorm(10) + model.encoder = nn.Module() + model.encoder.layers = nn.ModuleList([nn.Module() for _ in range(1)]) + model.encoder.layers[0].norm1 = nn.LayerNorm(10) + model.encoder.layers[0].self_attn = nn.Module() + + adapter = MockArchitectureAdapter() + adapter.component_mapping = { + "ln_final": NormalizationBridge(name="final_norm", config={}), + "blocks": BlockBridge( + name="encoder.layers", + submodules={ + "ln1": NormalizationBridge(name="norm1", config={}), + "attn": AttentionBridge(name="self_attn", config=SimpleNamespace(n_heads=1)), + }, + ), + } + return TransformerBridge(model, adapter, tokenizer=MagicMock()) + + def test_replay_captures_fires_hookpoints(self): + bridge = self._build_minimal_bridge() + # Pick any registered hook — test isn't coupled to specific naming. + assert bridge._hook_registry, "fixture must register at least one HookPoint" + target_name = next(iter(bridge._hook_registry)) + hp = bridge._hook_registry[target_name] + + recorded: list[torch.Tensor] = [] + hp.add_hook(lambda act, hook: recorded.append(act)) + + synthetic = torch.arange(6, dtype=torch.float32).reshape(2, 3) + bridge._replay_captures({target_name: synthetic}) + + assert len(recorded) == 1 + assert torch.equal(recorded[0], synthetic) + + def test_replay_drops_unknown_hook_names(self): + """Drivers may report names the bridge doesn't carry; silent-drop is the contract.""" + bridge = self._build_minimal_bridge() + bridge._replay_captures({"definitely.not.a.hook": torch.zeros(2)}) + # No exception, no side effect — the assertion is that we got here. + + def test_replay_converts_non_torch_tensors(self): + """Captures arriving as numpy / mlx / jax cross the to_torch boundary.""" + bridge = self._build_minimal_bridge() + target_name = next(iter(bridge._hook_registry)) + hp = bridge._hook_registry[target_name] + + recorded: list[torch.Tensor] = [] + hp.add_hook(lambda act, hook: recorded.append(act)) + + synthetic_np = np.arange(6, dtype=np.float32).reshape(2, 3) + bridge._replay_captures({target_name: synthetic_np}) + + assert len(recorded) == 1 + assert isinstance(recorded[0], torch.Tensor) + assert recorded[0].shape == (2, 3) + + +class TestBridgeToleratesWeirdLogits: + """Bridge tolerates non-torch.Tensor logits — audio CTC, encoder-only, etc.""" + + def _make_test_bridge(self): + from types import SimpleNamespace + from unittest.mock import MagicMock + + from tests.mocks.architecture_adapter import MockArchitectureAdapter + from transformer_lens.model_bridge.bridge import TransformerBridge + from transformer_lens.model_bridge.generalized_components import ( + AttentionBridge, + BlockBridge, + NormalizationBridge, + ) + + model = nn.Module() + model.final_norm = nn.LayerNorm(10) + model.encoder = nn.Module() + model.encoder.layers = nn.ModuleList([nn.Module() for _ in range(1)]) + model.encoder.layers[0].norm1 = nn.LayerNorm(10) + model.encoder.layers[0].self_attn = nn.Module() + + adapter = MockArchitectureAdapter() + adapter.component_mapping = { + "ln_final": NormalizationBridge(name="final_norm", config={}), + "blocks": BlockBridge( + name="encoder.layers", + submodules={ + "ln1": NormalizationBridge(name="norm1", config={}), + "attn": AttentionBridge(name="self_attn", config=SimpleNamespace(n_heads=1)), + }, + ), + } + return TransformerBridge(model, adapter, tokenizer=MagicMock()) + + def test_non_tensor_logits_passes_through_unchanged(self): + """Weird objects (HF dataclass, tuple-of-tuples) pass through; downstream + return_type branches do the strict typing.""" + from transformer_lens.model_bridge.driver_protocol import TensorLike + + class WeirdShape: + extra = "not a tensor" + + weird = WeirdShape() + assert not isinstance(weird, torch.Tensor) + assert not isinstance(weird, TensorLike) + # Mirrors the bridge boundary logic. + logits: object = weird + if isinstance(logits, torch.Tensor): + pass + elif logits is not None and isinstance(logits, TensorLike): + logits = to_torch(logits) + assert logits is weird + + def test_none_logits_passes_through(self): + """return_type=None is a legitimate ask.""" + from transformer_lens.model_bridge.driver_protocol import ForwardResult + + result = ForwardResult(logits=None) + logits: object = result.logits + if isinstance(logits, torch.Tensor): + pass + elif logits is not None and isinstance(logits, TensorLike): + logits = to_torch(logits) + assert logits is None + + def test_numpy_logits_converted_at_boundary(self): + """Non-torch TensorLike still converts.""" + arr = np.zeros((2, 3), dtype=np.float32) + logits: object = arr + if isinstance(logits, torch.Tensor): + pass + elif logits is not None and isinstance(logits, TensorLike): + logits = to_torch(logits) + assert isinstance(logits, torch.Tensor) + assert tuple(logits.shape) == (2, 3) + + +class TestDriverHookPointDeclaration: + """The driver tells the bridge which hooks it can fire.""" + + def _build_minimal_bridge(self): + from types import SimpleNamespace + from unittest.mock import MagicMock + + from tests.mocks.architecture_adapter import MockArchitectureAdapter + from transformer_lens.model_bridge.bridge import TransformerBridge + from transformer_lens.model_bridge.generalized_components import ( + AttentionBridge, + BlockBridge, + NormalizationBridge, + ) + + model = nn.Module() + model.final_norm = nn.LayerNorm(10) + model.encoder = nn.Module() + model.encoder.layers = nn.ModuleList([nn.Module() for _ in range(1)]) + model.encoder.layers[0].norm1 = nn.LayerNorm(10) + model.encoder.layers[0].self_attn = nn.Module() + + adapter = MockArchitectureAdapter() + adapter.component_mapping = { + "ln_final": NormalizationBridge(name="final_norm", config={}), + "blocks": BlockBridge( + name="encoder.layers", + submodules={ + "ln1": NormalizationBridge(name="norm1", config={}), + "attn": AttentionBridge(name="self_attn", config=SimpleNamespace(n_heads=1)), + }, + ), + } + return TransformerBridge(model, adapter, tokenizer=MagicMock()) + + def test_hf_driver_supports_full_registry(self): + """HF with eager attention fires every hook the bridge registers.""" + bridge = self._build_minimal_bridge() + assert bridge._driver.supported_hook_points == frozenset(bridge._hook_registry) + assert bridge._driver.non_fireable_hook_points == frozenset() + assert len(bridge._driver.supported_hook_points) > 0 # non-empty contract + + def test_non_fireable_subtracts_from_supported(self): + """A driver that declares fused-kernel hooks ends up with supported = registry - non_fireable.""" + from transformer_lens.model_bridge.sources.transformers_driver import ( + TransformersDriver, + ) + + bridge = self._build_minimal_bridge() + # Simulate a fused-kernel driver by re-declaring non_fireable, clearing + # supported, and re-running the backfill the bridge does in __init__. + sacrificed = next(iter(bridge._hook_registry)) + bridge._driver.non_fireable_hook_points = frozenset({sacrificed}) + bridge._driver.supported_hook_points = frozenset() + # Re-apply the bridge's __init__ backfill rule. + bridge._driver.supported_hook_points = ( + frozenset(bridge._hook_registry) - bridge._driver.non_fireable_hook_points + ) + + assert sacrificed not in bridge._driver.supported_hook_points + assert sacrificed in bridge._driver.non_fireable_hook_points + assert ( + bridge._driver.supported_hook_points | bridge._driver.non_fireable_hook_points + == frozenset(bridge._hook_registry) + ) + + def test_original_model_raises_for_non_torch_driver(self): + """A driver without ``underlying_model`` triggers the documented AttributeError.""" + from transformer_lens.model_bridge.sources._driver_base import DriverBase + + bridge = self._build_minimal_bridge() + + class NoUnderlyingModel(DriverBase): + def __init__(self, cfg) -> None: + super().__init__(cfg, tokenizer=None) + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult() + + bridge._driver = NoUnderlyingModel(_stub_adapter().cfg) + with pytest.raises(AttributeError, match="does not expose an nn.Module"): + _ = bridge.original_model + + def test_whitelist_driver_preserved(self): + """An Inspect-style driver that declares supported directly is not overwritten.""" + from transformer_lens.model_bridge.driver_protocol import ForwardResult + from transformer_lens.model_bridge.sources._driver_base import DriverBase + + cfg = _stub_adapter().cfg + + # Minimal whitelist driver stand-in for Inspect — declares supported + # before bridge construction and expects to keep that declaration. + class WhitelistDriver(DriverBase): + def __init__(self) -> None: + super().__init__(cfg, tokenizer=None) + # Inspect would declare its residual-stream subset here. + self.supported_hook_points = frozenset({"blocks.0.hook_resid_pre"}) + + def forward(self, *a, **kw): # type: ignore[override] + return ForwardResult() + + bridge = self._build_minimal_bridge() + whitelist = WhitelistDriver() + bridge._driver = whitelist + # Re-apply the backfill rule the bridge does in __init__; it should + # NOT overwrite a non-empty supported set. + if not bridge._driver.supported_hook_points: + bridge._driver.supported_hook_points = ( + frozenset(bridge._hook_registry) - bridge._driver.non_fireable_hook_points + ) + assert whitelist.supported_hook_points == frozenset({"blocks.0.hook_resid_pre"}) diff --git a/tests/unit/model_bridge/test_remote_bridge.py b/tests/unit/model_bridge/test_remote_bridge.py new file mode 100644 index 0000000000..1d0940b937 --- /dev/null +++ b/tests/unit/model_bridge/test_remote_bridge.py @@ -0,0 +1,312 @@ +"""RemoteBridge contract tests: same hook namespace as TransformerBridge, no +torch-only surface, no nn.Module parentage.""" +from __future__ import annotations + +import pytest +import torch.nn as nn + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.bridge_core import BridgeCore +from transformer_lens.model_bridge.driver_protocol import ForwardResult +from transformer_lens.model_bridge.remote_bridge import RemoteBridge +from transformer_lens.model_bridge.sources._driver_base import DriverBase + + +def _cfg() -> TransformerBridgeConfig: + return TransformerBridgeConfig( + d_model=4, + d_head=2, + n_layers=1, + n_ctx=8, + n_heads=2, + d_vocab=16, + d_mlp=8, + architecture="Mock", + ) + + +def _stub_adapter() -> ArchitectureAdapter: + adapter = ArchitectureAdapter(_cfg()) + adapter.component_mapping = {} # RemoteBridge doesn't walk it + return adapter + + +def _stub_driver(supported_hooks: frozenset[str] = frozenset({"blocks.0.hook_resid_pre"})): + """Minimal Driver-conformant stub usable inside a RemoteBridge.""" + + class StubDriver(DriverBase): + supported_hook_points = supported_hooks + _supported_features = frozenset() + + def __init__(self): + super().__init__(_cfg(), tokenizer=None) + self.forward_calls: list = [] + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + self.forward_calls.append({"input_ids": input_ids, "kwargs": kw}) + return ForwardResult(logits=None, captured={}) + + return StubDriver() + + +class TestRemoteBridgeContract: + """Type-shape promises distinguishing RemoteBridge from TransformerBridge.""" + + def test_is_bridge_core(self): + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + assert isinstance(bridge, BridgeCore) + + def test_is_not_nn_module(self): + """The load-bearing property that strips parameters/state_dict/etc.""" + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + assert not isinstance(bridge, nn.Module) + + def test_lacks_torch_specific_surface(self): + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + for missing in ( + "parameters", + "state_dict", + "load_state_dict", + "generate", + "enable_compatibility_mode", + "W_Q", + "W_K", + "W_V", + "W_O", + "W_in", + "W_out", + "_set_processed_weight_attributes", + ): + assert not hasattr(bridge, missing), ( + f"RemoteBridge unexpectedly has '{missing}' — " + "the type split is meant to strip torch-only surface." + ) + + +class TestRemoteBridgeConstruction: + """Driver must declare the hook namespace — no model to walk.""" + + def test_requires_driver_supported_hook_points(self): + empty_driver = _stub_driver(supported_hooks=frozenset()) + with pytest.raises(ValueError, match="supported_hook_points to be non-empty"): + RemoteBridge(_stub_adapter(), tokenizer=None, driver=empty_driver) + + def test_cfg_device_is_none(self): + """RemoteBridge has no local device — explicit None so tensor.to() is a no-op.""" + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + assert bridge.cfg.device is None + + def test_builds_hook_registry_from_driver_declarations(self): + hook_names = frozenset({"blocks.0.hook_resid_pre", "blocks.0.hook_resid_post"}) + bridge = RemoteBridge( + _stub_adapter(), tokenizer=None, driver=_stub_driver(supported_hooks=hook_names) + ) + assert frozenset(bridge._hook_registry) == hook_names + for name in hook_names: + assert bridge._hook_registry[name].name == name + + +class TestRemoteBridgeHookLifecycle: + """RemoteBridge has no nn.Module children walk — registry must be canonical.""" + + def test_reset_hooks_clears_registry_hooks(self): + """Without the registry path, reset_hooks is a silent no-op on RemoteBridge.""" + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + hp = bridge._hook_registry["blocks.0.hook_resid_pre"] + hp.add_hook(lambda act, hook: None) + assert len(hp.fwd_hooks) == 1 + bridge.reset_hooks() + assert len(hp.fwd_hooks) == 0 + + def test_list_hooks_works(self): + """HookIntrospectionMixin.list_hooks is framework-agnostic (uses hook_dict).""" + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + bridge.add_hook("blocks.0.hook_resid_pre", lambda act, hook: None) + listing = bridge.list_hooks() + assert "blocks.0.hook_resid_pre" in listing + assert len(listing["blocks.0.hook_resid_pre"]) == 1 + + def test_add_hook_by_name_uses_registry(self): + """RemoteBridge has no component tree; attribute walk would AttributeError.""" + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + bridge.add_hook("blocks.0.hook_resid_pre", lambda act, hook: None) + assert len(bridge._hook_registry["blocks.0.hook_resid_pre"].fwd_hooks) == 1 + + def test_input_device_returns_none_for_meta(self): + """Meta-device params (load_weights=False path) must not pull inputs onto meta.""" + import torch + + class MetaDriver(DriverBase): + supported_hook_points = frozenset({"x"}) + _supported_features = frozenset({"parameters"}) + + def __init__(self): + super().__init__(_cfg(), tokenizer=None) + + def parameters(self): + yield torch.empty(2, 2, device="meta") + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult() + + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=MetaDriver()) + assert bridge._input_device() is None + + def test_aliases_resolve_via_registry(self): + """Bridge-level aliases (hook_embed → embed.hook_out) work on RemoteBridge + when the driver declares the canonical target — registry lookup, not + attribute walk.""" + driver = _stub_driver(supported_hooks=frozenset({"embed.hook_out"})) + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=driver) + # hook_embed is in BridgeCore.hook_aliases as ["embed_ln.hook_out", "embed.hook_out"] + assert "hook_embed" in bridge.hook_dict + assert bridge.hook_dict["hook_embed"] is bridge._hook_registry["embed.hook_out"] + + +class TestRemoteBridgeForward: + """The driver-routed forward path.""" + + def test_forward_calls_driver(self): + driver = _stub_driver() + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=driver) + import torch + + bridge.forward(torch.tensor([[1, 2, 3]])) + assert len(driver.forward_calls) == 1 + + def test_forward_return_type_loss(self): + """return_type='loss' must compute loss, not silently return logits.""" + import torch + + class LogitsDriver(DriverBase): + supported_hook_points = frozenset({"x"}) + _supported_features = frozenset() + + def __init__(self): + super().__init__(_cfg(), tokenizer=None) + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult(logits=torch.randn(1, 3, 16), captured={}) + + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=LogitsDriver()) + loss = bridge.forward(torch.tensor([[1, 2, 3]]), return_type="loss") + assert isinstance(loss, torch.Tensor) and loss.dim() == 0 + + def test_forward_return_type_both(self): + import torch + + class LogitsDriver(DriverBase): + supported_hook_points = frozenset({"x"}) + _supported_features = frozenset() + + def __init__(self): + super().__init__(_cfg(), tokenizer=None) + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult(logits=torch.randn(1, 3, 16), captured={}) + + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=LogitsDriver()) + out = bridge.forward(torch.tensor([[1, 2, 3]]), return_type="both") + assert isinstance(out, tuple) and len(out) == 2 + logits, loss = out + assert isinstance(logits, torch.Tensor) and logits.shape == (1, 3, 16) + assert isinstance(loss, torch.Tensor) and loss.dim() == 0 + + def test_forward_invalid_return_type_raises(self): + """Unknown return_type used to silently return logits — must raise.""" + import torch + + class LogitsDriver(DriverBase): + supported_hook_points = frozenset({"x"}) + _supported_features = frozenset() + + def __init__(self): + super().__init__(_cfg(), tokenizer=None) + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult(logits=torch.randn(1, 3, 16), captured={}) + + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=LogitsDriver()) + with pytest.raises(ValueError, match="Invalid return_type"): + bridge.forward(torch.tensor([[1, 2, 3]]), return_type="nonsense") + + def test_forward_replays_captures(self): + import torch + + recorded: list = [] + + class CapturingDriver(DriverBase): + supported_hook_points = frozenset({"blocks.0.hook_resid_pre"}) + _supported_features = frozenset() + + def __init__(self): + super().__init__(_cfg(), tokenizer=None) + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult( + logits=None, + captured={"blocks.0.hook_resid_pre": torch.zeros(2, 3)}, + ) + + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=CapturingDriver()) + bridge._hook_registry["blocks.0.hook_resid_pre"].add_hook( + lambda act, hook: recorded.append(act) + ) + bridge.forward(torch.tensor([[1, 2]])) + assert len(recorded) == 1 + assert tuple(recorded[0].shape) == (2, 3) diff --git a/tests/unit/model_bridge/test_vllm_boot.py b/tests/unit/model_bridge/test_vllm_boot.py new file mode 100644 index 0000000000..cd0edaff03 --- /dev/null +++ b/tests/unit/model_bridge/test_vllm_boot.py @@ -0,0 +1,183 @@ +"""Orchestration tests for ``boot_vllm`` — mocks the HF / vLLM boundaries. + +Covers locked-kwarg rejection, dtype resolution, HF_TOKEN plumbing, env-var +override warning, plugin-config lifecycle, and the happy path. Driver-level +behavior is covered separately in test_vllm_driver.py. +""" +from __future__ import annotations + +import os +import sys +import warnings +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +import torch + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.remote_bridge import RemoteBridge +from transformer_lens.model_bridge.sources.vllm import plugin +from transformer_lens.model_bridge.sources.vllm.source import ( + _dtype_from_hf_config, + boot_vllm, +) + + +def _hf_config() -> SimpleNamespace: + return SimpleNamespace( + architectures=["LlamaForCausalLM"], + torch_dtype=torch.float16, + hidden_size=4, + vocab_size=16, + num_hidden_layers=2, + ) + + +def _cfg() -> TransformerBridgeConfig: + return TransformerBridgeConfig( + d_model=4, + d_head=2, + n_layers=2, + n_ctx=8, + n_heads=2, + d_vocab=16, + d_mlp=8, + architecture="LlamaForCausalLM", + ) + + +@pytest.fixture +def mocked_boot(monkeypatch): + """Mock every external boundary boot_vllm crosses; yield handles for assertions.""" + plugin._config.clear() + hf_config = _hf_config() + cfg = _cfg() + adapter = ArchitectureAdapter(cfg) + adapter.component_mapping = {} + + auto_config = MagicMock(return_value=hf_config) + auto_tokenizer = MagicMock(return_value=MagicMock(name="tokenizer")) + monkeypatch.setattr("transformers.AutoConfig.from_pretrained", auto_config) + monkeypatch.setattr("transformers.AutoTokenizer.from_pretrained", auto_tokenizer) + + fake_vllm = MagicMock() + fake_vllm.LLM = MagicMock(return_value=MagicMock(name="llm")) + monkeypatch.setitem(sys.modules, "vllm", fake_vllm) + + monkeypatch.setattr( + "transformer_lens.model_bridge.sources.vllm.source.get_hf_token", + lambda: "fake-token", + ) + monkeypatch.setattr( + "transformer_lens.model_bridge.sources.vllm.source.extract_hf_config", + lambda llm: hf_config, + ) + monkeypatch.setattr( + "transformer_lens.model_bridge.sources.vllm.source.build_bridge_config_from_hf", + lambda hf, arch, name, dt: cfg, + ) + monkeypatch.setattr( + "transformer_lens.model_bridge.sources.vllm.source" + ".ArchitectureAdapterFactory.select_architecture_adapter", + lambda c: adapter, + ) + # Skip the real monkey-patch of Worker.load_model — only entry-points discovery + # would import vllm.v1.worker.gpu_worker. Leave configure() unmocked so the + # clear() test exercises real state. + monkeypatch.setattr( + "transformer_lens.model_bridge.sources.vllm.source.plugin.register", + lambda: None, + ) + + yield { + "auto_config": auto_config, + "auto_tokenizer": auto_tokenizer, + "vllm_llm": fake_vllm.LLM, + "hf_config": hf_config, + } + + plugin._config.clear() + + +def test_rejects_locked_kwarg_override(): + """tensor_parallel_size != 1 fails fast — before any I/O.""" + with pytest.raises(ValueError, match="tensor_parallel_size"): + boot_vllm("any-model", tensor_parallel_size=2) + + +@pytest.mark.parametrize( + "raw, expected", + [ + (torch.float16, torch.float16), + (torch.bfloat16, torch.bfloat16), + ("bfloat16", torch.bfloat16), + ("nonexistent_dtype", torch.float16), + (None, torch.float16), + ], +) +def test_dtype_resolution(raw, expected): + assert _dtype_from_hf_config(SimpleNamespace(torch_dtype=raw)) == expected + + +def test_dtype_resolution_missing_attr(): + assert _dtype_from_hf_config(SimpleNamespace()) == torch.float16 + + +def test_happy_path_returns_remote_bridge(mocked_boot): + bridge = boot_vllm("any-model") + assert isinstance(bridge, RemoteBridge) + + +def test_hf_token_passed_to_both_hf_calls(mocked_boot): + boot_vllm("any-model") + assert mocked_boot["auto_config"].call_args.kwargs["token"] == "fake-token" + assert mocked_boot["auto_tokenizer"].call_args.kwargs["token"] == "fake-token" + + +def test_custom_tokenizer_skips_autotokenizer(mocked_boot): + boot_vllm("any-model", tokenizer=MagicMock(name="custom")) + mocked_boot["auto_tokenizer"].assert_not_called() + + +def test_env_var_override_warns(mocked_boot, monkeypatch): + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1") + with pytest.warns(UserWarning, match="VLLM_ENABLE_V1_MULTIPROCESSING"): + boot_vllm("any-model") + assert os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] == "0" + + +def test_env_var_zero_does_not_warn(mocked_boot, monkeypatch): + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + boot_vllm("any-model") + assert not any("VLLM_ENABLE_V1_MULTIPROCESSING" in str(w.message) for w in caught) + + +def test_plugin_config_cleared_after_boot(mocked_boot): + """No leak to non-TL vllm.LLM users in the same process.""" + boot_vllm("any-model") + assert plugin._config == {} + + +def test_llm_construction_kwargs(mocked_boot): + """Pin the kwargs boot_vllm passes to vllm.LLM(...). Catches regressions like + forgetting worker_extension_cls (collective_rpc methods unreachable), + max_num_batched_tokens (Dynamo symbolic-shape bound mismatch with buffer), + or max_logprobs (driver synthesizes logits via full-vocab logprobs).""" + boot_vllm("any-model", max_num_batched_tokens=1024) + kwargs = mocked_boot["vllm_llm"].call_args.kwargs + assert kwargs["model"] == "any-model" + assert kwargs["max_num_batched_tokens"] == 1024 + assert kwargs["worker_extension_cls"] == ( + "transformer_lens.model_bridge.sources.vllm.worker_extension.TLWorkerExtension" + ) + # Sized from the mocked hf_config.vocab_size in mocked_boot fixture. + assert kwargs["max_logprobs"] == mocked_boot["hf_config"].vocab_size + # Locked kwargs that must always reach LLM. + assert kwargs["tensor_parallel_size"] == 1 + assert kwargs["pipeline_parallel_size"] == 1 + assert kwargs["skip_tokenizer_init"] is True + assert kwargs["disable_log_stats"] is True diff --git a/tests/unit/model_bridge/test_vllm_driver.py b/tests/unit/model_bridge/test_vllm_driver.py new file mode 100644 index 0000000000..8a46a63938 --- /dev/null +++ b/tests/unit/model_bridge/test_vllm_driver.py @@ -0,0 +1,258 @@ +"""Unit tests for VLLMDriver — mocks vLLM's LLM so no vllm package install needed. + +GPU integration tests (real LLM, real captures) live separately as a Colab +notebook; the compiled-graph path can't be reached from mocked unit tests. +""" +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any +from unittest.mock import MagicMock + +import pytest +import torch + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.driver_protocol import ( + Driver, + ForwardResult, + validate_driver, +) +from transformer_lens.model_bridge.remote_bridge import RemoteBridge +from transformer_lens.model_bridge.sources.vllm.driver import VLLMDriver + + +def _hf_config(num_hidden_layers: int = 2, hidden_size: int = 4, vocab_size: int = 16) -> Any: + return SimpleNamespace( + num_hidden_layers=num_hidden_layers, + hidden_size=hidden_size, + vocab_size=vocab_size, + ) + + +def _overlay(specs=None, nonfiring=None): + """Minimal overlay stand-in — capture_specs + nonfiring_hooks.""" + return SimpleNamespace( + capture_specs=lambda hf_config: specs + or { + "embed.hook_out": ("model.embed_tokens", hf_config.hidden_size), + "blocks.0.hook_out": ("model.layers.0", hf_config.hidden_size), + "blocks.1.hook_out": ("model.layers.1", hf_config.hidden_size), + "unembed.hook_out": ("lm_head", hf_config.vocab_size), + }, + nonfiring_hooks=lambda: nonfiring + or [ + "blocks.{i}.attn.hook_pattern", + "blocks.{i}.attn.hook_attn_scores", + ], + ) + + +def _cfg() -> TransformerBridgeConfig: + return TransformerBridgeConfig( + d_model=4, + d_head=2, + n_layers=2, + n_ctx=8, + n_heads=2, + d_vocab=16, + d_mlp=8, + architecture="LlamaForCausalLM", + ) + + +def _adapter() -> ArchitectureAdapter: + adapter = ArchitectureAdapter(_cfg()) + adapter.component_mapping = {} + return adapter + + +def _fake_request_output(generated_token=None, top_logprobs=None): + """Build a vLLM RequestOutput-shaped mock for _synthesize_logits.""" + completion = MagicMock() + completion.token_ids = [generated_token] if generated_token is not None else [] + completion.logprobs = ( + [{tid: MagicMock(logprob=lp) for tid, lp in top_logprobs.items()}] + if top_logprobs is not None + else [] + ) + ro = MagicMock() + ro.outputs = [completion] + return ro + + +def _driver( + *, + captures=None, + hf_config=None, + max_num_batched_tokens=2048, + generated_token=None, + top_logprobs=None, +) -> VLLMDriver: + """Build a VLLMDriver. ``captures`` populates llm.collective_rpc; ``generated_token`` + and ``top_logprobs`` populate llm.generate's RequestOutput so _synthesize_logits + can be exercised on both code paths (logprobs preferred, token_id fallback).""" + llm = MagicMock() + if captures is not None: + llm.collective_rpc = MagicMock(return_value=[captures]) + llm.generate = MagicMock(return_value=[_fake_request_output(generated_token, top_logprobs)]) + return VLLMDriver( + llm=llm, + adapter=_adapter(), + tokenizer=None, + overlay=_overlay(), + hf_config=hf_config or _hf_config(), + max_num_batched_tokens=max_num_batched_tokens, + ) + + +class TestVLLMDriverProtocolConformance: + """VLLMDriver satisfies the Driver protocol and passes strict validation.""" + + def test_passes_validate_driver(self): + driver = _driver() + assert isinstance(driver, Driver) + validate_driver(driver) + + def test_no_torch_capability_flags(self): + """vLLM owns the model in a worker — no torch-specific capability surface.""" + driver = _driver() + for feature in ("parameters", "state_dict", "gradients", "weight_access"): + assert driver.supports(feature) is False, f"vLLM shouldn't support {feature!r}" + + def test_non_fireable_expanded_per_layer(self): + """``blocks.{i}.attn.hook_pattern`` template expands to one entry per layer.""" + assert _driver().non_fireable_hook_points == frozenset( + { + "blocks.0.attn.hook_pattern", + "blocks.1.attn.hook_pattern", + "blocks.0.attn.hook_attn_scores", + "blocks.1.attn.hook_attn_scores", + } + ) + + +class TestVLLMDriverForward: + """forward dispatches via llm.generate and surfaces captures via ForwardResult.""" + + def test_forward_logits_from_sampler_logprobs(self): + """vLLM bypasses lm_head; driver synthesizes logits from sampler logprobs. + Position -1 must carry the real next-token distribution.""" + pytest.importorskip("vllm") + result = _driver( + captures={"embed.hook_out": torch.randn(3, 4)}, + top_logprobs={7: 2.5, 3: 1.0, 11: -0.5}, + ).forward(torch.tensor([[1, 2, 3]])) + assert isinstance(result, ForwardResult) + assert tuple(result.captured["embed.hook_out"].shape) == (1, 3, 4) + assert result.logits is not None and tuple(result.logits.shape) == (1, 3, 16) + # Argmax = highest-logprob token. Values at non-listed positions are -inf. + assert int(result.logits[0, -1].argmax().item()) == 7 + assert float(result.logits[0, -1, 7].item()) == 2.5 + + def test_forward_logits_fallback_to_token_id(self): + """If logprobs are absent (e.g. return_logits=False elsewhere upstream), + _synthesize_logits falls back to the generated token id as a one-hot-ish.""" + pytest.importorskip("vllm") + result = _driver( + captures={"embed.hook_out": torch.randn(3, 4)}, + generated_token=9, + ).forward(torch.tensor([[1, 2, 3]])) + assert result.logits is not None + assert int(result.logits[0, -1].argmax().item()) == 9 + + def test_forward_rejects_batched_input(self): + """batch_size=1 only. Raises in _normalize_input_ids before any vllm import.""" + with pytest.raises(NotImplementedError, match="batch_size=1"): + _driver(captures={}).forward(torch.tensor([[1, 2], [3, 4]])) + + def test_forward_rejects_callable_interventions(self): + with pytest.raises(NotImplementedError, match="intervention specs"): + _driver(captures={}).forward( + torch.tensor([[1]]), intervene={"embed.hook_out": lambda a: a} + ) + + def test_forward_rejects_unsupported_intervention_op(self): + with pytest.raises(ValueError, match="Unsupported intervention op"): + _driver(captures={}).forward( + torch.tensor([[1]]), intervene={"embed.hook_out": {"op": "clamp", "value": 1.0}} + ) + + def test_forward_rejects_unknown_intervention_hook(self): + with pytest.raises(ValueError, match="not in supported_hook_points"): + _driver(captures={}).forward( + torch.tensor([[1]]), intervene={"blocks.99.hook_unknown": {"op": "suppress"}} + ) + + def test_forward_rejects_malformed_intervention_spec(self): + with pytest.raises(ValueError, match="must be a dict with 'op' key"): + _driver(captures={}).forward( + torch.tensor([[1]]), intervene={"embed.hook_out": {"no_op_key": True}} + ) + + def test_forward_rejects_scale_missing_factor(self): + with pytest.raises(ValueError, match="op='scale' requires 'factor'"): + _driver(captures={}).forward( + torch.tensor([[1]]), intervene={"embed.hook_out": {"op": "scale"}} + ) + + def test_forward_rejects_add_missing_value(self): + with pytest.raises(ValueError, match="op='add' requires 'value'"): + _driver(captures={}).forward( + torch.tensor([[1]]), intervene={"embed.hook_out": {"op": "add"}} + ) + + def test_forward_rejects_set_missing_value(self): + with pytest.raises(ValueError, match="op='set' requires 'value'"): + _driver(captures={}).forward( + torch.tensor([[1]]), intervene={"embed.hook_out": {"op": "set"}} + ) + + def test_forward_pushes_interventions_before_generate(self): + """The driver pushes spec dicts via collective_rpc('tl_set_interventions', ...).""" + pytest.importorskip("vllm") + driver = _driver(captures={"embed.hook_out": torch.zeros(3, 4)}) + driver.forward( + torch.tensor([[1, 2, 3]]), + intervene={"embed.hook_out": {"op": "suppress"}}, + ) + rpc_calls = [c.args for c in driver._llm.collective_rpc.call_args_list] + assert any( + args[0] == "tl_set_interventions" + and args[1] == ({"embed.hook_out": {"op": "suppress"}},) + for args in rpc_calls + ) + + def test_forward_always_pushes_interventions_for_reset(self): + """Even with intervene=None, push {} so stale state from prior forwards clears.""" + pytest.importorskip("vllm") + driver = _driver(captures={"embed.hook_out": torch.zeros(3, 4)}) + driver.forward(torch.tensor([[1, 2, 3]])) + rpc_calls = [c.args for c in driver._llm.collective_rpc.call_args_list] + assert any(args[0] == "tl_set_interventions" and args[1] == ({},) for args in rpc_calls) + + def test_forward_rejects_max_new_tokens_gt_one(self): + """Decode-step writes overwrite the prefill buffer — silent capture corruption.""" + with pytest.raises(NotImplementedError, match="max_new_tokens=1 only"): + _driver(captures={}).forward(torch.tensor([[1, 2]]), max_new_tokens=2) + + def test_forward_rejects_prompt_exceeding_buffer(self): + """Worker buffers silently clamp on overflow — driver must fail loud.""" + with pytest.raises(ValueError, match="exceeds max_num_batched_tokens"): + _driver(captures={}, max_num_batched_tokens=4).forward(torch.tensor([[1, 2, 3, 4, 5]])) + + +class TestVLLMDriverThroughBridge: + """End-to-end via RemoteBridge — drivers' captures flow into the HookPoint tree.""" + + def test_bridge_replays_vllm_captures(self): + pytest.importorskip("vllm") + driver = _driver(captures={"embed.hook_out": torch.randn(3, 4)}) + bridge = RemoteBridge(adapter=_adapter(), tokenizer=None, driver=driver) + + fired: list = [] + bridge.add_hook("embed.hook_out", lambda act, hook: fired.append(act)) + bridge.forward(torch.tensor([[1, 2, 3]])) + assert len(fired) == 1 + assert tuple(fired[0].shape) == (1, 3, 4) diff --git a/transformer_lens/model_bridge/__init__.py b/transformer_lens/model_bridge/__init__.py index 3e41974710..98108ee3de 100644 --- a/transformer_lens/model_bridge/__init__.py +++ b/transformer_lens/model_bridge/__init__.py @@ -3,7 +3,7 @@ This module provides functionality to bridge between different model architectures. """ from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter -from transformer_lens.model_bridge.bridge import TransformerBridge +from transformer_lens.model_bridge.bridge import BridgeCore, RemoteBridge, TransformerBridge from transformer_lens.model_bridge.component_setup import ( replace_remote_component, set_original_components, @@ -36,6 +36,8 @@ __all__ = [ "ArchitectureAdapter", + "BridgeCore", + "RemoteBridge", "TransformerBridge", "AttentionBridge", "BlockBridge", diff --git a/transformer_lens/model_bridge/bridge.py b/transformer_lens/model_bridge/bridge.py index 91e6852049..55e8937088 100644 --- a/transformer_lens/model_bridge/bridge.py +++ b/transformer_lens/model_bridge/bridge.py @@ -1,3751 +1,14 @@ -"""Bridge module for connecting different model architectures. +"""Compatibility re-export shim. -This module provides the bridge components that wrap remote model components and provide -a consistent interface for accessing their weights and performing operations. -""" -import logging -import re -import warnings -from collections.abc import Generator -from contextlib import contextmanager -from functools import lru_cache -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterator, - List, - Literal, - Optional, - Tuple, - Union, - cast, - overload, -) - -import einops -import numpy as np -import torch -import tqdm -from torch import nn - -from transformer_lens import utilities as utils -from transformer_lens.ActivationCache import ActivationCache -from transformer_lens.FactoredMatrix import FactoredMatrix -from transformer_lens.hook_points import HookIntrospectionMixin, HookPoint -from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter -from transformer_lens.model_bridge.component_setup import set_original_components -from transformer_lens.model_bridge.composition_scores import CompositionScores -from transformer_lens.model_bridge.exceptions import StopAtLayerException -from transformer_lens.model_bridge.generalized_components.base import ( - GeneralizedComponent, -) -from transformer_lens.model_bridge.generalized_components.block import ( - _BLOCK_INTERNAL_MODULES, - _NORM_PREFIXES, - _VARIANT_SUBMODULE_SET, - VARIANT_SUBMODULE_NAMES, -) -from transformer_lens.model_bridge.get_params_util import get_bridge_params -from transformer_lens.utilities.aliases import resolve_alias -from transformer_lens.utilities.devices import move_to_and_update_config -from transformer_lens.utilities.lm_utils import lm_cross_entropy_loss - -if TYPE_CHECKING: - from transformer_lens.ActivationCache import ActivationCache - -_BLOCK_PATTERN = re.compile("blocks\\.(\\d+)") - - -def _resolve_attr_path(obj: nn.Module, attr_path: str) -> torch.Tensor: - """Walk a dot-separated attribute path and return the final tensor.""" - result = obj - for attr in attr_path.split("."): - result = getattr(result, attr) - return cast(torch.Tensor, result) - - -def build_alias_to_canonical_map(hook_dict, prefix=""): - """Build a mapping from alias hook names to their canonical names. - - Args: - hook_dict: Dictionary mapping hook names to HookPoint objects - prefix: Prefix for nested keys - - Returns: - Dictionary mapping alias names to canonical names - - Example: - If hook_dict contains: - - "blocks.0.hook_q" -> HookPoint(name="blocks.0.attn.q.hook_out") - - Returns: - - {"blocks.0.hook_q": "blocks.0.attn.q.hook_out"} - """ - aliases = {} - for key, value in hook_dict.items(): - full_key = f"{prefix}.{key}" if prefix else key - if isinstance(value, dict): - aliases.update(build_alias_to_canonical_map(value, full_key)) - elif hasattr(value, "name"): - if key != value.name: - aliases[full_key] = value.name - return aliases - - -class TransformerBridge(HookIntrospectionMixin, nn.Module): - """Bridge between HuggingFace and TransformerLens models. - - This class provides a standardized interface to access components of a transformer - model, regardless of the underlying architecture. It uses an architecture adapter - to map between the TransformerLens and HuggingFace model structures. - - Tokenization notes - ------------------ - - :meth:`to_tokens`, :meth:`to_str_tokens`, :meth:`get_token_position`, - :meth:`forward` (string input), and :meth:`generate` accept ``prepend_bos`` - to control BOS prepending. Resolution: explicit arg → - ``cfg.default_prepend_bos`` (defaults ``True``, even for non-BOS-trained - models — attention heads tend to use position 0 as a resting state). - **Pass ``prepend_bos=False`` when tokenizing a fragment of a larger - prompt** — off-by-one position errors usually trace back here. - - Reconciliation with ``cfg.tokenizer_prepends_bos`` (tokenizers that add - BOS automatically) is handled internally — pass the value you want; - the bridge adds or strips manually as needed. When - ``cfg.tokenizer_appends_eos=True`` (OLMo, Apertus, etc.), - :meth:`to_tokens` also strips trailing EOS tokens so the model receives - a continuation rather than a terminated sequence; this path is - bridge-specific. - - BPE/SentencePiece tokenizers treat ``"hello"``, ``" hello"``, and - ``"Hello"`` as distinct tokens. Concatenated prompts may not tokenize - as the sum of parts — inspect with :meth:`to_str_tokens` when in doubt. - """ - - hook_aliases: Dict[str, Union[str, List[str]]] = { - # Prefer embed_ln.hook_out for post-LN models (Bloom, BERT) - "hook_embed": ["embed_ln.hook_out", "embed.hook_out"], - "hook_pos_embed": ["pos_embed.hook_out", "rotary_emb.hook_out"], - "hook_unembed": "unembed.hook_out", - } - - def __init__(self, model: nn.Module, adapter: ArchitectureAdapter, tokenizer: Any): - """Initialize the bridge. - - Args: - model: The model to bridge (must be a PyTorch nn.Module or PreTrainedModel) - adapter: The architecture adapter to use - tokenizer: The tokenizer to use (required) - """ - super().__init__() - self.__dict__["original_model"] = model - self.adapter = adapter - self.cfg = adapter.cfg - self.tokenizer = tokenizer - if self.cfg.d_vocab == -1 and self.tokenizer is not None: - if hasattr(self.tokenizer, "get_vocab"): - vocab = self.tokenizer.get_vocab() - self.cfg.d_vocab = max(vocab.values()) + 1 - elif hasattr(self.tokenizer, "vocab"): - self.cfg.d_vocab = max(self.tokenizer.vocab.values()) + 1 - else: - self.cfg.d_vocab = getattr(self.tokenizer, "vocab_size", 50257) - if self.cfg.d_vocab_out == -1: - self.cfg.d_vocab_out = self.cfg.d_vocab - self.compatibility_mode = False - self._hook_cache = None - self._hook_registry: Dict[str, HookPoint] = {} - self._hook_registry_initialized = False - self._hook_alias_registry: Dict[str, Union[str, List[str]]] = {} - self._property_alias_registry: Dict[str, str] = {} - # real_components maps TL keys to (remote_path, actual_instance) tuples - # For list components, actual_instance will be a list of component instances - self.real_components: Dict[str, tuple] = {} - if not hasattr(self.cfg, "device") or self.cfg.device is None: - try: - self.cfg.device = str(next(self.original_model.parameters()).device) - except StopIteration: - self.cfg.device = "cpu" - if not hasattr(adapter, "component_mapping") or adapter.component_mapping is None: - raise ValueError("Adapter must have a component_mapping attribute") - original_model = self.__dict__["original_model"] - set_original_components(self, self.adapter, original_model) - self._initialize_hook_registry() - self._register_aliases() - self._register_all_aliases_recursive() - self._setup_hook_compatibility() - self._initialize_hooks_to_cache() - self.processor = None - - @classmethod - def boot_transformers( - cls, - model_name: str, - hf_config_overrides: Optional[dict] = None, - device: Optional[Union[str, torch.device]] = None, - dtype: torch.dtype = torch.float32, - tokenizer: Optional[Any] = None, - load_weights: bool = True, - trust_remote_code: bool = False, - model_class: Optional[type] = None, - hf_model: Optional[Any] = None, - device_map: Optional[Union[str, Dict[str, Union[str, int]]]] = None, - n_devices: Optional[int] = None, - max_memory: Optional[Dict[Union[str, int], str]] = None, - n_ctx: Optional[int] = None, - revision: Optional[str] = None, - checkpoint_index: Optional[int] = None, - checkpoint_value: Optional[int] = None, - ) -> "TransformerBridge": - """Boot a model from HuggingFace (alias for sources.transformers.boot). - - Returns raw HF weights by default — logits/activations match HF, *not* - legacy ``HookedTransformer`` (which folds LayerNorm + centers weights). - Call ``enable_compatibility_mode()`` on the result for HookedTransformer- - equivalent numerics. Generation, argmax, and CE loss are unaffected. - - Attention implementation is forced to ``"eager"`` so hooks can capture scores - and patterns. For an apples-to-apples HF comparison, load the HF model with - ``attn_implementation="eager"`` too; comparing against the default ``"sdpa"`` - shows ~1e-3 fp32 drift from kernel-level op reordering, not a bridge bug. - - Args: - model_name: The name of the model to load. - hf_config_overrides: Optional overrides applied to the HuggingFace config before model load. - device: The device to use. If None, will be determined automatically. Mutually exclusive - with ``device_map``. - dtype: The dtype to use for the model. - tokenizer: Optional pre-initialized tokenizer to use; if not provided one will be created. - load_weights: If False, load model without weights (on meta device) for config inspection only. - trust_remote_code: Whether to trust remote code for custom model architectures. - model_class: Optional HuggingFace model class to use instead of the default - auto-detected class (e.g., BertForNextSentencePrediction). - hf_model: Optional pre-loaded HuggingFace model to use instead of loading one. Useful - for models loaded with custom configurations (e.g., quantization via - BitsAndBytesConfig). When provided, load_weights is ignored. If the pre-loaded - model was built with a ``device_map``, ``cfg.device`` and ``cfg.n_devices`` are - derived from its ``hf_device_map`` automatically. - device_map: HuggingFace-style device map for multi-GPU inference. Pass ``"auto"``, - ``"balanced"``, ``"sequential"``, or an explicit ``{submodule_path: device}`` dict. - Mutually exclusive with ``device``. - n_devices: Convenience shortcut: split the model across this many CUDA devices. - Translated to a ``max_memory`` dict over devices 0..n_devices-1 and passed as - ``device_map`` to HF. Requires CUDA with at least this many visible devices. - max_memory: Optional per-device memory budget, passed through to HF's dispatcher. - Only used when ``device_map`` or ``n_devices`` is in effect. - n_ctx: Optional context length override. Writes to the appropriate HF config field - for this model automatically (callers don't need to know the field name). - Warns if larger than the model's default context length. - revision: Optional HF revision (branch, tag, or commit). Forwarded to the underlying - ``AutoConfig.from_pretrained`` and ``AutoModelForCausalLM.from_pretrained`` calls. - Mutually exclusive with ``checkpoint_index`` / ``checkpoint_value``. - checkpoint_index: Index into the available training checkpoints for the model family - (currently ``EleutherAI/pythia*`` and ``stanford-crfm/*``). Resolved to a revision - string via known per-family naming conventions. - checkpoint_value: Training step or token count of the desired checkpoint. Alternative - to ``checkpoint_index``; must match an entry in the family's checkpoint label list. - - Returns: - The bridge to the loaded model. - """ - from transformer_lens.model_bridge.sources.transformers import boot - - return boot( - model_name=model_name, - hf_config_overrides=hf_config_overrides, - device=device, - dtype=dtype, - tokenizer=tokenizer, - load_weights=load_weights, - trust_remote_code=trust_remote_code, - model_class=model_class, - hf_model=hf_model, - device_map=device_map, - n_devices=n_devices, - max_memory=max_memory, - n_ctx=n_ctx, - revision=revision, - checkpoint_index=checkpoint_index, - checkpoint_value=checkpoint_value, - ) - - @property - def original_model(self) -> nn.Module: - """Get the original model.""" - if "original_model" not in self.__dict__: - raise AttributeError("original_model has not been set") - return self.__dict__["original_model"] - - @original_model.setter - def original_model(self, value: nn.Module) -> None: - """Set the original model.""" - self.__dict__["original_model"] = value - - def _register_aliases(self) -> None: - """Register bridge-level aliases. - - This is called at the END of __init__ when all components are set up. - It registers the top-level bridge aliases (hook_embed, hook_pos_embed, etc.) - and creates direct attribute references. - """ - if self.hook_aliases: - self._hook_alias_registry.update(self.hook_aliases) - for alias_name, target_path in self.hook_aliases.items(): - try: - if isinstance(target_path, list): - for single_target in target_path: - try: - target_obj = self - for part in single_target.split("."): - target_obj = getattr(target_obj, part) - object.__setattr__(self, alias_name, target_obj) - break - except AttributeError: - continue - else: - target_obj = self - for part in target_path.split("."): - target_obj = getattr(target_obj, part) - object.__setattr__(self, alias_name, target_obj) - except AttributeError: - pass - - def _set_processed_weight_attributes(self) -> None: - """Create 3D processed weight attributes for attention components. - - For each attention component, if it has 2D weights (q.weight, k.weight, v.weight), - reshape them to 3D format [n_heads, d_model, d_head] and set as: - - _processed_W_Q - - _processed_W_K - - _processed_W_V - - _processed_b_Q - - _processed_b_K - - _processed_b_V - - This allows property aliases (W_Q, W_K, W_V) to return 3D format for - HookedTransformer compatibility while keeping 2D format for calculations. - """ - - n_heads = self.cfg.n_heads - d_head = self.cfg.d_head - d_model = self.cfg.d_model - if not hasattr(self, "blocks"): - return - for block in self.blocks: - if "attn" not in block._modules: - continue - attn = block.attn - if not (hasattr(attn, "q") and hasattr(attn.q, "weight")): - continue - try: - w_q_2d = attn.q.weight.data - w_k_2d = attn.k.weight.data - w_v_2d = attn.v.weight.data - attn._processed_W_Q = einops.rearrange( - w_q_2d, "m (i h) -> i m h", i=n_heads, h=d_head - ) - attn._processed_W_K = einops.rearrange( - w_k_2d, "m (i h) -> i m h", i=n_heads, h=d_head - ) - attn._processed_W_V = einops.rearrange( - w_v_2d, "m (i h) -> i m h", i=n_heads, h=d_head - ) - if hasattr(attn.q, "bias") and attn.q.bias is not None: - b_q_2d = attn.q.bias.data - b_k_2d = attn.k.bias.data - b_v_2d = attn.v.bias.data - attn._processed_b_Q = einops.rearrange( - b_q_2d, "(i h) -> i h", i=n_heads, h=d_head - ) - attn._processed_b_K = einops.rearrange( - b_k_2d, "(i h) -> i h", i=n_heads, h=d_head - ) - attn._processed_b_V = einops.rearrange( - b_v_2d, "(i h) -> i h", i=n_heads, h=d_head - ) - if hasattr(attn, "o") and hasattr(attn.o, "weight"): - w_o_2d = attn.o.weight.data - w_o_transposed = w_o_2d.T - attn._processed_W_O = einops.rearrange( - w_o_transposed, "m (i h) -> i h m", i=n_heads, h=d_head - ) - if hasattr(attn.o, "bias") and attn.o.bias is not None: - attn._processed_b_O = attn.o.bias.data - except Exception: - pass - - def _register_all_aliases_recursive(self) -> None: - """Recursively register aliases on all bridge components. - - This walks through all components and calls _register_aliases() on each one. - Used after weight processing to ensure aliases point to processed weights. - """ - if hasattr(self, "_register_aliases"): - self._register_aliases() - for module in self.modules(): - if module is not self and hasattr(module, "_register_aliases"): - getattr(module, "_register_aliases")() - - def __setattr__(self, name: str, value: Any) -> None: - """Override setattr to track HookPoint objects dynamically.""" - super().__setattr__(name, value) - if isinstance(value, HookPoint): - value.name = name - self._hook_registry[name] = value - elif hasattr(value, "get_hooks") and callable(getattr(value, "get_hooks")): - component_hooks = value.get_hooks() - for hook_name, hook in component_hooks.items(): - full_name = f"{name}.{hook_name}" - hook.name = full_name - self._hook_registry[full_name] = hook - - def _initialize_hook_registry(self) -> None: - """Initialize the hook registry by scanning existing components.""" - if self._hook_registry_initialized: - return - self._scan_existing_hooks(self, "") - self._hook_registry_initialized = True - - def _collect_component_aliases(self, component_mapping, prefix=""): - """Recursively collect aliases from components.""" - aliases = {} - if isinstance(component_mapping, dict): - for name, component in component_mapping.items(): - sub_prefix = f"{prefix}.{name}" if prefix else name - aliases.update(self._collect_component_aliases(component, sub_prefix)) - else: - if hasattr(component_mapping, "hook_aliases") and component_mapping.hook_aliases: - for alias_name, target in component_mapping.hook_aliases.items(): - full_alias = f"{prefix}.{alias_name}" if prefix else alias_name - full_target = f"{prefix}.{target}" if prefix else target - aliases[full_alias] = full_target - if hasattr(component_mapping, "submodules") and component_mapping.submodules: - for sub_name, sub_component in component_mapping.submodules.items(): - sub_prefix = f"{prefix}.{sub_name}" if prefix else sub_name - aliases.update(self._collect_component_aliases(sub_component, sub_prefix)) - return aliases - - @staticmethod - @lru_cache(maxsize=128) - def _compute_hook_aliases_cached( - hook_names_tuple: Tuple[str, ...], component_aliases_tuple: Tuple[Tuple[str, str], ...] - ) -> Tuple[Tuple[str, str], ...]: - """Cached computation of hook aliases. Takes immutable inputs for caching.""" - aliases = {} - component_aliases = dict(component_aliases_tuple) - for hook_name in hook_names_tuple: - for alias_pattern, target_pattern in component_aliases.items(): - if "blocks." in target_pattern and "blocks." in hook_name: - block_match = _BLOCK_PATTERN.search(hook_name) - if block_match: - block_num = block_match.group(1) - dynamic_alias_pattern = alias_pattern.replace( - "blocks.", f"blocks.{block_num}." - ) - dynamic_target_pattern = target_pattern.replace( - "blocks.", f"blocks.{block_num}." - ) - if hook_name.endswith(dynamic_target_pattern): - target_len = len(dynamic_target_pattern) - alias_name = hook_name[:-target_len] + dynamic_alias_pattern - aliases[alias_name] = hook_name - elif hook_name.endswith(target_pattern): - target_len = len(target_pattern) - alias_name = hook_name[:-target_len] + alias_pattern - aliases[alias_name] = hook_name - return tuple(aliases.items()) - - def _collect_hook_aliases_from_registry(self): - """Collect aliases based on existing hooks in the registry.""" - if hasattr(self.adapter, "component_mapping"): - component_aliases = self._collect_component_aliases(self.adapter.component_mapping) - hook_names_tuple = tuple(sorted(self._hook_registry.keys())) - component_aliases_tuple = tuple(sorted(component_aliases.items())) # type: ignore[operator] - aliases_tuple = self._compute_hook_aliases_cached( - hook_names_tuple, component_aliases_tuple - ) - return dict(aliases_tuple) - return {} - - def _add_aliases_to_hooks(self, hooks: Dict[str, HookPoint]) -> None: - """Add aliases to hooks in place.""" - component_aliases = self._collect_hook_aliases_from_registry() - all_aliases = {**self.hook_aliases, **component_aliases} - if not all_aliases: - return - for alias_name, target in all_aliases.items(): - if isinstance(target, list): - for single_target in target: - try: - target_hook = resolve_alias(self, alias_name, {alias_name: single_target}) - if target_hook is not None: - hooks[alias_name] = target_hook - break - except AttributeError: - continue - else: - try: - target_hook = resolve_alias(self, alias_name, {alias_name: target}) - if target_hook is not None: - hooks[alias_name] = target_hook - except AttributeError: - continue - - def _scan_existing_hooks(self, module: nn.Module, prefix: str = "") -> None: - """Scan existing modules for hooks and add them to registry.""" - visited = set() - # Protect canonical HookPoint names from alias overwrites - named_hook_ids: set = set() - - def scan_module(mod: nn.Module, path: str = "") -> None: - obj_id = id(mod) - if obj_id in visited: - return - visited.add(obj_id) - if hasattr(mod, "get_hooks") and callable(getattr(mod, "get_hooks")): - component_hooks = mod.get_hooks() # type: ignore[operator] - if isinstance(component_hooks, dict): - hooks_dict = cast(Dict[str, HookPoint], component_hooks) - for hook_name, hook in hooks_dict.items(): - full_name = f"{path}.{hook_name}" if path else hook_name - hook_id = id(hook) - if hook_id not in named_hook_ids: - hook.name = full_name - named_hook_ids.add(hook_id) - self._hook_registry[full_name] = hook - for attr_name in dir(mod): - if attr_name.startswith("_"): - continue - if attr_name == "original_component" or attr_name == "original_model": - continue - if attr_name in [ - "OV", - "QK", - "W_V", - "W_O", - "W_Q", - "W_K", - "W_in", - "W_gate", - "W_out", - "b_V", - "b_O", - "b_Q", - "b_K", - "b_in", - "b_out", - ]: - continue - try: - attr = getattr(mod, attr_name) - except (AttributeError, NameError, RuntimeError, TypeError): - continue - name = f"{path}.{attr_name}" if path else attr_name - if isinstance(attr, HookPoint): - hook_id = id(attr) - if hook_id not in named_hook_ids: - attr.name = name - named_hook_ids.add(hook_id) - self._hook_registry[name] = attr - for child_name, child_module in mod.named_children(): - if ( - child_name == "original_component" - or child_name == "_original_component" - or child_name == "original_model" - ): - continue - child_path = f"{path}.{child_name}" if path else child_name - scan_module(child_module, child_path) - - scan_module(module, prefix) - - @property - def hook_dict(self) -> dict[str, HookPoint]: - """Get all HookPoint objects in the model for compatibility with TransformerLens.""" - hooks = self._hook_registry.copy() - self._add_aliases_to_hooks(hooks) - return hooks - - @property - def n_params_total(self) -> int: - """Total number of parameters in the model, including embeddings, biases, - and layer norm weights. - - Mirrors :attr:`HookedTransformer.n_params_total`. Use this when you want - the actual parameter count for memory budgeting, comparison with - HuggingFace's ``model.num_parameters()``, or alignment with reported - model sizes in papers (e.g. the Pythia suite). - - Returns: - int: ``sum(p.numel() for p in self.parameters())`` - """ - return sum(p.numel() for p in self.parameters()) - - def clear_hook_registry(self) -> None: - """Clear the hook registry and force re-initialization.""" - self._hook_registry.clear() - self._hook_registry_initialized = False - - def _initialize_hooks_to_cache(self) -> None: - """Initialize the hooks to cache when running the model with cache.""" - self.hooks_to_cache = {} - default_cached_hooks_names = [ - "embed.hook_in", - "embed.hook_out", - "pos_embed.hook_in", - "pos_embed.hook_out", - "rotary_embed.hook_in", - "rotary_embed.hook_out", - "ln_final.hook_in", - "ln_final.hook_scale", - "ln_final.hook_normalized", - "ln_final.hook_out", - "unembed.hook_in", - "unembed.hook_out", - ] - for block_idx in range(self.cfg.n_layers): - default_cached_hooks_names.append(f"blocks.{block_idx}.hook_in") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln1.hook_in") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln1.hook_scale") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln1.hook_normalized") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln1.hook_out") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln1_post.hook_in") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln1_post.hook_scale") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln1_post.hook_normalized") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln1_post.hook_out") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.hook_in") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.q.hook_in") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.q.hook_out") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.q_norm.hook_in") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.q_norm.hook_out") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.k.hook_in") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.k.hook_out") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.k_norm.hook_in") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.k_norm.hook_out") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.v.hook_in") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.v.hook_out") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.o.hook_in") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.o.hook_out") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.hook_attn_scores") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.hook_pattern") # type: ignore[operator] - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.hook_hidden_states") - default_cached_hooks_names.append(f"blocks.{block_idx}.attn.hook_out") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln2.hook_in") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln2.hook_scale") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln2.hook_normalized") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln2.hook_out") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln2_post.hook_in") # type: ignore[operator] - default_cached_hooks_names.append(f"blocks.{block_idx}.ln2_post.hook_scale") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln2_post.hook_normalized") - default_cached_hooks_names.append(f"blocks.{block_idx}.ln2_post.hook_out") - default_cached_hooks_names.append(f"blocks.{block_idx}.mlp.hook_in") # type: ignore[operator] - default_cached_hooks_names.append(f"blocks.{block_idx}.mlp.in.hook_in") - default_cached_hooks_names.append(f"blocks.{block_idx}.mlp.in.hook_out") # type: ignore[operator] - default_cached_hooks_names.append(f"blocks.{block_idx}.mlp.out.hook_in") - default_cached_hooks_names.append(f"blocks.{block_idx}.mlp.out.hook_out") - default_cached_hooks_names.append(f"blocks.{block_idx}.mlp.gate.hook_in") - default_cached_hooks_names.append(f"blocks.{block_idx}.mlp.gate.hook_out") - default_cached_hooks_names.append(f"blocks.{block_idx}.mlp.hook_out") - default_cached_hooks_names.append(f"blocks.{block_idx}.hook_out") - for hook_name in default_cached_hooks_names: - if hook_name in self._hook_registry: - self.hooks_to_cache[hook_name] = self._hook_registry[hook_name] # type: ignore[arg-type] - - def __getattr__(self, name: str) -> Any: - """Provide a clear error message for missing attributes.""" - if name in self.__dict__: # type: ignore[arg-type] - return self.__dict__[name] - # Use __dict__ directly to avoid recursion - if "_modules" in self.__dict__ and name in self.__dict__["_modules"]: # type: ignore[arg-type] - return self.__dict__["_modules"][name] - if "original_model" in self.__dict__ and self.__dict__["original_model"] is not None: - try: - name_split = name.split(".") - if len(name_split) > 1: - current = getattr(self.__dict__["original_model"], name_split[0]) - for part in name_split[1:]: # type: ignore[operator] - current = getattr(current, part) - return current - else: - return getattr(self.__dict__["original_model"], name) - except AttributeError: - pass # type: ignore[operator,assignment] - raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") - - def __str__(self) -> str: - """Get a string representation of the bridge. - # type: ignore[operator] - Returns: - A string describing the bridge's components # type: ignore[operator] - """ - lines = ["TransformerBridge:"] - mapping = self.adapter.get_component_mapping() - lines.extend(self._format_component_mapping(mapping, indent=1)) - return "\n".join(lines) - - def enable_compatibility_mode( - self, - disable_warnings: bool = False, - no_processing: bool = False, - fold_ln: bool = True, - center_writing_weights: bool = True, - center_unembed: bool = True, - fold_value_biases: bool = True, - refactor_factored_attn_matrices: bool = False, - ) -> None: - """Apply HookedTransformer-equivalent weight processing and legacy hook compatibility. - - Defaults match HookedTransformer's load-time processing (fold_ln + weight - centering) — required for analyses that reason in HookedTransformer's - post-processed coordinate system: logit lens, direct logit attribution, - residual-stream norms. Also enables legacy hook/component name aliases. - - Args: - disable_warnings: Whether to disable warnings about legacy components/hooks - no_processing: Whether to disable ALL pre-processing steps of the model. - If True, overrides fold_ln, center_writing_weights, and center_unembed to False. - fold_ln: Whether to fold layer norm weights into the subsequent linear layers. - Default: True. Ignored if no_processing=True. - center_writing_weights: Whether to center the writing weights (W_out in attention and MLPs). - Default: True. Ignored if no_processing=True. - center_unembed: Whether to center the unembedding matrix. - Default: True. Ignored if no_processing=True. - fold_value_biases: Whether to fold value biases into output bias. - Default: True. Ignored if no_processing=True. - refactor_factored_attn_matrices: Whether to refactor factored attention matrices. - Default: False. Ignored if no_processing=True. - """ - from transformer_lens.utilities.bridge_components import ( - apply_fn_to_all_components, - ) - - self.compatibility_mode = True - - def set_compatibility_mode(component: Any) -> None: - """Set compatibility mode on a component.""" - component.compatibility_mode = True - component.disable_warnings = disable_warnings - - apply_fn_to_all_components(self, set_compatibility_mode) - self.clear_hook_registry() - try: - if not no_processing: - self.process_weights( - fold_ln=fold_ln, - center_writing_weights=center_writing_weights, - center_unembed=center_unembed, - fold_value_biases=fold_value_biases, - refactor_factored_attn_matrices=refactor_factored_attn_matrices, - ) - finally: - # Re-initialize hooks even on failure so bridge stays usable - self._initialize_hook_registry() - self._setup_hook_compatibility() - self._register_all_aliases_recursive() - - def _setup_hook_compatibility(self) -> None: - """Setup hook compatibility transformations to match HookedTransformer behavior. - - This method sets up hook conversions and wrappers that ensure Bridge hooks - have the same shapes and behavior as HookedTransformer hooks. This includes: - 1. hook_z reshaping from [batch, seq, d_model] to [batch, seq, n_heads, d_head] - 2. Wrapping HF attention forward to inject position embeddings/attention masks - 3. Architecture-specific setup (e.g., rotary embedding references) - - This is called during __init__ and should always be run, regardless of whether - compatibility mode or weight processing is enabled. - - Note: This method is idempotent - can be called multiple times safely. - """ - if hasattr(self.adapter, "setup_hook_compatibility"): - self.adapter.setup_hook_compatibility(self) - elif hasattr(self.adapter, "setup_no_processing_hooks"): - self.adapter.setup_no_processing_hooks(self) - blocks_to_process = [] - if hasattr(self, "blocks"): - blocks_to_process.extend(self.blocks) - if hasattr(self, "encoder_blocks"): - blocks_to_process.extend(self.encoder_blocks) - if hasattr(self, "decoder_blocks"): - blocks_to_process.extend(self.decoder_blocks) - for block in blocks_to_process: - for attn_name in ["attn", "self_attn", "cross_attn"]: - if hasattr(block, attn_name): - attn = getattr(block, attn_name) - if hasattr(attn, "setup_hook_compatibility"): - attn.setup_hook_compatibility() - elif hasattr(attn, "setup_no_processing_hooks"): - attn.setup_no_processing_hooks() - - def process_weights( - self, - verbose: bool = False, - fold_ln: bool = True, - center_writing_weights: bool = True, - center_unembed: bool = True, - fold_value_biases: bool = True, - refactor_factored_attn_matrices: bool = False, - ) -> None: - """Process weights directly using ProcessWeights and architecture adapter. - - This method applies weight processing transformations to improve model interpretability - without requiring a reference HookedTransformer model. Works with all architectures - supported by TransformerBridge, including GPT-OSS and other new models. - - Args: - verbose: If True, print detailed progress messages. Default: False - fold_ln: Fold LayerNorm weights/biases into subsequent layers. Default: True - center_writing_weights: Center weights that write to residual stream. Default: True - center_unembed: Center unembedding weights (translation invariant). Default: True - fold_value_biases: Fold value biases into output bias. Default: True - refactor_factored_attn_matrices: Experimental QK/OV factorization. Default: False - """ - from transformer_lens.weight_processing import ProcessWeights - - if verbose: - print(f"Processing weights for {self.cfg.model_name}...") - - # Soft capping (tanh) is not translation-invariant; centering would change output. - if center_unembed and getattr(self.cfg, "output_logits_soft_cap", -1.0) > 0.0: - import logging - - logging.warning( - "center_unembed=True is incompatible with logit softcapping " - "(output_logits_soft_cap=%.1f). Disabling center_unembed.", - self.cfg.output_logits_soft_cap, - ) - center_unembed = False - - if verbose: - print(" Extracting state dict from existing model...") - state_dict = self.state_dict() - adapter = self.adapter - - # Untie embed/unembed weights (GPT-2) so centering affects only unembed - embed_key = "embed.weight" - unembed_key = "unembed.weight" - - if embed_key in state_dict and unembed_key in state_dict: - # Check if they point to the same tensor (weight tying) - if state_dict[embed_key].data_ptr() == state_dict[unembed_key].data_ptr(): - if verbose: - print(" Breaking weight tying between embed and unembed in state dict...") - # Clone the unembed weight to break the tie - state_dict[unembed_key] = state_dict[unembed_key].clone() - - if adapter and hasattr(adapter, "preprocess_weights"): - adapter._fold_ln_requested = fold_ln # type: ignore[union-attr] - state_dict = adapter.preprocess_weights(state_dict) - - # Use unified ProcessWeights.process_weights() like HookedTransformer does. - # Float32 upcasting for precision is handled centrally in process_weights(). - if verbose: - print(" Processing weights (fold_ln, center_writing_weights, etc.)...") - state_dict = ProcessWeights.process_weights( - state_dict, - self.cfg, - fold_ln=fold_ln, - center_writing_weights=center_writing_weights, - center_unembed=center_unembed, - fold_value_biases=fold_value_biases, - refactor_factored_attn_matrices=refactor_factored_attn_matrices, - adapter=adapter, - ) - - # Normalize HF-prefix keys to TL format for weight routing - import re - - hf_to_tl_prefix = {} - for tl_name, (remote_path, _component) in self.real_components.items(): - if remote_path and remote_path != tl_name: - hf_to_tl_prefix[remote_path] = tl_name - - normalized_state_dict = {} - for key, value in state_dict.items(): - new_key = key - for hf_prefix, tl_prefix in hf_to_tl_prefix.items(): - if key.startswith(hf_prefix + "."): - suffix = key[len(hf_prefix) + 1 :] - new_key = f"{tl_prefix}.{suffix}" - break - normalized_state_dict[new_key] = value - state_dict = normalized_state_dict - - if verbose: - print(" Distributing weights to generalized components...") - ProcessWeights.distribute_weights_to_components( - state_dict=state_dict, - component_mapping=self.real_components, - ) - - def _calculate_loss(self, logits, tokens, loss_per_token=False): - """Calculate cross-entropy loss.""" - shift_logits = logits[..., :-1, :].contiguous() - shift_labels = tokens[..., 1:].contiguous() - loss_fct = torch.nn.CrossEntropyLoss(reduction="none" if loss_per_token else "mean") - flat_logits = shift_logits.view(-1, shift_logits.size(-1)) - flat_labels = shift_labels.view(-1) - loss = loss_fct(flat_logits, flat_labels) - if loss_per_token: - return loss.view(shift_labels.shape) - else: - return loss - - def _extract_hf_weights(self): - """Extract weights from the original HuggingFace model.""" - hf_state_dict = self.state_dict() - for layer_idx in range(self.cfg.n_layers): - combined_qkv_key = f"transformer.h.{layer_idx}.attn.c_attn.weight" - combined_qkv_bias_key = f"transformer.h.{layer_idx}.attn.c_attn.bias" - if combined_qkv_key in hf_state_dict: - separate_keys_to_remove = [ - f"transformer.h.{layer_idx}.attn.q.weight", - f"transformer.h.{layer_idx}.attn.q.bias", - f"transformer.h.{layer_idx}.attn.k.weight", - f"transformer.h.{layer_idx}.attn.k.bias", - f"transformer.h.{layer_idx}.attn.v.weight", - f"transformer.h.{layer_idx}.attn.v.bias", - ] - for key_to_remove in separate_keys_to_remove: - if key_to_remove in hf_state_dict: - del hf_state_dict[key_to_remove] - return hf_state_dict - - def to_tokens( - self, - input: Union[str, List[str]], - prepend_bos: Optional[bool] = None, - padding_side: Optional[str] = None, - move_to_device: bool = True, - truncate: bool = True, - ) -> torch.Tensor: - """Converts a string to a tensor of tokens. - - See the class-level "Tokenization notes" for full ``prepend_bos`` - semantics, the ``default_prepend_bos`` / - ``tokenizer_prepends_bos`` interaction, and the whitespace- - sensitivity gotcha. **Pass ``prepend_bos=False`` whenever you're - tokenizing only part of a prompt.** - - Args: - input: The input to tokenize. - prepend_bos: Overrides ``self.cfg.default_prepend_bos``. Defaults - to ``None`` (use the cfg setting). Pass ``True`` or ``False`` - to override locally. - padding_side: Which side to pad on when tokenizing multiple - strings of different lengths. Defaults to the tokenizer's - ``padding_side``. - move_to_device: Whether to move the result to ``cfg.device``. - truncate: Whether to truncate inputs longer than ``cfg.n_ctx``. - - Returns: - Token tensor of shape ``[batch, pos]``. - """ - assert self.tokenizer is not None, "Cannot use to_tokens without a tokenizer" - if prepend_bos is None: - prepend_bos = getattr(self.cfg, "default_prepend_bos", True) - if padding_side is None: - padding_side = getattr(self.tokenizer, "padding_side", "right") - tokenizer_prepends_bos = getattr(self.cfg, "tokenizer_prepends_bos", True) - if prepend_bos and (not tokenizer_prepends_bos): - input = utils.get_input_with_manually_prepended_bos(self.tokenizer.bos_token, input) - if isinstance(input, str): - input = [input] - tokens = self.tokenizer( - input, - return_tensors="pt", - padding=True, - truncation=truncate, - max_length=self.cfg.n_ctx if truncate else None, - )["input_ids"] - # Strip auto-appended EOS tokens (e.g., OLMo) - if ( - getattr(self.cfg, "tokenizer_appends_eos", False) - and self.tokenizer.eos_token_id is not None - ): - # Remove trailing EOS, keep at least 1 token - while tokens.shape[-1] > 1 and (tokens[:, -1] == self.tokenizer.eos_token_id).all(): - tokens = tokens[:, :-1] - if not prepend_bos and tokenizer_prepends_bos: - tokens = utils.get_tokens_with_bos_removed(self.tokenizer, tokens) - if move_to_device: - tokens = tokens.to(self.cfg.device) - return tokens - - def to_string( - self, tokens: Union[List[int], torch.Tensor, np.ndarray] - ) -> Union[str, List[str]]: - """Convert tokens to string(s). - - Args: - tokens: Tokens to convert - - Returns: - Decoded string(s) - """ - if not isinstance(tokens, torch.Tensor): - tokens = torch.tensor(tokens) - if len(tokens.shape) == 2: - return self.tokenizer.batch_decode(tokens, clean_up_tokenization_spaces=False) - elif len(tokens.shape) <= 1: - return self.tokenizer.decode(tokens, clean_up_tokenization_spaces=False) - else: - raise ValueError(f"Invalid shape passed in: {tokens.shape}") - - def to_str_tokens( - self, - input: Union[str, torch.Tensor, np.ndarray, List], - prepend_bos: Optional[bool] = None, - padding_side: Optional[str] = None, - ) -> Union[List[str], List[List[str]]]: - """Map text or tokens to a list of tokens as strings. - - See the class-level "Tokenization notes" for full ``prepend_bos`` - semantics. **Pass ``prepend_bos=False`` whenever you're tokenizing - only part of a prompt.** When ``input`` is already a tensor or - array, ``prepend_bos`` and ``padding_side`` are ignored. - - Args: - input: A string, list of strings, or tensor/array of token IDs. - prepend_bos: Overrides ``self.cfg.default_prepend_bos``. Only - applies when ``input`` is a string. Defaults to ``None`` - (use the cfg setting). - padding_side: Which side to pad on. Only applies when ``input`` - is a string. - - Returns: - List of token strings. - """ - if isinstance(input, list): - return cast( - List[List[str]], - [self.to_str_tokens(item, prepend_bos, padding_side) for item in input], - ) - elif isinstance(input, str): - tokens = self.to_tokens(input, prepend_bos=prepend_bos, padding_side=padding_side)[0] - elif isinstance(input, torch.Tensor): - tokens = input.squeeze() - if tokens.dim() == 0: - tokens = tokens.unsqueeze(0) - assert ( - tokens.dim() == 1 - ), f"Invalid tokens input to to_str_tokens, has shape: {tokens.shape}" - elif isinstance(input, np.ndarray): - tokens_np = input.squeeze() - if tokens_np.ndim == 0: - tokens_np = np.expand_dims(tokens_np, axis=0) - assert ( - tokens_np.ndim == 1 - ), f"Invalid tokens input to to_str_tokens, has shape: {tokens_np.shape}" - tokens = torch.tensor(tokens_np) - else: - raise ValueError(f"Invalid input type to to_str_tokens: {type(input)}") - # v5 compat: wrap each token so batch_decode decodes them individually - tokens_list = [[int(t)] for t in tokens.tolist()] - str_tokens = self.tokenizer.batch_decode(tokens_list, clean_up_tokenization_spaces=False) - return str_tokens - - def to_single_token(self, string: str) -> int: - """Map a string that makes up a single token to the id for that token. - - Args: - string: The string to convert - - Returns: - Token ID - - Raises: - AssertionError: If string is not a single token - """ - token = self.to_tokens(string, prepend_bos=False).squeeze() - if token.numel() != 1: - raise AssertionError(f"Input string: {string} is not a single token!") - return int(token.item()) - - def get_token_position( - self, - single_token: Union[str, int], - input: Union[str, torch.Tensor], - mode="first", - prepend_bos: Optional[Union[bool, None]] = None, - padding_side: Optional[Union[Literal["left", "right"], None]] = None, - ): - """Get the position of a single_token in a string or sequence of tokens. - - Raises an error if the token is not present. - - When ``input`` is a string it's tokenized internally — see the - class-level "Tokenization notes" for ``prepend_bos`` semantics. - Off-by-one position errors usually mean ``prepend_bos`` is on - when it shouldn't be (or vice versa); pass ``prepend_bos=False`` - when ``input`` is a fragment of a larger prompt. - - Args: - single_token (Union[str, int]): The token to search for. Can - be a token index, or a string (but the string must correspond to a single token). - input (Union[str, torch.Tensor]): The sequence to - search in. Can be a string or a rank 1 tensor of tokens or a rank 2 tensor of tokens - with a dummy batch dimension. - mode (str, optional): If there are multiple matches, which match to return. Supports - "first" or "last". Defaults to "first". - prepend_bos (bool, optional): Overrides ``self.cfg.default_prepend_bos``. Only - applies when ``input`` is a string. Defaults to ``None`` (use the cfg setting). - padding_side (Union[Literal["left", "right"], None], optional): Specifies which - side to pad when tokenizing multiple strings of different lengths. - """ - if isinstance(input, str): - tokens = self.to_tokens(input, prepend_bos=prepend_bos, padding_side=padding_side) - else: - tokens = input - if len(tokens.shape) == 2: - assert ( - tokens.shape[0] == 1 - ), f"If tokens are rank two, they must have shape [1, seq_len], not {tokens.shape}" - tokens = tokens[0] - if isinstance(single_token, str): - single_token = self.to_single_token(single_token) - elif isinstance(single_token, torch.Tensor): - single_token = single_token.item() - indices = torch.arange(len(tokens), device=tokens.device)[tokens == single_token] - assert len(indices) > 0, "The token does not occur in the prompt" - if mode == "first": - return indices[0].item() - elif mode == "last": - return indices[-1].item() - else: - raise ValueError(f"mode must be 'first' or 'last', not {mode}") - - def to_single_str_token(self, int_token: int) -> str: - """Get the single token corresponding to an int in string form. - - Args: - int_token: The token ID - - Returns: - The token string - """ - assert isinstance(int_token, int) - token = self.to_str_tokens(torch.tensor([int_token])) - if isinstance(token, list) and len(token) == 1: - return str(token[0]) - raise AssertionError("Expected a single string token.") - - def blocks_with(self, submodule: str) -> List[Tuple[int, "GeneralizedComponent"]]: - """Return (index, block) pairs for blocks with the named bridged submodule. - - Checks _modules (not hasattr) so HF-internal attrs don't match. - Use instead of assuming blocks[0] is representative on hybrid models. - """ - if not hasattr(self, "blocks"): - return [] - return [(i, block) for i, block in enumerate(self.blocks) if submodule in block._modules] - - def stack_params_for( - self, submodule: str, attr_path: str, reshape_fn: Optional[Callable] = None - ) -> Tuple[List[int], torch.Tensor]: - """Stack a parameter across matching blocks only. Returns (layer_indices, tensor). - - Use for hybrid models where not all blocks have the submodule. - """ - matching = self.blocks_with(submodule) - if not matching: - raise ValueError( - f"No blocks have submodule '{submodule}'. " - f"Available submodules can be checked with blocks_with()." - ) - indices: List[int] = [] - weights: List[torch.Tensor] = [] - for idx, block in matching: - w = _resolve_attr_path(block, attr_path) - if reshape_fn is not None: - w = reshape_fn(w) - weights.append(w) - indices.append(idx) - return indices, torch.stack(weights, dim=0) - - def _stack_block_params( - self, attr_path: str, reshape_fn: Optional[Callable] = None - ) -> torch.Tensor: - """Stack a parameter across all blocks; falls back to matching-only on hybrids. - - On hybrid models, logs a warning about index mapping and returns only - blocks that have the submodule. First path segment is checked against - _modules; deeper segments resolve via getattr (intentional — W_Q etc. - are exposed via __getattr__ delegation). - """ - first_attr = attr_path.split(".")[0] - matching_blocks = [ - (i, block) for i, block in enumerate(self.blocks) if first_attr in block._modules - ] - - if len(matching_blocks) == 0: - raise AttributeError( - f"No blocks have submodule '{first_attr}'. " - f"Use bridge.blocks_with('{first_attr}') to check availability." - ) - - if len(matching_blocks) < len(self.blocks): - indices = [i for i, _ in matching_blocks] - logging.warning( - "Hybrid model: only %d/%d blocks have '%s'. Returning stacked tensor " - "for layers %s only. Tensor index i corresponds to original layer " - "indices[i], not layer i. For explicit index mapping, use " - "bridge.stack_params_for('%s', '%s').", - len(matching_blocks), - len(self.blocks), - first_attr, - indices, - first_attr, - attr_path, - ) - - weights: List[torch.Tensor] = [] - for _, block in matching_blocks: - w = _resolve_attr_path(block, attr_path) - if reshape_fn is not None: - w = reshape_fn(w) - weights.append(w) - # Under a device_map split, per-block tensors live on different devices. - # torch.stack requires a common device; gather onto cfg.device (the embedding / - # input device — a natural "home" for cross-layer reductions). - if getattr(self.cfg, "n_devices", 1) > 1 and weights and self.cfg.device is not None: - target_device = torch.device(self.cfg.device) - weights = [w.to(target_device) for w in weights] - return torch.stack(weights, dim=0) - - def _reshape_qkv(self, w: torch.Tensor) -> torch.Tensor: - """Reshape 2D [d_model, d_model] QKV weight to 3D [n_heads, d_model, d_head].""" - if w.shape == (self.cfg.d_model, self.cfg.d_model): - d_head = self.cfg.d_model // self.cfg.n_heads - return w.reshape(self.cfg.n_heads, self.cfg.d_model, d_head) - return w - - def _reshape_o(self, w: torch.Tensor) -> torch.Tensor: - """Reshape 2D [d_model, d_model] O weight to 3D [n_heads, d_head, d_model].""" - if w.shape == (self.cfg.d_model, self.cfg.d_model): - d_head = self.cfg.d_model // self.cfg.n_heads - return w.reshape(self.cfg.n_heads, d_head, self.cfg.d_model) - return w - - @property - def W_K(self) -> torch.Tensor: - """Stack the key weights across all layers.""" - return self._stack_block_params("attn.W_K", self._reshape_qkv) - - @property - def W_Q(self) -> torch.Tensor: - """Stack the query weights across all layers.""" - return self._stack_block_params("attn.W_Q", self._reshape_qkv) - - @property - def W_V(self) -> torch.Tensor: - """Stack the value weights across all layers.""" - return self._stack_block_params("attn.W_V", self._reshape_qkv) - - @property - def W_O(self) -> torch.Tensor: - """Stack the attn output weights across all layers.""" - return self._stack_block_params("attn.W_O", self._reshape_o) - - @property - def W_in(self) -> torch.Tensor: - """Stack the MLP input weights across all layers.""" - return self._stack_block_params("mlp.W_in") - - @property - def W_gate(self) -> Union[torch.Tensor, None]: - """Stack the MLP gate weights across all layers (gated MLPs only).""" - if getattr(self.cfg, "gated_mlp", False): - return self._stack_block_params("mlp.W_gate") - return None - - @property - def W_out(self) -> torch.Tensor: - """Stack the MLP output weights across all layers.""" - return self._stack_block_params("mlp.W_out") - - @property - def b_K(self) -> torch.Tensor: - """Stack the key biases across all layers.""" - return self._stack_block_params("attn.b_K") - - @property - def b_Q(self) -> torch.Tensor: - """Stack the query biases across all layers.""" - return self._stack_block_params("attn.b_Q") - - @property - def b_V(self) -> torch.Tensor: - """Stack the value biases across all layers.""" - return self._stack_block_params("attn.b_V") - - @property - def b_O(self) -> torch.Tensor: - """Stack the attn output biases across all layers.""" - return self._stack_block_params("attn.b_O") - - @property - def b_in(self) -> torch.Tensor: - """Stack the MLP input biases across all layers.""" - return self._stack_block_params("mlp.b_in") - - @property - def b_out(self) -> torch.Tensor: - """Stack the MLP output biases across all layers.""" - return self._stack_block_params("mlp.b_out") - - @property - def W_U(self) -> torch.Tensor: - """Unembedding matrix (d_model, d_vocab). Maps residual stream to logits.""" - return self.unembed.W_U - - @property - def b_U(self) -> torch.Tensor: - """Unembedding bias (d_vocab).""" - return self.unembed.b_U - - @property - def W_E(self) -> torch.Tensor: - """Token embedding matrix (d_vocab, d_model).""" - return self.embed.W_E - - @property - def QK(self): - """QK circuit. On hybrids, returns attn layers only (with warning). See QK_for_attn_layers().""" - return FactoredMatrix(self.W_Q, self.W_K.transpose(-2, -1)) - - @property - def OV(self): - """OV circuit. On hybrids, returns attn layers only (with warning). See OV_for_attn_layers().""" - return FactoredMatrix(self.W_V, self.W_O) - - def QK_for_attn_layers(self) -> Tuple[List[int], FactoredMatrix]: - """QK circuit for attention layers only. Returns (layer_indices, FactoredMatrix).""" - q_indices, W_Q = self.stack_params_for("attn", "attn.W_Q", self._reshape_qkv) - _, W_K = self.stack_params_for("attn", "attn.W_K", self._reshape_qkv) - return q_indices, FactoredMatrix(W_Q, W_K.transpose(-2, -1)) - - def OV_for_attn_layers(self) -> Tuple[List[int], FactoredMatrix]: - """OV circuit for attention layers only. Returns (layer_indices, FactoredMatrix).""" - v_indices, W_V = self.stack_params_for("attn", "attn.W_V", self._reshape_qkv) - _, W_O = self.stack_params_for("attn", "attn.W_O", self._reshape_o) - return v_indices, FactoredMatrix(W_V, W_O) - - # ------------------------------------------------------------------ - # Mechanistic interpretability analysis methods - # ------------------------------------------------------------------ - - def tokens_to_residual_directions( - self, - tokens: Union[str, int, torch.Tensor], - ) -> torch.Tensor: - """Map tokens to their unembedding vectors (residual stream directions). - - Returns the columns of W_U corresponding to the given tokens — i.e. the - directions in the residual stream that the model dots with to produce the - logit for each token. - - WARNING: If you use this without folding in LayerNorm (compatibility mode), - the results will be misleading because LN weights change the unembed map. - - Args: - tokens: A single token (str, int, or scalar tensor), a 1-D tensor of - token IDs, or a 2-D batch of token IDs. - - Returns: - Tensor of unembedding vectors with shape matching the input token shape - plus a trailing d_model dimension. - """ - if isinstance(tokens, torch.Tensor) and tokens.numel() > 1: - residual_directions = self.W_U[:, tokens] - residual_directions = einops.rearrange( - residual_directions, "d_model ... -> ... d_model" - ) - return residual_directions - else: - if isinstance(tokens, str): - token = self.to_single_token(tokens) - elif isinstance(tokens, int): - token = tokens - elif isinstance(tokens, torch.Tensor) and tokens.numel() == 1: - token = int(tokens.item()) - else: - raise ValueError(f"Invalid token type: {type(tokens)}") - residual_direction = self.W_U[:, token] - return residual_direction - - # Variant → attr paths for the output bias that feeds the residual stream. - _VARIANT_OUTPUT_BIAS_ATTRS: Dict[str, tuple] = { - "attn": ("b_O",), - "linear_attn": ("out_proj.bias",), - "mamba": ("out_proj.bias",), - "mixer": ("out_proj.bias",), - "ssm": ("out_proj.bias",), - } - - def _get_block_variant_bias(self, block: "GeneralizedComponent") -> Optional[torch.Tensor]: - """Return the output bias from this block's variant submodule, or None.""" - for name in VARIANT_SUBMODULE_NAMES: - if name not in block._modules: - continue - variant = block._modules[name] - for attr_path in self._VARIANT_OUTPUT_BIAS_ATTRS.get(name, ()): - obj = variant - try: - for attr in attr_path.split("."): - obj = getattr(obj, attr) - except AttributeError: - continue - if obj is not None and isinstance(obj, torch.Tensor): - return obj - return None - - def accumulated_bias( - self, - layer: int, - mlp_input: bool = False, - include_mlp_biases: bool = True, - ) -> torch.Tensor: - """Sum of variant + MLP output biases through the residual stream up to `layer`. - - Includes all layer types (attn, SSM, linear-attn). Set mlp_input=True - to include the variant bias of the target layer itself. - """ - accumulated = torch.zeros(self.cfg.d_model, device=self.cfg.device) - for i in range(layer): - block = self.blocks[i] - b_O = self._get_block_variant_bias(block) - if b_O is not None: - accumulated = accumulated + b_O.to(accumulated.device) - if include_mlp_biases and "mlp" in block._modules: - b_out = getattr(block.mlp, "b_out", None) - if b_out is not None: - accumulated = accumulated + b_out.to(accumulated.device) - if mlp_input: - assert layer < self.cfg.n_layers, "Cannot include attn_bias from beyond the final layer" - block = self.blocks[layer] - b_O = self._get_block_variant_bias(block) - if b_O is not None: - accumulated = accumulated + b_O.to(accumulated.device) - return accumulated - - def all_composition_scores(self, mode: str) -> CompositionScores: - """Composition scores for all attention head pairs. Returns CompositionScores. - - See https://transformer-circuits.pub/2021/framework/index.html - On hybrid models, only attention layers are included; layer_indices - maps tensor position i to original layer number. - """ - attn_blocks = self.blocks_with("attn") - if not attn_blocks: - raise ValueError("No attention layers found — cannot compute composition scores.") - - indices = [idx for idx, _ in attn_blocks] - blocks_list = [block for _, block in attn_blocks] - - def _stack(attr_path: str, reshape_fn: Optional[Callable] = None) -> torch.Tensor: - weights: List[torch.Tensor] = [] - for block in blocks_list: - w = _resolve_attr_path(block, attr_path) - if reshape_fn is not None: - w = reshape_fn(w) - weights.append(w) - # See _stack_block_params: gather per-block tensors onto cfg.device when split. - if getattr(self.cfg, "n_devices", 1) > 1 and weights and self.cfg.device is not None: - target_device = torch.device(self.cfg.device) - weights = [w.to(target_device) for w in weights] - return torch.stack(weights, dim=0) - - W_V = _stack("attn.W_V", self._reshape_qkv) - W_O = _stack("attn.W_O", self._reshape_o) - left = FactoredMatrix(W_V, W_O) - - if mode == "Q": - W_Q = _stack("attn.W_Q", self._reshape_qkv) - W_K = _stack("attn.W_K", self._reshape_qkv) - right = FactoredMatrix(W_Q, W_K.transpose(-2, -1)) - elif mode == "K": - W_Q = _stack("attn.W_Q", self._reshape_qkv) - W_K = _stack("attn.W_K", self._reshape_qkv) - right = FactoredMatrix(W_Q, W_K.transpose(-2, -1)).T - elif mode == "V": - right = left - else: - raise ValueError(f"mode must be one of ['Q', 'K', 'V'] not {mode}") - - scores = utils.composition_scores(left, right, broadcast_dims=True) - n_attn = len(indices) - idx_tensor = torch.arange(n_attn, device=self.cfg.device) - mask = idx_tensor[:, None, None, None] < idx_tensor[None, None, :, None] - scores = torch.where(mask, scores, torch.zeros_like(scores)) - - labels = [f"L{l}H{h}" for l in indices for h in range(self.cfg.n_heads)] - return CompositionScores(scores=scores, layer_indices=indices, head_labels=labels) - - def composition_layer_indices(self) -> List[int]: - """Original layer indices for attention layers (maps composition score positions).""" - return [idx for idx, _ in self.blocks_with("attn")] - - def block_hooks(self, layer_idx: int) -> List[str]: - """Sorted hook names available on block `layer_idx` (block-relative paths).""" - prefix = f"blocks.{layer_idx}." - return sorted(name[len(prefix) :] for name in self.hook_dict if name.startswith(prefix)) - - def block_submodules(self, layer_idx: int) -> List[str]: - """Return bridged submodule names on block `layer_idx`.""" - block = self.blocks[layer_idx] - return [name for name in block._modules if name not in _BLOCK_INTERNAL_MODULES] - - def layer_types(self) -> List[str]: - """Per-block type labels, e.g. ["attn+mlp", "ssm+mlp", ...]. Deterministic order.""" - types = [] - for block in self.blocks: - variants = [n for n in VARIANT_SUBMODULE_NAMES if n in block._modules] - universals = sorted( - n - for n in block._modules - if n not in _VARIANT_SUBMODULE_SET - and n not in _BLOCK_INTERNAL_MODULES - and not n.startswith(_NORM_PREFIXES) - ) - parts = variants + universals - types.append("+".join(parts) if parts else "unknown") - return types - - @property - def all_head_labels(self) -> list[str]: - """Human-readable labels for all attention heads, e.g. ['L0H0', 'L0H1', ...].""" - return [f"L{l}H{h}" for l in range(self.cfg.n_layers) for h in range(self.cfg.n_heads)] - - @property - def attn_head_labels(self) -> list[str]: - """Head labels for attention layers only — matches all_composition_scores() dims.""" - return [ - f"L{l}H{h}" for l in self.composition_layer_indices() for h in range(self.cfg.n_heads) - ] - - def parameters(self, recurse: bool = True) -> Iterator[nn.Parameter]: - """Returns parameters following standard PyTorch semantics. - - This method delegates to the underlying HuggingFace model's parameters(). - For TransformerLens-style parameter generator, use tl_parameters() instead. - - Args: - recurse: If True, yields parameters of this module and all submodules - - Returns: - Iterator of nn.Parameter objects - """ - return self.original_model.parameters(recurse=recurse) - - def named_parameters( - self, prefix: str = "", recurse: bool = True, remove_duplicate: bool = True - ) -> Iterator[tuple[str, nn.Parameter]]: - """Returns named parameters following standard PyTorch semantics. - - This method delegates to the underlying HuggingFace model's named_parameters(). - For TransformerLens-style generator, use tl_named_parameters() instead. - - Args: - prefix: Prefix to prepend to all parameter names - recurse: If True, yields parameters of this module and all submodules - remove_duplicate: If True, removes duplicate parameters - - Returns: - Iterator of (name, parameter) tuples - """ - return self.original_model.named_parameters(prefix, recurse, remove_duplicate) - - def tl_parameters(self) -> dict[str, torch.Tensor]: - """Returns TransformerLens-style parameter dictionary. - - Parameter names follow TransformerLens conventions (e.g., 'blocks.0.attn.W_Q') and may - include processed weights (non-leaf tensors). This format is expected by SVDInterpreter - among other analysis tools. - - Returns: - Dictionary mapping TransformerLens parameter names to tensors - - Example: - >>> bridge = TransformerBridge.boot_transformers("gpt2") - >>> tl_params = bridge.tl_parameters() - >>> W_Q = tl_params["blocks.0.attn.W_Q"] # Shape: [n_heads, d_model, d_head] - """ - return self.get_params() - - def tl_named_parameters(self) -> Iterator[tuple[str, torch.Tensor]]: - """Returns iterator of TransformerLens-style named parameters. - - This provides the same parameters as tl_parameters() but as an iterator - for consistency with PyTorch's named_parameters() API pattern. - - Returns: - Iterator of (name, tensor) tuples with TransformerLens naming conventions - - Example: - >>> bridge = TransformerBridge.boot_transformers("gpt2") - >>> for name, param in bridge.tl_named_parameters(): - ... if "attn.W_Q" in name: - ... print(f"{name}: {param.shape}") # doctest: +ELLIPSIS - blocks.0.attn.W_Q: torch.Size([12, 768, 64]) - ... - """ - return iter(self.get_params().items()) - - def forward( - self, - input: Union[str, List[str], torch.Tensor], - return_type: Optional[str] = "logits", - loss_per_token: bool = False, - prepend_bos: Optional[bool] = None, - padding_side: Optional[str] = None, - attention_mask: Optional[torch.Tensor] = None, - start_at_layer: Optional[int] = None, - stop_at_layer: Optional[int] = None, - pixel_values: Optional[torch.Tensor] = None, - input_values: Optional[torch.Tensor] = None, - **kwargs, - ) -> Any: - """Forward pass through the model. - - Args: - input: Input to the model - return_type: Type of output to return ('logits', 'loss', 'both', 'predictions', None) - loss_per_token: Whether to return loss per token - prepend_bos: Whether to prepend BOS token - padding_side: Which side to pad on - start_at_layer: Not implemented in TransformerBridge. The bridge delegates - to HuggingFace's model.forward() which owns the layer iteration loop, - making start_at_layer infeasible without monkey-patching HF internals - (fragile across HF versions) or exception-based layer skipping (corrupts - model state). Raises NotImplementedError if a non-None value is passed. - stop_at_layer: Layer to stop forward pass at - pixel_values: Optional image tensor for multimodal models (e.g., LLaVA, Gemma3). - The tensor is passed directly to the underlying HuggingFace model. - Only valid when cfg.is_multimodal is True. - input_values: Optional audio waveform tensor for audio models (e.g., HuBERT). - The tensor is passed directly to the underlying HuggingFace model. - Only valid when cfg.is_audio_model is True. - **kwargs: Additional arguments passed to model - - Returns: - Model output based on return_type - """ - - if start_at_layer is not None: - raise NotImplementedError( - "start_at_layer is not supported in TransformerBridge. " - "The bridge delegates to HuggingFace's model.forward() which controls " - "the layer iteration loop. See the TransformerBridge review plan for a " - "detailed analysis of implementation approaches and their tradeoffs." - ) - - # Set stop_at_layer flag on all blocks if requested - if stop_at_layer is not None and hasattr(self, "blocks"): - for block in self.blocks: - block._stop_at_layer_idx = stop_at_layer - - # Map HookedEncoderDecoder-style kwargs to HF-compatible names - if "decoder_input" in kwargs: - kwargs["decoder_input_ids"] = kwargs.pop("decoder_input") - if "one_zero_attention_mask" in kwargs: - if attention_mask is None: - attention_mask = kwargs.pop("one_zero_attention_mask") - else: - kwargs.pop("one_zero_attention_mask") - - # Detect batched list input that will need padding. For this case we force - # left-padding internally and auto-compute attention_mask + position_ids - # (unless the caller passed them explicitly) so pad tokens don't contaminate - # attention or position embeddings. - _is_batched_list = ( - isinstance(input, list) - and len(input) > 1 - and not getattr(self.cfg, "is_audio_model", False) - ) - - try: - if isinstance(input, (str, list)): - if getattr(self.cfg, "is_audio_model", False): - raise ValueError( - "Audio models require tensor input (raw waveform), not text. " - "Pass a torch.Tensor or use the input_values parameter." - ) - if _is_batched_list and padding_side is None: - # Force left-padding so real tokens are flush-right. - _orig_padding_side = self.tokenizer.padding_side - self.tokenizer.padding_side = "left" - try: - input_ids = self.to_tokens( - input, prepend_bos=prepend_bos, padding_side=padding_side - ) - finally: - self.tokenizer.padding_side = _orig_padding_side - else: - input_ids = self.to_tokens( - input, prepend_bos=prepend_bos, padding_side=padding_side - ) - else: - input_ids = input - # Promote 1D integer token tensors to 2D [batch=1, seq] to match - # HookedTransformer's contract. Float tensors (inputs_embeds, - # audio waveforms) are passed through unchanged. - if ( - isinstance(input_ids, torch.Tensor) - and input_ids.ndim == 1 - and not input_ids.is_floating_point() - ): - input_ids = input_ids.unsqueeze(0) - - # Detect inputs_embeds: if the tensor is floating point, it's pre-computed - # embeddings (e.g., from multimodal models) rather than token IDs. - _is_inputs_embeds = ( - isinstance(input_ids, torch.Tensor) and input_ids.is_floating_point() - ) - - # Auto-compute attention_mask + position_ids for batched list input - # when the caller didn't supply them. Matches HF generation convention. - if ( - _is_batched_list - and attention_mask is None - and self.tokenizer is not None - and self.tokenizer.pad_token_id is not None - and not _is_inputs_embeds - ): - _prev_side = self.tokenizer.padding_side - self.tokenizer.padding_side = "left" - try: - attention_mask = utils.get_attention_mask( - self.tokenizer, - input_ids, - prepend_bos=getattr(self.cfg, "default_prepend_bos", True), - ).to(self.cfg.device) - finally: - self.tokenizer.padding_side = _prev_side - if "position_ids" not in kwargs: - position_ids = attention_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attention_mask == 0, 1) - kwargs["position_ids"] = position_ids - - if attention_mask is not None: - kwargs["attention_mask"] = attention_mask - if kwargs.pop("use_past_kv_cache", False) or kwargs.get("use_cache", False): - kwargs["use_cache"] = True - # Auto-generate decoder_input_ids for encoder-decoder models - if ( - "decoder_input_ids" not in kwargs - and hasattr(self.original_model, "config") - and getattr(self.original_model.config, "is_encoder_decoder", False) - ): - decoder_start_token_id = getattr( - self.original_model.config, "decoder_start_token_id", None - ) - if decoder_start_token_id is not None: - shifted = input_ids[:, :-1] - start_tokens = torch.full( - (input_ids.shape[0], 1), - decoder_start_token_id, - dtype=input_ids.dtype, - device=input_ids.device, - ) - kwargs["decoder_input_ids"] = torch.cat([start_tokens, shifted], dim=1) - else: - kwargs["decoder_input_ids"] = input_ids - - # Tell PosEmbedBridge to expand batch=1 position_ids to full batch. - if hasattr(self, "pos_embed"): - self.pos_embed._current_batch_size = input_ids.shape[0] - - # Handle pixel_values for multimodal models - if pixel_values is not None: - if not getattr(self.cfg, "is_multimodal", False): - raise ValueError( - "pixel_values can only be passed to multimodal models " - "(cfg.is_multimodal must be True)" - ) - kwargs["pixel_values"] = pixel_values - - # Handle input_values for audio models - if input_values is not None: - if not getattr(self.cfg, "is_audio_model", False): - raise ValueError( - "input_values can only be passed to audio models " - "(cfg.is_audio_model must be True)" - ) - kwargs["input_values"] = input_values - - # Audio models use input_values (waveform), not input_ids - if getattr(self.cfg, "is_audio_model", False): - if input_values is not None: - output = self.original_model(**kwargs) - elif isinstance(input, torch.Tensor): - kwargs["input_values"] = input - output = self.original_model(**kwargs) - else: - raise ValueError( - "Audio models require tensor input (raw waveform). " - "Pass a torch.Tensor or use input_values parameter." - ) - elif _is_inputs_embeds: - output = self.original_model(inputs_embeds=input_ids, **kwargs) - else: - output = self.original_model(input_ids, **kwargs) - # Stash only the cache object (not the full output) for generate(). - if getattr(self, "_capture_hf_cache", False): - self._last_hf_cache = getattr(output, "past_key_values", None) - if hasattr(output, "logits"): - logits = output.logits - elif isinstance(output, tuple) and len(output) > 0: - logits = output[0] - else: - logits = output - if return_type == "logits": - return logits - elif return_type == "loss": - if getattr(self.cfg, "is_audio_model", False): - raise ValueError( - "Audio models do not support return_type='loss'. " - "CTC loss requires aligned frame-level labels." - ) - if _is_inputs_embeds: - raise ValueError( - "Cannot compute loss with inputs_embeds — token IDs required for labels." - ) - # Always use self.loss_fn for consistency with HT's formula - # (log_softmax + gather). HF's output.loss uses F.cross_entropy - # which gives different results in bfloat16. - assert isinstance( - logits, torch.Tensor - ), f"Expected logits tensor, got {type(logits)}" - return self.loss_fn(logits, input_ids, per_token=loss_per_token) - elif return_type == "both": - if getattr(self.cfg, "is_audio_model", False): - raise ValueError( - "Audio models do not support return_type='both'. " - "CTC loss requires aligned frame-level labels." - ) - if _is_inputs_embeds: - raise ValueError( - "Cannot compute loss with inputs_embeds — token IDs required for labels." - ) - assert isinstance( - logits, torch.Tensor - ), f"Expected logits tensor, got {type(logits)}" - loss = self.loss_fn(logits, input_ids, per_token=loss_per_token) - return (logits, loss) - elif return_type == "predictions": - assert ( - self.tokenizer is not None - ), "Must have a tokenizer to use return_type='predictions'" - if logits.shape[-1] == 2: - # Next Sentence Prediction — 2-class output - logprobs = logits.log_softmax(dim=-1) - predictions = [ - "The sentences are sequential", - "The sentences are NOT sequential", - ] - return predictions[logprobs.argmax(dim=-1).item()] - else: - # Masked Language Modeling — decode [MASK] tokens - logprobs = logits[input_ids == self.tokenizer.mask_token_id].log_softmax(dim=-1) - predictions = self.tokenizer.decode(logprobs.argmax(dim=-1)) - if " " in predictions: - predictions = predictions.split(" ") - predictions = [f"Prediction {i}: {p}" for i, p in enumerate(predictions)] - return predictions - elif return_type is None: - return None - else: - raise ValueError(f"Invalid return_type: {return_type}") - except StopAtLayerException as e: - # Execution stopped at the requested layer - return e.layer_output - finally: - # Clean up state that may be inconsistent after StopAtLayerException - if stop_at_layer is not None and hasattr(self, "blocks"): - # Reset the stop flag on all blocks - for block in self.blocks: - block._stop_at_layer_idx = None - - # Clear any stale KV cache — layers after the stop point didn't - # execute, so the cache is incomplete and would corrupt subsequent - # generate() calls that expect a full cache. - if hasattr(self, "_last_hf_cache"): - del self._last_hf_cache - - def get_hook_point(self, hook_name: str) -> Optional[HookPoint]: - """Get a hook point by name from the bridge's hook system.""" - if hook_name in self._hook_registry: - return self._hook_registry[hook_name] - try: - parts = hook_name.split(".") - current = self - for part in parts: - current = getattr(current, part) - if isinstance(current, HookPoint): - return current - except AttributeError: - pass - return None +Historically, :class:`TransformerBridge` lived in this module. The class was +split into three (``BridgeCore`` framework-agnostic parent, ``TransformerBridge`` +torch-backed, ``RemoteBridge`` non-torch) across their own files; this shim +preserves the original import path: - def loss_fn( - self, - logits: torch.Tensor, - tokens: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - per_token: bool = False, - ) -> torch.Tensor: - """Calculate cross-entropy loss. - - Uses the same formula as HookedTransformer (log_softmax + gather) to ensure - numerically identical results when logits match. - - Args: - logits: Model logits - tokens: Target tokens - attention_mask: Optional attention mask for padding - per_token: Whether to return per-token loss - - Returns: - Loss tensor - """ - if tokens.device != logits.device: - tokens = tokens.to(logits.device) - return lm_cross_entropy_loss(logits, tokens, attention_mask, per_token) - - @overload - def run_with_cache( - self, - input: Union[str, List[str], torch.Tensor], - return_cache_object: Literal[True] = True, - remove_batch_dim: bool = False, - **kwargs, - ) -> Tuple[Any, ActivationCache]: - """Run with cache - placeholder implementation.""" - pass - - @overload - def run_with_cache( - self, - input: Union[str, List[str], torch.Tensor], - return_cache_object: Literal[False], - remove_batch_dim: bool = False, - **kwargs, - ) -> Tuple[Any, Dict[str, torch.Tensor]]: - """Run with cache - placeholder implementation.""" - pass - - def run_with_cache( - self, - input: Union[str, List[str], torch.Tensor], - return_cache_object: bool = True, - remove_batch_dim: bool = False, - names_filter: Optional[Union[str, List[str], Callable[[str], bool]]] = None, - stop_at_layer: Optional[int] = None, - **kwargs, - ) -> Tuple[Any, Union[ActivationCache, Dict[str, torch.Tensor]]]: - """Run the model and cache all activations. - - Args: - input: Input to the model - return_cache_object: Whether to return ActivationCache object - remove_batch_dim: Whether to remove batch dimension - names_filter: Filter for which activations to cache (str, list of str, or callable) - stop_at_layer: Layer to stop forward pass at (uses StopAtLayerException; cleans up KV cache on stop) - **kwargs: Additional arguments - # type: ignore[name-defined] - Returns: - Tuple of (output, cache) - """ - aliases = build_alias_to_canonical_map(self.hook_dict) - - def create_names_filter_fn(filter_input): - if filter_input is None: - return lambda name: True - elif isinstance(filter_input, str): - mapped_name = aliases.get(filter_input, None) - if mapped_name: - return lambda name: name == mapped_name or name == filter_input - else: - return lambda name: name == filter_input - elif isinstance(filter_input, list): - mapped_list = [] - for item in filter_input: - mapped_list.append(item) - mapped_name = aliases.get(item, None) - if mapped_name: - mapped_list.append(mapped_name) - return lambda name: name in mapped_list - elif callable(filter_input): - return filter_input - else: - raise ValueError("names_filter must be a string, list of strings, or callable") - - names_filter_fn = create_names_filter_fn(names_filter) - cache: Dict[str, torch.Tensor] = {} - hooks: List[Tuple[HookPoint, str]] = [] - visited: set[int] = set() - - # None → no-op .to(None), tensors stay on their current device. - cache_device = kwargs.pop("device", None) - - def make_cache_hook(name: str): - def cache_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: - if tensor is None: - cache[name] = None - elif isinstance(tensor, torch.Tensor): - cache[name] = tensor.detach().to(cache_device) - elif isinstance(tensor, tuple): - if len(tensor) > 0 and isinstance(tensor[0], torch.Tensor): - cache[name] = tensor[0].detach().to(cache_device) - else: - pass - else: - try: - if hasattr(tensor, "detach"): - cache[name] = tensor.detach().to(cache_device) - except: - pass - return tensor - - return cache_hook - - hook_dict = self.hook_dict - effective_stop_layer = None - if stop_at_layer is not None and hasattr(self, "blocks"): - if stop_at_layer < 0: - effective_stop_layer = len(self.blocks) + stop_at_layer - else: - effective_stop_layer = stop_at_layer - for hook_name, hook in hook_dict.items(): - if names_filter_fn(hook_name): - if effective_stop_layer is not None: - if hook_name.startswith("blocks."): - try: - layer_num = int(hook_name.split(".")[1]) - if layer_num >= effective_stop_layer: - continue - except (IndexError, ValueError): - pass - hooks.append((hook, hook_name)) - for hp, name in hooks: - hp.add_hook(make_cache_hook(name)) - processed_args = [input] - if processed_args and isinstance(processed_args[0], str): - assert self.tokenizer is not None, "Tokenizer must be set to pass string input." - input_ids = self.to_tokens(processed_args[0]) - input_ids = input_ids.to(next(self.original_model.parameters()).device) - kwargs["input_ids"] = input_ids - processed_args = processed_args[1:] - elif "input" in kwargs and isinstance(kwargs["input"], str): - assert self.tokenizer is not None, "Tokenizer must be set to pass string input." - input_ids = self.to_tokens(kwargs["input"]) - input_ids = input_ids.to(next(self.original_model.parameters()).device) - kwargs["input_ids"] = input_ids - del kwargs["input"] - if stop_at_layer is not None and hasattr(self, "blocks"): - if stop_at_layer < 0: - stop_at_layer = len(self.blocks) + stop_at_layer - last_layer_to_process = stop_at_layer - 1 - - def stop_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: - raise StopAtLayerException(tensor) - - if stop_at_layer >= 0 and stop_at_layer < len(self.blocks): - # Stop at the beginning of the specified block, not at the end of the previous block - block_hook_name = f"blocks.{stop_at_layer}.hook_in" - hook_dict = self.hook_dict - if block_hook_name in hook_dict: - hook_dict[block_hook_name].add_hook(stop_hook) - hooks.append((hook_dict[block_hook_name], block_hook_name)) - filtered_kwargs = kwargs.copy() - if cache_device is not None: - if getattr(self.cfg, "n_devices", 1) > 1: - # Moving a dispatched model to a single device collapses accelerate's - # split and breaks its routing hooks. The cache will stay spread across - # the per-layer devices; callers can .to(cache_device) on cache entries - # after the fact if they need a single-device cache. - warnings.warn( - f"run_with_cache(device={cache_device!r}) ignored: model is dispatched " - f"across {self.cfg.n_devices} devices via device_map. Cached activations " - "will remain on their per-layer devices.", - stacklevel=2, - ) - else: - self.original_model = self.original_model.to(cache_device) - if processed_args and isinstance(processed_args[0], torch.Tensor): - processed_args = [processed_args[0].to(cache_device)] + list(processed_args[1:]) - for key, value in filtered_kwargs.items(): - if isinstance(value, torch.Tensor): - filtered_kwargs[key] = value.to(cache_device) - try: - if "output_attentions" not in filtered_kwargs: - filtered_kwargs["output_attentions"] = True - if processed_args: - output = self.forward(processed_args[0], **filtered_kwargs) - elif "input_ids" in filtered_kwargs: - output = self.forward( - filtered_kwargs["input_ids"], - **{k: v for k, v in filtered_kwargs.items() if k != "input_ids"}, - ) - else: - output = self.forward(**filtered_kwargs) - if hasattr(output, "logits"): - output = output.logits - except StopAtLayerException as e: - output = e.layer_output - except Exception as e: - raise e - finally: - for hp, _ in hooks: - hp.remove_hooks() - if self.compatibility_mode == True: - reverse_aliases = {} - for old_name, new_name in aliases.items(): - if isinstance(new_name, list): - for single_new_name in new_name: - reverse_aliases[single_new_name] = old_name - else: - reverse_aliases[new_name] = old_name - cache_items_to_add = {} - for cache_name, cached_value in cache.items(): - for new_name, old_name in reverse_aliases.items(): - if cache_name == new_name: - cache_items_to_add[old_name] = cached_value - break - cache.update(cache_items_to_add) - for alias_name, target_name in aliases.items(): - if isinstance(target_name, list): - for single_target in target_name: - if single_target in cache and alias_name not in cache: - cache[alias_name] = cache[single_target] - break - elif target_name in cache and alias_name not in cache: - cache[alias_name] = cache[target_name] - if return_cache_object: - activation_cache = ActivationCache(cache, self, has_batch_dim=True) - if remove_batch_dim: - activation_cache.remove_batch_dim() - return (output, activation_cache) - else: - if remove_batch_dim: - for key in cache: - if cache[key] is not None and isinstance(cache[key], torch.Tensor): - if cache[key].size(0) == 1: - cache[key] = cache[key][0] - return (output, cache) - - def run_with_hooks( - self, - input: Union[str, List[str], torch.Tensor], - fwd_hooks: List[Tuple[Union[str, Callable], Callable]] = [], - bwd_hooks: List[Tuple[Union[str, Callable], Callable]] = [], - reset_hooks_end: bool = True, - clear_contexts: bool = False, - return_type: Optional[str] = "logits", - names_filter: Optional[Union[str, List[str], Callable[[str], bool]]] = None, - stop_at_layer: Optional[int] = None, - remove_batch_dim: bool = False, - **kwargs, - ) -> Any: - """Run the model with specified forward and backward hooks. - - Args: - input: Input to the model - fwd_hooks: Forward hooks to apply - bwd_hooks: Backward hooks to apply - reset_hooks_end: Whether to reset hooks at the end - clear_contexts: Whether to clear hook contexts - return_type: What to return ("logits", "loss", etc.) - names_filter: Filter for hook names (not used directly, for compatibility) - stop_at_layer: Layer to stop at (uses StopAtLayerException; cleans up KV cache on stop) - remove_batch_dim: Whether to remove batch dimension from hook inputs (only works for batch_size==1) - **kwargs: Additional arguments - - Returns: - Model output - """ - added_hooks: List[Tuple[HookPoint, str]] = [] - effective_stop_layer = None - if stop_at_layer is not None and hasattr(self, "blocks"): - if stop_at_layer < 0: - effective_stop_layer = len(self.blocks) + stop_at_layer - else: - effective_stop_layer = stop_at_layer - - def add_hook_to_point( - hook_point: HookPoint, hook_fn: Callable, name: str, dir: Literal["fwd", "bwd"] = "fwd" - ): - if effective_stop_layer is not None and name.startswith("blocks."): - try: - layer_num = int(name.split(".")[1]) - if layer_num >= effective_stop_layer: - return - except (IndexError, ValueError): - pass - if self.compatibility_mode and name != hook_point.name: - alias_names_list: list[str] = [] - if hook_point.name is not None: - alias_names_list.append(hook_point.name) - alias_names_list.append(name) - hook_point.add_hook(hook_fn, dir=dir, alias_names=alias_names_list) - else: - hook_point.add_hook(hook_fn, dir=dir) - added_hooks.append((hook_point, name)) - - if stop_at_layer is not None and hasattr(self, "blocks"): - if stop_at_layer < 0: - stop_at_layer = len(self.blocks) + stop_at_layer - last_layer_to_process = stop_at_layer - 1 - - def stop_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: - raise StopAtLayerException(tensor) - - if stop_at_layer >= 0 and stop_at_layer < len(self.blocks): - # Stop at the beginning of the specified block, not at the end of the previous block - block_hook_name = f"blocks.{stop_at_layer}.hook_in" - hook_dict = self.hook_dict - if block_hook_name in hook_dict: - add_hook_to_point(hook_dict[block_hook_name], stop_hook, block_hook_name, "fwd") - - def apply_hooks(hooks: List[Tuple[Union[str, Callable], Callable]], is_fwd: bool): - direction: Literal["fwd", "bwd"] = "fwd" if is_fwd else "bwd" - aliases = build_alias_to_canonical_map(self.hook_dict) - for hook_name_or_filter, hook_fn in hooks: - if remove_batch_dim: - original_hook_fn = hook_fn - - # Default arg captures hook_fn by value (avoids closure issue) - def wrapped_hook_fn(tensor, hook, _orig_fn=original_hook_fn): - if tensor.shape[0] == 1: - tensor_no_batch = tensor.squeeze(0) - result = _orig_fn(tensor_no_batch, hook) - if result.dim() == tensor_no_batch.dim(): - result = result.unsqueeze(0) - return result - else: - return _orig_fn(tensor, hook) - - hook_fn = wrapped_hook_fn - if isinstance(hook_name_or_filter, str): - hook_dict = self.hook_dict - actual_hook_name = hook_name_or_filter - if hook_name_or_filter in aliases: - actual_hook_name = aliases[hook_name_or_filter] - if actual_hook_name in hook_dict: - add_hook_to_point( - hook_dict[actual_hook_name], hook_fn, actual_hook_name, direction - ) - else: - hook_dict = self.hook_dict - seen_hooks = set() - for name, hook_point in hook_dict.items(): - if hook_name_or_filter(name): - hook_id = id(hook_point) - if hook_id in seen_hooks: - continue - seen_hooks.add(hook_id) - hook_name_to_use = hook_point.name if hook_point.name else name - add_hook_to_point(hook_point, hook_fn, hook_name_to_use, direction) - - try: - apply_hooks(fwd_hooks, True) - apply_hooks(bwd_hooks, False) - try: - output = self.forward( - input, return_type=return_type, stop_at_layer=stop_at_layer, **kwargs - ) - except StopAtLayerException as e: - output = e.layer_output - return output - finally: - if reset_hooks_end: - for hook_point, name in added_hooks: - hook_point.remove_hooks() - - def _generate_tokens( - self, - current_tokens: torch.Tensor, - input_tokens: torch.Tensor, - batch_size: int, - *, - max_new_tokens: int, - do_sample: bool, - top_k: Optional[int], - top_p: Optional[float], - temperature: float, - freq_penalty: float, - repetition_penalty: float, - stop_at_eos: bool, - stop_tokens: List[int], - eos_token_for_padding: int, - finished_sequences: torch.Tensor, - use_past_kv_cache: bool, - use_stateful_cache: bool, - mamba_cache: Any, - mamba_conv_kernel: int, - is_encoder_decoder: bool, - _is_batched_list: bool, - _generate_from_embeds: bool, - encoder_input: Optional[torch.Tensor], - decoder_tokens: Optional[torch.Tensor], - generated_token_ids: Optional[List[torch.Tensor]], - pixel_values: Optional[torch.Tensor], - multimodal_kwargs: Dict[str, Any], - verbose: bool, - ) -> Generator[Tuple[torch.Tensor, torch.Tensor, bool], None, None]: - """Core generation loop. Yields (sampled_tokens, final_logits, all_finished) per step. - - Owns the forward pass, sampling, EOS handling, token accumulation, and - KV cache management. Callers are responsible for try/finally cleanup of - ``_capture_hf_cache``. - """ - _hf_kv_cache = None - - for gen_step_idx in tqdm.tqdm(range(max_new_tokens), disable=not verbose): - with torch.no_grad(): - if is_encoder_decoder: - logits = self( - encoder_input, - return_type="logits", - decoder_input=decoder_tokens, - ) - else: - forward_kwargs: Dict[str, Any] = {} - # Compute attention mask and position_ids for batched - # inputs with padding. - if ( - _is_batched_list - and self.tokenizer is not None - and self.tokenizer.pad_token_id is not None - ): - _prev_side = self.tokenizer.padding_side - self.tokenizer.padding_side = "left" - attn_mask = utils.get_attention_mask( - self.tokenizer, - current_tokens, - prepend_bos=getattr(self.cfg, "default_prepend_bos", True), - ).to(self.cfg.device) - self.tokenizer.padding_side = _prev_side - forward_kwargs["attention_mask"] = attn_mask - position_ids = attn_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attn_mask == 0, 1) - forward_kwargs["position_ids"] = position_ids - if gen_step_idx == 0: - if pixel_values is not None: - forward_kwargs["pixel_values"] = pixel_values - if multimodal_kwargs: - forward_kwargs.update(multimodal_kwargs) - if use_stateful_cache: - forward_kwargs["cache_params"] = mamba_cache - forward_kwargs["use_cache"] = True - if gen_step_idx == 0: - cache_position = torch.arange( - 0, mamba_conv_kernel, device=self.cfg.device - ) - forward_kwargs["cache_position"] = cache_position - logits = self( - current_tokens, - return_type="logits", - **forward_kwargs, - ) - else: - input_seq_pos = input_tokens.shape[1] + gen_step_idx - 1 - cache_position = torch.tensor([input_seq_pos], device=self.cfg.device) - forward_kwargs["cache_position"] = cache_position - if "position_ids" in forward_kwargs: - forward_kwargs["position_ids"] = forward_kwargs["position_ids"][ - :, -1: - ] - logits = self( - current_tokens[:, -1:], - return_type="logits", - **forward_kwargs, - ) - elif use_past_kv_cache: - forward_kwargs["use_cache"] = True - if _hf_kv_cache is not None: - forward_kwargs["past_key_values"] = _hf_kv_cache - if "position_ids" in forward_kwargs: - forward_kwargs["position_ids"] = forward_kwargs["position_ids"][ - :, -1: - ] - logits = self( - current_tokens[:, -1:], - return_type="logits", - **forward_kwargs, - ) - else: - logits = self( - current_tokens, - return_type="logits", - **forward_kwargs, - ) - else: - logits = self(current_tokens, return_type="logits", **forward_kwargs) - if use_past_kv_cache and hasattr(self, "_last_hf_cache"): - _hf_kv_cache = self._last_hf_cache or _hf_kv_cache - del self._last_hf_cache - final_logits = logits[:, -1, :] - - # Sample next token - penalty_tokens = ( - torch.stack(generated_token_ids, dim=1) - if _generate_from_embeds and generated_token_ids - else None - ) - if do_sample: - sampled_tokens = utils.sample_logits( - final_logits, - top_k=top_k, - top_p=top_p, - temperature=temperature, - freq_penalty=freq_penalty, - repetition_penalty=repetition_penalty, - tokens=penalty_tokens - if _generate_from_embeds - else (decoder_tokens if is_encoder_decoder else current_tokens), - ).to(self.cfg.device) - else: - sampled_tokens = utils.sample_logits( - final_logits, - temperature=0.0, - repetition_penalty=repetition_penalty, - tokens=penalty_tokens - if _generate_from_embeds - else (decoder_tokens if is_encoder_decoder else current_tokens), - ).to(self.cfg.device) - - # Handle EOS - if stop_at_eos: - sampled_tokens[finished_sequences] = eos_token_for_padding - finished_sequences.logical_or_( - torch.isin( - sampled_tokens.to(self.cfg.device), - torch.tensor(stop_tokens).to(self.cfg.device), - ) - ) - - # Update token sequences - if is_encoder_decoder: - assert decoder_tokens is not None - decoder_tokens = torch.cat([decoder_tokens, sampled_tokens.unsqueeze(1)], dim=1) - elif _generate_from_embeds: - assert generated_token_ids is not None - generated_token_ids.append(sampled_tokens) - embed_fn = self.original_model.get_input_embeddings() # type: ignore[operator] - assert embed_fn is not None - new_embed = embed_fn(sampled_tokens.unsqueeze(1)).to(current_tokens.dtype) - current_tokens = torch.cat([current_tokens, new_embed], dim=1) - else: - current_tokens = torch.cat([current_tokens, sampled_tokens.unsqueeze(1)], dim=1) - - all_finished = bool(stop_at_eos and finished_sequences.all().item()) - - yield sampled_tokens, final_logits, all_finished - - if all_finished: - return - - def generate( - self, - input: Union[str, List[str], torch.Tensor] = "", - max_new_tokens: int = 10, - stop_at_eos: bool = True, - eos_token_id: Optional[int] = None, - do_sample: bool = True, - top_k: Optional[int] = None, - top_p: Optional[float] = None, - temperature: float = 1.0, - freq_penalty: float = 0.0, - repetition_penalty: float = 1.0, - use_past_kv_cache: bool = True, - prepend_bos: Optional[bool] = None, - padding_side: Optional[str] = None, - return_type: Optional[str] = "input", - verbose: bool = True, - output_logits: bool = False, - pixel_values: Optional[torch.Tensor] = None, - **multimodal_kwargs, - ) -> str | list[str] | torch.Tensor | Any: # Any for transformers.utils.ModelOutput - # Any: beartype forward ref limitation (beartype#546) - """Sample tokens from the model. - - Sample tokens from the model until the model outputs eos_token or max_new_tokens is reached. - This implementation is based on HookedTransformer.generate() to ensure consistent behavior. - - Args: - input: Text string, list of strings, or tensor of tokens - max_new_tokens: Maximum number of tokens to generate - stop_at_eos: If True, stop generating tokens when the model outputs eos_token - eos_token_id: The token ID to use for end of sentence - do_sample: If True, sample from the model's output distribution. Otherwise, use greedy search - top_k: Number of tokens to sample from. If None, sample from all tokens - top_p: Probability mass to sample from. If 1.0, sample from all tokens - temperature: Temperature for sampling. Higher values will make the model more random - freq_penalty: Frequency penalty for sampling - how much to penalise previous tokens - repetition_penalty: HuggingFace-style repetition penalty. Values > 1.0 discourage - repetition by dividing positive logits and multiplying negative logits for - previously seen tokens. Default 1.0 (no penalty). - use_past_kv_cache: If True, use KV caching for faster generation - prepend_bos: Accepted for API compatibility but not applied during generation. - The HF model expects tokens in its native format (tokenizer defaults). - Overriding BOS can silently degrade generation quality. - padding_side: Which side to pad when tokenizing multiple strings of different - lengths. For batched list inputs, left-padding is forced internally for - correct generation behavior. Defaults to None (tokenizer default). - return_type: The type of output to return - 'input', 'str', or 'tokens' - verbose: Not used in Bridge (kept for API compatibility) - output_logits: If True, return a ModelOutput with sequences and logits tuple - pixel_values: Optional image tensor for multimodal models. Only passed on the - first generation step (the vision encoder processes the image once, then - embeddings are part of the token sequence for subsequent steps). - - Returns: - Generated sequence as string, list of strings, or tensor depending on input type and return_type. - If output_logits=True, returns a ModelOutput-like object with 'sequences' and 'logits' attributes. - """ - # prepend_bos is intentionally not applied during generation. - # The HF model expects tokens in its native format. Overriding BOS can silently - # degrade quality. - if prepend_bos is not None: - import warnings - - warnings.warn( - "prepend_bos is ignored during TransformerBridge.generate(). " - "The HF model expects tokens with the tokenizer's default BOS handling. " - "To control BOS, tokenize with to_tokens(prepend_bos=...) and pass the " - "resulting tensor to generate().", - stacklevel=2, - ) - # padding_side is handled internally: for batched list inputs, left-padding - # is forced to ensure correct generation. See _is_batched_list logic below. - - # Stateful dispatch is decided after input parsing so we can fall back - # to hf_generate() for input types the stateful loop doesn't handle. - is_stateful_model = getattr(self.cfg, "is_stateful", False) - - _is_batched_list = isinstance(input, list) and len(input) > 1 - - _generate_from_embeds = False - if isinstance(input, str): - input_tokens = self.to_tokens(input, move_to_device=True, truncate=False) - input_type = "str" - elif isinstance(input, list): - # Force left-padding for batched generation so real tokens are - # flush-right and logits[:, -1, :] is always the last real token. - if _is_batched_list: - _orig_padding_side = self.tokenizer.padding_side - self.tokenizer.padding_side = "left" - input_tokens = self.to_tokens(input, move_to_device=True, truncate=False) - if _is_batched_list: - self.tokenizer.padding_side = _orig_padding_side - input_type = "list" - elif isinstance(input, torch.Tensor) and input.is_floating_point(): - # inputs_embeds: pre-computed embeddings (e.g., from multimodal models) - input_tokens = input.to(self.cfg.device) - input_type = "embeds" - _generate_from_embeds = True - else: - input_tokens = input.to(self.cfg.device) - input_type = "tokens" - - # Determine return type - if return_type == "input": - if input_type in ["str", "list"]: - return_type = "str" - elif input_type == "embeds": - return_type = "tokens" - else: - return_type = "tokens" - - batch_size = input_tokens.shape[0] - - # Setup EOS token handling - stop_tokens = [] - eos_token_for_padding = 0 - if stop_at_eos: - tokenizer_has_eos_token = ( - self.tokenizer is not None and self.tokenizer.eos_token_id is not None - ) - if eos_token_id is None: - assert ( - tokenizer_has_eos_token - ), "Must pass eos_token_id if stop_at_eos is True and tokenizer is None or has no eos_token_id" - assert self.tokenizer is not None - eos_token_id = self.tokenizer.eos_token_id - - if isinstance(eos_token_id, int): - stop_tokens = [eos_token_id] - eos_token_for_padding = eos_token_id - else: - stop_tokens = list(eos_token_id) - if tokenizer_has_eos_token: - assert self.tokenizer is not None - eos_token_for_padding = self.tokenizer.eos_token_id - else: - eos_token_for_padding = eos_token_id[0] - - # Track which sequences have finished - finished_sequences = torch.zeros(batch_size, dtype=torch.bool, device=self.cfg.device) - - # Optionally collect logits at each generation step for downstream tooling/tests - logits_seq_list: list[torch.Tensor] | None = [] if output_logits else None - - # Detect encoder-decoder models (T5, BART, etc.) - is_encoder_decoder = hasattr(self.original_model, "config") and getattr( - self.original_model.config, "is_encoder_decoder", False - ) - - # HF cache flows opaquely through the component chain via - # _reconstruct_attention() → _update_kv_cache() on each layer. - _hf_kv_cache = None - if use_past_kv_cache and is_encoder_decoder: - # Encoder-decoder models (T5, BART) don't support the opaque - # cache path — silently disable rather than crash, since - # use_past_kv_cache=True is the default. - use_past_kv_cache = False - - # SSMs (Mamba/Mamba-2) run through a dedicated cache path so hooks - # fire on every step. Unsupported input types fall back to hf_generate(). - use_stateful_cache = ( - is_stateful_model - and use_past_kv_cache - and not is_encoder_decoder - and not _generate_from_embeds - and pixel_values is None - and not multimodal_kwargs - ) - if is_stateful_model and not use_stateful_cache: - hf_kwargs: dict[str, Any] = { - "max_new_tokens": max_new_tokens, - "do_sample": do_sample, - "temperature": temperature, - } - if top_k is not None: - hf_kwargs["top_k"] = top_k - if top_p is not None: - hf_kwargs["top_p"] = top_p - if eos_token_id is not None: - hf_kwargs["eos_token_id"] = eos_token_id - return self.hf_generate(input, **hf_kwargs) - - # SSM cache is built once and mutated in place across forward calls. - # Adapter owns the cache-type choice; new SSMs just override - # create_stateful_cache(). - mamba_cache: Any = None - mamba_conv_kernel: int = 0 - if use_stateful_cache: - hf_model: Any = self.original_model - mamba_conv_kernel = int(getattr(hf_model.config, "conv_kernel", 4)) - cache_dtype = self.cfg.dtype or torch.float32 - mamba_cache = self.adapter.create_stateful_cache( - hf_model=hf_model, - batch_size=batch_size, - device=self.cfg.device, - dtype=cache_dtype, - ) - - if use_past_kv_cache and not use_stateful_cache: - self._capture_hf_cache = True # Signal forward() to stash cache - - # Generate tokens - current_tokens = input_tokens.clone() - # For inputs_embeds generation, also track generated token IDs for decoding - if _generate_from_embeds: - generated_token_ids: list[torch.Tensor] = [] - sampled_tokens_list = [] - - # For encoder-decoder models, keep encoder input fixed and grow decoder input - if is_encoder_decoder: - encoder_input = input_tokens.clone() - decoder_start_token_id = getattr( - self.original_model.config, "decoder_start_token_id", 0 - ) - decoder_tokens = torch.full( - (batch_size, 1), - decoder_start_token_id, - dtype=input_tokens.dtype, - device=self.cfg.device, - ) - - try: - for sampled_tokens, final_logits, all_finished in self._generate_tokens( - current_tokens, - input_tokens, - batch_size, - max_new_tokens=max_new_tokens, - do_sample=do_sample, - top_k=top_k, - top_p=top_p, - temperature=temperature, - freq_penalty=freq_penalty, - repetition_penalty=repetition_penalty, - stop_at_eos=stop_at_eos, - stop_tokens=stop_tokens, - eos_token_for_padding=eos_token_for_padding, - finished_sequences=finished_sequences, - use_past_kv_cache=use_past_kv_cache, - use_stateful_cache=use_stateful_cache, - mamba_cache=mamba_cache, - mamba_conv_kernel=mamba_conv_kernel, - is_encoder_decoder=is_encoder_decoder, - _is_batched_list=_is_batched_list, - _generate_from_embeds=_generate_from_embeds, - encoder_input=encoder_input if is_encoder_decoder else None, - decoder_tokens=decoder_tokens if is_encoder_decoder else None, - generated_token_ids=generated_token_ids if _generate_from_embeds else None, - pixel_values=pixel_values, - multimodal_kwargs=multimodal_kwargs if multimodal_kwargs else {}, - verbose=verbose, - ): - sampled_tokens_list.append(sampled_tokens.unsqueeze(1)) - if logits_seq_list is not None: - logits_seq_list.append(final_logits.clone()) - if all_finished: - break - finally: - self._capture_hf_cache = False - if hasattr(self, "_last_hf_cache"): - del self._last_hf_cache - - # Concatenate all sampled tokens - sampled_tokens = torch.cat(sampled_tokens_list, dim=1) - if is_encoder_decoder: - # Reconstruct full decoder sequence: start token + generated tokens - output_tokens = torch.cat([decoder_tokens[:, :1], sampled_tokens], dim=1) - elif _generate_from_embeds: - # For inputs_embeds, we only have the generated token IDs (no input token IDs) - output_tokens = sampled_tokens - else: - output_tokens = torch.cat([input_tokens, sampled_tokens], dim=1) - - # Return ModelOutput if output_logits was requested - if output_logits and logits_seq_list is not None: - from transformers.utils import ModelOutput # type: ignore - - def _logits_to_tuple(logits_list: list[torch.Tensor]) -> tuple[torch.Tensor, ...]: - assert logits_list is not None - # Convert list of [batch, vocab] tensors to tuple - return tuple(logits_list) - - try: - from transformers.generation.utils import GenerateDecoderOnlyOutput - - # Return a HF-compatible ModelOutput structure - # GenerateDecoderOnlyOutput expects: sequences, scores (optional), logits (optional) - return GenerateDecoderOnlyOutput( - sequences=cast(torch.LongTensor, output_tokens), - # HF's type hint says tuple[FloatTensor] but should be tuple[FloatTensor, ...] - # (variable-length tuple with one element per generated token) - logits=_logits_to_tuple(logits_seq_list), # type: ignore[arg-type] - ) - except (ImportError, AttributeError): - # Fallback if GenerateDecoderOnlyOutput not available in this transformers version - return ModelOutput( - sequences=output_tokens, - logits=_logits_to_tuple(logits_seq_list), - ) - - # Format output - if return_type == "str": - assert self.tokenizer is not None - if input_type == "str": - return self.tokenizer.decode(output_tokens[0], skip_special_tokens=True) - else: - decoded_texts = [ - self.tokenizer.decode(tokens, skip_special_tokens=True) - for tokens in output_tokens - ] - return decoded_texts[0] if len(decoded_texts) == 1 else decoded_texts - else: # return_type == "tokens" - return output_tokens - - @torch.no_grad() - def generate_stream( - self, - input: Union[str, List[str], torch.Tensor] = "", - max_new_tokens: int = 10, - max_tokens_per_yield: int = 25, - stop_at_eos: bool = True, - eos_token_id: Optional[int] = None, - do_sample: bool = True, - top_k: Optional[int] = None, - top_p: Optional[float] = None, - temperature: float = 1.0, - freq_penalty: float = 0.0, - repetition_penalty: float = 1.0, - use_past_kv_cache: bool = True, - prepend_bos: Optional[bool] = None, - padding_side: Optional[str] = None, - return_type: Optional[str] = "input", - verbose: bool = True, - ) -> Generator[Union[torch.Tensor, str], None, None]: - """Stream tokens from the model as they are generated. - - Yields batches of tokens progressively during generation rather than - waiting for the entire sequence. Uses the same core loop as generate(). - - Args: - input: Text string, list of strings, or tensor of tokens. - max_new_tokens: Maximum number of tokens to generate. - max_tokens_per_yield: Yield accumulated tokens every this many steps. - stop_at_eos: If True, stop when eos_token is produced. - eos_token_id: Token ID(s) for end of sentence. Defaults to tokenizer's. - do_sample: If True, sample; otherwise greedy. - top_k: Top-k sampling. None means no filtering. - top_p: Nucleus sampling threshold. - temperature: Sampling temperature. - freq_penalty: Frequency penalty for previous tokens. - repetition_penalty: HF-style repetition penalty (>1.0 discourages repeats). - use_past_kv_cache: Use KV caching for faster generation. - prepend_bos: Not applied (API compatibility). See generate() docstring. - padding_side: Which side to pad for batched list inputs. Left-padding - is forced internally for batched generation. - return_type: 'input' (match input type), 'str', or 'tokens'. - verbose: Show progress bar. - - Yields: - Token tensors [batch, seq_len] or strings, accumulated up to - max_tokens_per_yield tokens between yields. First yield includes - the input tokens; subsequent yields contain only new tokens. - """ - if prepend_bos is not None: - warnings.warn( - "prepend_bos is ignored during TransformerBridge.generate_stream(). " - "The HF model expects tokens with the tokenizer's default BOS handling.", - stacklevel=2, - ) - - # --- Input parsing (mirrors generate()) --- - _is_batched_list = isinstance(input, list) and len(input) > 1 - - if isinstance(input, str): - input_tokens = self.to_tokens(input, move_to_device=True, truncate=False) - input_type = "str" - elif isinstance(input, list): - if _is_batched_list: - _orig_ps = self.tokenizer.padding_side - self.tokenizer.padding_side = "left" - try: - input_tokens = self.to_tokens(input, move_to_device=True, truncate=False) - finally: - if _is_batched_list: - self.tokenizer.padding_side = _orig_ps - input_type = "list" - else: - input_tokens = input.to(self.cfg.device) - input_type = "tokens" - - if return_type == "input": - return_type = "str" if input_type in ["str", "list"] else "tokens" - - batch_size = input_tokens.shape[0] - - # --- EOS setup --- - stop_tokens: List[int] = [] - eos_token_for_padding = 0 - if stop_at_eos: - tokenizer_has_eos_token = ( - self.tokenizer is not None and self.tokenizer.eos_token_id is not None - ) - if eos_token_id is None: - assert ( - tokenizer_has_eos_token - ), "Must pass eos_token_id if stop_at_eos is True and tokenizer is None or has no eos_token_id" - assert self.tokenizer is not None - eos_token_id = self.tokenizer.eos_token_id - if isinstance(eos_token_id, int): - stop_tokens = [eos_token_id] - eos_token_for_padding = eos_token_id - else: - stop_tokens = list(eos_token_id) - if tokenizer_has_eos_token: - assert self.tokenizer is not None - eos_token_for_padding = self.tokenizer.eos_token_id - else: - eos_token_for_padding = eos_token_id[0] - - finished_sequences = torch.zeros(batch_size, dtype=torch.bool, device=self.cfg.device) - - # --- Cache setup --- - if use_past_kv_cache: - self._capture_hf_cache = True - - current_tokens = input_tokens.clone() - - # --- Streaming loop --- - # All yields are token tensors [batch, seq_len]. Each yield contains - # only the newly generated tokens since the previous yield (the first - # yield additionally prepends the input tokens for context). - accumulated_tokens: Optional[torch.Tensor] = None - tokens_since_last_yield = 0 - - def _maybe_decode( - tokens: torch.Tensor, - ) -> Union[torch.Tensor, str]: - if return_type == "str": - assert self.tokenizer is not None - return self.tokenizer.decode(tokens[0], skip_special_tokens=True) - return tokens - - try: - for step_idx, (sampled_tokens, _, all_finished) in enumerate( - self._generate_tokens( - current_tokens, - input_tokens, - batch_size, - max_new_tokens=max_new_tokens, - do_sample=do_sample, - top_k=top_k, - top_p=top_p, - temperature=temperature, - freq_penalty=freq_penalty, - repetition_penalty=repetition_penalty, - stop_at_eos=stop_at_eos, - stop_tokens=stop_tokens, - eos_token_for_padding=eos_token_for_padding, - finished_sequences=finished_sequences, - use_past_kv_cache=use_past_kv_cache, - use_stateful_cache=False, - mamba_cache=None, - mamba_conv_kernel=0, - is_encoder_decoder=False, - _is_batched_list=_is_batched_list, - _generate_from_embeds=False, - encoder_input=None, - decoder_tokens=None, - generated_token_ids=None, - pixel_values=None, - multimodal_kwargs={}, - verbose=verbose, - ) - ): - new_tokens = sampled_tokens.unsqueeze(-1) - - if step_idx == 0: - accumulated_tokens = torch.cat([input_tokens, new_tokens], dim=-1) - tokens_since_last_yield = accumulated_tokens.shape[1] - else: - if accumulated_tokens is None: - accumulated_tokens = new_tokens - else: - accumulated_tokens = torch.cat([accumulated_tokens, new_tokens], dim=-1) - tokens_since_last_yield += 1 - - if tokens_since_last_yield >= max_tokens_per_yield: - yield _maybe_decode(accumulated_tokens) - tokens_since_last_yield = 0 - accumulated_tokens = None - - if all_finished: - if accumulated_tokens is not None: - yield _maybe_decode(accumulated_tokens) - break - - # Yield remainder after loop completes without break - if accumulated_tokens is not None: - yield _maybe_decode(accumulated_tokens) - finally: - self._capture_hf_cache = False - if hasattr(self, "_last_hf_cache"): - del self._last_hf_cache - - def hf_generate( - self, - input: str | list[str] | torch.Tensor = "", - max_new_tokens: int = 10, - stop_at_eos: bool = True, - eos_token_id: int | None = None, - do_sample: bool = True, - top_k: int | None = None, - top_p: float | None = None, - temperature: float = 1.0, - use_past_kv_cache: bool = True, - return_type: str | None = "input", - pixel_values: torch.Tensor | None = None, - **generation_kwargs, - ) -> str | list[str] | torch.Tensor | Any: # Any for HF ModelOutput types - # Any: beartype forward ref limitation (beartype#546) - """Generate text using the underlying HuggingFace model with full HF API support. - - This method provides direct access to HuggingFace's generation API, forwarding all - generation parameters (including output_scores, output_logits, output_attentions, - output_hidden_states) directly to the underlying HF model. Use this when you need - full HuggingFace generation features not supported by the standard generate() method. - - For standard generation compatible with HookedTransformer, use generate() instead. - - Args: - input: Text string, list of strings, or tensor of tokens - max_new_tokens: Maximum number of tokens to generate - stop_at_eos: If True, stop generating tokens when the model outputs eos_token - eos_token_id: The token ID to use for end of sentence - do_sample: If True, sample from the model's output distribution - top_k: Number of tokens to sample from - top_p: Probability mass to sample from - temperature: Temperature for sampling - use_past_kv_cache: If True, use KV caching for faster generation - return_type: The type of output to return - 'input', 'str', or 'tokens' - **generation_kwargs: Additional HuggingFace generation parameters including: - - output_scores: Return generation scores - - output_logits: Return generation logits - - output_attentions: Return attention weights - - output_hidden_states: Return hidden states - - return_dict_in_generate: Return ModelOutput object - - And any other HF generation parameters - - Returns: - Generated sequence as string, list of strings, tensor, or HF ModelOutput - depending on input type, return_type, and generation_kwargs. - - Example:: - - # Get full HF ModelOutput with logits and attentions - from transformer_lens import HookedTransformer - model = HookedTransformer.from_pretrained("tiny-stories-1M") - result = model.hf_generate( - "Hello world", - max_new_tokens=5, - output_logits=True, - output_attentions=True, - return_dict_in_generate=True - ) - print(result.sequences) # Generated tokens - print(result.logits) # Logits for each generation step - print(result.attentions) # Attention weights - """ - # Handle string input by tokenizing it - if isinstance(input, str): - inputs = self.tokenizer(input, return_tensors="pt", padding=False, truncation=False).to( - self.cfg.device - ) - input_ids = inputs["input_ids"] - input_type = "str" - elif isinstance(input, list): - inputs = self.tokenizer(input, return_tensors="pt", padding=True, truncation=False).to( - self.cfg.device - ) - input_ids = inputs["input_ids"] - input_type = "list" - else: - input_ids = input - if input_ids.device != self.cfg.device: - input_ids = input_ids.to(self.cfg.device) - input_type = "tokens" - - # Build generation_kwargs from explicit args and kwargs - generation_kwargs = dict(generation_kwargs) if generation_kwargs is not None else {} - generation_kwargs.update( - { - "max_new_tokens": max_new_tokens, - "do_sample": do_sample, - "temperature": temperature, - "pad_token_id": self.tokenizer.eos_token_id, - } - ) - - if top_k is not None: - generation_kwargs["top_k"] = top_k - if top_p is not None: - generation_kwargs["top_p"] = top_p - if eos_token_id is not None: - generation_kwargs["eos_token_id"] = eos_token_id - elif stop_at_eos and self.tokenizer.eos_token_id is not None: - generation_kwargs["eos_token_id"] = self.tokenizer.eos_token_id - - if pixel_values is not None: - generation_kwargs["pixel_values"] = pixel_values - - if use_past_kv_cache: - generation_kwargs["use_cache"] = True - - # HF dict flags that trigger ModelOutput returns - hf_dict_flags = ( - "output_scores", - "output_logits", - "output_attentions", - "output_hidden_states", - ) - - # If any HF-style output flags are provided, ensure return_dict_in_generate is set - any_flag_set = False - for f in hf_dict_flags: - if generation_kwargs.get(f) is not None: - generation_kwargs[f] = bool(generation_kwargs[f]) - any_flag_set = True - - if any_flag_set: - generation_kwargs.setdefault("return_dict_in_generate", True) - - # Generate using the original HuggingFace model - with torch.no_grad(): - outputs = self.original_model.generate(input_ids, **generation_kwargs) # type: ignore[operator] - - # Check if output is a ModelOutput - try: - from transformers.utils import ModelOutput # type: ignore - - is_model_output = isinstance(outputs, ModelOutput) - except Exception: - is_model_output = False - - # Return based on return_type and input format - if return_type == "input" or return_type is None: - if input_type == "str": - # Decode the full output back to string - if is_model_output and hasattr(outputs, "sequences"): - return self.tokenizer.decode(outputs.sequences[0], skip_special_tokens=True) - return self.tokenizer.decode(outputs[0], skip_special_tokens=True) - elif input_type == "list": - # Decode each sequence in the batch - if is_model_output and hasattr(outputs, "sequences"): - return [ - self.tokenizer.decode(seq, skip_special_tokens=True) - for seq in outputs.sequences - ] - return [self.tokenizer.decode(seq, skip_special_tokens=True) for seq in outputs] - else: - # Return the full token sequence including input - return outputs - elif return_type == "tokens": - return outputs - else: - # For other return types, default to the decoded text - if input_type == "str": - if is_model_output and hasattr(outputs, "sequences"): - return self.tokenizer.decode(outputs.sequences[0], skip_special_tokens=True) - return self.tokenizer.decode(outputs[0], skip_special_tokens=True) - elif input_type == "list": - if is_model_output and hasattr(outputs, "sequences"): - return [ - self.tokenizer.decode(seq, skip_special_tokens=True) - for seq in outputs.sequences - ] - return [self.tokenizer.decode(seq, skip_special_tokens=True) for seq in outputs] - else: - return outputs - - def prepare_multimodal_inputs( - self, - text: Union[str, List[str]], - images: Optional[Any] = None, - ) -> Dict[str, torch.Tensor]: - """Prepare multimodal inputs using the model's processor. - - Converts text and images into model-ready tensors (input_ids, pixel_values, - attention_mask, etc.) using the HuggingFace processor loaded during boot(). - - Args: - text: Text prompt(s), typically containing image placeholder tokens - (e.g., "" for LLaVA). - images: PIL Image or list of PIL Images to process. Pass None for - text-only inputs on a multimodal model. - - Returns: - Dictionary with 'input_ids', 'pixel_values', 'attention_mask', etc. - All tensors are moved to the model's device. - - Raises: - ValueError: If model is not multimodal or processor is not available. - """ - if not getattr(self.cfg, "is_multimodal", False): - raise ValueError( - "prepare_multimodal_inputs() requires a multimodal model " - "(cfg.is_multimodal must be True)" - ) - if self.processor is None: - raise ValueError( - "No processor available. Load model with boot_transformers() or " - "set bridge.processor = AutoProcessor.from_pretrained(...) manually." - ) - inputs = self.processor(text=text, images=images, return_tensors="pt") - return {k: v.to(self.cfg.device) if hasattr(v, "to") else v for k, v in inputs.items()} - - def to(self, *args, **kwargs) -> "TransformerBridge": - """Move model to device and/or change dtype. - - Args: - args: Positional arguments for nn.Module.to - kwargs: Keyword arguments for nn.Module.to - print_details: Whether to print details about device/dtype changes (default: True) - - Returns: - Self for chaining - """ - # Extract print_details if provided - print_details = kwargs.pop("print_details", True) - - # Handle both device and dtype changes - # torch.nn.Module.to() supports: to(device), to(dtype), to(device, dtype), - # to(device=...), to(dtype=...), to(device=..., dtype=...) - target_device, target_dtype = None, None - - if len(args) >= 1: - first_arg = args[0] - if isinstance(first_arg, (torch.device, str)): - target_device = first_arg - elif isinstance(first_arg, torch.dtype): - target_dtype = first_arg - if len(args) >= 2: - second_arg = args[1] - if isinstance(second_arg, torch.dtype): - target_dtype = second_arg - - # these override positional args - if "device" in kwargs: - target_device = kwargs["device"] - if "dtype" in kwargs: - target_dtype = kwargs["dtype"] - - # Moving a multi-device (device_map-dispatched) model to a single device would - # collapse the split and break accelerate's hook routing. Warn and drop the - # device move; still honor dtype changes. - if target_device is not None and getattr(self.cfg, "n_devices", 1) > 1: - warnings.warn( - f"TransformerBridge.to({target_device!r}) ignored: model is dispatched " - f"across {self.cfg.n_devices} devices via device_map. Reload with " - "device=... (and no device_map/n_devices) to move to a single device.", - stacklevel=2, - ) - target_device = None - - if target_device is not None: - move_to_and_update_config(self, target_device, print_details) - if target_dtype is not None: - move_to_and_update_config(self, target_dtype, print_details) - - # Move the original model with all original args/kwargs (with print_details removed). - # When we've nulled target_device for multi-GPU safety, strip device args so the - # underlying module isn't moved either. - if target_device is None and (len(args) > 0 or "device" in kwargs): - kwargs.pop("device", None) - # Filter positional args: drop devices/strings, keep dtypes. - args = tuple(a for a in args if not isinstance(a, (torch.device, str))) - self.original_model = self.original_model.to(*args, **kwargs) - return self - - def cuda(self, device: Optional[Union[int, torch.device]] = None) -> "TransformerBridge": - """Move model to CUDA. - - Args: - device: CUDA device - - Returns: - Self for chaining - """ - if isinstance(device, int): - return self.to(f"cuda:{device}") - elif device is None: - return self.to("cuda") - else: - return self.to(device) - - def cpu(self) -> "TransformerBridge": - """Move model to CPU. - - Returns: - Self for chaining - """ - return self.to(torch.device("cpu")) - - def mps(self) -> "TransformerBridge": - """Move model to MPS. - - Returns: - Self for chaining - """ - return self.to(torch.device("mps")) - - def add_hook( - self, - name: Union[str, Callable[[str], bool]], - hook_fn, - dir="fwd", - is_permanent=False, - ): - """Add a hook to a specific component or to all components matching a filter. - - Args: - name: Either a string hook point name (e.g. "blocks.0.attn.hook_q") - or a callable filter ``(str) -> bool`` that is applied to every - hook point name; the hook is added to each point where the filter - returns True. - hook_fn: The hook function ``(activation, hook) -> activation | None``. - dir: Hook direction, ``"fwd"`` or ``"bwd"``. - is_permanent: If True the hook survives ``reset_hooks()`` calls. - """ - if callable(name) and not isinstance(name, str): - hook_dict = self.hook_dict - seen_hooks: set[int] = set() - for hook_name, hook_point in hook_dict.items(): - if name(hook_name): - hook_id = id(hook_point) - if hook_id in seen_hooks: - continue - seen_hooks.add(hook_id) - hook_point.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) - return - - component = self - parts = name.split(".") - for part in parts[:-1]: - if hasattr(component, part): - component = getattr(component, part) - else: - raise AttributeError(f"Component path '{'.'.join(parts[:-1])}' not found") - hook_name = parts[-1] - if hasattr(component, hook_name): - hook_point = getattr(component, hook_name) - if isinstance(hook_point, HookPoint): - hook_point.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) - else: - raise AttributeError( - f"'{hook_name}' is not a hook point. Found object of type: {type(hook_point)} with value: {hook_point}" - ) - else: - raise AttributeError(f"Hook point '{hook_name}' not found on component") - - def reset_hooks(self, clear_contexts=True): - """Remove all hooks from the model.""" - - def remove_hooks_recursive(module): - if isinstance(module, GeneralizedComponent): - module.remove_hooks() - for child in module.children(): - remove_hooks_recursive(child) - - remove_hooks_recursive(self) - - def hooks(self, fwd_hooks=[], bwd_hooks=[], reset_hooks_end=True, clear_contexts=False): - """Context manager for temporarily adding hooks. - - Args: - fwd_hooks: List of (hook_name, hook_fn) tuples for forward hooks - bwd_hooks: List of (hook_name, hook_fn) tuples for backward hooks - reset_hooks_end: If True, removes hooks when context exits - clear_contexts: Unused (for compatibility with HookedTransformer) - - Example: - with model.hooks(fwd_hooks=[("hook_embed", my_hook)]): - output = model("Hello world") - """ - - @contextmanager - def _hooks_context(): - added_hooks: List[Tuple[HookPoint, str]] = [] - - def add_hook_to_point( - hook_point: HookPoint, - hook_fn: Callable, - name: str, - dir: Literal["fwd", "bwd"] = "fwd", - ): - if self.compatibility_mode and name != hook_point.name: - alias_names_list: list[str] = [] - if hook_point.name is not None: - alias_names_list.append(hook_point.name) - alias_names_list.append(name) - hook_point.add_hook(hook_fn, dir=dir, alias_names=alias_names_list) - else: - hook_point.add_hook(hook_fn, dir=dir) - added_hooks.append((hook_point, name)) - - def apply_hooks(hooks: List[Tuple[Union[str, Callable], Callable]], is_fwd: bool): - direction: Literal["fwd", "bwd"] = "fwd" if is_fwd else "bwd" - aliases = build_alias_to_canonical_map(self.hook_dict) - for hook_name_or_filter, hook_fn in hooks: - if isinstance(hook_name_or_filter, str): - hook_dict = self.hook_dict - actual_hook_name = hook_name_or_filter - if hook_name_or_filter in aliases: - actual_hook_name = aliases[hook_name_or_filter] - if actual_hook_name in hook_dict: - add_hook_to_point( - hook_dict[actual_hook_name], hook_fn, actual_hook_name, direction - ) - else: - hook_dict = self.hook_dict - seen_hooks = set() - for name, hook_point in hook_dict.items(): - if hook_name_or_filter(name): - hook_id = id(hook_point) - if hook_id in seen_hooks: - continue - seen_hooks.add(hook_id) - hook_name_to_use = hook_point.name if hook_point.name else name - add_hook_to_point(hook_point, hook_fn, hook_name_to_use, direction) - - try: - apply_hooks(fwd_hooks, True) - apply_hooks(bwd_hooks, False) - yield self - finally: - if reset_hooks_end: - for hook_point, name in added_hooks: - hook_point.remove_hooks() - - return _hooks_context() - - def set_use_attn_result(self, use_attn_result: bool): - """Toggle whether to explicitly calculate and expose the result for each attention head. - - Useful for interpretability but can easily burn through GPU memory. - """ - if use_attn_result: - self._validate_attention_fork_supported("use_attn_result") - self.cfg.use_attn_result = use_attn_result - self._propagate_attention_flag("use_attn_result", use_attn_result) - - def set_use_split_qkv_input(self, use_split_qkv_input: bool): - """Toggle independent residual copies for Q/K/V so each path can be patched alone. - - Mutually exclusive with `use_attn_in` — set that flag off first if it's on. - """ - if use_split_qkv_input: - if bool(getattr(self.cfg, "use_attn_in", False)): - raise ValueError( - "use_split_qkv_input and use_attn_in are mutually exclusive. " - "Call set_use_attn_in(False) before enabling use_split_qkv_input." - ) - self._validate_attention_fork_supported("use_split_qkv_input") - self.cfg.use_split_qkv_input = use_split_qkv_input - self._propagate_attention_flag("use_split_qkv_input", use_split_qkv_input) - - def set_use_attn_in(self, use_attn_in: bool): - """Toggle a single 4D residual copy feeding all three Q/K/V projections. - - Mutually exclusive with `use_split_qkv_input` — set that flag off first - if it's on. When on, `hook_attn_in` fires at - `[batch, pos, n_heads, d_model]`, enabling coarse-grained interventions - on the residual-stream copy shared across Q/K/V. - """ - if use_attn_in: - if bool(getattr(self.cfg, "use_split_qkv_input", False)): - raise ValueError( - "use_attn_in and use_split_qkv_input are mutually exclusive. " - "Call set_use_split_qkv_input(False) before enabling use_attn_in." - ) - self._validate_attention_fork_supported("use_attn_in") - self.cfg.use_attn_in = use_attn_in - self._propagate_attention_flag("use_attn_in", use_attn_in) - - def _propagate_attention_flag(self, flag_name: str, value: bool) -> None: - """Mirror `bridge.cfg.` onto every block's attention config. - - Some adapters (Llama family) deep-copy the block template during - `setup_blocks_bridge`, cloning the attention bridge's config along - with it. Others (Pythia, GPT-2) override `__deepcopy__` to share the - config. Setting the flag only on `self.cfg` silently misses the - cloned-config case. Propagating explicitly keeps both patterns - honest — a no-op when configs are shared, a correctness fix when - they aren't. - """ - if not hasattr(self, "blocks"): - return - for block in self.blocks: - attn = block._modules.get("attn") if hasattr(block, "_modules") else None - if attn is None: - continue - attn_cfg = getattr(attn, "config", None) - if attn_cfg is not None and attn_cfg is not self.cfg: - try: - setattr(attn_cfg, flag_name, value) - except Exception: - # Some cfg objects may be frozen/immutable. Skip silently — - # the block simply won't honor the flag, which is the - # same outcome as before this fix. - pass - - def _validate_attention_fork_supported(self, flag_name: str) -> None: - """Raise / warn if the model can't honor a fine-grained attention flag. - - The post-ln1 fork path lives on JointQKVAttentionBridge and - PositionEmbeddingsAttentionBridge. Plain AttentionBridge delegates to - HF and exposes no fork point; we raise rather than setting the flag - silently. For hybrid models (some attention layers, some not), we warn - and list which layers will honor the flag. - """ - # Deferred imports: tight circular dependency with bridge setup. - from transformer_lens.model_bridge.generalized_components.joint_qkv_attention import ( - JointQKVAttentionBridge, - ) - from transformer_lens.model_bridge.generalized_components.position_embeddings_attention import ( - PositionEmbeddingsAttentionBridge, - ) - - if not hasattr(self, "blocks"): - raise NotImplementedError( - f"{flag_name}: this bridge has no `blocks` attribute, so no " - "attention bridges to apply the flag to." - ) - supported_classes = (JointQKVAttentionBridge, PositionEmbeddingsAttentionBridge) - supporting_layers: list[int] = [] - attn_classes: set[str] = set() - total_with_attn = 0 - for idx, block in enumerate(self.blocks): - attn = block._modules.get("attn") if hasattr(block, "_modules") else None - if attn is None: - continue - total_with_attn += 1 - attn_classes.add(type(attn).__name__) - if isinstance(attn, supported_classes): - supporting_layers.append(idx) - if total_with_attn == 0: - raise NotImplementedError(f"{flag_name}: no attention bridges found on self.blocks.") - if not supporting_layers: - raise NotImplementedError( - f"{flag_name}: none of this model's attention bridges support " - "the fine-grained Q/K/V hook fork. Found attention classes: " - f"{sorted(attn_classes)}. Supported classes: " - f"{[c.__name__ for c in supported_classes]}. Plain " - "AttentionBridge delegates to HuggingFace and exposes no hook " - "point before the Q/K/V projection." - ) - if len(supporting_layers) < total_with_attn: - skipped = total_with_attn - len(supporting_layers) - warnings.warn( - f"{flag_name}: {skipped} of {total_with_attn} attention layers " - "use an attention-bridge class that cannot honor this flag " - f"(attention classes present: {sorted(attn_classes)}). " - f"The flag will affect layers: {supporting_layers}.", - stacklevel=3, - ) - - def _is_valid_bridge_path(self, hf_path: str) -> bool: - """Check if a HuggingFace path corresponds to a valid bridge component. - - This validates that the path follows the bridge component structure and doesn't - contain nested HuggingFace components that should have been wrapped. - - Args: - hf_path: HuggingFace path after removing _original_component - - Returns: - True if the path is valid, False if it contains nested HF components - """ - # Split the path into parts - parts = hf_path.split(".") - - # Get the component mapping for validation - component_mapping = self.adapter.component_mapping - if not component_mapping: - return True # If no mapping, accept all keys - - # Walk through the path and check if each level is a registered bridge component - # For example, transformer.h.0.mlp.in.weight should be valid - # but transformer.h.0.mlp.c_fc.weight should be invalid (c_fc is nested HF component) - - # Start from the root - current_component = None - idx = 0 - - # Find which top-level component this belongs to - for tl_name, component in component_mapping.items(): - if component.name and hf_path.startswith(component.name + "."): - current_component = component - # Skip past the HF prefix - remaining_path = hf_path[len(component.name) + 1 :] - parts = remaining_path.split(".") - idx = 0 - break - - if current_component is None: - return True # Path doesn't match any component, let it through - - # Special handling for blocks - if hasattr(current_component, "is_list_item") and current_component.is_list_item: - # Skip the layer index - if idx < len(parts) and parts[idx].isdigit(): - idx += 1 - - # Now validate the rest of the path against submodules - while idx < len(parts): - part = parts[idx] - - # If we hit 'weight' or 'bias', we're at a parameter - this is valid - if part in ("weight", "bias"): - return True - - # Check if this part is a registered submodule - if hasattr(current_component, "submodules") and current_component.submodules: - if part in current_component.submodules: - current_component = current_component.submodules[part] - idx += 1 - continue - else: - # This part is not a registered bridge component - # It's likely a nested HF component (like c_fc, c_proj, c_attn) - return False - else: - # No submodules to check, but not at a parameter yet - # Check if next is weight/bias - if idx + 1 < len(parts) and parts[idx + 1] in ("weight", "bias"): - return True - # Otherwise this is likely a nested HF component - return False - - idx += 1 - - return True - - def _normalize_bridge_key_to_hf(self, key: str) -> str: - """Normalize a key that uses bridge attribute names to use HF module names. - - PyTorch's state_dict uses the Python attribute names (e.g., 'ln1') - but the conversion logic expects HF module names (e.g., 'ln_1'). This - function only replaces non-nested component names, leaving bridge - subcomponents (like 'in', 'out', 'q', 'k', 'v') unchanged since they're - handled by the component structure. - - Args: - key: Key that may use bridge attribute names - - Returns: - Key with attribute names replaced by module names where needed - """ - component_mapping = self.adapter.component_mapping - if not component_mapping: - return key - - # Build a mapping of only the direct module attribute names to HF names - # We only care about top-level and block-level component names, NOT subcomponents - attr_to_hf = {} - - # Map top-level components - for tl_name, component in component_mapping.items(): - if component.name and tl_name != "blocks": - # Skip if TL name is already a suffix of the HF path (avoids doubling). - if tl_name != component.name and not component.name.endswith("." + tl_name): - attr_to_hf[tl_name] = component.name - - # Map block-level components (ln1, ln2, attn, mlp) - blocks_component = component_mapping.get("blocks") - if blocks_component and hasattr(blocks_component, "submodules"): - for tl_subname, subcomponent in blocks_component.submodules.items(): - if subcomponent.name: - # Only map if the names differ (e.g., ln1 -> ln_1, but attn -> attn) - if tl_subname != subcomponent.name: - attr_to_hf[tl_subname] = subcomponent.name - - # Replace only these specific attribute names in the key - # We need to be careful to only replace whole path components, not substrings - parts = key.split(".") - result_parts = [] - - for part in parts: - if part in attr_to_hf: - result_parts.append(attr_to_hf[part]) - else: - result_parts.append(part) - - return ".".join(result_parts) - - def state_dict(self, destination=None, prefix="", keep_vars=False): - """Get state dict with TransformerLens format keys. - - Converts HuggingFace format keys to TransformerLens format and filters out - _original_component references and nested HuggingFace components. - - This returns a clean state dict with only bridge component paths converted to TL format, - excluding nested HF components (like c_fc, c_proj, c_attn) that exist inside - original_component modules. - - Args: - destination: Optional dict to store state dict in - prefix: Optional prefix to add to all keys - keep_vars: Whether to keep variables as Variables instead of tensors - - Returns: - Dict containing the state dict with TransformerLens format keys - """ - if destination is not None: - raw_state_dict = self.original_model.state_dict( - destination=destination, prefix=prefix, keep_vars=keep_vars - ) - else: - raw_state_dict = self.original_model.state_dict(prefix=prefix, keep_vars=keep_vars) - - # Clean _original_component references and convert to TL format - # Also filter out nested HuggingFace components that are wrapped by bridge components - tl_state_dict = {} - - for key, value in raw_state_dict.items(): - # Skip _original_component keys - if key == "_original_component" or key.startswith("_original_component."): - continue - - # Remove all _original_component from the key - clean_key = key.replace("._original_component", "") - - # Check if this is a valid bridge path (not a nested HF component) - if not self._is_valid_bridge_path(clean_key): - continue - - # Normalize bridge component names to HF names for conversion - # (e.g., 'ln1' -> 'ln_1', 'mlp.in' -> 'mlp.c_fc') - hf_key = self._normalize_bridge_key_to_hf(clean_key) - - # Convert to TL format - this uses the adapter's component_mapping - tl_key = self.adapter.convert_hf_key_to_tl_key(hf_key) - - # Only add if we haven't seen this TL key yet (handles duplicates) - if tl_key not in tl_state_dict: - tl_state_dict[tl_key] = value - - return tl_state_dict - - def load_state_dict(self, state_dict, strict=True, assign=False): - """Load state dict into the model, handling both clean keys and original keys with _original_component references. - - Args: - state_dict: Dictionary containing a whole state of the module - strict: Whether to strictly enforce that the keys in state_dict match the keys returned by this module's state_dict() function - assign: Whether to assign items in the state dictionary to their corresponding keys in the module instead of copying them - - Returns: - NamedTuple with missing_keys and unexpected_keys fields - """ - current_state_dict = self.original_model.state_dict() - clean_to_actual = {} - actual_to_clean = {} - for actual_key in current_state_dict.keys(): - if actual_key != "_original_component": - clean_key = actual_key.replace("._original_component", "") - clean_to_actual[clean_key] = actual_key - actual_to_clean[actual_key] = clean_key - mapped_state_dict = {} - for input_key, value in state_dict.items(): - if input_key in current_state_dict: - mapped_state_dict[input_key] = value - else: - if input_key in clean_to_actual: - actual_key = clean_to_actual[input_key] - mapped_state_dict[actual_key] = value - else: - mapped_state_dict[input_key] = value - effective_strict = strict and len(mapped_state_dict) == len(current_state_dict) - return self.original_model.load_state_dict( - mapped_state_dict, strict=effective_strict, assign=assign - ) - - def get_params(self): - """Access to model parameters in the format expected by SVDInterpreter. - - For missing weights, returns zero tensors of appropriate shape instead of raising exceptions. - This ensures compatibility across different model architectures. - - Returns: - dict: Dictionary of parameter tensors with TransformerLens naming convention - - Raises: - ValueError: If configuration is inconsistent (e.g., cfg.n_layers != len(blocks)) - """ - return get_bridge_params(self) + from transformer_lens.model_bridge.bridge import TransformerBridge +""" +from transformer_lens.model_bridge.bridge_core import BridgeCore +from transformer_lens.model_bridge.remote_bridge import RemoteBridge +from transformer_lens.model_bridge.transformer_bridge import TransformerBridge - # NOTE: list_supported_models and check_model_support are attached to this class - # dynamically by transformer_lens.model_bridge.sources.transformers module. - # These are HuggingFace-specific methods that belong in the transformers source module. +__all__ = ["BridgeCore", "RemoteBridge", "TransformerBridge"] diff --git a/transformer_lens/model_bridge/bridge_core.py b/transformer_lens/model_bridge/bridge_core.py new file mode 100644 index 0000000000..301432dbfe --- /dev/null +++ b/transformer_lens/model_bridge/bridge_core.py @@ -0,0 +1,865 @@ +"""Framework-agnostic bridge surface shared by TransformerBridge and RemoteBridge.""" +from __future__ import annotations + +import re +import warnings +from contextlib import contextmanager +from functools import lru_cache +from typing import ( + Any, + Callable, + Dict, + Iterator, + List, + Literal, + Mapping, + Optional, + Tuple, + Union, + overload, +) + +import torch + +from transformer_lens.ActivationCache import ActivationCache +from transformer_lens.hook_points import HookPoint +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.driver_protocol import to_torch +from transformer_lens.model_bridge.exceptions import StopAtLayerException +from transformer_lens.utilities.aliases import resolve_alias +from transformer_lens.utilities.lm_utils import lm_cross_entropy_loss + +_BLOCK_PATTERN = re.compile("blocks\\.(\\d+)") + + +def build_alias_to_canonical_map(hook_dict: Any, prefix: str = "") -> dict: + """Map alias hook names to their canonical names (where ``.name`` differs from the key).""" + aliases: dict = {} + for key, value in hook_dict.items(): + full_key = f"{prefix}.{key}" if prefix else key + if isinstance(value, dict): + aliases.update(build_alias_to_canonical_map(value, full_key)) + elif hasattr(value, "name"): + if key != value.name: + aliases[full_key] = value.name + return aliases + + +class BridgeCore: + """Framework-agnostic bridge surface: hooks, cache, run_with_*, driver wiring. + + Holds state shared by every bridge (adapter, cfg, tokenizer, driver, hook + registries). Subclasses add framework-specific state — ``TransformerBridge`` + walks the wrapped ``nn.Module``; ``RemoteBridge`` builds components from + adapter metadata. + """ + + hook_aliases: Dict[str, Union[str, List[str]]] = { + # Prefer embed_ln.hook_out for post-LN models (Bloom, BERT) + "hook_embed": ["embed_ln.hook_out", "embed.hook_out"], + "hook_pos_embed": ["pos_embed.hook_out", "rotary_emb.hook_out"], + "hook_unembed": "unembed.hook_out", + } + + def __init__( + self, + adapter: ArchitectureAdapter, + tokenizer: Any, + driver: Any, + ) -> None: + """Subclasses call this AFTER ``nn.Module.__init__`` (if applicable), + then do framework-specific setup.""" + self.adapter = adapter + self.cfg = adapter.cfg + self.tokenizer = tokenizer + if self.cfg.d_vocab == -1 and self.tokenizer is not None: + if hasattr(self.tokenizer, "get_vocab"): + vocab = self.tokenizer.get_vocab() + self.cfg.d_vocab = max(vocab.values()) + 1 + elif hasattr(self.tokenizer, "vocab"): + self.cfg.d_vocab = max(self.tokenizer.vocab.values()) + 1 + else: + self.cfg.d_vocab = getattr(self.tokenizer, "vocab_size", 50257) + if self.cfg.d_vocab_out == -1: + self.cfg.d_vocab_out = self.cfg.d_vocab + self.compatibility_mode = False + self._hook_cache = None + self._hook_registry: Dict[str, HookPoint] = {} + self._hook_registry_initialized = False + self._hook_alias_registry: Dict[str, Union[str, List[str]]] = {} + self._property_alias_registry: Dict[str, str] = {} + self._driver = driver + if not hasattr(adapter, "component_mapping") or adapter.component_mapping is None: + raise ValueError("Adapter must have a component_mapping attribute") + + # ---- hook registry ---- + + def _initialize_hook_registry(self) -> None: + """Initialize the hook registry by scanning existing components.""" + if self._hook_registry_initialized: + return + self._scan_existing_hooks(self, "") + self._hook_registry_initialized = True + + def _scan_existing_hooks(self, module: Any, prefix: str = "") -> None: + """Walk components for HookPoint instances. Framework-specific.""" + raise NotImplementedError( + f"{type(self).__name__} must implement _scan_existing_hooks " + "(walks the component tree to find HookPoint instances)." + ) + + def clear_hook_registry(self) -> None: + """Clear the hook registry and force re-initialization.""" + self._hook_registry.clear() + self._hook_registry_initialized = False + + @property + def hook_dict(self) -> dict[str, HookPoint]: + """All HookPoint objects, including aliases — TransformerLens-compatible.""" + hooks = self._hook_registry.copy() + self._add_aliases_to_hooks(hooks) + return hooks + + # ---- alias registry ---- + + def _register_aliases(self) -> None: + """Register bridge-level aliases (hook_embed, hook_pos_embed, etc.) by + resolving each alias target path and installing the target HookPoint + as a direct attribute under the alias name.""" + if self.hook_aliases: + self._hook_alias_registry.update(self.hook_aliases) + for alias_name, target_path in self.hook_aliases.items(): + try: + if isinstance(target_path, list): + for single_target in target_path: + try: + target_obj = self + for part in single_target.split("."): + target_obj = getattr(target_obj, part) + object.__setattr__(self, alias_name, target_obj) + break + except AttributeError: + continue + else: + target_obj = self + for part in target_path.split("."): + target_obj = getattr(target_obj, part) + object.__setattr__(self, alias_name, target_obj) + except AttributeError: + pass + + def _collect_component_aliases(self, component_mapping: Any, prefix: str = "") -> dict: + """Recursively collect aliases from components.""" + aliases: dict = {} + if isinstance(component_mapping, dict): + for name, component in component_mapping.items(): + sub_prefix = f"{prefix}.{name}" if prefix else name + aliases.update(self._collect_component_aliases(component, sub_prefix)) + else: + if hasattr(component_mapping, "hook_aliases") and component_mapping.hook_aliases: + for alias_name, target in component_mapping.hook_aliases.items(): + full_alias = f"{prefix}.{alias_name}" if prefix else alias_name + full_target = f"{prefix}.{target}" if prefix else target + aliases[full_alias] = full_target + if hasattr(component_mapping, "submodules") and component_mapping.submodules: + for sub_name, sub_component in component_mapping.submodules.items(): + sub_prefix = f"{prefix}.{sub_name}" if prefix else sub_name + aliases.update(self._collect_component_aliases(sub_component, sub_prefix)) + return aliases + + @staticmethod + @lru_cache(maxsize=128) + def _compute_hook_aliases_cached( + hook_names_tuple: Tuple[str, ...], + component_aliases_tuple: Tuple[Tuple[str, str], ...], + ) -> Tuple[Tuple[str, str], ...]: + """Cached computation of hook aliases.""" + aliases: dict = {} + component_aliases = dict(component_aliases_tuple) + for hook_name in hook_names_tuple: + for alias_pattern, target_pattern in component_aliases.items(): + if "blocks." in target_pattern and "blocks." in hook_name: + block_match = _BLOCK_PATTERN.search(hook_name) + if block_match: + block_num = block_match.group(1) + dynamic_alias_pattern = alias_pattern.replace( + "blocks.", f"blocks.{block_num}." + ) + dynamic_target_pattern = target_pattern.replace( + "blocks.", f"blocks.{block_num}." + ) + if hook_name.endswith(dynamic_target_pattern): + target_len = len(dynamic_target_pattern) + alias_name = hook_name[:-target_len] + dynamic_alias_pattern + aliases[alias_name] = hook_name + elif hook_name.endswith(target_pattern): + target_len = len(target_pattern) + alias_name = hook_name[:-target_len] + alias_pattern + aliases[alias_name] = hook_name + return tuple(aliases.items()) + + def _collect_hook_aliases_from_registry(self) -> dict: + """Collect aliases based on existing hooks in the registry.""" + if hasattr(self.adapter, "component_mapping"): + component_aliases = self._collect_component_aliases(self.adapter.component_mapping) + hook_names_tuple = tuple(sorted(self._hook_registry.keys())) + component_aliases_tuple = tuple(sorted(component_aliases.items())) + aliases_tuple = self._compute_hook_aliases_cached( + hook_names_tuple, component_aliases_tuple + ) + return dict(aliases_tuple) + return {} + + def _add_aliases_to_hooks(self, hooks: Dict[str, HookPoint]) -> None: + """Add aliases to hooks in place. Registry-first so RemoteBridge works.""" + component_aliases = self._collect_hook_aliases_from_registry() + all_aliases = {**self.hook_aliases, **component_aliases} + if not all_aliases: + return + for alias_name, target in all_aliases.items(): + targets = target if isinstance(target, list) else [target] + for t in targets: + hp = self._hook_registry.get(t) + if hp is not None: + hooks[alias_name] = hp + break + # Fall back to attribute walk for TransformerBridge nested paths + # not directly keyed in the registry. + try: + target_hook = resolve_alias(self, alias_name, {alias_name: t}) + if target_hook is not None: + hooks[alias_name] = target_hook + break + except AttributeError: + continue + + # ---- captures from driver ---- + + def _replay_captures(self, captured: Mapping[str, Any]) -> None: + """Fire driver-delivered captures through the registry. Unknown names dropped silently.""" + for hook_name, activation in captured.items(): + hp = self._hook_registry.get(hook_name) + if hp is None: + continue + hp(to_torch(activation)) + + # ---- driver dispatch (subclasses concrete-override) ---- + + def forward(self, *args: Any, **kwargs: Any) -> Any: + """Subclasses implement how the driver gets called.""" + raise NotImplementedError(f"{type(self).__name__} must implement forward()") + + def to_tokens(self, *args: Any, **kwargs: Any) -> Any: + """Subclasses implement against their tokenizer surface.""" + raise NotImplementedError(f"{type(self).__name__} must implement to_tokens()") + + def close(self) -> None: + """Release driver-managed resources. Idempotent — safe to call multiple times.""" + self._driver.close() + + def _input_device(self) -> Any: + """Driver's expected device for inputs; None for remote / meta / dispatched drivers.""" + if not self._driver.supports("parameters"): + return None + params = getattr(self._driver, "parameters", None) + if not callable(params): + return None + try: + device = next(params()).device + except (StopIteration, NotImplementedError, RuntimeError): + return None + # Meta device → inputs would silently become meta tensors with no data. + if device.type == "meta": + return None + return device + + def loss_fn( + self, + logits: torch.Tensor, + tokens: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + per_token: bool = False, + ) -> torch.Tensor: + """Cross-entropy loss matching HookedTransformer's formula (log_softmax + gather).""" + if tokens.device != logits.device: + tokens = tokens.to(logits.device) + return lm_cross_entropy_loss(logits, tokens, attention_mask, per_token) + + def _finalize_return( + self, + return_type: Optional[str], + logits: Optional[torch.Tensor], + input_ids: Optional[torch.Tensor], + *, + is_audio_model: bool = False, + inputs_embeds_was_used: bool = False, + loss_per_token: bool = False, + ) -> Any: + """Post-process driver output into the user's requested return_type.""" + if return_type == "logits": + return logits + if return_type is None: + return None + if return_type == "loss": + if is_audio_model: + raise ValueError( + "Audio models do not support return_type='loss'. " + "CTC loss requires aligned frame-level labels." + ) + if inputs_embeds_was_used: + raise ValueError( + "Cannot compute loss with inputs_embeds — token IDs required for labels." + ) + assert isinstance(logits, torch.Tensor), f"Expected logits tensor, got {type(logits)}" + assert input_ids is not None, "input_ids required for return_type='loss'" + return self.loss_fn(logits, input_ids, per_token=loss_per_token) + if return_type == "both": + if is_audio_model: + raise ValueError( + "Audio models do not support return_type='both'. " + "CTC loss requires aligned frame-level labels." + ) + if inputs_embeds_was_used: + raise ValueError( + "Cannot compute loss with inputs_embeds — token IDs required for labels." + ) + assert isinstance(logits, torch.Tensor), f"Expected logits tensor, got {type(logits)}" + assert input_ids is not None, "input_ids required for return_type='both'" + loss = self.loss_fn(logits, input_ids, per_token=loss_per_token) + return (logits, loss) + if return_type == "predictions": + assert self.tokenizer is not None, "Tokenizer required for return_type='predictions'" + assert isinstance(logits, torch.Tensor), f"Expected logits tensor, got {type(logits)}" + if logits.shape[-1] == 2: + # Next Sentence Prediction — 2-class output + logprobs = logits.log_softmax(dim=-1) + predictions = [ + "The sentences are sequential", + "The sentences are NOT sequential", + ] + return predictions[int(logprobs.argmax(dim=-1).item())] + else: + # Masked Language Modeling — decode [MASK] tokens + assert input_ids is not None, "input_ids required for MLM predictions" + logprobs = logits[input_ids == self.tokenizer.mask_token_id].log_softmax(dim=-1) + preds = self.tokenizer.decode(logprobs.argmax(dim=-1)) + if " " in preds: + parts = preds.split(" ") + return [f"Prediction {i}: {p}" for i, p in enumerate(parts)] + return preds + raise ValueError(f"Invalid return_type: {return_type}") + + # ---- hook lookup / mutation ---- + + def get_hook_point(self, hook_name: str) -> Optional[HookPoint]: + """Get a hook point by name from the bridge's hook system.""" + if hook_name in self._hook_registry: + return self._hook_registry[hook_name] + try: + parts = hook_name.split(".") + current: Any = self + for part in parts: + current = getattr(current, part) + if isinstance(current, HookPoint): + return current + except AttributeError: + pass + return None + + def add_hook( + self, + name: Union[str, Callable[[str], bool]], + hook_fn: Any, + dir: Literal["fwd", "bwd"] = "fwd", + is_permanent: bool = False, + ) -> None: + """Add a hook to a specific component or to all components matching a filter. + + Args: + name: Either a string hook point name (e.g. "blocks.0.attn.hook_q") + or a callable filter ``(str) -> bool`` that is applied to every + hook point name; the hook is added to each point where the filter + returns True. + hook_fn: The hook function ``(activation, hook) -> activation | None``. + dir: Hook direction, ``"fwd"`` or ``"bwd"``. + is_permanent: If True the hook survives ``reset_hooks()`` calls. + """ + if callable(name) and not isinstance(name, str): + hook_dict = self.hook_dict + seen_hooks: set = set() + for hook_name, hook_point in hook_dict.items(): + if name(hook_name): + hook_id = id(hook_point) + if hook_id in seen_hooks: + continue + seen_hooks.add(hook_id) + hook_point.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) + return + + # Registry-first: works for any bridge (RemoteBridge has no component tree). + registry_hp = self._hook_registry.get(name) + if registry_hp is not None: + registry_hp.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) + return + + component: Any = self + parts = name.split(".") + for part in parts[:-1]: + if hasattr(component, part): + component = getattr(component, part) + else: + raise AttributeError(f"Component path '{'.'.join(parts[:-1])}' not found") + hook_name = parts[-1] + if hasattr(component, hook_name): + hook_point = getattr(component, hook_name) + if isinstance(hook_point, HookPoint): + hook_point.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) + else: + raise AttributeError( + f"'{hook_name}' is not a hook point. Found object of type: {type(hook_point)} with value: {hook_point}" + ) + else: + raise AttributeError(f"Hook point '{hook_name}' not found on component") + + def reset_hooks(self, clear_contexts: bool = True) -> None: + """Remove all hooks. Registry is canonical; nn.Module children walked additively.""" + for hp in self._hook_registry.values(): + hp.remove_hooks() + if hasattr(self, "children"): + from transformer_lens.model_bridge.generalized_components.base import ( + GeneralizedComponent, + ) + + def remove_hooks_recursive(module: Any) -> None: + if isinstance(module, GeneralizedComponent): + module.remove_hooks() + if hasattr(module, "children"): + for child in module.children(): + remove_hooks_recursive(child) + + remove_hooks_recursive(self) + + def hooks( + self, + fwd_hooks: List = [], + bwd_hooks: List = [], + reset_hooks_end: bool = True, + clear_contexts: bool = False, + ) -> Any: + """Context manager for temporarily adding hooks. + + Example: + with model.hooks(fwd_hooks=[("hook_embed", my_hook)]): + output = model("Hello world") + """ + + @contextmanager + def _hooks_context() -> Iterator["BridgeCore"]: + added_hooks: List[Tuple[HookPoint, str]] = [] + + def add_hook_to_point( + hook_point: HookPoint, + hook_fn: Callable, + name: str, + dir: Literal["fwd", "bwd"] = "fwd", + ) -> None: + if self.compatibility_mode and name != hook_point.name: + alias_names_list: list = [] + if hook_point.name is not None: + alias_names_list.append(hook_point.name) + alias_names_list.append(name) + hook_point.add_hook(hook_fn, dir=dir, alias_names=alias_names_list) + else: + hook_point.add_hook(hook_fn, dir=dir) + added_hooks.append((hook_point, name)) + + def apply_hooks(hook_list: List[Tuple[Any, Callable]], is_fwd: bool) -> None: + direction: Literal["fwd", "bwd"] = "fwd" if is_fwd else "bwd" + aliases = build_alias_to_canonical_map(self.hook_dict) + for hook_name_or_filter, hook_fn in hook_list: + if isinstance(hook_name_or_filter, str): + hook_dict = self.hook_dict + actual_hook_name = hook_name_or_filter + if hook_name_or_filter in aliases: + actual_hook_name = aliases[hook_name_or_filter] + if actual_hook_name in hook_dict: + add_hook_to_point( + hook_dict[actual_hook_name], hook_fn, actual_hook_name, direction + ) + else: + hook_dict = self.hook_dict + seen_hooks = set() + for n, hook_point in hook_dict.items(): + if hook_name_or_filter(n): + hook_id = id(hook_point) + if hook_id in seen_hooks: + continue + seen_hooks.add(hook_id) + hook_name_to_use = hook_point.name if hook_point.name else n + add_hook_to_point(hook_point, hook_fn, hook_name_to_use, direction) + + try: + apply_hooks(fwd_hooks, True) + apply_hooks(bwd_hooks, False) + yield self + finally: + if reset_hooks_end: + for hook_point, _ in added_hooks: + hook_point.remove_hooks() + + return _hooks_context() + + # ---- high-level execution: run_with_hooks ---- + + def run_with_hooks( + self, + input: Any, + fwd_hooks: List[Tuple[Union[str, Callable], Callable]] = [], + bwd_hooks: List[Tuple[Union[str, Callable], Callable]] = [], + reset_hooks_end: bool = True, + clear_contexts: bool = False, + return_type: Optional[str] = "logits", + names_filter: Optional[Union[str, List[str], Callable[[str], bool]]] = None, + stop_at_layer: Optional[int] = None, + remove_batch_dim: bool = False, + **kwargs: Any, + ) -> Any: + """Run the model with specified forward and backward hooks. + + ``stop_at_layer`` raises :class:`StopAtLayerException` to stop early + (KV cache cleaned up on stop). ``remove_batch_dim`` squeezes/unsqueezes + the batch dim around hook callbacks (batch_size==1 only). + """ + added_hooks: List[Tuple[HookPoint, str]] = [] + effective_stop_layer = None + if stop_at_layer is not None and hasattr(self, "blocks"): + if stop_at_layer < 0: + effective_stop_layer = len(self.blocks) + stop_at_layer + else: + effective_stop_layer = stop_at_layer + + def add_hook_to_point( + hook_point: HookPoint, + hook_fn: Callable, + name: str, + dir: Literal["fwd", "bwd"] = "fwd", + ) -> None: + if effective_stop_layer is not None and name.startswith("blocks."): + try: + layer_num = int(name.split(".")[1]) + if layer_num >= effective_stop_layer: + return + except (IndexError, ValueError): + pass + if self.compatibility_mode and name != hook_point.name: + alias_names_list: list = [] + if hook_point.name is not None: + alias_names_list.append(hook_point.name) + alias_names_list.append(name) + hook_point.add_hook(hook_fn, dir=dir, alias_names=alias_names_list) + else: + hook_point.add_hook(hook_fn, dir=dir) + added_hooks.append((hook_point, name)) + + if stop_at_layer is not None and hasattr(self, "blocks"): + if stop_at_layer < 0: + stop_at_layer = len(self.blocks) + stop_at_layer + if stop_at_layer >= 0 and stop_at_layer < len(self.blocks): + + def stop_hook(tensor: Any, *, hook: Any) -> Any: + raise StopAtLayerException(tensor) + + # Stop at the beginning of the specified block, not at the end of the previous block + block_hook_name = f"blocks.{stop_at_layer}.hook_in" + hook_dict = self.hook_dict + if block_hook_name in hook_dict: + add_hook_to_point(hook_dict[block_hook_name], stop_hook, block_hook_name, "fwd") + + def apply_hooks( + hook_list: List[Tuple[Union[str, Callable], Callable]], is_fwd: bool + ) -> None: + direction: Literal["fwd", "bwd"] = "fwd" if is_fwd else "bwd" + aliases = build_alias_to_canonical_map(self.hook_dict) + for hook_name_or_filter, hook_fn in hook_list: + if remove_batch_dim: + original_hook_fn = hook_fn + + # Default arg captures hook_fn by value (avoids closure issue) + def wrapped_hook_fn(tensor, hook, _orig_fn=original_hook_fn): + if tensor.shape[0] == 1: + tensor_no_batch = tensor.squeeze(0) + result = _orig_fn(tensor_no_batch, hook) + if result.dim() == tensor_no_batch.dim(): + result = result.unsqueeze(0) + return result + else: + return _orig_fn(tensor, hook) + + hook_fn = wrapped_hook_fn + if isinstance(hook_name_or_filter, str): + hook_dict = self.hook_dict + actual_hook_name = hook_name_or_filter + if hook_name_or_filter in aliases: + actual_hook_name = aliases[hook_name_or_filter] + if actual_hook_name in hook_dict: + add_hook_to_point( + hook_dict[actual_hook_name], hook_fn, actual_hook_name, direction + ) + else: + hook_dict = self.hook_dict + seen_hooks: set = set() + for n, hook_point in hook_dict.items(): + if hook_name_or_filter(n): + hook_id = id(hook_point) + if hook_id in seen_hooks: + continue + seen_hooks.add(hook_id) + hook_name_to_use = hook_point.name if hook_point.name else n + add_hook_to_point(hook_point, hook_fn, hook_name_to_use, direction) + + try: + apply_hooks(fwd_hooks, True) + apply_hooks(bwd_hooks, False) + try: + output = self.forward( + input, return_type=return_type, stop_at_layer=stop_at_layer, **kwargs + ) + except StopAtLayerException as e: + output = e.layer_output + return output + finally: + if reset_hooks_end: + for hook_point, _ in added_hooks: + hook_point.remove_hooks() + + # ---- high-level execution: run_with_cache ---- + + @overload + def run_with_cache( + self, + input: Union[str, List[str], torch.Tensor], + return_cache_object: Literal[True] = True, + remove_batch_dim: bool = False, + **kwargs, + ) -> Tuple[Any, ActivationCache]: + """Run with cache - placeholder implementation.""" + pass + + @overload + def run_with_cache( + self, + input: Union[str, List[str], torch.Tensor], + return_cache_object: Literal[False], + remove_batch_dim: bool = False, + **kwargs, + ) -> Tuple[Any, Dict[str, torch.Tensor]]: + """Run with cache - placeholder implementation.""" + pass + + def run_with_cache( + self, + input: Union[str, List[str], torch.Tensor], + return_cache_object: bool = True, + remove_batch_dim: bool = False, + names_filter: Optional[Union[str, List[str], Callable[[str], bool]]] = None, + stop_at_layer: Optional[int] = None, + **kwargs, + ) -> Tuple[Any, Union[ActivationCache, Dict[str, torch.Tensor]]]: + """Run the model and cache activations. Returns ``(output, cache)``. + + ``stop_at_layer`` raises :class:`StopAtLayerException` to stop early. + """ + aliases = build_alias_to_canonical_map(self.hook_dict) + + def create_names_filter_fn(filter_input): + if filter_input is None: + return lambda name: True + elif isinstance(filter_input, str): + mapped_name = aliases.get(filter_input, None) + if mapped_name: + return lambda name: name == mapped_name or name == filter_input + else: + return lambda name: name == filter_input + elif isinstance(filter_input, list): + mapped_list = [] + for item in filter_input: + mapped_list.append(item) + mapped_name = aliases.get(item, None) + if mapped_name: + mapped_list.append(mapped_name) + return lambda name: name in mapped_list + elif callable(filter_input): + return filter_input + else: + raise ValueError("names_filter must be a string, list of strings, or callable") + + names_filter_fn = create_names_filter_fn(names_filter) + cache: Dict[str, torch.Tensor] = {} + hooks: List[Tuple[HookPoint, str]] = [] + visited: set[int] = set() + + # None → no-op .to(None), tensors stay on their current device. + cache_device = kwargs.pop("device", None) + + def make_cache_hook(name: str): + def cache_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: + if tensor is None: + cache[name] = None + elif isinstance(tensor, torch.Tensor): + cache[name] = tensor.detach().to(cache_device) + elif isinstance(tensor, tuple): + if len(tensor) > 0 and isinstance(tensor[0], torch.Tensor): + cache[name] = tensor[0].detach().to(cache_device) + else: + pass + else: + try: + if hasattr(tensor, "detach"): + cache[name] = tensor.detach().to(cache_device) + except: + pass + return tensor + + return cache_hook + + hook_dict = self.hook_dict + effective_stop_layer = None + if stop_at_layer is not None and hasattr(self, "blocks"): + if stop_at_layer < 0: + effective_stop_layer = len(self.blocks) + stop_at_layer + else: + effective_stop_layer = stop_at_layer + for hook_name, hook in hook_dict.items(): + if names_filter_fn(hook_name): + if effective_stop_layer is not None: + if hook_name.startswith("blocks."): + try: + layer_num = int(hook_name.split(".")[1]) + if layer_num >= effective_stop_layer: + continue + except (IndexError, ValueError): + pass + hooks.append((hook, hook_name)) + for hp, name in hooks: + hp.add_hook(make_cache_hook(name)) + processed_args = [input] + # Driver-aware input placement: torch drivers move input_ids to the model's + # device; remote drivers (no local parameters) leave them as-is. + target_device = self._input_device() + if processed_args and isinstance(processed_args[0], str): + assert self.tokenizer is not None, "Tokenizer must be set to pass string input." + input_ids = self.to_tokens(processed_args[0]) + if target_device is not None: + input_ids = input_ids.to(target_device) + kwargs["input_ids"] = input_ids + processed_args = processed_args[1:] + elif "input" in kwargs and isinstance(kwargs["input"], str): + assert self.tokenizer is not None, "Tokenizer must be set to pass string input." + input_ids = self.to_tokens(kwargs["input"]) + if target_device is not None: + input_ids = input_ids.to(target_device) + kwargs["input_ids"] = input_ids + del kwargs["input"] + if stop_at_layer is not None and hasattr(self, "blocks"): + if stop_at_layer < 0: + stop_at_layer = len(self.blocks) + stop_at_layer + last_layer_to_process = stop_at_layer - 1 + + def stop_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: + raise StopAtLayerException(tensor) + + if stop_at_layer >= 0 and stop_at_layer < len(self.blocks): + # Stop at the beginning of the specified block, not at the end of the previous block + block_hook_name = f"blocks.{stop_at_layer}.hook_in" + hook_dict = self.hook_dict + if block_hook_name in hook_dict: + hook_dict[block_hook_name].add_hook(stop_hook) + hooks.append((hook_dict[block_hook_name], block_hook_name)) + filtered_kwargs = kwargs.copy() + if cache_device is not None: + if getattr(self.cfg, "n_devices", 1) > 1: + # Moving a dispatched model to a single device collapses accelerate's + # split and breaks its routing hooks. The cache will stay spread across + # the per-layer devices; callers can .to(cache_device) on cache entries + # after the fact if they need a single-device cache. + warnings.warn( + f"run_with_cache(device={cache_device!r}) ignored: model is dispatched " + f"across {self.cfg.n_devices} devices via device_map. Cached activations " + "will remain on their per-layer devices.", + stacklevel=2, + ) + else: + try: + # original_model / its setter exist on TransformerBridge; on + # non-torch bridges the property raises AttributeError. + underlying = getattr(self, "original_model") + setattr(self, "original_model", underlying.to(cache_device)) + except AttributeError: + # Non-torch driver: cache_device doesn't apply to the model. + warnings.warn( + f"run_with_cache(device={cache_device!r}) ignored: driver does not " + "expose a local model. Cached activations will stay on driver-managed " + "devices.", + stacklevel=2, + ) + if processed_args and isinstance(processed_args[0], torch.Tensor): + processed_args = [processed_args[0].to(cache_device)] + list(processed_args[1:]) + for key, value in filtered_kwargs.items(): + if isinstance(value, torch.Tensor): + filtered_kwargs[key] = value.to(cache_device) + try: + if "output_attentions" not in filtered_kwargs: + filtered_kwargs["output_attentions"] = True + if processed_args: + output = self.forward(processed_args[0], **filtered_kwargs) + elif "input_ids" in filtered_kwargs: + output = self.forward( + filtered_kwargs["input_ids"], + **{k: v for k, v in filtered_kwargs.items() if k != "input_ids"}, + ) + else: + output = self.forward(**filtered_kwargs) + if hasattr(output, "logits"): + output = output.logits + except StopAtLayerException as e: + output = e.layer_output + except Exception as e: + raise e + finally: + for hp, _ in hooks: + hp.remove_hooks() + if self.compatibility_mode == True: + reverse_aliases = {} + for old_name, new_name in aliases.items(): + if isinstance(new_name, list): + for single_new_name in new_name: + reverse_aliases[single_new_name] = old_name + else: + reverse_aliases[new_name] = old_name + cache_items_to_add = {} + for cache_name, cached_value in cache.items(): + for new_name, old_name in reverse_aliases.items(): + if cache_name == new_name: + cache_items_to_add[old_name] = cached_value + break + cache.update(cache_items_to_add) + for alias_name, target_name in aliases.items(): + if isinstance(target_name, list): + for single_target in target_name: + if single_target in cache and alias_name not in cache: + cache[alias_name] = cache[single_target] + break + elif target_name in cache and alias_name not in cache: + cache[alias_name] = cache[target_name] + if return_cache_object: + activation_cache = ActivationCache(cache, self, has_batch_dim=True) + if remove_batch_dim: + activation_cache.remove_batch_dim() + return (output, activation_cache) + else: + if remove_batch_dim: + for key in cache: + if cache[key] is not None and isinstance(cache[key], torch.Tensor): + if cache[key].size(0) == 1: + cache[key] = cache[key][0] + return (output, cache) diff --git a/transformer_lens/model_bridge/driver_protocol.py b/transformer_lens/model_bridge/driver_protocol.py new file mode 100644 index 0000000000..ffe7a4862e --- /dev/null +++ b/transformer_lens/model_bridge/driver_protocol.py @@ -0,0 +1,192 @@ +"""Driver protocol: the contract every model-execution backend satisfies.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable, Mapping, Protocol, Union, runtime_checkable + +import numpy as np +import torch + +from transformer_lens.config import TransformerBridgeConfig + + +@runtime_checkable +class TensorLike(Protocol): + """Quacks like a tensor: ``__array__`` + ``shape`` + ``dtype``.""" + + # Attribute types stay loose — torch.Size, plain tuple, and numpy's shape + # don't share a Protocol-strict supertype. + @property + def shape(self) -> Any: + ... + + @property + def dtype(self) -> Any: + ... + + def __array__(self, dtype: Any = None) -> np.ndarray: + ... + + +InterventionFn = Callable[[TensorLike], TensorLike] +InterventionSpec = Mapping[str, Any] +# Drivers that can dispatch Python at the engine boundary (HF) accept +# InterventionFn; drivers that can't (vLLM under compile, remote APIs) accept +# InterventionSpec only. +Intervention = Union[InterventionFn, InterventionSpec] + + +@dataclass(frozen=True) +class ForwardResult: + """One forward call's outputs. Tensors are native to the driver's framework.""" + + logits: TensorLike | None = None + captured: Mapping[str, TensorLike] = field(default_factory=dict) + new_tokens: TensorLike | None = None + # Driver's native return value (HF CausalLMOutputWithPast, vLLM + # RequestOutput, ...). Bridge reads driver-specific extras here. + raw_output: Any = None + + +@runtime_checkable +class Driver(Protocol): + """The forward-pass contract. Hook installation is the driver's problem.""" + + architecture: str + bridge_config: TransformerBridgeConfig + tokenizer: Any + supported_hook_points: frozenset[str] + non_fireable_hook_points: frozenset[str] + + def forward( + self, + input_ids: TensorLike | None = None, + *, + capture: tuple[str, ...] = (), + intervene: Mapping[str, Intervention] | None = None, + max_new_tokens: int = 1, + return_logits: bool = True, + **kwargs: Any, + ) -> ForwardResult: + ... + + def close(self) -> None: + ... + + def supports(self, feature: str) -> bool: + """Capability flag. Known features: gradients, parameters, state_dict, + generate_streaming, weight_access, intervention_callbacks.""" + ... + + # Note: torch-specific surface (parameters, named_parameters, state_dict, + # weight access) is NOT in the protocol. Drivers that can serve those + # methods provide them as implementation details, gated by supports("..."). + + +def to_torch(t: TensorLike, *, dtype: torch.dtype | None = None) -> torch.Tensor: + """Convert any TensorLike to torch.Tensor at the bridge boundary. + + Order: torch passthrough → DLPack (jax/mlx/tf/cupy/numpy≥1.22, preserves + device) → ``__array__`` + ``from_numpy`` (CPU only). + """ + if isinstance(t, torch.Tensor): + return t.to(dtype) if dtype is not None else t + + if hasattr(t, "__dlpack__"): + try: + out = torch.from_dlpack(t) + return out.to(dtype) if dtype is not None else out + except (BufferError, RuntimeError, ValueError, TypeError, AttributeError): + # Fall through on stream-sync, missing __dlpack_device__, or + # version-skew failures; the numpy path either succeeds or raises + # informatively. + pass + + arr = np.asarray(t) + out = torch.from_numpy(arr) + return out.to(dtype) if dtype is not None else out + + +# Parameter names a conforming driver's forward() must accept; missing names +# get silently swallowed by **kwargs and break the contract. +_DRIVER_FORWARD_REQUIRED_PARAMS = frozenset( + ("input_ids", "capture", "intervene", "max_new_tokens", "return_logits") +) + + +def validate_driver(driver: Any, *, after_bridge_construction: bool = False) -> None: + """Stronger than ``isinstance(driver, Driver)``: checks types, signatures, + and (optionally) post-construction state. + + Args: + after_bridge_construction: when True, also requires at least one of + ``supported_hook_points`` / ``non_fireable_hook_points`` non-empty + (the bridge backfills the former, so empty-on-both means the + driver silently degrades to "supports nothing"). + + Raises: + TypeError: with a message naming the contract violation. + """ + _expect_attr_type(driver, "architecture", str) + _expect_attr_type(driver, "bridge_config", TransformerBridgeConfig) + if not hasattr(driver, "tokenizer"): + raise TypeError("Driver missing required attribute: 'tokenizer'") + _expect_attr_type(driver, "supported_hook_points", frozenset) + _expect_attr_type(driver, "non_fireable_hook_points", frozenset) + + overlap = driver.supported_hook_points & driver.non_fireable_hook_points + if overlap: + raise TypeError( + f"Driver.supported_hook_points and Driver.non_fireable_hook_points " + f"overlap on {sorted(overlap)[:3]}; a hook is either fireable or not." + ) + for name in driver.supported_hook_points | driver.non_fireable_hook_points: + if not isinstance(name, str): + raise TypeError(f"Hook-point names must be str; got {type(name).__name__}: {name!r}") + + forward = getattr(driver, "forward", None) + if not callable(forward): + raise TypeError("Driver.forward must be callable") + import inspect + + sig = inspect.signature(forward) + params = sig.parameters + has_var_keyword = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values()) + missing = [p for p in _DRIVER_FORWARD_REQUIRED_PARAMS if p not in params] + if missing and not has_var_keyword: + raise TypeError( + f"Driver.forward must accept parameters {sorted(_DRIVER_FORWARD_REQUIRED_PARAMS)}; " + f"missing {sorted(missing)} (and no **kwargs to absorb them)." + ) + + if not callable(getattr(driver, "close", None)): + raise TypeError("Driver.close must be callable") + + if after_bridge_construction: + if not driver.supported_hook_points and not driver.non_fireable_hook_points: + raise TypeError( + "Driver has empty supported_hook_points AND non_fireable_hook_points " + "after bridge construction. Drivers must declare at least one — the " + "bridge backfills supported from registry minus non_fireable, but " + "empty-on-both means there's no contract for downstream code." + ) + + +def _expect_attr_type(obj: Any, name: str, expected: type) -> None: + if not hasattr(obj, name): + raise TypeError(f"Driver missing required attribute: {name!r}") + value = getattr(obj, name) + if not isinstance(value, expected): + raise TypeError(f"Driver.{name} must be {expected.__name__}; got {type(value).__name__}.") + + +__all__ = [ + "Driver", + "ForwardResult", + "Intervention", + "InterventionFn", + "InterventionSpec", + "TensorLike", + "to_torch", + "validate_driver", +] diff --git a/transformer_lens/model_bridge/remote_bridge.py b/transformer_lens/model_bridge/remote_bridge.py new file mode 100644 index 0000000000..b41fa10352 --- /dev/null +++ b/transformer_lens/model_bridge/remote_bridge.py @@ -0,0 +1,100 @@ +"""Non-torch bridge: vLLM workers, Inspect remote providers.""" +from __future__ import annotations + +from typing import Any + +from transformer_lens.hook_points import HookIntrospectionMixin, HookPoint +from transformer_lens.model_bridge.bridge_core import BridgeCore +from transformer_lens.model_bridge.driver_protocol import ( + ForwardResult, + TensorLike, + to_torch, + validate_driver, +) + + +class RemoteBridge(BridgeCore, HookIntrospectionMixin): + """Bridge for backends with no local ``nn.Module`` (vLLM, Inspect). + + No nn.Module parentage strips the torch-only surface; driver pre-declares + ``supported_hook_points`` (no model to walk). + """ + + def __init__( + self, + adapter: Any, + tokenizer: Any, + driver: Any, + ) -> None: + if not driver.supported_hook_points: + raise ValueError( + "RemoteBridge requires driver.supported_hook_points to be " + "non-empty: non-torch drivers own the hook namespace because " + "there is no local model for the bridge to walk." + ) + BridgeCore.__init__(self, adapter, tokenizer, driver) + # No local device; tensor.to(None) is a no-op so downstream patterns degrade cleanly. + self.cfg.device = None + # HookPoint is nn.Module-backed but RemoteBridge isn't an nn.Module — + # named_modules() walks don't apply; only registry lookup matters. + for name in driver.supported_hook_points: + hp = HookPoint() + hp.name = name + self._hook_registry[name] = hp + self._hook_registry_initialized = True + validate_driver(self._driver, after_bridge_construction=True) + + @staticmethod + def boot_vllm(*args: Any, **kwargs: Any) -> "RemoteBridge": + """Boot a model via vLLM. Returns a RemoteBridge wrapping a VLLMDriver. + + Mirrors ``TransformerBridge.boot_transformers``. Lazy import so + ``remote_bridge`` itself stays vLLM-agnostic — only callers of this + method need vLLM installed. See :func:`sources.vllm.boot_vllm` for kwargs. + """ + from .sources.vllm import boot_vllm as _boot_vllm + + return _boot_vllm(*args, **kwargs) + + def _scan_existing_hooks(self, module: Any, prefix: str = "") -> None: + """No-op: registry built from driver declarations in __init__.""" + + def forward( + self, + input: Any = None, + *, + return_type: str | None = "logits", + loss_per_token: bool = False, + **kwargs: Any, + ) -> Any: + """Tokenize → driver.forward → replay captures → finalize per return_type.""" + if isinstance(input, str): + assert self.tokenizer is not None, "Tokenizer must be set for string input." + tokens = self.tokenizer.encode(input, return_tensors="pt") + kwargs["input_ids"] = tokens + elif input is not None: + kwargs["input_ids"] = input + + result: ForwardResult = self._driver.forward(**kwargs) + if result.captured: + self._replay_captures(result.captured) + + logits: Any = result.logits + if logits is not None and not isinstance(logits, TensorLike): + return logits # weird shape — let caller handle + if logits is not None: + logits = to_torch(logits) + + return self._finalize_return( + return_type, + logits, + kwargs.get("input_ids"), + is_audio_model=getattr(self.cfg, "is_audio_model", False), + loss_per_token=loss_per_token, + ) + + def to_tokens(self, text: Any, *args: Any, **kwargs: Any) -> Any: + """Tokenize via ``self.tokenizer``. BOS/padding handling lives on + :class:`TransformerBridge`.""" + assert self.tokenizer is not None, "Tokenizer must be set." + return self.tokenizer.encode(text, return_tensors="pt") diff --git a/transformer_lens/model_bridge/sources/__init__.py b/transformer_lens/model_bridge/sources/__init__.py index c3e54e2364..2dabe400ba 100644 --- a/transformer_lens/model_bridge/sources/__init__.py +++ b/transformer_lens/model_bridge/sources/__init__.py @@ -8,9 +8,11 @@ check_model_support, list_supported_models, ) +from transformer_lens.model_bridge.sources.vllm import boot_vllm __all__ = [ "boot", + "boot_vllm", "list_supported_models", "check_model_support", ] diff --git a/transformer_lens/model_bridge/sources/_bridge_builder.py b/transformer_lens/model_bridge/sources/_bridge_builder.py new file mode 100644 index 0000000000..152671d790 --- /dev/null +++ b/transformer_lens/model_bridge/sources/_bridge_builder.py @@ -0,0 +1,193 @@ +"""Loader-agnostic helpers for building a TransformerBridge around a pre-loaded model.""" +from __future__ import annotations + +from typing import Any, Callable, Optional + +import torch +from torch import nn + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.factories.architecture_adapter_factory import ( + ArchitectureAdapterFactory, +) +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.bridge import TransformerBridge +from transformer_lens.model_bridge.sources._hf_format import ( + map_default_transformer_lens_config, + setup_tokenizer, +) + +# Architecture-agnostic; do not extend per-architecture. +_HF_PASSTHROUGH_ATTRS = [ + # OPT + "is_gated_act", + "word_embed_proj_dim", + "do_layer_norm_before", + # Granite + "position_embedding_type", + # Falcon + "parallel_attn", + "multi_query", + "new_decoder_architecture", + "alibi", + "num_ln_in_parallel_attn", + # Mamba (SSM config) + "state_size", + "conv_kernel", + "expand", + "time_step_rank", + "intermediate_size", + # Mamba-2 (additional SSM config) + "n_groups", + "chunk_size", + # Multimodal + "vision_config", +] + + +def build_bridge_config_from_hf( + hf_config: Any, + architecture: str, + model_name: str, + dtype: torch.dtype, +) -> TransformerBridgeConfig: + """Translate an HF config into a :class:`TransformerBridgeConfig`.""" + tl_config = map_default_transformer_lens_config(hf_config) + config_dict = dict(tl_config.__dict__) + # HF's attribute_map remaps num_experts → num_local_experts; restore the TL name. + if "num_local_experts" in config_dict and "num_experts" not in config_dict: + config_dict["num_experts"] = config_dict["num_local_experts"] + bridge_config = TransformerBridgeConfig.from_dict(config_dict) + bridge_config.architecture = architecture + bridge_config.model_name = model_name + bridge_config.dtype = dtype + + for attr in _HF_PASSTHROUGH_ATTRS: + val = getattr(hf_config, attr, None) + if val is not None: + setattr(bridge_config, attr, val) + + # Gemma2: HF softcap field names differ from TL's. + final_logit_softcapping = getattr(hf_config, "final_logit_softcapping", None) + if final_logit_softcapping is not None: + bridge_config.output_logits_soft_cap = float(final_logit_softcapping) + attn_logit_softcapping = getattr(hf_config, "attn_logit_softcapping", None) + if attn_logit_softcapping is not None: + bridge_config.attn_scores_soft_cap = float(attn_logit_softcapping) + + return bridge_config + + +def detect_tokenizer_bos_eos(tokenizer: Any) -> tuple[bool, bool]: + """Detect whether the tokenizer prepends BOS and/or appends EOS.""" + # Non-empty test string — "" is unreliable with token aliasing. + encoded_test = tokenizer.encode("a") + prepends_bos = ( + len(encoded_test) > 1 + and tokenizer.bos_token_id is not None + and encoded_test[0] == tokenizer.bos_token_id + ) + appends_eos = ( + len(encoded_test) > 1 + and tokenizer.eos_token_id is not None + and encoded_test[-1] == tokenizer.eos_token_id + ) + return prepends_bos, appends_eos + + +def build_bridge_from_module( + model: nn.Module, + architecture: str, + hf_config: Optional[Any] = None, + tl_config: Optional[TransformerBridgeConfig] = None, + tokenizer: Optional[Any] = None, + dtype: Optional[torch.dtype] = None, + device: Optional[Any] = None, + model_name: str = "external", + post_adapter_hook: Optional[Callable[[ArchitectureAdapter], None]] = None, +) -> TransformerBridge: + """Build a :class:`TransformerBridge` around a pre-loaded model. + + The bridge never moves, casts, or mutates the supplied model. + + Args: + model: Any ``nn.Module`` whose submodule tree matches the adapter's + expected dot-paths for ``architecture``. + architecture: Architecture identifier (e.g. ``"LlamaForCausalLM"``). + hf_config: Optional HF-style config; translated via + :func:`build_bridge_config_from_hf`. Mutually exclusive with ``tl_config``. + tl_config: Optional pre-built :class:`TransformerBridgeConfig`; bypasses + HF translation. Mutually exclusive with ``hf_config``. + tokenizer: Optional tokenizer. If supplied, passes through + ``setup_tokenizer`` and detects BOS/EOS behavior. + dtype: Recorded on ``cfg.dtype``. Default ``None`` reads from the model's + first parameter; explicit values override. + device: Recorded on ``cfg.device``. Default ``None`` reads from the + model's first parameter. + model_name: Recorded on ``cfg.model_name``. + post_adapter_hook: Optional callback invoked after adapter selection and + before :meth:`adapter.prepare_model`. Source-specific overlays mutate + ``component_mapping`` here. + + Returns: + A :class:`TransformerBridge` wrapping the supplied model. + """ + if hf_config is None and tl_config is None: + raise ValueError( + "build_bridge_from_module requires exactly one of hf_config or " + "tl_config — the bridge needs config fields (d_model, n_heads, " + "n_layers, ...) that can't be inferred from the model alone." + ) + if hf_config is not None and tl_config is not None: + raise ValueError( + "build_bridge_from_module got both hf_config and tl_config; supply " + "exactly one. hf_config triggers HF→bridge translation; tl_config " + "bypasses it." + ) + + # Reading dtype from the model avoids silently lying about a bf16 model. + if dtype is None: + try: + dtype = next(model.parameters()).dtype + except StopIteration: + dtype = torch.float32 + + if tl_config is not None: + bridge_config = tl_config + bridge_config.architecture = architecture + # Explicit kwarg wins over whatever tl_config carries; default only fills a gap. + if model_name != "external" or not getattr(bridge_config, "model_name", None): + bridge_config.model_name = model_name + bridge_config.dtype = dtype + else: + bridge_config = build_bridge_config_from_hf(hf_config, architecture, model_name, dtype) + + adapter = ArchitectureAdapterFactory.select_architecture_adapter(bridge_config) + + if post_adapter_hook is not None: + post_adapter_hook(adapter) + + if device is not None: + adapter.cfg.device = str(device) + else: + try: + adapter.cfg.device = str(next(model.parameters()).device) + except StopIteration: + adapter.cfg.device = "cpu" + + adapter.prepare_model(model) + + if tokenizer is not None: + default_padding_side = getattr(adapter.cfg, "default_padding_side", None) + tokenizer = setup_tokenizer(tokenizer, default_padding_side=default_padding_side) + ( + adapter.cfg.tokenizer_prepends_bos, + adapter.cfg.tokenizer_appends_eos, + ) = detect_tokenizer_bos_eos(tokenizer) + + from transformer_lens.model_bridge.sources.transformers_driver import ( + TransformersDriver, + ) + + driver = TransformersDriver(model, adapter, tokenizer) + return TransformerBridge(model, adapter, tokenizer, driver=driver) diff --git a/transformer_lens/model_bridge/sources/_driver_base.py b/transformer_lens/model_bridge/sources/_driver_base.py new file mode 100644 index 0000000000..175d3fee24 --- /dev/null +++ b/transformer_lens/model_bridge/sources/_driver_base.py @@ -0,0 +1,67 @@ +"""Optional base class for :class:`Driver` implementations. + +The protocol is duck-typed; inheriting is convenient, not required. +""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any, Mapping + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge.driver_protocol import ( + ForwardResult, + Intervention, + TensorLike, +) + + +class DriverBase(ABC): + """Defaults for the optional Driver members. Subclasses implement ``forward`` + and override the rest only when they can do better.""" + + architecture: str = "" + # The bridge overwrites this slot at construction (registry − non_fireable) + # when it's empty. Whitelist-semantic drivers (e.g. Inspect) declare a + # non-empty set in the subclass to keep it. + supported_hook_points: frozenset[str] = frozenset() + non_fireable_hook_points: frozenset[str] = frozenset() + + # Subclasses override with the capability strings they actually serve. + _supported_features: frozenset[str] = frozenset() + + def __init__( + self, + bridge_config: TransformerBridgeConfig, + tokenizer: Any, + *, + architecture: str | None = None, + ) -> None: + self.bridge_config = bridge_config + self.tokenizer = tokenizer + # Resolution order: explicit kwarg > bridge_config field > class default. + self.architecture = ( + architecture or getattr(bridge_config, "architecture", "") or self.architecture + ) + + @abstractmethod + def forward( + self, + input_ids: TensorLike | None = None, + *, + capture: tuple[str, ...] = (), + intervene: Mapping[str, Intervention] | None = None, + max_new_tokens: int = 1, + return_logits: bool = True, + **kwargs: Any, + ) -> ForwardResult: + ... + + def close(self) -> None: + """No-op default. Override when the driver owns releasable resources.""" + + def supports(self, feature: str) -> bool: + return feature in self._supported_features + + # Torch-specific surface (parameters, named_parameters, state_dict, weight + # access) is NOT defined here. Drivers that serve those methods provide + # them directly; callers route via supports("...") + hasattr/getattr. diff --git a/transformer_lens/model_bridge/sources/_hf_format.py b/transformer_lens/model_bridge/sources/_hf_format.py new file mode 100644 index 0000000000..b0e7395a4d --- /dev/null +++ b/transformer_lens/model_bridge/sources/_hf_format.py @@ -0,0 +1,261 @@ +"""Shared HF-format utilities used by every source whose backend produces an +HF-shaped config or ``PreTrainedTokenizerBase`` tokenizer. + +The transformers source loads HF objects directly; the vLLM source extracts the +same HF-shaped config via ``llm.llm_engine.model_config.hf_config`` because vLLM +re-uses the ``transformers`` config and tokenizer libraries internally. This +module is loader-agnostic — it speaks HF format, not HF loading. +""" +from __future__ import annotations + +import copy + +from transformers import PreTrainedTokenizerBase + +from transformer_lens.factories.architecture_adapter_factory import ( + SUPPORTED_ARCHITECTURES, +) +from transformer_lens.utilities import get_tokenizer_with_bos + + +def map_default_transformer_lens_config(hf_config): + """Map HuggingFace config fields to TransformerLens config format. + + Standardized mapping from various HuggingFace config field names to the + consistent TransformerLens naming convention. For multimodal models (LLaVA, + Gemma3ForConditionalGeneration), the language model dimensions are nested + under ``text_config``; we extract from there first. + + Args: + hf_config: The HuggingFace config object + + Returns: + A copy of hf_config with additional TransformerLens fields + """ + source_config = hf_config + if hasattr(hf_config, "text_config") and hf_config.text_config is not None: + source_config = hf_config.text_config + + tl_config = copy.deepcopy(hf_config) + if hasattr(source_config, "n_embd"): + tl_config.d_model = source_config.n_embd + elif hasattr(source_config, "hidden_size"): + tl_config.d_model = source_config.hidden_size + elif hasattr(source_config, "model_dim"): + tl_config.d_model = source_config.model_dim + elif hasattr(source_config, "d_model"): + tl_config.d_model = source_config.d_model + if hasattr(source_config, "n_head"): + tl_config.n_heads = source_config.n_head + elif hasattr(source_config, "num_attention_heads"): + n_heads = source_config.num_attention_heads + if isinstance(n_heads, list): + n_heads = max(n_heads) + tl_config.n_heads = n_heads + elif hasattr(source_config, "num_heads"): + tl_config.n_heads = source_config.num_heads + elif hasattr(source_config, "num_query_heads") and isinstance( + source_config.num_query_heads, list + ): + tl_config.n_heads = max(source_config.num_query_heads) + if ( + hasattr(source_config, "num_key_value_heads") + and source_config.num_key_value_heads is not None + ): + try: + num_kv_heads = source_config.num_key_value_heads + # Per-layer lists (e.g., OpenELM) collapse to the max. + if isinstance(num_kv_heads, list): + num_kv_heads = max(num_kv_heads) + if hasattr(num_kv_heads, "item"): + num_kv_heads = num_kv_heads.item() + num_kv_heads = int(num_kv_heads) + num_heads = tl_config.n_heads + if hasattr(num_heads, "item"): + num_heads = num_heads.item() + num_heads = int(num_heads) + if num_kv_heads != num_heads: + tl_config.n_key_value_heads = num_kv_heads + except (TypeError, ValueError, AttributeError): + pass + elif hasattr(source_config, "num_kv_heads") and source_config.num_kv_heads is not None: + try: + num_kv_heads = source_config.num_kv_heads + if isinstance(num_kv_heads, list): + num_kv_heads = max(num_kv_heads) + if hasattr(num_kv_heads, "item"): + num_kv_heads = num_kv_heads.item() + num_kv_heads = int(num_kv_heads) + num_heads = tl_config.n_heads + if hasattr(num_heads, "item"): + num_heads = num_heads.item() + num_heads = int(num_heads) + if num_kv_heads != num_heads: + tl_config.n_key_value_heads = num_kv_heads + except (TypeError, ValueError, AttributeError): + pass + if hasattr(source_config, "n_layer"): + tl_config.n_layers = source_config.n_layer + elif hasattr(source_config, "num_hidden_layers"): + tl_config.n_layers = source_config.num_hidden_layers + elif hasattr(source_config, "num_transformer_layers"): + tl_config.n_layers = source_config.num_transformer_layers + elif hasattr(source_config, "num_layers"): + tl_config.n_layers = source_config.num_layers + if hasattr(source_config, "vocab_size") and isinstance(source_config.vocab_size, int): + tl_config.d_vocab = source_config.vocab_size + if hasattr(source_config, "n_positions"): + tl_config.n_ctx = source_config.n_positions + elif hasattr(source_config, "max_position_embeddings"): + tl_config.n_ctx = source_config.max_position_embeddings + elif hasattr(source_config, "max_context_length"): + tl_config.n_ctx = source_config.max_context_length + elif hasattr(source_config, "max_length"): + tl_config.n_ctx = source_config.max_length + elif hasattr(source_config, "seq_length"): + tl_config.n_ctx = source_config.seq_length + else: + # ALiBi models (Bloom) have no context length field; 2048 is a safe fallback. + tl_config.n_ctx = 2048 + if hasattr(source_config, "n_inner"): + tl_config.d_mlp = source_config.n_inner + elif hasattr(source_config, "intermediate_size"): + tl_config.d_mlp = source_config.intermediate_size + elif hasattr(tl_config, "d_model"): + tl_config.d_mlp = getattr(source_config, "n_inner", 4 * tl_config.d_model) + if hasattr(source_config, "head_dim") and source_config.head_dim is not None: + tl_config.d_head = source_config.head_dim + elif hasattr(tl_config, "d_model") and hasattr(tl_config, "n_heads"): + tl_config.d_head = tl_config.d_model // tl_config.n_heads + elif hasattr(tl_config, "d_model"): + # Attention-less architectures (Mamba SSMs): set d_head = d_model so + # __post_init__ computes n_heads = 1. Values are nominal. + tl_config.d_head = tl_config.d_model + if hasattr(source_config, "activation_function"): + tl_config.act_fn = source_config.activation_function + elif hasattr(source_config, "hidden_act"): + tl_config.act_fn = source_config.hidden_act + # LayerNorm / RMSNorm epsilon — HF uses 3 different field names. + if hasattr(source_config, "rms_norm_eps"): + tl_config.eps = source_config.rms_norm_eps + elif hasattr(source_config, "layer_norm_eps"): + tl_config.eps = source_config.layer_norm_eps + elif hasattr(source_config, "layer_norm_epsilon"): + tl_config.eps = source_config.layer_norm_epsilon + if hasattr(source_config, "num_local_experts"): + tl_config.num_experts = source_config.num_local_experts + if hasattr(source_config, "num_experts_per_tok"): + tl_config.experts_per_token = source_config.num_experts_per_tok + if hasattr(source_config, "sliding_window") and source_config.sliding_window is not None: + tl_config.sliding_window = source_config.sliding_window + if getattr(hf_config, "use_parallel_residual", False): + tl_config.parallel_attn_mlp = True + # GPT-J and CodeGen run parallel attn+MLP but don't set use_parallel_residual. + arch_classes = getattr(hf_config, "architectures", []) or [] + if any(a in ("GPTJForCausalLM", "CodeGenForCausalLM") for a in arch_classes): + tl_config.parallel_attn_mlp = True + tl_config.default_prepend_bos = True + return tl_config + + +def determine_architecture_from_hf_config(hf_config): + """Determine the architecture name from HuggingFace config. + + Returns: + str: The architecture name (e.g., "GPT2LMHeadModel", "LlamaForCausalLM") + + Raises: + ValueError: If architecture cannot be determined + """ + architectures = [] + if hasattr(hf_config, "original_architecture"): + architectures.append(hf_config.original_architecture) + if hasattr(hf_config, "architectures") and hf_config.architectures: + architectures.extend(hf_config.architectures) + if hasattr(hf_config, "model_type"): + model_type = hf_config.model_type + model_type_mappings = { + "apertus": "ApertusForCausalLM", + "gpt2": "GPT2LMHeadModel", + "hubert": "HubertModel", + "llama": "LlamaForCausalLM", + "mamba": "MambaForCausalLM", + "mamba2": "Mamba2ForCausalLM", + "mistral": "MistralForCausalLM", + "mixtral": "MixtralForCausalLM", + "gemma": "GemmaForCausalLM", + "gemma2": "Gemma2ForCausalLM", + "gemma3": "Gemma3ForCausalLM", + "bert": "BertForMaskedLM", + "bloom": "BloomForCausalLM", + "codegen": "CodeGenForCausalLM", + "gptj": "GPTJForCausalLM", + "gpt_neo": "GPTNeoForCausalLM", + "gpt_neox": "GPTNeoXForCausalLM", + "opt": "OPTForCausalLM", + "phi": "PhiForCausalLM", + "phi3": "Phi3ForCausalLM", + "qwen": "QwenForCausalLM", + "qwen2": "Qwen2ForCausalLM", + "qwen3": "Qwen3ForCausalLM", + # qwen3_5 is the top-level multimodal config type; qwen3_5_text is + # the text-only sub-config. Both map to the text-only adapter so + # Qwen3.5 checkpoints (which report qwen3_5 even when loaded as + # text-only) are routed to Qwen3_5ForCausalLM. + "qwen3_5": "Qwen3_5ForCausalLM", + "qwen3_5_text": "Qwen3_5ForCausalLM", + "openelm": "OpenELMForCausalLM", + "stablelm": "StableLmForCausalLM", + "t5": "T5ForConditionalGeneration", + "mt5": "MT5ForConditionalGeneration", + } + if model_type in model_type_mappings: + architectures.append(model_type_mappings[model_type]) + + for arch in architectures: + if arch in SUPPORTED_ARCHITECTURES: + return arch + raise ValueError( + f"Could not determine supported architecture from config. Available architectures: " + f"{list(SUPPORTED_ARCHITECTURES.keys())}, Config architectures: {architectures}, " + f"Model type: {getattr(hf_config, 'model_type', None)}" + ) + + +def setup_tokenizer(tokenizer, default_padding_side=None): + """Normalize a HuggingFace tokenizer for use with the bridge. + + Args: + tokenizer: A ``PreTrainedTokenizer`` or ``PreTrainedTokenizerFast``. + default_padding_side: ``"right"`` or ``"left"``; sets ``tokenizer.padding_side``. + """ + assert isinstance( + tokenizer, PreTrainedTokenizerBase + ), f"{type(tokenizer)} is not a supported tokenizer; use PreTrainedTokenizer or PreTrainedTokenizerFast" + assert default_padding_side in [ + "right", + "left", + None, + ], f"padding_side must be 'right', 'left' or None, got {default_padding_side}" + tokenizer = get_tokenizer_with_bos(tokenizer) + assert tokenizer is not None + if default_padding_side is not None: + tokenizer.padding_side = default_padding_side + if tokenizer.padding_side is None: + tokenizer.padding_side = "right" + if tokenizer.eos_token is None: + tokenizer.eos_token = "<|endoftext|>" + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + if tokenizer.bos_token is None: + tokenizer.bos_token = tokenizer.eos_token + + # Some vocabularies lack default IDs for these tokens; register them. + if tokenizer.pad_token is not None and tokenizer.pad_token_id is None: + tokenizer.add_special_tokens({"pad_token": tokenizer.pad_token}) + if tokenizer.eos_token is not None and tokenizer.eos_token_id is None: + tokenizer.add_special_tokens({"eos_token": tokenizer.eos_token}) + if tokenizer.bos_token is not None and tokenizer.bos_token_id is None: + tokenizer.add_special_tokens({"bos_token": tokenizer.bos_token}) + + return tokenizer diff --git a/transformer_lens/model_bridge/sources/transformers.py b/transformer_lens/model_bridge/sources/transformers.py deleted file mode 100644 index e30a022f41..0000000000 --- a/transformer_lens/model_bridge/sources/transformers.py +++ /dev/null @@ -1,936 +0,0 @@ -"""Transformers module for TransformerLens. - -This module provides functionality to load and convert models from HuggingFace to TransformerLens format. -""" -import contextlib -import copy -import logging -import os -import warnings -from typing import Any - -import torch -from transformers import ( - AutoConfig, - AutoModelForCausalLM, - AutoModelForMaskedLM, - AutoModelForSeq2SeqLM, - AutoTokenizer, - PreTrainedTokenizerBase, -) - -from transformer_lens.config import TransformerBridgeConfig -from transformer_lens.factories.architecture_adapter_factory import ( - SUPPORTED_ARCHITECTURES, - ArchitectureAdapterFactory, -) -from transformer_lens.model_bridge.bridge import TransformerBridge -from transformer_lens.supported_models import MODEL_ALIASES -from transformer_lens.utilities import get_device, get_tokenizer_with_bos - -# Suppress transformers warnings that go to stderr -# This prevents notebook tests from failing due to unexpected stderr output -warnings.filterwarnings("ignore", message=".*generation flags.*not valid.*") -logging.getLogger("transformers").setLevel(logging.ERROR) - - -def map_default_transformer_lens_config(hf_config): - """Map HuggingFace config fields to TransformerLens config format. - - This function provides a standardized mapping from various HuggingFace config - field names to the consistent TransformerLens naming convention. - - For multimodal models (LLaVA, Gemma3ForConditionalGeneration), the language - model dimensions are nested under text_config. We extract from text_config - first, then apply the standard mapping. - - Args: - hf_config: The HuggingFace config object - - Returns: - A copy of hf_config with additional TransformerLens fields - """ - # Extract language model config from text_config for multimodal models - source_config = hf_config - if hasattr(hf_config, "text_config") and hf_config.text_config is not None: - source_config = hf_config.text_config - - tl_config = copy.deepcopy(hf_config) - if hasattr(source_config, "n_embd"): - tl_config.d_model = source_config.n_embd - elif hasattr(source_config, "hidden_size"): - tl_config.d_model = source_config.hidden_size - elif hasattr(source_config, "model_dim"): - tl_config.d_model = source_config.model_dim - elif hasattr(source_config, "d_model"): - tl_config.d_model = source_config.d_model - if hasattr(source_config, "n_head"): - tl_config.n_heads = source_config.n_head - elif hasattr(source_config, "num_attention_heads"): - n_heads = source_config.num_attention_heads - if isinstance(n_heads, list): - n_heads = max(n_heads) - tl_config.n_heads = n_heads - elif hasattr(source_config, "num_heads"): - tl_config.n_heads = source_config.num_heads - elif hasattr(source_config, "num_query_heads") and isinstance( - source_config.num_query_heads, list - ): - tl_config.n_heads = max(source_config.num_query_heads) - if ( - hasattr(source_config, "num_key_value_heads") - and source_config.num_key_value_heads is not None - ): - try: - num_kv_heads = source_config.num_key_value_heads - # Handle per-layer lists (e.g., OpenELM) by taking the max - if isinstance(num_kv_heads, list): - num_kv_heads = max(num_kv_heads) - if hasattr(num_kv_heads, "item"): - num_kv_heads = num_kv_heads.item() - num_kv_heads = int(num_kv_heads) - num_heads = tl_config.n_heads - if hasattr(num_heads, "item"): - num_heads = num_heads.item() - num_heads = int(num_heads) - if num_kv_heads != num_heads: - tl_config.n_key_value_heads = num_kv_heads - except (TypeError, ValueError, AttributeError): - pass - elif hasattr(source_config, "num_kv_heads") and source_config.num_kv_heads is not None: - try: - num_kv_heads = source_config.num_kv_heads - if isinstance(num_kv_heads, list): - num_kv_heads = max(num_kv_heads) - if hasattr(num_kv_heads, "item"): - num_kv_heads = num_kv_heads.item() - num_kv_heads = int(num_kv_heads) - num_heads = tl_config.n_heads - if hasattr(num_heads, "item"): - num_heads = num_heads.item() - num_heads = int(num_heads) - if num_kv_heads != num_heads: - tl_config.n_key_value_heads = num_kv_heads - except (TypeError, ValueError, AttributeError): - pass - if hasattr(source_config, "n_layer"): - tl_config.n_layers = source_config.n_layer - elif hasattr(source_config, "num_hidden_layers"): - tl_config.n_layers = source_config.num_hidden_layers - elif hasattr(source_config, "num_transformer_layers"): - tl_config.n_layers = source_config.num_transformer_layers - elif hasattr(source_config, "num_layers"): - tl_config.n_layers = source_config.num_layers - if hasattr(source_config, "vocab_size") and isinstance(source_config.vocab_size, int): - tl_config.d_vocab = source_config.vocab_size - if hasattr(source_config, "n_positions"): - tl_config.n_ctx = source_config.n_positions - elif hasattr(source_config, "max_position_embeddings"): - tl_config.n_ctx = source_config.max_position_embeddings - elif hasattr(source_config, "max_context_length"): - tl_config.n_ctx = source_config.max_context_length - elif hasattr(source_config, "max_length"): - tl_config.n_ctx = source_config.max_length - elif hasattr(source_config, "seq_length"): - tl_config.n_ctx = source_config.seq_length - else: - # Models like Bloom use ALiBi (no positional embeddings) and have no - # context length field. Default to 2048 as a reasonable fallback. - tl_config.n_ctx = 2048 - if hasattr(source_config, "n_inner"): - tl_config.d_mlp = source_config.n_inner - elif hasattr(source_config, "intermediate_size"): - tl_config.d_mlp = source_config.intermediate_size - elif hasattr(tl_config, "d_model"): - tl_config.d_mlp = getattr(source_config, "n_inner", 4 * tl_config.d_model) - if hasattr(source_config, "head_dim") and source_config.head_dim is not None: - tl_config.d_head = source_config.head_dim - elif hasattr(tl_config, "d_model") and hasattr(tl_config, "n_heads"): - tl_config.d_head = tl_config.d_model // tl_config.n_heads - elif hasattr(tl_config, "d_model"): - # Models without attention (e.g., Mamba SSMs) have no n_heads or head_dim. - # Set d_head = d_model so TransformerLensConfig.__post_init__ computes - # n_heads = 1. These values are nominal and have no functional meaning - # for attention-less architectures. - tl_config.d_head = tl_config.d_model - if hasattr(source_config, "activation_function"): - tl_config.act_fn = source_config.activation_function - elif hasattr(source_config, "hidden_act"): - tl_config.act_fn = source_config.hidden_act - # Layer norm / RMS norm epsilon — HF uses 3 different field names - if hasattr(source_config, "rms_norm_eps"): - tl_config.eps = source_config.rms_norm_eps - elif hasattr(source_config, "layer_norm_eps"): - tl_config.eps = source_config.layer_norm_eps - elif hasattr(source_config, "layer_norm_epsilon"): - tl_config.eps = source_config.layer_norm_epsilon - if hasattr(source_config, "num_local_experts"): - tl_config.num_experts = source_config.num_local_experts - if hasattr(source_config, "num_experts_per_tok"): - tl_config.experts_per_token = source_config.num_experts_per_tok - if hasattr(source_config, "sliding_window") and source_config.sliding_window is not None: - tl_config.sliding_window = source_config.sliding_window - if getattr(hf_config, "use_parallel_residual", False): - tl_config.parallel_attn_mlp = True - # GPT-J and CodeGen: parallel attn+MLP but missing use_parallel_residual in HF config - arch_classes = getattr(hf_config, "architectures", []) or [] - if any(a in ("GPTJForCausalLM", "CodeGenForCausalLM") for a in arch_classes): - tl_config.parallel_attn_mlp = True - tl_config.default_prepend_bos = True - return tl_config - - -def determine_architecture_from_hf_config(hf_config): - """Determine the architecture name from HuggingFace config. - - Args: - hf_config: The HuggingFace config object - - Returns: - str: The architecture name (e.g., "GPT2LMHeadModel", "LlamaForCausalLM") - - Raises: - ValueError: If architecture cannot be determined - """ - architectures = [] - if hasattr(hf_config, "original_architecture"): - architectures.append(hf_config.original_architecture) - if hasattr(hf_config, "architectures") and hf_config.architectures: - architectures.extend(hf_config.architectures) - if hasattr(hf_config, "model_type"): - model_type = hf_config.model_type - model_type_mappings = { - "apertus": "ApertusForCausalLM", - "gpt2": "GPT2LMHeadModel", - "hubert": "HubertModel", - "llama": "LlamaForCausalLM", - "mamba": "MambaForCausalLM", - "mamba2": "Mamba2ForCausalLM", - "mistral": "MistralForCausalLM", - "mixtral": "MixtralForCausalLM", - "gemma": "GemmaForCausalLM", - "gemma2": "Gemma2ForCausalLM", - "gemma3": "Gemma3ForCausalLM", - "bert": "BertForMaskedLM", - "bloom": "BloomForCausalLM", - "codegen": "CodeGenForCausalLM", - "gptj": "GPTJForCausalLM", - "gpt_neo": "GPTNeoForCausalLM", - "gpt_neox": "GPTNeoXForCausalLM", - "opt": "OPTForCausalLM", - "phi": "PhiForCausalLM", - "phi3": "Phi3ForCausalLM", - "qwen": "QwenForCausalLM", - "qwen2": "Qwen2ForCausalLM", - "qwen3": "Qwen3ForCausalLM", - # qwen3_5 is the top-level multimodal config type; qwen3_5_text is - # the text-only sub-config. Both map to the text-only adapter so - # Qwen3.5 checkpoints (which report qwen3_5 even when loaded as - # text-only) are routed to Qwen3_5ForCausalLM. - "qwen3_5": "Qwen3_5ForCausalLM", - "qwen3_5_text": "Qwen3_5ForCausalLM", - "openelm": "OpenELMForCausalLM", - "stablelm": "StableLmForCausalLM", - "t5": "T5ForConditionalGeneration", - "mt5": "MT5ForConditionalGeneration", - } - if model_type in model_type_mappings: - architectures.append(model_type_mappings[model_type]) - - for arch in architectures: - if arch in SUPPORTED_ARCHITECTURES: - return arch - raise ValueError( - f"Could not determine supported architecture from config. Available architectures: {list(SUPPORTED_ARCHITECTURES.keys())}, Config architectures: {architectures}, Model type: {getattr(hf_config, 'model_type', None)}" - ) - - -def get_hf_model_class_for_architecture(architecture: str): - """Determine the correct HuggingFace AutoModel class for loading. - - Uses centralized architecture sets from utilities.architectures. - """ - from transformer_lens.utilities.architectures import ( - AUDIO_ARCHITECTURES, - MASKED_LM_ARCHITECTURES, - MULTIMODAL_ARCHITECTURES, - SEQ2SEQ_ARCHITECTURES, - ) - - if architecture in SEQ2SEQ_ARCHITECTURES: - return AutoModelForSeq2SeqLM - elif architecture in MASKED_LM_ARCHITECTURES: - return AutoModelForMaskedLM - elif architecture in MULTIMODAL_ARCHITECTURES: - from transformers import AutoModelForImageTextToText - - return AutoModelForImageTextToText - elif architecture in AUDIO_ARCHITECTURES: - if "ForCTC" in architecture: - from transformers import AutoModelForCTC - - return AutoModelForCTC - from transformers import AutoModel - - return AutoModel - else: - return AutoModelForCausalLM - - -# Known training-checkpoint revision conventions on HF. -_CHECKPOINT_REVISION_FORMATS: dict[str, str] = { - "EleutherAI/pythia": "step{value}", - "stanford-crfm": "checkpoint-{value}", -} - - -def _resolve_checkpoint_to_revision( - model_name: str, - checkpoint_index: int | None, - checkpoint_value: int | None, -) -> str: - """Convert a checkpoint index/value into an HF revision string, validated against ``get_checkpoint_labels``.""" - if checkpoint_index is None and checkpoint_value is None: - raise ValueError("Must specify either checkpoint_index or checkpoint_value.") - - format_str: str | None = None - for prefix, fmt in _CHECKPOINT_REVISION_FORMATS.items(): - if model_name.startswith(prefix): - format_str = fmt - break - if format_str is None: - raise ValueError( - f"Model {model_name!r} does not have a known checkpoint revision convention. " - f"Pass revision= directly if your model uses HF revisions. Known checkpoint " - f"families: {list(_CHECKPOINT_REVISION_FORMATS.keys())}." - ) - - from transformer_lens.loading_from_pretrained import get_checkpoint_labels - - labels, _ = get_checkpoint_labels(model_name) - if checkpoint_value is not None: - if checkpoint_value not in labels: - raise ValueError( - f"checkpoint_value={checkpoint_value} not in available checkpoints for " - f"{model_name!r}. {len(labels)} labels available, " - f"first/last: {labels[0]}..{labels[-1]}." - ) - else: - assert checkpoint_index is not None # narrowed by initial guard - if not 0 <= checkpoint_index < len(labels): - raise ValueError( - f"checkpoint_index={checkpoint_index} out of range [0, {len(labels)}) " - f"for {model_name!r}." - ) - checkpoint_value = labels[checkpoint_index] - return format_str.format(value=checkpoint_value) - - -def boot( - model_name: str, - hf_config_overrides: dict | None = None, - device: str | torch.device | None = None, - dtype: torch.dtype = torch.float32, - tokenizer: PreTrainedTokenizerBase | None = None, - load_weights: bool = True, - trust_remote_code: bool = False, - model_class: Any | None = None, - hf_model: Any | None = None, - n_ctx: int | None = None, - revision: str | None = None, - checkpoint_index: int | None = None, - checkpoint_value: int | None = None, - # Experimental – Have not been fully tested on multi-gpu devices - # Use at your own risk, report any issues here: https://github.com/TransformerLensOrg/TransformerLens/issues - device_map: str | dict[str, str | int] | None = None, - n_devices: int | None = None, - max_memory: dict[str | int, str] | None = None, -) -> TransformerBridge: - """Boot a model from HuggingFace. - - Args: - model_name: The name of the model to load. - hf_config_overrides: Optional overrides applied to the HuggingFace config before model load. - device: The device to use. If None, will be determined automatically. Mutually exclusive - with ``device_map``. - dtype: The dtype to use for the model. - tokenizer: Optional pre-initialized tokenizer to use; if not provided one will be created. - load_weights: If False, load model without weights (on meta device) for config inspection only. - model_class: Optional HuggingFace model class to use instead of the default auto-detected - class. When the class name matches a key in SUPPORTED_ARCHITECTURES, the corresponding - adapter is selected automatically (e.g., BertForNextSentencePrediction). - hf_model: Optional pre-loaded HuggingFace model to use instead of loading one. Useful for - models loaded with custom configurations (e.g., quantization via BitsAndBytesConfig). - When provided, load_weights is ignored. - device_map: HuggingFace-style device map (``"auto"``, ``"balanced"``, dict, etc.) for - multi-GPU inference. Passed straight to ``from_pretrained``. Mutually exclusive - with ``device``. - n_devices: Convenience: split the model across this many CUDA devices (translated to a - ``max_memory`` dict internally). Requires CUDA with at least this many visible devices. - max_memory: Optional per-device memory budget for HF's dispatcher. - n_ctx: Optional context length override. The bridge normally uses the model's documented - max context from the HF config. Setting this writes to whichever HF field the model - uses (n_positions / max_position_embeddings / etc.), so callers don't need to know - the field name. If larger than the model's default, a warning is emitted — quality - may degrade past the trained length for rotary models. - revision: Optional HF revision string (branch, tag, or commit). Forwarded to - ``AutoConfig.from_pretrained`` and ``AutoModelForCausalLM.from_pretrained``. - Mutually exclusive with ``checkpoint_index`` and ``checkpoint_value``. - checkpoint_index: Index into the available training checkpoints for the model family. - Convenience over ``revision`` for checkpointed models like EleutherAI/pythia* and - stanford-crfm/*. Resolved to a revision string via the known per-family naming - conventions (``step{value}`` for Pythia, ``checkpoint-{value}`` for stanford-crfm). - checkpoint_value: Training step or token count of the desired checkpoint. Alternative to - ``checkpoint_index``; must be one of the labels returned by ``get_checkpoint_labels``. - - Returns: - The bridge to the loaded model. - """ - for official_name, aliases in MODEL_ALIASES.items(): - if model_name in aliases: - logging.warning( - f"DEPRECATED: You are using a deprecated, model_name alias '{model_name}'. TransformerLens will now load the official transformers model name, '{official_name}' instead.\n Please update your code to use the official name by changing model_name from '{model_name}' to '{official_name}'.\nSince TransformerLens v3, all model names should be the official transformers model names.\nThe aliases will be removed in the next version of TransformerLens, so please do the update now." - ) - model_name = official_name - break - if checkpoint_index is not None or checkpoint_value is not None: - if revision is not None: - raise ValueError( - "Specify either revision= or checkpoint_index/checkpoint_value, not both." - ) - revision = _resolve_checkpoint_to_revision(model_name, checkpoint_index, checkpoint_value) - # Pass HF token for gated model access (e.g. meta-llama/*) - from transformer_lens.utilities.hf_utils import get_hf_token - - _hf_token = get_hf_token() - if hf_model is not None: - # Reuse the pre-loaded model's config to avoid a Hub call when model_name - # is a Hub repo ID, but the model is already loaded locally. - hf_config = copy.deepcopy(hf_model.config) - else: - hf_config = AutoConfig.from_pretrained( - model_name, - output_attentions=True, - trust_remote_code=trust_remote_code, - token=_hf_token, - revision=revision, - ) - _n_ctx_field: str | None = None - if n_ctx is not None: - # Validation (#2): reject non-positive values before doing anything else. - if n_ctx <= 0: - raise ValueError(f"n_ctx must be a positive integer, got n_ctx={n_ctx}.") - # Resolve n_ctx to whichever HF config field this model uses. Mirrors - # the order in map_default_transformer_lens_config so the TL config - # derivation picks up the override. - for _field in ( - "n_positions", - "max_position_embeddings", - "max_context_length", - "max_length", - "seq_length", - ): - if hasattr(hf_config, _field): - _n_ctx_field = _field - break - if _n_ctx_field is None: - raise ValueError( - f"Cannot apply n_ctx={n_ctx}: no recognized context-length field on " - f"HF config for {model_name}. Use hf_config_overrides instead." - ) - _default_n_ctx = getattr(hf_config, _n_ctx_field) - if _default_n_ctx is not None and n_ctx > _default_n_ctx: - logging.warning( - "Setting n_ctx=%d which is larger than the model's default " - "context length of %d. The model was not trained on sequences " - "this long and may produce unreliable results (especially for " - "rotary models without RoPE scaling).", - n_ctx, - _default_n_ctx, - ) - # Conflict detection (#4): warn if the caller also set the same field - # via hf_config_overrides — explicit n_ctx wins but users should know. - if hf_config_overrides and _n_ctx_field in hf_config_overrides: - _conflicting_value = hf_config_overrides[_n_ctx_field] - if _conflicting_value != n_ctx: - logging.warning( - "Both n_ctx=%d and hf_config_overrides['%s']=%s were provided. " - "The explicit n_ctx takes precedence.", - n_ctx, - _n_ctx_field, - _conflicting_value, - ) - # Explicit n_ctx wins over hf_config_overrides for the resolved field. - hf_config_overrides = dict(hf_config_overrides or {}) - hf_config_overrides[_n_ctx_field] = n_ctx - if hf_config_overrides: - hf_config.__dict__.update(hf_config_overrides) - tl_config = map_default_transformer_lens_config(hf_config) - architecture = determine_architecture_from_hf_config(hf_config) - config_dict = dict(tl_config.__dict__) - # Restore TL attribute names that HF remaps via attribute_map - if "num_local_experts" in config_dict and "num_experts" not in config_dict: - config_dict["num_experts"] = config_dict["num_local_experts"] - bridge_config = TransformerBridgeConfig.from_dict(config_dict) - bridge_config.architecture = architecture - bridge_config.model_name = model_name - bridge_config.dtype = dtype - # Propagate HF-specific config attributes that adapters may need. - # Any attribute present on the HF config and not None is copied to bridge_config. - # This is architecture-agnostic — new architectures don't need changes here. - _HF_PASSTHROUGH_ATTRS = [ - # OPT - "is_gated_act", - "word_embed_proj_dim", - "do_layer_norm_before", - # Granite - "position_embedding_type", - # Falcon - "parallel_attn", - "multi_query", - "new_decoder_architecture", - "alibi", - "num_ln_in_parallel_attn", - # Mamba (SSM config) - "state_size", - "conv_kernel", - "expand", - "time_step_rank", - "intermediate_size", - # Mamba-2 (additional SSM config) - "n_groups", - "chunk_size", - # Multimodal - "vision_config", - ] - for attr in _HF_PASSTHROUGH_ATTRS: - val = getattr(hf_config, attr, None) - if val is not None: - setattr(bridge_config, attr, val) - - # Gemma2 softcapping: HF names differ from TL names, need explicit mapping - final_logit_softcapping = getattr(hf_config, "final_logit_softcapping", None) - if final_logit_softcapping is not None: - bridge_config.output_logits_soft_cap = float(final_logit_softcapping) - attn_logit_softcapping = getattr(hf_config, "attn_logit_softcapping", None) - if attn_logit_softcapping is not None: - bridge_config.attn_scores_soft_cap = float(attn_logit_softcapping) - adapter = ArchitectureAdapterFactory.select_architecture_adapter(bridge_config) - # Pre-loaded models carry their own weight placement (possibly set by the caller via - # device_map). Passing device_map / n_devices / max_memory alongside hf_model= is - # ambiguous and would silently be ignored, so fail loudly. - if hf_model is not None and ( - device_map is not None or n_devices is not None or max_memory is not None - ): - raise ValueError( - "device_map / n_devices / max_memory are only supported when the bridge loads " - "the HF model itself. When passing hf_model=..., apply device_map via " - "AutoModel.from_pretrained before handing the model to the bridge." - ) - # Stateful/SSM (e.g. Mamba) models keep a per-layer recurrent cache that must live on - # that layer's device. The bridge currently allocates the stateful cache on a single - # cfg.device, so cross-device splits would silently misplace the cache. Block this - # combination until a v2 addresses per-layer stateful cache placement. - if (n_devices is not None and n_devices > 1) or device_map is not None: - if getattr(bridge_config, "is_stateful", False): - raise ValueError( - "Multi-device splits are not yet supported for stateful (SSM / Mamba) " - "architectures: the stateful cache allocation is single-device. " - "Load on one device, or wait for v2 support." - ) - # Resolve device_map before defaulting `device` — the two are mutually exclusive, and - # the resolver raises on conflict. If n_devices>1 is passed, it's translated into a - # device_map + max_memory pair here so downstream code only needs to check the - # resolved values. - from transformer_lens.utilities.multi_gpu import ( - count_unique_devices, - find_embedding_device, - resolve_device_map, - ) - - resolved_device_map, resolved_max_memory = resolve_device_map( - n_devices, device_map, device, max_memory - ) - if resolved_device_map is None: - if device is None: - device = get_device() - adapter.cfg.device = str(device) - else: - # cfg.device will be set from hf_device_map after the model is loaded. - # Provisionally keep it None; find_embedding_device fills it in below. - adapter.cfg.device = None - if model_class is None: - model_class = get_hf_model_class_for_architecture(architecture) - # Ensure pad_token_id exists (v5 raises AttributeError if missing) - if not hasattr(hf_config, "pad_token_id") or "pad_token_id" not in hf_config.__dict__: - fallback_pad = getattr(hf_config, "eos_token_id", None) - # eos_token_id can be a list (e.g., Gemma3 uses [1, 106]); take the first. - if isinstance(fallback_pad, list): - fallback_pad = fallback_pad[0] if fallback_pad else None - hf_config.pad_token_id = fallback_pad - model_kwargs = {"config": hf_config, "torch_dtype": dtype} - if _hf_token: - model_kwargs["token"] = _hf_token - if trust_remote_code: - model_kwargs["trust_remote_code"] = True - if revision is not None: - model_kwargs["revision"] = revision - if resolved_device_map is not None: - model_kwargs["device_map"] = resolved_device_map - if resolved_max_memory is not None: - model_kwargs["max_memory"] = resolved_max_memory - if hasattr(adapter.cfg, "attn_implementation") and adapter.cfg.attn_implementation is not None: - model_kwargs["attn_implementation"] = adapter.cfg.attn_implementation - else: - # Default to eager (required for output_attentions hooks) - model_kwargs["attn_implementation"] = "eager" - adapter.prepare_loading(model_name, model_kwargs) - if hf_model is not None: - # Use the pre-loaded model as-is (e.g., quantized models with custom device_map) - pass - elif not load_weights: - from_config_kwargs = {} - if trust_remote_code: - from_config_kwargs["trust_remote_code"] = True - prepared_config = model_kwargs.get("config", hf_config) - with contextlib.redirect_stdout(None): - hf_model = model_class.from_config(prepared_config, **from_config_kwargs) - else: - try: - hf_model = model_class.from_pretrained(model_name, **model_kwargs) - except RuntimeError as e: - # #5: HF refuses to load when positional-weight shapes don't match. - # If the user requested an n_ctx that conflicts with the saved weights - # (common for learned-pos-embed models like GPT-2), re-raise with a - # clearer message pointing them at the likely cause. - if n_ctx is not None and "ignore_mismatched_sizes" in str(e): - raise RuntimeError( - f"Failed to load {model_name} with n_ctx={n_ctx}: the pretrained " - f"weights' positional-embedding shape does not match the requested " - f"context length. This affects models with learned positional " - f"embeddings (e.g. GPT-2, OPT). Options: (1) use the model's " - f"default n_ctx, (2) pass load_weights=False if you only need " - f"config inspection, or (3) choose a rotary-embedding model " - f"(e.g. Llama, Mistral) which supports n_ctx changes without " - f"weight mismatch." - ) from e - raise - # Skip explicit .to(device) when accelerate has placed weights via device_map. - if resolved_device_map is None and device is not None: - hf_model = hf_model.to(device) - # Cast params to dtype; preserve float32 buffers (e.g., RotaryEmbedding.inv_freq) - for param in hf_model.parameters(): - if param.is_floating_point() and param.dtype != dtype: - param.data = param.data.to(dtype=dtype) - # Derive cfg.device / cfg.n_devices from hf_device_map when present. This covers: - # - fresh loads with a resolved device_map (set above) - # - pre-loaded hf_model that the caller dispatched themselves (e.g., device_map="auto") - hf_device_map_post = getattr(hf_model, "hf_device_map", None) - if hf_device_map_post: - # Pre-loaded path can still smuggle CPU/disk offload in; validate here too. - offload_values = {str(v).lower() for v in hf_device_map_post.values() if isinstance(v, str)} - forbidden = offload_values & {"cpu", "disk", "meta"} - if forbidden and ((n_devices is not None and n_devices > 1) or device_map is not None): - # Fresh-load path: we set the device_map ourselves, so this shouldn't happen — - # but if the user asked for n_devices>1 and somehow got CPU offload, surface it. - raise ValueError( - f"hf_device_map contains unsupported offload targets: {sorted(forbidden)}. " - "v1 multi-device support is GPU-only." - ) - embedding_device = find_embedding_device(hf_model) - if embedding_device is not None: - adapter.cfg.device = str(embedding_device) - adapter.cfg.n_devices = count_unique_devices(hf_model) - elif adapter.cfg.device is None: - # Pre-loaded single-device model with no hf_device_map — fall back to first param. - try: - adapter.cfg.device = str(next(hf_model.parameters()).device) - except StopIteration: - adapter.cfg.device = "cpu" - # #7: Verify the n_ctx override actually took effect on the loaded model. - # If HF's config class silently dropped or normalized the value, warn so - # the user doesn't get misled into thinking longer sequences are supported. - if n_ctx is not None and _n_ctx_field is not None and hf_model is not None: - _actual = getattr(hf_model.config, _n_ctx_field, None) - if _actual != n_ctx: - logging.warning( - "n_ctx=%d was requested but hf_model.config.%s=%s after load. " - "The override may not have taken effect; the model may not " - "accept sequences longer than %s.", - n_ctx, - _n_ctx_field, - _actual, - _actual, - ) - adapter.prepare_model(hf_model) - tokenizer = tokenizer - default_padding_side = getattr(adapter.cfg, "default_padding_side", None) - use_fast = getattr(adapter.cfg, "use_fast", True) - # Audio models use feature extractors, not text tokenizers - _is_audio = getattr(adapter.cfg, "is_audio_model", False) - if _is_audio and tokenizer is None: - tokenizer = None # Skip tokenizer loading for audio models - elif tokenizer is not None: - tokenizer = setup_tokenizer(tokenizer, default_padding_side=default_padding_side) - else: - token_arg = get_hf_token() - # Use adapter's tokenizer_name if model lacks one (e.g., OpenELM) - tokenizer_source = model_name - if hasattr(adapter.cfg, "tokenizer_name") and adapter.cfg.tokenizer_name is not None: - tokenizer_source = adapter.cfg.tokenizer_name - # Try to load tokenizer with add_bos_token=True first - # (encoder-decoder models like T5 don't have BOS tokens and will raise ValueError) - try: - base_tokenizer = AutoTokenizer.from_pretrained( - tokenizer_source, - add_bos_token=True, - use_fast=use_fast, - token=token_arg, - trust_remote_code=trust_remote_code, - ) - except ValueError: - # Model doesn't have a BOS token, load without add_bos_token - base_tokenizer = AutoTokenizer.from_pretrained( - tokenizer_source, - use_fast=use_fast, - token=token_arg, - trust_remote_code=trust_remote_code, - ) - tokenizer = setup_tokenizer( - base_tokenizer, - default_padding_side=default_padding_side, - ) - if tokenizer is not None: - # Detect BOS/EOS behavior (use non-empty string; empty is unreliable with token aliasing) - encoded_test = tokenizer.encode("a") - adapter.cfg.tokenizer_prepends_bos = ( - len(encoded_test) > 1 - and tokenizer.bos_token_id is not None - and encoded_test[0] == tokenizer.bos_token_id - ) - adapter.cfg.tokenizer_appends_eos = ( - len(encoded_test) > 1 - and tokenizer.eos_token_id is not None - and encoded_test[-1] == tokenizer.eos_token_id - ) - bridge = TransformerBridge(hf_model, adapter, tokenizer) - - # Load processor for multimodal models (needed for image preprocessing) - if getattr(adapter.cfg, "is_multimodal", False): - try: - from transformers import AutoProcessor - - huggingface_token = os.environ.get("HF_TOKEN", "") - token_arg = huggingface_token if len(huggingface_token) > 0 else None - bridge.processor = AutoProcessor.from_pretrained( - model_name, - token=token_arg, - trust_remote_code=trust_remote_code, - ) - except Exception: - # Some processors need torchvision (e.g., LlavaOnevision); install if needed - _torchvision_available = False - try: - import torchvision # noqa: F401 - - _torchvision_available = True - except Exception: - # Install/reinstall torchvision if missing or broken - import shutil - import subprocess - import sys - - try: - if shutil.which("uv"): - subprocess.check_call( - ["uv", "pip", "install", "torchvision", "-q"], - ) - else: - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "torchvision", "-q"], - ) - import importlib - - importlib.invalidate_caches() - _torchvision_available = True - except Exception: - pass # torchvision install failed; processor will be unavailable - - if _torchvision_available: - try: - from transformers import AutoProcessor - - huggingface_token = os.environ.get("HF_TOKEN", "") - token_arg = huggingface_token if len(huggingface_token) > 0 else None - bridge.processor = AutoProcessor.from_pretrained( - model_name, - token=token_arg, - trust_remote_code=trust_remote_code, - ) - except Exception: - pass # Processor not available; user can set bridge.processor manually - - # Load feature extractor for audio models (needed for audio preprocessing) - if getattr(adapter.cfg, "is_audio_model", False): - try: - from transformers import AutoFeatureExtractor - - huggingface_token = os.environ.get("HF_TOKEN", "") - token_arg = huggingface_token if len(huggingface_token) > 0 else None - bridge.processor = AutoFeatureExtractor.from_pretrained( - model_name, - token=token_arg, - trust_remote_code=trust_remote_code, - ) - except Exception: - pass # Feature extractor not available; user can set bridge.processor manually - - return bridge - - -def setup_tokenizer(tokenizer, default_padding_side=None): - """Set's up the tokenizer. - - Args: - tokenizer (PreTrainedTokenizer): a pretrained HuggingFace tokenizer. - default_padding_side (str): "right" or "left", which side to pad on. - - """ - assert isinstance( - tokenizer, PreTrainedTokenizerBase - ), f"{type(tokenizer)} is not a supported tokenizer, please use PreTrainedTokenizer or PreTrainedTokenizerFast" - assert default_padding_side in [ - "right", - "left", - None, - ], f"padding_side must be 'right', 'left' or 'None', got {default_padding_side}" - tokenizer_with_bos = get_tokenizer_with_bos(tokenizer) - tokenizer = tokenizer_with_bos - assert tokenizer is not None - if default_padding_side is not None: - tokenizer.padding_side = default_padding_side - if tokenizer.padding_side is None: - tokenizer.padding_side = "right" - if tokenizer.eos_token is None: - tokenizer.eos_token = "<|endoftext|>" - if tokenizer.pad_token is None: - tokenizer.pad_token = tokenizer.eos_token - if tokenizer.bos_token is None: - tokenizer.bos_token = tokenizer.eos_token - - # Ensure special tokens resolve to valid IDs (some vocabularies lack defaults) - if tokenizer.pad_token is not None and tokenizer.pad_token_id is None: - tokenizer.add_special_tokens({"pad_token": tokenizer.pad_token}) - if tokenizer.eos_token is not None and tokenizer.eos_token_id is None: - tokenizer.add_special_tokens({"eos_token": tokenizer.eos_token}) - if tokenizer.bos_token is not None and tokenizer.bos_token_id is None: - tokenizer.add_special_tokens({"bos_token": tokenizer.bos_token}) - - return tokenizer - - -def list_supported_models( - architecture: str | None = None, - verified_only: bool = False, -) -> list[str]: - """List all models supported by TransformerLens. - - This function provides convenient access to the model registry API - for discovering which HuggingFace models can be loaded. - - Args: - architecture: Filter by architecture ID (e.g., "GPT2LMHeadModel"). - If None, returns all supported models. - verified_only: If True, only return models that have been verified - to work with TransformerLens. - - Returns: - List of model IDs (e.g., ["gpt2", "gpt2-medium", ...]) - - Example: - >>> from transformer_lens.model_bridge.sources.transformers import list_supported_models - >>> models = list_supported_models() - >>> gpt2_models = list_supported_models(architecture="GPT2LMHeadModel") - """ - try: - from transformer_lens.tools.model_registry import api - - models = api.get_supported_models(architecture=architecture, verified_only=verified_only) - return [m.model_id for m in models] - except ImportError: - return [] - except Exception: - return [] - - -def check_model_support(model_id: str) -> dict: - """Check if a model is supported and get detailed support info. - - This function provides detailed information about a model's compatibility - with TransformerLens, including architecture type and verification status. - - Args: - model_id: The HuggingFace model ID to check (e.g., "gpt2") - - Returns: - Dictionary with support information: - - is_supported: bool - Whether the model is supported - - architecture_id: str | None - The architecture type if supported - - verified: bool - Whether the model has been verified to work - - suggestion: str | None - Suggested alternative if not supported - - Example: - >>> from transformer_lens.model_bridge.sources.transformers import check_model_support # doctest: +SKIP - >>> info = check_model_support("openai-community/gpt2") # doctest: +SKIP - >>> info["is_supported"] # doctest: +SKIP - True - """ - try: - from transformer_lens.tools.model_registry import api - - is_supported = api.is_model_supported(model_id) - - if is_supported: - model_info = api.get_model_info(model_id) - return { - "is_supported": True, - "architecture_id": model_info.architecture_id, - "status": model_info.status, - "verified_date": ( - model_info.verified_date.isoformat() if model_info.verified_date else None - ), - "suggestion": None, - } - else: - suggestion = api.suggest_similar_model(model_id) - return { - "is_supported": False, - "architecture_id": None, - "verified": False, - "verified_date": None, - "suggestion": suggestion, - } - except ImportError: - return { - "is_supported": None, - "architecture_id": None, - "verified": False, - "verified_date": None, - "suggestion": None, - "error": "Model registry not available", - } - except Exception as e: - return { - "is_supported": None, - "architecture_id": None, - "verified": False, - "verified_date": None, - "suggestion": None, - "error": str(e), - } - - -# Attach functions to TransformerBridge as static methods -setattr(TransformerBridge, "boot_transformers", staticmethod(boot)) -setattr(TransformerBridge, "list_supported_models", staticmethod(list_supported_models)) -setattr(TransformerBridge, "check_model_support", staticmethod(check_model_support)) diff --git a/transformer_lens/model_bridge/sources/transformers/__init__.py b/transformer_lens/model_bridge/sources/transformers/__init__.py new file mode 100644 index 0000000000..1da07bc4b6 --- /dev/null +++ b/transformer_lens/model_bridge/sources/transformers/__init__.py @@ -0,0 +1,47 @@ +"""HuggingFace ``transformers`` source for TransformerBridge.""" +from __future__ import annotations + +# Re-exported so external code that patches ``AutoConfig.from_pretrained`` / +# ``AutoTokenizer.from_pretrained`` via this module path keeps working after the +# package split. Class-method monkey-patches reach the same class objects that +# ``source.py`` imports directly, so this re-export keeps tests stable. +from transformers import AutoConfig, AutoTokenizer + +from transformer_lens.model_bridge.bridge import TransformerBridge + +# Re-export shared HF-format utilities at the historical path for backward compatibility +# with `from transformer_lens.model_bridge.sources.transformers import ...` callers. +from transformer_lens.model_bridge.sources._hf_format import ( + determine_architecture_from_hf_config, + map_default_transformer_lens_config, + setup_tokenizer, +) + +from .helpers import ( + _CHECKPOINT_REVISION_FORMATS, + _resolve_checkpoint_to_revision, + check_model_support, + get_hf_model_class_for_architecture, + list_supported_models, +) +from .source import boot + +# Attach functions to TransformerBridge as static methods. +setattr(TransformerBridge, "boot_transformers", staticmethod(boot)) +setattr(TransformerBridge, "list_supported_models", staticmethod(list_supported_models)) +setattr(TransformerBridge, "check_model_support", staticmethod(check_model_support)) + + +__all__ = [ + "AutoConfig", + "AutoTokenizer", + "boot", + "check_model_support", + "determine_architecture_from_hf_config", + "get_hf_model_class_for_architecture", + "list_supported_models", + "map_default_transformer_lens_config", + "setup_tokenizer", + "_CHECKPOINT_REVISION_FORMATS", + "_resolve_checkpoint_to_revision", +] diff --git a/transformer_lens/model_bridge/sources/transformers/helpers.py b/transformer_lens/model_bridge/sources/transformers/helpers.py new file mode 100644 index 0000000000..922c30f5f8 --- /dev/null +++ b/transformer_lens/model_bridge/sources/transformers/helpers.py @@ -0,0 +1,157 @@ +"""HF-loader-specific helpers: model-class selection, checkpoint revision resolution, registry discovery.""" +from __future__ import annotations + +from transformers import ( + AutoModelForCausalLM, + AutoModelForMaskedLM, + AutoModelForSeq2SeqLM, +) + + +def get_hf_model_class_for_architecture(architecture: str): + """Pick the correct HuggingFace ``AutoModel*`` class for the architecture.""" + from transformer_lens.utilities.architectures import ( + AUDIO_ARCHITECTURES, + MASKED_LM_ARCHITECTURES, + MULTIMODAL_ARCHITECTURES, + SEQ2SEQ_ARCHITECTURES, + ) + + if architecture in SEQ2SEQ_ARCHITECTURES: + return AutoModelForSeq2SeqLM + elif architecture in MASKED_LM_ARCHITECTURES: + return AutoModelForMaskedLM + elif architecture in MULTIMODAL_ARCHITECTURES: + from transformers import AutoModelForImageTextToText + + return AutoModelForImageTextToText + elif architecture in AUDIO_ARCHITECTURES: + if "ForCTC" in architecture: + from transformers import AutoModelForCTC + + return AutoModelForCTC + from transformers import AutoModel + + return AutoModel + else: + return AutoModelForCausalLM + + +# Known training-checkpoint revision conventions on HF Hub. +_CHECKPOINT_REVISION_FORMATS: dict[str, str] = { + "EleutherAI/pythia": "step{value}", + "stanford-crfm": "checkpoint-{value}", +} + + +def _resolve_checkpoint_to_revision( + model_name: str, + checkpoint_index: int | None, + checkpoint_value: int | None, +) -> str: + """Convert a checkpoint index/value into an HF revision string, validated against ``get_checkpoint_labels``.""" + if checkpoint_index is None and checkpoint_value is None: + raise ValueError("Must specify either checkpoint_index or checkpoint_value.") + + format_str: str | None = None + for prefix, fmt in _CHECKPOINT_REVISION_FORMATS.items(): + if model_name.startswith(prefix): + format_str = fmt + break + if format_str is None: + raise ValueError( + f"Model {model_name!r} does not have a known checkpoint revision convention. " + f"Pass revision= directly if your model uses HF revisions. Known checkpoint " + f"families: {list(_CHECKPOINT_REVISION_FORMATS.keys())}." + ) + + from transformer_lens.loading_from_pretrained import get_checkpoint_labels + + labels, _ = get_checkpoint_labels(model_name) + if checkpoint_value is not None: + if checkpoint_value not in labels: + raise ValueError( + f"checkpoint_value={checkpoint_value} not in available checkpoints for " + f"{model_name!r}. {len(labels)} labels available, " + f"first/last: {labels[0]}..{labels[-1]}." + ) + else: + assert checkpoint_index is not None # narrowed by initial guard + if not 0 <= checkpoint_index < len(labels): + raise ValueError( + f"checkpoint_index={checkpoint_index} out of range [0, {len(labels)}) " + f"for {model_name!r}." + ) + checkpoint_value = labels[checkpoint_index] + return format_str.format(value=checkpoint_value) + + +def list_supported_models( + architecture: str | None = None, + verified_only: bool = False, +) -> list[str]: + """List all models supported by TransformerLens. + + Args: + architecture: Filter by architecture ID (e.g., "GPT2LMHeadModel"). + verified_only: If True, only return verified-to-work models. + + Returns: + List of model IDs. + """ + try: + from transformer_lens.tools.model_registry import api + + models = api.get_supported_models(architecture=architecture, verified_only=verified_only) + return [m.model_id for m in models] + except ImportError: + return [] + except Exception: + return [] + + +def check_model_support(model_id: str) -> dict: + """Detailed support info for a model: ``is_supported``, ``architecture_id``, ``verified``, ``suggestion``.""" + try: + from transformer_lens.tools.model_registry import api + + is_supported = api.is_model_supported(model_id) + + if is_supported: + model_info = api.get_model_info(model_id) + return { + "is_supported": True, + "architecture_id": model_info.architecture_id, + "status": model_info.status, + "verified_date": ( + model_info.verified_date.isoformat() if model_info.verified_date else None + ), + "suggestion": None, + } + else: + suggestion = api.suggest_similar_model(model_id) + return { + "is_supported": False, + "architecture_id": None, + "verified": False, + "verified_date": None, + "suggestion": suggestion, + } + except ImportError: + return { + "is_supported": None, + "architecture_id": None, + "verified": False, + "verified_date": None, + "suggestion": None, + "error": "Model registry not available", + } + except Exception as e: + return { + "is_supported": None, + "architecture_id": None, + "verified": False, + "verified_date": None, + "suggestion": None, + "error": str(e), + } diff --git a/transformer_lens/model_bridge/sources/transformers/source.py b/transformer_lens/model_bridge/sources/transformers/source.py new file mode 100644 index 0000000000..ce0c7def5b --- /dev/null +++ b/transformer_lens/model_bridge/sources/transformers/source.py @@ -0,0 +1,435 @@ +"""``boot`` — load a model via HuggingFace ``transformers`` and wrap it in a TransformerBridge.""" +from __future__ import annotations + +import contextlib +import copy +import logging +import os +import warnings +from typing import Any + +import torch +from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase + +from transformer_lens.factories.architecture_adapter_factory import ( + ArchitectureAdapterFactory, +) +from transformer_lens.model_bridge.bridge import TransformerBridge +from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_config_from_hf, + detect_tokenizer_bos_eos, +) +from transformer_lens.model_bridge.sources._hf_format import ( + determine_architecture_from_hf_config, + setup_tokenizer, +) +from transformer_lens.supported_models import MODEL_ALIASES +from transformer_lens.utilities import get_device + +from .helpers import ( + _resolve_checkpoint_to_revision, + get_hf_model_class_for_architecture, +) + +# Suppress transformers warnings that go to stderr; otherwise notebook tests fail +# on unexpected stderr output. +warnings.filterwarnings("ignore", message=".*generation flags.*not valid.*") +logging.getLogger("transformers").setLevel(logging.ERROR) + + +def boot( + model_name: str, + hf_config_overrides: dict | None = None, + device: str | torch.device | None = None, + dtype: torch.dtype = torch.float32, + tokenizer: PreTrainedTokenizerBase | None = None, + load_weights: bool = True, + trust_remote_code: bool = False, + model_class: Any | None = None, + hf_model: Any | None = None, + n_ctx: int | None = None, + revision: str | None = None, + checkpoint_index: int | None = None, + checkpoint_value: int | None = None, + # Experimental – Have not been fully tested on multi-gpu devices + # Use at your own risk, report any issues here: https://github.com/TransformerLensOrg/TransformerLens/issues + device_map: str | dict[str, str | int] | None = None, + n_devices: int | None = None, + max_memory: dict[str | int, str] | None = None, +) -> TransformerBridge: + """Boot a model from HuggingFace. + + Args: + model_name: The name of the model to load. + hf_config_overrides: Optional overrides applied to the HuggingFace config before model load. + device: The device to use. If None, will be determined automatically. Mutually exclusive + with ``device_map``. + dtype: The dtype to use for the model. + tokenizer: Optional pre-initialized tokenizer to use; if not provided one will be created. + load_weights: If False, load model without weights (on meta device) for config inspection only. + model_class: Optional HuggingFace model class to use instead of the default auto-detected + class. When the class name matches a key in SUPPORTED_ARCHITECTURES, the corresponding + adapter is selected automatically (e.g., BertForNextSentencePrediction). + hf_model: Optional pre-loaded HuggingFace model to use instead of loading one. Useful for + models loaded with custom configurations (e.g., quantization via BitsAndBytesConfig). + When provided, load_weights is ignored. + device_map: HuggingFace-style device map (``"auto"``, ``"balanced"``, dict, etc.) for + multi-GPU inference. Passed straight to ``from_pretrained``. Mutually exclusive + with ``device``. + n_devices: Convenience: split the model across this many CUDA devices (translated to a + ``max_memory`` dict internally). Requires CUDA with at least this many visible devices. + max_memory: Optional per-device memory budget for HF's dispatcher. + n_ctx: Optional context length override. The bridge normally uses the model's documented + max context from the HF config. Setting this writes to whichever HF field the model + uses (n_positions / max_position_embeddings / etc.), so callers don't need to know + the field name. If larger than the model's default, a warning is emitted — quality + may degrade past the trained length for rotary models. + revision: Optional HF revision string (branch, tag, or commit). Forwarded to + ``AutoConfig.from_pretrained`` and ``AutoModelForCausalLM.from_pretrained``. + Mutually exclusive with ``checkpoint_index`` and ``checkpoint_value``. + checkpoint_index: Index into the available training checkpoints for the model family. + Convenience over ``revision`` for checkpointed models like EleutherAI/pythia* and + stanford-crfm/*. Resolved to a revision string via the known per-family naming + conventions (``step{value}`` for Pythia, ``checkpoint-{value}`` for stanford-crfm). + checkpoint_value: Training step or token count of the desired checkpoint. Alternative to + ``checkpoint_index``; must be one of the labels returned by ``get_checkpoint_labels``. + + Returns: + The bridge to the loaded model. + """ + for official_name, aliases in MODEL_ALIASES.items(): + if model_name in aliases: + logging.warning( + f"DEPRECATED: You are using a deprecated, model_name alias '{model_name}'. TransformerLens will now load the official transformers model name, '{official_name}' instead.\n Please update your code to use the official name by changing model_name from '{model_name}' to '{official_name}'.\nSince TransformerLens v3, all model names should be the official transformers model names.\nThe aliases will be removed in the next version of TransformerLens, so please do the update now." + ) + model_name = official_name + break + if checkpoint_index is not None or checkpoint_value is not None: + if revision is not None: + raise ValueError( + "Specify either revision= or checkpoint_index/checkpoint_value, not both." + ) + revision = _resolve_checkpoint_to_revision(model_name, checkpoint_index, checkpoint_value) + # Pass HF token for gated model access (e.g. meta-llama/*) + from transformer_lens.utilities.hf_utils import get_hf_token + + _hf_token = get_hf_token() + if hf_model is not None: + # Reuse the pre-loaded model's config to avoid a Hub call when model_name + # is a Hub repo ID but the model is already loaded locally. + hf_config = copy.deepcopy(hf_model.config) + else: + hf_config = AutoConfig.from_pretrained( + model_name, + output_attentions=True, + trust_remote_code=trust_remote_code, + token=_hf_token, + revision=revision, + ) + _n_ctx_field: str | None = None + if n_ctx is not None: + if n_ctx <= 0: + raise ValueError(f"n_ctx must be a positive integer, got n_ctx={n_ctx}.") + # Resolve n_ctx to whichever HF config field this model uses. Mirrors the order in + # map_default_transformer_lens_config so the TL config derivation picks up the override. + for _field in ( + "n_positions", + "max_position_embeddings", + "max_context_length", + "max_length", + "seq_length", + ): + if hasattr(hf_config, _field): + _n_ctx_field = _field + break + if _n_ctx_field is None: + raise ValueError( + f"Cannot apply n_ctx={n_ctx}: no recognized context-length field on " + f"HF config for {model_name}. Use hf_config_overrides instead." + ) + _default_n_ctx = getattr(hf_config, _n_ctx_field) + if _default_n_ctx is not None and n_ctx > _default_n_ctx: + logging.warning( + "Setting n_ctx=%d which is larger than the model's default " + "context length of %d. The model was not trained on sequences " + "this long and may produce unreliable results (especially for " + "rotary models without RoPE scaling).", + n_ctx, + _default_n_ctx, + ) + # Warn if the caller also set the same field via hf_config_overrides — explicit n_ctx wins. + if hf_config_overrides and _n_ctx_field in hf_config_overrides: + _conflicting_value = hf_config_overrides[_n_ctx_field] + if _conflicting_value != n_ctx: + logging.warning( + "Both n_ctx=%d and hf_config_overrides['%s']=%s were provided. " + "The explicit n_ctx takes precedence.", + n_ctx, + _n_ctx_field, + _conflicting_value, + ) + hf_config_overrides = dict(hf_config_overrides or {}) + hf_config_overrides[_n_ctx_field] = n_ctx + if hf_config_overrides: + hf_config.__dict__.update(hf_config_overrides) + architecture = determine_architecture_from_hf_config(hf_config) + bridge_config = build_bridge_config_from_hf(hf_config, architecture, model_name, dtype) + adapter = ArchitectureAdapterFactory.select_architecture_adapter(bridge_config) + # Pre-loaded models carry their own weight placement (possibly set by the caller via + # device_map). Passing device_map / n_devices / max_memory alongside hf_model= is ambiguous + # and would silently be ignored, so fail loudly. + if hf_model is not None and ( + device_map is not None or n_devices is not None or max_memory is not None + ): + raise ValueError( + "device_map / n_devices / max_memory are only supported when the bridge loads " + "the HF model itself. When passing hf_model=..., apply device_map via " + "AutoModel.from_pretrained before handing the model to the bridge." + ) + # Stateful/SSM (Mamba) models keep a per-layer recurrent cache that must live on that + # layer's device. The bridge allocates the stateful cache on a single cfg.device, so + # cross-device splits would silently misplace the cache. Blocked until v2. + if (n_devices is not None and n_devices > 1) or device_map is not None: + if getattr(bridge_config, "is_stateful", False): + raise ValueError( + "Multi-device splits are not yet supported for stateful (SSM / Mamba) " + "architectures: the stateful cache allocation is single-device. " + "Load on one device, or wait for v2 support." + ) + # Resolve device_map before defaulting `device` — the two are mutually exclusive and the + # resolver raises on conflict. If n_devices>1 is passed it's translated into a device_map + + # max_memory pair here so downstream code only needs to check the resolved values. + from transformer_lens.utilities.multi_gpu import ( + count_unique_devices, + find_embedding_device, + resolve_device_map, + ) + + resolved_device_map, resolved_max_memory = resolve_device_map( + n_devices, device_map, device, max_memory + ) + if resolved_device_map is None: + if device is None: + device = get_device() + adapter.cfg.device = str(device) + else: + # cfg.device set from hf_device_map after the model is loaded; provisionally None. + adapter.cfg.device = None + if model_class is None: + model_class = get_hf_model_class_for_architecture(architecture) + # Ensure pad_token_id exists (v5 raises AttributeError if missing). + if not hasattr(hf_config, "pad_token_id") or "pad_token_id" not in hf_config.__dict__: + fallback_pad = getattr(hf_config, "eos_token_id", None) + # eos_token_id can be a list (Gemma3 uses [1, 106]); take the first. + if isinstance(fallback_pad, list): + fallback_pad = fallback_pad[0] if fallback_pad else None + hf_config.pad_token_id = fallback_pad + model_kwargs = {"config": hf_config, "torch_dtype": dtype} + if _hf_token: + model_kwargs["token"] = _hf_token + if trust_remote_code: + model_kwargs["trust_remote_code"] = True + if revision is not None: + model_kwargs["revision"] = revision + if resolved_device_map is not None: + model_kwargs["device_map"] = resolved_device_map + if resolved_max_memory is not None: + model_kwargs["max_memory"] = resolved_max_memory + if hasattr(adapter.cfg, "attn_implementation") and adapter.cfg.attn_implementation is not None: + model_kwargs["attn_implementation"] = adapter.cfg.attn_implementation + else: + # Eager is required for output_attentions hooks. + model_kwargs["attn_implementation"] = "eager" + adapter.prepare_loading(model_name, model_kwargs) + if hf_model is not None: + # Use the pre-loaded model as-is (quantized models with custom device_map, etc). + pass + elif not load_weights: + from_config_kwargs = {} + if trust_remote_code: + from_config_kwargs["trust_remote_code"] = True + # adapter.prepare_loading may have replaced model_kwargs["config"] (e.g. Qwen3.5 + # text-only extraction); honor that here so the no-weights path uses the + # same config the load-weights path would. + prepared_config = model_kwargs.get("config", hf_config) + with contextlib.redirect_stdout(None): + hf_model = model_class.from_config(prepared_config, **from_config_kwargs) + else: + try: + hf_model = model_class.from_pretrained(model_name, **model_kwargs) + except RuntimeError as e: + # HF refuses to load when positional-weight shapes don't match. If the user + # requested an n_ctx that conflicts with the saved weights (common for + # learned-pos-embed models like GPT-2), re-raise with a clearer message. + if n_ctx is not None and "ignore_mismatched_sizes" in str(e): + raise RuntimeError( + f"Failed to load {model_name} with n_ctx={n_ctx}: the pretrained " + f"weights' positional-embedding shape does not match the requested " + f"context length. This affects models with learned positional " + f"embeddings (e.g. GPT-2, OPT). Options: (1) use the model's " + f"default n_ctx, (2) pass load_weights=False if you only need " + f"config inspection, or (3) choose a rotary-embedding model " + f"(e.g. Llama, Mistral) which supports n_ctx changes without " + f"weight mismatch." + ) from e + raise + # Skip explicit .to(device) when accelerate has placed weights via device_map. + if resolved_device_map is None and device is not None: + hf_model = hf_model.to(device) + # Cast params to dtype; preserve float32 buffers (e.g. RotaryEmbedding.inv_freq). + for param in hf_model.parameters(): + if param.is_floating_point() and param.dtype != dtype: + param.data = param.data.to(dtype=dtype) + # Derive cfg.device / cfg.n_devices from hf_device_map when present. Covers fresh loads + # with a resolved device_map AND pre-loaded models with caller-dispatched device_map="auto". + hf_device_map_post = getattr(hf_model, "hf_device_map", None) + if hf_device_map_post: + # Pre-loaded path can smuggle CPU/disk offload in; validate. + offload_values = {str(v).lower() for v in hf_device_map_post.values() if isinstance(v, str)} + forbidden = offload_values & {"cpu", "disk", "meta"} + if forbidden and ((n_devices is not None and n_devices > 1) or device_map is not None): + raise ValueError( + f"hf_device_map contains unsupported offload targets: {sorted(forbidden)}. " + "v1 multi-device support is GPU-only." + ) + embedding_device = find_embedding_device(hf_model) + if embedding_device is not None: + adapter.cfg.device = str(embedding_device) + adapter.cfg.n_devices = count_unique_devices(hf_model) + elif adapter.cfg.device is None: + # Pre-loaded single-device model with no hf_device_map — fall back to first param. + try: + adapter.cfg.device = str(next(hf_model.parameters()).device) + except StopIteration: + adapter.cfg.device = "cpu" + # Verify the n_ctx override actually took effect on the loaded model. If HF's config class + # silently dropped or normalized the value, warn so the user isn't misled. + if n_ctx is not None and _n_ctx_field is not None and hf_model is not None: + _actual = getattr(hf_model.config, _n_ctx_field, None) + if _actual != n_ctx: + logging.warning( + "n_ctx=%d was requested but hf_model.config.%s=%s after load. " + "The override may not have taken effect; the model may not " + "accept sequences longer than %s.", + n_ctx, + _n_ctx_field, + _actual, + _actual, + ) + adapter.prepare_model(hf_model) + default_padding_side = getattr(adapter.cfg, "default_padding_side", None) + use_fast = getattr(adapter.cfg, "use_fast", True) + # Audio models use feature extractors, not text tokenizers. + _is_audio = getattr(adapter.cfg, "is_audio_model", False) + if _is_audio and tokenizer is None: + tokenizer = None + elif tokenizer is not None: + tokenizer = setup_tokenizer(tokenizer, default_padding_side=default_padding_side) + else: + token_arg = get_hf_token() + # Some adapters override tokenizer source (e.g. OpenELM has no tokenizer of its own). + tokenizer_source = model_name + if hasattr(adapter.cfg, "tokenizer_name") and adapter.cfg.tokenizer_name is not None: + tokenizer_source = adapter.cfg.tokenizer_name + # Encoder-decoder models like T5 don't have a BOS token and raise on add_bos_token=True. + try: + base_tokenizer = AutoTokenizer.from_pretrained( + tokenizer_source, + add_bos_token=True, + use_fast=use_fast, + token=token_arg, + trust_remote_code=trust_remote_code, + ) + except ValueError: + base_tokenizer = AutoTokenizer.from_pretrained( + tokenizer_source, + use_fast=use_fast, + token=token_arg, + trust_remote_code=trust_remote_code, + ) + tokenizer = setup_tokenizer( + base_tokenizer, + default_padding_side=default_padding_side, + ) + if tokenizer is not None: + ( + adapter.cfg.tokenizer_prepends_bos, + adapter.cfg.tokenizer_appends_eos, + ) = detect_tokenizer_bos_eos(tokenizer) + from transformer_lens.model_bridge.sources.transformers_driver import ( + TransformersDriver, + ) + + driver = TransformersDriver(hf_model, adapter, tokenizer) + bridge = TransformerBridge(hf_model, adapter, tokenizer, driver=driver) + + # Multimodal models: load the image preprocessor. + if getattr(adapter.cfg, "is_multimodal", False): + try: + from transformers import AutoProcessor + + huggingface_token = os.environ.get("HF_TOKEN", "") + token_arg = huggingface_token if len(huggingface_token) > 0 else None + bridge.processor = AutoProcessor.from_pretrained( + model_name, + token=token_arg, + trust_remote_code=trust_remote_code, + ) + except Exception: + # Some processors need torchvision (e.g. LlavaOnevision); install if missing. + _torchvision_available = False + try: + import torchvision # noqa: F401 + + _torchvision_available = True + except Exception: + import shutil + import subprocess + import sys + + try: + if shutil.which("uv"): + subprocess.check_call( + ["uv", "pip", "install", "torchvision", "-q"], + ) + else: + subprocess.check_call( + [sys.executable, "-m", "pip", "install", "torchvision", "-q"], + ) + import importlib + + importlib.invalidate_caches() + _torchvision_available = True + except Exception: + pass + + if _torchvision_available: + try: + from transformers import AutoProcessor + + huggingface_token = os.environ.get("HF_TOKEN", "") + token_arg = huggingface_token if len(huggingface_token) > 0 else None + bridge.processor = AutoProcessor.from_pretrained( + model_name, + token=token_arg, + trust_remote_code=trust_remote_code, + ) + except Exception: + pass + + # Audio models: load the feature extractor. + if getattr(adapter.cfg, "is_audio_model", False): + try: + from transformers import AutoFeatureExtractor + + huggingface_token = os.environ.get("HF_TOKEN", "") + token_arg = huggingface_token if len(huggingface_token) > 0 else None + bridge.processor = AutoFeatureExtractor.from_pretrained( + model_name, + token=token_arg, + trust_remote_code=trust_remote_code, + ) + except Exception: + pass + + return bridge diff --git a/transformer_lens/model_bridge/sources/transformers_driver.py b/transformer_lens/model_bridge/sources/transformers_driver.py new file mode 100644 index 0000000000..c745788e2b --- /dev/null +++ b/transformer_lens/model_bridge/sources/transformers_driver.py @@ -0,0 +1,75 @@ +"""HuggingFace transformers Driver.""" +from __future__ import annotations + +from typing import Any, Iterator, Mapping + +import torch +from torch import nn + +from transformer_lens.model_bridge.driver_protocol import ( + ForwardResult, + Intervention, + TensorLike, +) +from transformer_lens.model_bridge.sources._driver_base import DriverBase + + +class TransformersDriver(DriverBase): + """Wraps an HF ``nn.Module``. PyTorch hooks fire via module replacement during the + real forward; this driver just runs the engine and threads the native output back.""" + + _supported_features = frozenset( + {"gradients", "parameters", "state_dict", "weight_access", "intervention_callbacks"} + ) + + def __init__(self, model: nn.Module, adapter: Any, tokenizer: Any) -> None: + super().__init__(adapter.cfg, tokenizer) + self._model = model + self._adapter = adapter + + def forward( + self, + input_ids: TensorLike | None = None, + *, + capture: tuple[str, ...] = (), + intervene: Mapping[str, Intervention] | None = None, + max_new_tokens: int = 1, + return_logits: bool = True, + **kwargs: Any, + ) -> ForwardResult: + if input_ids is not None: + raw = self._model(input_ids, **kwargs) + else: + raw = self._model(**kwargs) + + logits = None + if return_logits: + if hasattr(raw, "logits"): + logits = raw.logits + elif isinstance(raw, tuple) and len(raw) > 0: + logits = raw[0] + else: + logits = raw + + return ForwardResult(logits=logits, raw_output=raw) + + def parameters(self) -> Iterator[torch.Tensor]: + return self._model.parameters() + + def named_parameters( + self, + prefix: str = "", + recurse: bool = True, + remove_duplicate: bool = True, + ) -> Iterator[tuple[str, torch.Tensor]]: + return self._model.named_parameters(prefix, recurse, remove_duplicate) + + @property + def underlying_model(self) -> nn.Module: + """Escape hatch for code that needs the raw HF module. Driver-specific.""" + return self._model + + def set_underlying_model(self, value: nn.Module) -> None: + """Used by weight-processing paths that move the model to a different + device. Non-torch drivers don't implement this.""" + self._model = value diff --git a/transformer_lens/model_bridge/sources/vllm/__init__.py b/transformer_lens/model_bridge/sources/vllm/__init__.py new file mode 100644 index 0000000000..ee059a6e69 --- /dev/null +++ b/transformer_lens/model_bridge/sources/vllm/__init__.py @@ -0,0 +1,12 @@ +"""vLLM source for TransformerBridge. + +Provides :func:`boot_vllm`, which constructs a vLLM ``LLM`` and wraps its +inner ``nn.Module`` in a :class:`TransformerBridge`. vLLM drives the forward +pass (PagedAttention, ``torch.compile``, CUDA graphs); the bridge surface is +populated from GPU buffers written by hooks the plugin installs pre-compile. +""" +from __future__ import annotations + +from .source import boot_vllm + +__all__ = ["boot_vllm"] diff --git a/transformer_lens/model_bridge/sources/vllm/driver.py b/transformer_lens/model_bridge/sources/vllm/driver.py new file mode 100644 index 0000000000..e6733c3ef1 --- /dev/null +++ b/transformer_lens/model_bridge/sources/vllm/driver.py @@ -0,0 +1,207 @@ +"""vLLM Driver: forward dispatches via ``llm.generate``; captures via ``collective_rpc``.""" +from __future__ import annotations + +import logging +from typing import Any, Mapping + +import torch + +from transformer_lens.model_bridge.driver_protocol import ( + ForwardResult, + Intervention, + TensorLike, +) +from transformer_lens.model_bridge.sources._driver_base import DriverBase + +from .intervention_specs import SUPPORTED_OPS + + +class VLLMDriver(DriverBase): + """Driver wrapping a vLLM ``LLM``; captures via ``collective_rpc``.""" + + # vLLM owns the model in a worker — no torch surface (parameters/state_dict/grads). + _supported_features = frozenset() + + def __init__( + self, + llm: Any, + adapter: Any, + tokenizer: Any, + overlay: Any, + hf_config: Any, + max_num_batched_tokens: int, + ) -> None: + super().__init__(adapter.cfg, tokenizer) + self._llm = llm + self._max_num_batched_tokens = max_num_batched_tokens + + self.supported_hook_points = frozenset(overlay.capture_specs(hf_config).keys()) + + n_layers = getattr(hf_config, "num_hidden_layers", 0) + nonfiring: list[str] = [] + for tmpl in overlay.nonfiring_hooks(): + if "{i}" in tmpl and isinstance(n_layers, int) and n_layers > 0: + nonfiring.extend(tmpl.replace("{i}", str(i)) for i in range(n_layers)) + else: + nonfiring.append(tmpl) + self.non_fireable_hook_points = frozenset(nonfiring) + + def forward( + self, + input_ids: TensorLike | None = None, + *, + capture: tuple[str, ...] = (), + intervene: Mapping[str, Intervention] | None = None, + max_new_tokens: int = 1, + return_logits: bool = True, + **kwargs: Any, + ) -> ForwardResult: + if input_ids is None: + raise ValueError("VLLMDriver requires input_ids") + if int(max_new_tokens) != 1: + raise NotImplementedError( + "VLLMDriver supports max_new_tokens=1 only — decode-step writes " + "overwrite the prefill buffer; multi-step capture is multi-buffer work." + ) + intervene_specs = self._validate_interventions(intervene or {}) + + ids_list = self._normalize_input_ids(input_ids) + if len(ids_list) > self._max_num_batched_tokens: + # Worker buffers silently clamp on overflow — fail loud here instead. + raise ValueError( + f"Prompt length {len(ids_list)} exceeds max_num_batched_tokens=" + f"{self._max_num_batched_tokens}; raise the boot_vllm kwarg or " + "shorten the prompt." + ) + + from vllm import SamplingParams + from vllm.inputs import TokensPrompt + + # Push intervention state (possibly empty) before generate — this also + # resets stale interventions from prior forwards. + self._llm.collective_rpc("tl_set_interventions", args=(intervene_specs,)) + # Request full-vocab logprobs so the driver can populate position -1 of + # the synthesized logits with the real next-token distribution. vLLM's + # ``max_logprobs`` was set to d_vocab at boot to make this legal. + outputs = self._llm.generate( + prompts=[TokensPrompt(prompt_token_ids=ids_list)], + sampling_params=SamplingParams( + max_tokens=int(max_new_tokens), + temperature=0.0, + logprobs=self.bridge_config.d_vocab if return_logits else None, + ), + ) + + n_tokens = len(ids_list) + # collective_rpc returns one result per worker; single-rank, so [0]. + worker_captures = self._llm.collective_rpc("tl_read_captures", args=([n_tokens],))[0] + # Add batch dim: vLLM hands back (n_tokens, width); bridge expects (1, n_tokens, width). + captured = {name: t.unsqueeze(0) for name, t in worker_captures.items()} + + logits: torch.Tensor | None = None + if return_logits: + logits = self._synthesize_logits(outputs[0], n_tokens, self.bridge_config.d_vocab) + + return ForwardResult(logits=logits, captured=captured, raw_output=outputs[0]) + + def close(self) -> None: + # Detach hooks before dropping the LLM so they don't stay registered on + # worker modules for the life of the process (long-running notebooks). + log = logging.getLogger("transformer_lens.vllm") + if self._llm is not None: + try: + self._llm.collective_rpc("tl_remove_hooks") + except Exception as e: + # Best-effort: engine may already be torn down or the RPC surface + # gone. Log so hook-leak debugging has a thread to pull. + log.debug("tl_remove_hooks failed during close(): %s", e) + self._llm = None + # vLLM 0.20.2 has no LLM.shutdown() — model weights and KV cache stay + # resident until process exit unless we explicitly tear down the + # distributed environment vLLM set up at construction. Both calls are + # best-effort: they're no-ops if there's no distributed state. After + # this, the caller still needs gc.collect() + torch.cuda.empty_cache() + # to drop PyTorch's caching allocator entries. + try: + from vllm.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + ) + + destroy_model_parallel() + destroy_distributed_environment() + except Exception as e: + log.debug("vLLM distributed teardown failed during close(): %s", e) + + @staticmethod + def _synthesize_logits(request_output: Any, n_tokens: int, d_vocab: int) -> torch.Tensor: + """Build a (1, n_tokens, d_vocab) logits-like tensor from vLLM's sampler output. + + vLLM's lm_head bypass means our hook never fires; the sampler returns + full-vocab logprobs (we set ``max_logprobs=d_vocab`` at boot to allow this). + Position -1 — the input's last token, = next-token prediction — is populated + from those logprobs. Earlier positions stay at ``-inf`` so any argmax there + is loud rather than silently misleading; populating them would need + ``prompt_logprobs`` requested per call (much more expensive). + """ + logits = torch.full((1, n_tokens, d_vocab), float("-inf"), dtype=torch.float16) + gen = request_output.outputs[0] if request_output.outputs else None + if gen is None: + return logits + # Prefer real logprobs; fall back to the generated token id (one-hot-ish) + # if logprobs weren't requested (e.g. return_logits=False elsewhere). + if gen.logprobs: + for token_id, lp_obj in gen.logprobs[0].items(): + logits[0, -1, int(token_id)] = float(lp_obj.logprob) + elif gen.token_ids: + logits[0, -1, int(gen.token_ids[0])] = 0.0 + return logits + + def _validate_interventions(self, intervene: Mapping[str, Any]) -> dict: + """Reject callables, validate spec format and hook names; return a plain dict.""" + out: dict = {} + for hook_name, spec in intervene.items(): + if callable(spec): + raise NotImplementedError( + "VLLMDriver requires intervention specs (dict), not callables. " + "Supported ops: suppress, scale (factor: float), add (value: scalar or width-shaped), " + "set (value: scalar or width-shaped)." + ) + if not isinstance(spec, Mapping) or "op" not in spec: + raise ValueError( + f"Intervention spec for {hook_name!r} must be a dict with 'op' key; got {spec!r}" + ) + op = spec["op"] + if op not in SUPPORTED_OPS: + raise ValueError( + f"Unsupported intervention op {op!r} for {hook_name!r}. " + f"Supported: {sorted(SUPPORTED_OPS)}." + ) + if op == "scale" and "factor" not in spec: + raise ValueError( + f"Intervention {hook_name!r}: op='scale' requires 'factor' (float)." + ) + if op in ("add", "set") and "value" not in spec: + raise ValueError( + f"Intervention {hook_name!r}: op={op!r} requires 'value' " + "(scalar or width-shaped tensor/list)." + ) + if hook_name not in self.supported_hook_points: + raise ValueError( + f"Cannot intervene on {hook_name!r}: not in supported_hook_points." + ) + out[hook_name] = dict(spec) + return out + + @staticmethod + def _normalize_input_ids(input_ids: Any) -> list: + """Coerce input_ids to a flat list[int] for ``TokensPrompt``; batch_size=1 only.""" + if isinstance(input_ids, torch.Tensor): + ids_list = input_ids.tolist() + else: + ids_list = list(input_ids) + if ids_list and isinstance(ids_list[0], list): + if len(ids_list) != 1: + raise NotImplementedError("VLLMDriver supports batch_size=1 only.") + ids_list = ids_list[0] + return ids_list diff --git a/transformer_lens/model_bridge/sources/vllm/internals.py b/transformer_lens/model_bridge/sources/vllm/internals.py new file mode 100644 index 0000000000..bdabf5455a --- /dev/null +++ b/transformer_lens/model_bridge/sources/vllm/internals.py @@ -0,0 +1,24 @@ +"""Single chokepoint for vLLM internal API access. + +vLLM rearranges its internal class paths every 4-6 weeks. Centralize every +``llm.llm_engine.…`` walk here so version drift is patched in one place. + +**Validated against ``vllm==0.20.2``** (also the version pinned in +``demos/vLLM_Bridge_Integration_Test.ipynb``). The patched-load-model path in +``plugin.py`` and the ``hf_config`` walk below have been confirmed on that +release; newer releases may move attributes — re-validate before bumping. +""" +from __future__ import annotations + +from typing import Any + + +def extract_hf_config(llm: Any) -> Any: + """Return the HF config that vLLM loaded the model from.""" + try: + return llm.llm_engine.model_config.hf_config + except AttributeError as e: + raise RuntimeError( + "Could not locate hf_config under llm.llm_engine.model_config. " + "vLLM may have moved it; update extract_hf_config() to match." + ) from e diff --git a/transformer_lens/model_bridge/sources/vllm/intervention_specs.py b/transformer_lens/model_bridge/sources/vllm/intervention_specs.py new file mode 100644 index 0000000000..ab35dac72d --- /dev/null +++ b/transformer_lens/model_bridge/sources/vllm/intervention_specs.py @@ -0,0 +1,9 @@ +"""Intervention op vocabulary shared by the driver (spec validation) and the +worker extension (spec → buffer translation). + +Adding a new op requires (a) listing it here, (b) handling it in +``worker_extension._apply_intervention``. +""" +from __future__ import annotations + +SUPPORTED_OPS = frozenset({"suppress", "scale", "add", "set"}) diff --git a/transformer_lens/model_bridge/sources/vllm/overlays/__init__.py b/transformer_lens/model_bridge/sources/vllm/overlays/__init__.py new file mode 100644 index 0000000000..37d3650d75 --- /dev/null +++ b/transformer_lens/model_bridge/sources/vllm/overlays/__init__.py @@ -0,0 +1,21 @@ +"""vLLM overlay registry. + +One :class:`DecoderOnlyOverlay` handles every vLLM decoder-only model +(Llama / Qwen / Mistral / Gemma / Phi3 / Qwen3 / Kimi / GLM / …). It's the +default for any architecture; per-architecture overlays would only land if a +model breaks vLLM's conventional decoder-only shape. +""" +from __future__ import annotations + +from .base import AdapterOverlay +from .decoder_only import DecoderOnlyOverlay + +DEFAULT_VLLM_OVERLAY: AdapterOverlay = DecoderOnlyOverlay() + + +def get_overlay(architecture: str) -> AdapterOverlay: + """Return the overlay for an architecture; falls back to the decoder-only default.""" + return DEFAULT_VLLM_OVERLAY + + +__all__ = ["AdapterOverlay", "DEFAULT_VLLM_OVERLAY", "DecoderOnlyOverlay", "get_overlay"] diff --git a/transformer_lens/model_bridge/sources/vllm/overlays/base.py b/transformer_lens/model_bridge/sources/vllm/overlays/base.py new file mode 100644 index 0000000000..e59bad5392 --- /dev/null +++ b/transformer_lens/model_bridge/sources/vllm/overlays/base.py @@ -0,0 +1,25 @@ +"""Base class for vLLM overlays.""" +from __future__ import annotations + +from typing import Any, Dict, List, Tuple + + +class AdapterOverlay: + """A vLLM overlay for a single architecture family. + + :meth:`capture_specs` is called BEFORE ``LLM(...)`` to register the dot-paths + and output widths the plugin should pre-allocate GPU buffers for. The plugin + reads these during ``Worker.load_model`` so capture hooks are present when + ``torch.compile`` traces the model. + + :meth:`nonfiring_hooks` enumerates hooks that vLLM's fused kernels prevent + from firing; surfaced as a single boot-time warning. + """ + + def capture_specs(self, hf_config: Any) -> Dict[str, Tuple[str, int]]: + """Return ``{canonical_hook_name: (dot_path_in_vllm_model, output_width)}``.""" + raise NotImplementedError + + def nonfiring_hooks(self) -> List[str]: + """Canonical hook names that vLLM's fused kernels cannot expose.""" + return [] diff --git a/transformer_lens/model_bridge/sources/vllm/overlays/decoder_only.py b/transformer_lens/model_bridge/sources/vllm/overlays/decoder_only.py new file mode 100644 index 0000000000..fbed07c8c4 --- /dev/null +++ b/transformer_lens/model_bridge/sources/vllm/overlays/decoder_only.py @@ -0,0 +1,65 @@ +"""Generic overlay for any decoder-only model vLLM supports. + +vLLM's decoder-only models all share the same internal structure: +``model.embed_tokens`` / ``model.layers.{i}`` (each with ``self_attn`` and +``mlp`` submodules) / ``model.norm`` / ``lm_head``. This overlay hooks that +shared abstraction, so one file works for Llama, Qwen, Mistral, Gemma, Phi3, +Qwen3, Kimi, GLM, and every other model that inherits the standard shape. + +Non-decoder-only architectures (Mamba SSM, T5 encoder-decoder, BERT, MoE +per-expert) break the convention and would need their own overlays. + +Two hooks capture different points than HF/HookedTransformer: + +- ``blocks.{i}.hook_out``: vLLM's layer returns ``(mlp_delta, residual)`` + separately (fused-residual). The plugin's hook materializes the sum so the + captured value matches HF's "post-MLP residual stream". +- ``ln_final.hook_normalized``: vLLM exposes ``x * rsqrt(var+eps) * weight``; + HF/HT exposes the pre-weight value. Divide the vLLM capture by + ``model.norm.weight`` (or ``1 + weight`` for Gemma) for HT-equivalent output. + +Non-decoder-only architectures (Mamba SSM, T5 encoder-decoder, BERT, MoE +per-expert) break the convention and would need their own overlays. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Tuple + +from .base import AdapterOverlay + + +class DecoderOnlyOverlay(AdapterOverlay): + """Default overlay for vLLM decoder-only models.""" + + def capture_specs(self, hf_config: Any) -> Dict[str, Tuple[str, int]]: + d_model = hf_config.hidden_size + n_layers = hf_config.num_hidden_layers + # unembed.hook_out is intentionally NOT captured here — vLLM's sampler + # computes logits via a direct matmul on the final hidden state and + # never invokes lm_head.__call__, so register_forward_hook on lm_head + # would install but never fire. See nonfiring_hooks() below; the + # driver synthesizes the next-token logits from vLLM's sampler output. + specs: Dict[str, Tuple[str, int]] = { + "embed.hook_out": ("model.embed_tokens", d_model), + "ln_final.hook_normalized": ("model.norm", d_model), + } + for i in range(n_layers): + specs[f"blocks.{i}.hook_out"] = (f"model.layers.{i}", d_model) + specs[f"blocks.{i}.attn.hook_out"] = (f"model.layers.{i}.self_attn", d_model) + specs[f"blocks.{i}.mlp.hook_out"] = (f"model.layers.{i}.mlp", d_model) + return specs + + def nonfiring_hooks(self) -> List[str]: + # vLLM's universal fused-kernel limitations — PagedAttention fuses the + # QK^T → softmax → attn-weight path; QKVParallelLinear fuses RoPE inside + # the projection. Same restriction on every decoder-only model. + return [ + "blocks.{i}.attn.hook_pattern", + "blocks.{i}.attn.hook_attn_scores", + "blocks.{i}.attn.hook_rot_q", + "blocks.{i}.attn.hook_rot_k", + # vLLM's sampler bypasses lm_head.__call__ — capture-via-forward-hook + # never fires; driver synthesizes argmax-matching logits from the + # sampler's returned token. + "unembed.hook_out", + ] diff --git a/transformer_lens/model_bridge/sources/vllm/plugin.py b/transformer_lens/model_bridge/sources/vllm/plugin.py new file mode 100644 index 0000000000..02ddf9a586 --- /dev/null +++ b/transformer_lens/model_bridge/sources/vllm/plugin.py @@ -0,0 +1,170 @@ +"""vLLM plugin entry point. + +Monkey-patches ``Worker.load_model`` to install capture hooks after weights +load and before ``compile_or_warm_up_model`` — the only window where hooks +make it into the compiled FX graph (PyTorch #117758). + +Hook body invariants under ``torch.compile``: in-place writes to a +pre-allocated GPU tensor; no ``.cpu()`` (illegal during CUDA-graph capture); +SymInt-indexed slicing only (Python ``.shape`` access forces specialization). + +Interventions ride the same hook. Each hook applies an affine transform +``output = output * scale_buf + bias_buf`` before capturing. Defaults are +``scale=ones`` / ``bias=zeros`` (identity). The driver swaps buffer contents +between forwards via ``tl_set_interventions`` — the FX graph references the +buffers, so swaps take effect on the next dispatch without recompiling. + +Memory cost: the affine transform allocates a transient output-shape tensor +per hook per forward, even in identity mode. Peak forward memory is roughly +1.5× the prior capture-only design — the caching allocator reuses the slot +but the peak pressure rises. Branching the hook to skip the affine in +identity mode would defeat the swap-via-buffer trick and break the FX graph. +""" +from __future__ import annotations + +import re +from typing import Any, Dict, Tuple + +import torch + +# Matches dot-paths like "model.layers.0", "model.layers.15" — vLLM's decoder +# layers, which return the fused-residual (mlp_delta, residual) 2-tuple. Hooks +# on these need to materialize the sum so the capture matches HF's +# blocks.{i}.hook_out semantics (full residual stream). +_DECODER_LAYER_PATH = re.compile(r"^model\.layers\.\d+$") + +# Transient signal driver → worker during LLM construction. Per-Worker buffers +# live on Worker instances so concurrent boot_vllm calls don't collide. +_config: Dict[str, Any] = {} +_install_patched = False +_orig_load_model = None + + +def configure( + capture_specs: Dict[str, Tuple[str, int]], + max_num_batched_tokens: int, + dtype: torch.dtype, +) -> None: + """Set capture specs, buffer length, and dtype before ``LLM(...)``.""" + _config["capture_specs"] = capture_specs + _config["max_num_batched_tokens"] = max_num_batched_tokens + _config["dtype"] = dtype + + +def register() -> None: + """Idempotent monkey-patch of ``Worker.load_model``. + + vLLM calls ``register()`` once per process at entry-points discovery. Idempotent + so re-imports (notebook restarts, repeated ``boot_vllm`` in the same process) + don't double-wrap. + + No ``unregister()`` symmetry: the patch stays for process lifetime. Benign + because ``patched_load_model`` no-ops when ``_config["capture_specs"]`` is + absent — and ``boot_vllm`` clears ``_config`` after each ``LLM(...)``, so + any subsequent non-TL ``LLM(...)`` in the same process hits the no-op path. + """ + global _install_patched, _orig_load_model + if _install_patched: + return + from vllm.v1.worker.gpu_worker import Worker + + _orig_load_model = Worker.load_model + + def patched_load_model(self): + _orig_load_model(self) + specs = _config.get("capture_specs") + if not specs: + return # not a TL-driven LLM; no hooks to install + max_n = _config["max_num_batched_tokens"] + dtype = _config["dtype"] + device = next(self.model_runner.model.parameters()).device + + # Detach prior handles before reassigning — vLLM doesn't double-load + # today, but unconditional reassignment would orphan hooks if it ever did. + for handle in getattr(self, "_tl_hook_handles", []): + handle.remove() + self._tl_buffers = {} + self._tl_scale_buffers = {} + self._tl_bias_buffers = {} + self._tl_hook_handles = [] + for canonical_name, (dot_path, width) in specs.items(): + target = self.model_runner.model + for seg in dot_path.split("."): + target = target[int(seg)] if seg.isdigit() else getattr(target, seg) + capture_buf = torch.zeros(max_n, width, device=device, dtype=dtype) + # Affine identity at install. Driver swaps via tl_set_interventions + # to enable suppress/scale/add/set ops between forwards. + scale_buf = torch.ones(width, device=device, dtype=dtype) + bias_buf = torch.zeros(width, device=device, dtype=dtype) + self._tl_buffers[canonical_name] = capture_buf + self._tl_scale_buffers[canonical_name] = scale_buf + self._tl_bias_buffers[canonical_name] = bias_buf + # Decoder layers return vLLM's (mlp_delta, residual) tuple; the + # hook materializes their sum so the capture semantically matches + # HF's full residual stream. Other modules use the default path. + materialize = bool(_DECODER_LAYER_PATH.match(dot_path)) + handle = target.register_forward_hook( + _make_capture_hook(capture_buf, scale_buf, bias_buf, materialize=materialize) + ) + self._tl_hook_handles.append(handle) + + Worker.load_model = patched_load_model + _install_patched = True + + +def _make_capture_hook( + capture_buf: torch.Tensor, + scale_buf: torch.Tensor, + bias_buf: torch.Tensor, + *, + materialize: bool = False, +): + """GPU-only, dynamic-shape-safe affine + capture into pre-allocated buffers. + + When ``materialize=True`` (decoder layers), treat the module's output as + vLLM's fused-residual ``(mlp_delta, residual)`` tuple: capture + ``mlp_delta + residual`` (the full residual stream, matching HF's + blocks.{i}.hook_out semantics) and return ``(modified - residual, residual)`` + so the next layer's input_layernorm sees the same fused sum. Mutations + propagate through both the capture and the downstream graph. + """ + + @torch.no_grad() + def hook(_module, _inputs, output): + if materialize and isinstance(output, tuple) and len(output) == 2: + hidden, residual = output + if isinstance(hidden, torch.Tensor) and isinstance(residual, torch.Tensor): + t = hidden + residual + modified = t * scale_buf + bias_buf + n = t.shape[0] + capture_buf.narrow(0, 0, n).copy_(modified) + # ``(modified - residual) + residual`` reconstructs ``modified`` in + # the next layer's fused input_layernorm. Identity case is exact; + # for interventions, fp16 precision loss is bounded by one ulp at + # the residual stream's magnitude — small relative to mutation effect. + return (modified - residual, residual) + + tuple_tail: tuple = () + if isinstance(output, tuple): + t = output[0] + tuple_tail = output[1:] + else: + t = output + if not isinstance(t, torch.Tensor): + return None + # Affine transform; default scale=1 / bias=0 means identity. Driver + # swaps buffer contents to enable interventions. + modified = t * scale_buf + bias_buf + # ``narrow`` produces an explicit-shape view ``(n, width)`` that Dynamo + # traces correctly under dynamic shapes; the ``[:n]`` getitem form gets + # erased under fake-tensor tracing and ``copy_`` then sees the full + # buffer as destination, raising "expand s72 -> max_n". + n = t.shape[0] + capture_buf.narrow(0, 0, n).copy_(modified) + # Preserve the input wrapping — a 1-tuple input must come back as a 1-tuple + # (``tuple_tail`` is falsy in that case, so don't gate on it). + if isinstance(output, tuple): + return (modified,) + tuple_tail + return modified + + return hook diff --git a/transformer_lens/model_bridge/sources/vllm/source.py b/transformer_lens/model_bridge/sources/vllm/source.py new file mode 100644 index 0000000000..e5b7c083fe --- /dev/null +++ b/transformer_lens/model_bridge/sources/vllm/source.py @@ -0,0 +1,207 @@ +"""``boot_vllm`` — construct a vLLM LLM, wrap it in a RemoteBridge via VLLMDriver.""" +from __future__ import annotations + +import logging +import os +import warnings +from typing import Any, Dict, Optional + +import torch + +from transformer_lens.factories.architecture_adapter_factory import ( + ArchitectureAdapterFactory, +) +from transformer_lens.model_bridge.remote_bridge import RemoteBridge +from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_config_from_hf, +) +from transformer_lens.utilities.hf_utils import get_hf_token + +from . import plugin +from .driver import VLLMDriver +from .internals import extract_hf_config +from .overlays import get_overlay + +# Forced LLM(...) kwargs that the capture-hook design depends on. Caller override → ValueError. +_LOCKED_KWARGS = { + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "skip_tokenizer_init": True, + "disable_log_stats": True, +} + + +def boot_vllm( + model_name: str, + tokenizer: Optional[Any] = None, + dtype: Optional[torch.dtype] = None, + gpu_memory_utilization: float = 0.5, + max_model_len: Optional[int] = None, + max_num_batched_tokens: int = 2048, + **vllm_kwargs: Any, +) -> RemoteBridge: + """Boot a model via vLLM and wrap it in a :class:`RemoteBridge` via :class:`VLLMDriver`. + + vLLM drives the forward pass (PagedAttention + ``torch.compile`` + CUDA graphs). + Capture buffers are populated by hooks the plugin installs pre-compile inside + the worker; they come back via ``collective_rpc`` and replay through the + bridge's HookPoint tree. + + **Scope vs vllm-lens:** vllm-lens is observation-only. This source extends to + observation + spec-vocabulary *mutation* — each capture hook also applies an + affine transform ``output = output * scale + bias`` (default identity), so + interventions (``suppress`` / ``scale`` / ``add`` / ``set``) propagate to + downstream layers. The hook's return value replaces the module output per + PyTorch ``register_forward_hook`` semantics. **The mutation path under + torch.compile + CUDA graphs is verified end-to-end by** + ``demos/vLLM_Bridge_Integration_Test.ipynb``; unit tests cover the dispatch + protocol only. + + Some captures use vLLM-native conventions that differ from HF/HT; see + :mod:`transformer_lens.model_bridge.sources.vllm.overlays.decoder_only` for + which hooks diverge and the conversion to apply for HT-equivalent values. + + GPU memory cost: each capture buffer is ``max_num_batched_tokens × width`` at + the model's dtype. For Llama-3.2-1B at fp16 with ``max_num_batched_tokens=2048``, + the unembed buffer alone is ~525 MB (2048 × 128256 × 2 bytes); residual-stream + buffers add ~8 MB per hook. The affine intervention hook also allocates a + transient output-shape tensor per forward (even in identity mode), so peak + forward memory is ~1.5× the capture buffers' resident size. + + KV-cache footprint: vLLM reserves KV cache sized for ``max_model_len`` × layers + × heads × head_dim. If ``max_model_len`` is left as ``None``, vLLM uses the + model's native context (e.g. 131072 for Llama-3.2-1B) — easily 4+ GiB even + on a 1B model. Pass an explicit ``max_model_len`` (e.g. ``2048`` for typical + mech-interp prompts) to keep the budget on smaller GPUs. + """ + _reject_locked_overrides(vllm_kwargs) + + from transformers import AutoConfig, AutoTokenizer + + # Resolve architecture WITHOUT loading weights so we can tell the plugin + # which dot-paths to hook before LLM(...) constructs the worker. + hf_token = get_hf_token() + hf_config_preview = AutoConfig.from_pretrained(model_name, token=hf_token) + architecture = hf_config_preview.architectures[0] + overlay = get_overlay(architecture) + + resolved_dtype = dtype or _dtype_from_hf_config(hf_config_preview) + plugin.configure( + capture_specs=overlay.capture_specs(hf_config_preview), + max_num_batched_tokens=max_num_batched_tokens, + dtype=resolved_dtype, + ) + plugin.register() + + # The plugin's _config singleton must be visible to the worker, which only + # holds with single-process execution. Multi-GPU is unsupported. Override + # any user setting — silent capture failure otherwise. + existing_mp = os.environ.get("VLLM_ENABLE_V1_MULTIPROCESSING") + if existing_mp not in (None, "0"): + warnings.warn( + f"VLLM_ENABLE_V1_MULTIPROCESSING={existing_mp!r} overridden to '0' — " + "boot_vllm needs single-process execution for capture hooks to install. " + "Multi-GPU vLLM is unsupported.", + UserWarning, + stacklevel=2, + ) + os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0" + + from vllm import LLM + + llm = LLM( + model=model_name, + gpu_memory_utilization=gpu_memory_utilization, + max_model_len=max_model_len, + # Critical: vLLM defaults max_num_batched_tokens to 8192 for chunked prefill. + # We size our capture buffer to this same value, so vLLM must compile for + # the matching dynamic-shape range — otherwise Dynamo's symbolic-shape + # hint exceeds the buffer dim and narrow() fails at compile time. + max_num_batched_tokens=max_num_batched_tokens, + # Register TLWorkerExtension so its tl_* methods are reachable via + # collective_rpc. Passed as a dotted path; vLLM imports the class at + # worker construction and mixes it into the Worker via multiple + # inheritance (asserts no attribute name collisions — hence the tl_ prefix). + worker_extension_cls="transformer_lens.model_bridge.sources.vllm.worker_extension.TLWorkerExtension", + # Allow full-vocab logprobs so the driver can synthesize real logits for + # the generated position (vLLM caps logprobs to this value; default 20 is + # too small for mech-interp). One ~512 KB buffer per call; negligible. + max_logprobs=hf_config_preview.vocab_size, + dtype=str(resolved_dtype).replace("torch.", "") if dtype is not None else "auto", + **_LOCKED_KWARGS, + **vllm_kwargs, + ) + + # Capture-path validity is enforced inside patched_load_model during + # LLM(...) above — any missing dot-path raises AttributeError there. By + # the time we reach this line every spec has already been walked successfully. + hf_config = extract_hf_config(llm) + # _config has been consumed into per-Worker state; clear so a non-TL + # vllm.LLM(...) in the same process doesn't inherit our specs. + plugin._config.clear() + if tokenizer is None: + tokenizer = AutoTokenizer.from_pretrained(model_name, token=hf_token) + + # Build the adapter (RemoteBridge skips adapter.prepare_model — there's no + # local model tree to walk). Use the shared HF→TL config builder so + # bridge_config is a real TransformerBridgeConfig with all dataclass + # defaults (d_vocab_out=-1, etc.) — not a deep-copied HF config with + # extra fields, which is missing the TL-only attributes BridgeCore reads. + bridge_config = build_bridge_config_from_hf(hf_config, architecture, model_name, resolved_dtype) + adapter = ArchitectureAdapterFactory.select_architecture_adapter(bridge_config) + + driver = VLLMDriver( + llm=llm, + adapter=adapter, + tokenizer=tokenizer, + overlay=overlay, + hf_config=hf_config, + max_num_batched_tokens=max_num_batched_tokens, + ) + bridge = RemoteBridge(adapter=adapter, tokenizer=tokenizer, driver=driver) + _log_hook_summary(model_name, architecture, driver) + return bridge + + +def _reject_locked_overrides(vllm_kwargs: Dict[str, Any]) -> None: + for key, locked in _LOCKED_KWARGS.items(): + if key in vllm_kwargs and vllm_kwargs[key] != locked: + raise ValueError( + f"boot_vllm forces {key}={locked}; caller passed {key}={vllm_kwargs[key]}. " + "Multi-device / continuous batching / vLLM-owned tokenizer are unsupported." + ) + + +def _dtype_from_hf_config(hf_config: Any) -> torch.dtype: + raw = getattr(hf_config, "torch_dtype", None) + if isinstance(raw, torch.dtype): + return raw + if isinstance(raw, str): + return getattr(torch, raw, torch.float16) + return torch.float16 + + +def _log_hook_summary(model_name: str, architecture: str, driver: VLLMDriver) -> None: + """Log the fireable and non-fireable hook sets so users don't have to grep.""" + log = logging.getLogger("transformer_lens.vllm") + fireable = sorted(driver.supported_hook_points) + log.info( + "vLLM source on %s (%s) captures %d hook(s): %s", + model_name, + architecture, + len(fireable), + ", ".join(fireable), + ) + nonfiring = sorted(driver.non_fireable_hook_points) + if nonfiring: + log.info( + "vLLM source on %s (%s) cannot fire %d hook(s) (vLLM fuses these): %s. " + "Use boot_transformers() if you need them.", + model_name, + architecture, + len(nonfiring), + ", ".join(nonfiring), + ) + + +__all__ = ["boot_vllm"] diff --git a/transformer_lens/model_bridge/sources/vllm/worker_extension.py b/transformer_lens/model_bridge/sources/vllm/worker_extension.py new file mode 100644 index 0000000000..70a98ad39a --- /dev/null +++ b/transformer_lens/model_bridge/sources/vllm/worker_extension.py @@ -0,0 +1,101 @@ +"""Worker extension exposed to collective_rpc for capture-buffer reads and +intervention-buffer writes. + +Hook *installation* lives in :mod:`plugin` (must happen pre-compile). This class +only exposes the post-compile read/write surface. Buffers are per-Worker +(``self._tl_buffers`` / ``self._tl_scale_buffers`` / ``self._tl_bias_buffers``) +so concurrent ``boot_vllm`` calls don't collide. All methods prefixed ``tl_`` +to avoid colliding with vLLM ``Worker`` attributes. +""" +from __future__ import annotations + +from typing import Any, Dict, List + +import torch + +from .intervention_specs import SUPPORTED_OPS + + +class TLWorkerExtension: + """Mixed into vLLM's ``Worker`` via ``worker_extension_cls``.""" + + _tl_hook_handles: list + _tl_buffers: Dict[str, torch.Tensor] + _tl_scale_buffers: Dict[str, torch.Tensor] + _tl_bias_buffers: Dict[str, torch.Tensor] + + def tl_read_captures(self, prompt_lens: List[int]) -> Dict[str, torch.Tensor]: + """Slice each capture buffer back to ``sum(prompt_lens)`` rows; CPU copies. + + Caller (VLLMDriver.forward) gates ``sum(prompt_lens) <= max_num_batched_tokens`` + before the RPC, so ``total`` is always within buffer bounds. + """ + total = sum(prompt_lens) + buffers: Dict[str, torch.Tensor] = getattr(self, "_tl_buffers", {}) + return {name: buf[:total].detach().cpu().clone() for name, buf in buffers.items()} + + def tl_set_interventions(self, specs: Dict[str, Dict[str, Any]]) -> None: + """Reset all affine buffers to identity, then apply each spec. + + Driver pushes the full spec set every forward (or ``{}`` to reset). + Spec format: ``{hook_name: {"op": , ...op-specific params>}}``. + Supported ops: suppress, scale (``factor``: float), add and set + (``value``: scalar broadcast across width, or 1-D shape ``(width,)``). + """ + scale_bufs: Dict[str, torch.Tensor] = getattr(self, "_tl_scale_buffers", {}) + bias_bufs: Dict[str, torch.Tensor] = getattr(self, "_tl_bias_buffers", {}) + # Reset every hook to identity first — clears any stale state from + # the previous forward. + for name, sb in scale_bufs.items(): + sb.fill_(1.0) + bias_bufs[name].zero_() + for hook_name, spec in specs.items(): + if hook_name not in scale_bufs: + raise KeyError(f"Unknown hook for intervention: {hook_name!r}") + _apply_intervention(scale_bufs[hook_name], bias_bufs[hook_name], spec) + + def tl_remove_hooks(self) -> None: + """Detach all capture hooks and drop buffer references. Idempotent.""" + for handle in getattr(self, "_tl_hook_handles", []): + handle.remove() + self._tl_hook_handles = [] + self._tl_buffers = {} + self._tl_scale_buffers = {} + self._tl_bias_buffers = {} + + +def _apply_intervention( + scale_buf: torch.Tensor, bias_buf: torch.Tensor, spec: Dict[str, Any] +) -> None: + """Translate a spec dict to in-place buffer writes.""" + op = spec.get("op") + if op not in SUPPORTED_OPS: + raise ValueError(f"Unsupported intervention op: {op!r}. Supported: {sorted(SUPPORTED_OPS)}") + if op == "suppress": + scale_buf.zero_() + bias_buf.zero_() + return + if op == "scale": + scale_buf.fill_(float(spec["factor"])) + bias_buf.zero_() + return + value = torch.as_tensor(spec["value"], device=bias_buf.device, dtype=bias_buf.dtype) + # 0-d broadcasts across width (e.g. "shift all dims by 0.5"); width-shaped + # writes element-wise (e.g. SAE steering vector). Anything else is an error. + if value.ndim == 0: + bias_buf.fill_(value.item()) + elif value.shape == bias_buf.shape: + bias_buf.copy_(value) + else: + raise ValueError( + f"Intervention 'value' must be a scalar or shape {tuple(bias_buf.shape)}; " + f"got shape {tuple(value.shape)}" + ) + if op == "add": + scale_buf.fill_(1.0) + elif op == "set": + scale_buf.zero_() + else: + raise RuntimeError( + f"op {op!r} is in SUPPORTED_OPS but _apply_intervention has no branch for it." + ) diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py new file mode 100644 index 0000000000..ae3830a911 --- /dev/null +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -0,0 +1,2991 @@ +"""Bridge module for connecting different model architectures. + +This module provides the bridge components that wrap remote model components and provide +a consistent interface for accessing their weights and performing operations. +""" +import logging +import re +import warnings +from collections.abc import Generator +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterator, + List, + Literal, + Optional, + Tuple, + Union, + cast, +) + +import einops +import numpy as np +import torch +import tqdm +from torch import nn + +from transformer_lens import utilities as utils +from transformer_lens.FactoredMatrix import FactoredMatrix +from transformer_lens.hook_points import HookIntrospectionMixin, HookPoint +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.bridge_core import BridgeCore +from transformer_lens.model_bridge.component_setup import set_original_components +from transformer_lens.model_bridge.composition_scores import CompositionScores +from transformer_lens.model_bridge.driver_protocol import ( + TensorLike, + to_torch, + validate_driver, +) +from transformer_lens.model_bridge.exceptions import StopAtLayerException +from transformer_lens.model_bridge.generalized_components.base import ( + GeneralizedComponent, +) +from transformer_lens.model_bridge.generalized_components.block import ( + _BLOCK_INTERNAL_MODULES, + _NORM_PREFIXES, + _VARIANT_SUBMODULE_SET, + VARIANT_SUBMODULE_NAMES, +) +from transformer_lens.model_bridge.get_params_util import get_bridge_params +from transformer_lens.utilities.devices import move_to_and_update_config + +if TYPE_CHECKING: + pass + +_BLOCK_PATTERN = re.compile("blocks\\.(\\d+)") + + +def _resolve_attr_path(obj: nn.Module, attr_path: str) -> torch.Tensor: + """Walk a dot-separated attribute path and return the final tensor.""" + result = obj + for attr in attr_path.split("."): + result = getattr(result, attr) + return cast(torch.Tensor, result) + + +# build_alias_to_canonical_map lives in bridge_core.py; re-import for the module's +# internal use (run_with_cache, hooks() context manager in this file). +from transformer_lens.model_bridge.bridge_core import ( # noqa: E402 + build_alias_to_canonical_map, +) + + +class TransformerBridge(BridgeCore, HookIntrospectionMixin, nn.Module): + """Torch-backed bridge: HF, vLLM-via-torch, anything that wraps an ``nn.Module``. + + Provides a standardized interface to access components of a transformer + model, regardless of the underlying architecture. It uses an architecture adapter + to map between the TransformerLens and HuggingFace model structures. + + Tokenization notes + ------------------ + + :meth:`to_tokens`, :meth:`to_str_tokens`, :meth:`get_token_position`, + :meth:`forward` (string input), and :meth:`generate` accept ``prepend_bos`` + to control BOS prepending. Resolution: explicit arg → + ``cfg.default_prepend_bos`` (defaults ``True``, even for non-BOS-trained + models — attention heads tend to use position 0 as a resting state). + **Pass ``prepend_bos=False`` when tokenizing a fragment of a larger + prompt** — off-by-one position errors usually trace back here. + + Reconciliation with ``cfg.tokenizer_prepends_bos`` (tokenizers that add + BOS automatically) is handled internally — pass the value you want; + the bridge adds or strips manually as needed. When + ``cfg.tokenizer_appends_eos=True`` (OLMo, Apertus, etc.), + :meth:`to_tokens` also strips trailing EOS tokens so the model receives + a continuation rather than a terminated sequence; this path is + bridge-specific. + + BPE/SentencePiece tokenizers treat ``"hello"``, ``" hello"``, and + ``"Hello"`` as distinct tokens. Concatenated prompts may not tokenize + as the sum of parts — inspect with :meth:`to_str_tokens` when in doubt. + """ + + # hook_aliases inherited from BridgeCore + + def __init__( + self, + model: nn.Module, + adapter: ArchitectureAdapter, + tokenizer: Any, + *, + driver: Any = None, + ): + """Initialize the bridge. + + Args: + model: The model to bridge (must be a PyTorch nn.Module or PreTrainedModel) + adapter: The architecture adapter to use + tokenizer: The tokenizer to use (required) + driver: Optional pre-built :class:`Driver`. Sources that construct + exotic drivers (vLLM, Inspect) pass them here. When ``None``, + a :class:`TransformersDriver` is built from the supplied + ``model``/``adapter``/``tokenizer`` — kept for backward + compatibility with direct ``TransformerBridge(...)`` callers. + """ + nn.Module.__init__(self) + self.__dict__["original_model"] = model + # Production sources construct their Driver and pass it via ``driver=``. + # The fallback covers tests / direct callers with a hand-rolled triple. + if driver is None: + from transformer_lens.model_bridge.sources.transformers_driver import ( + TransformersDriver, + ) + + driver = TransformersDriver(model, adapter, tokenizer) + BridgeCore.__init__(self, adapter, tokenizer, driver) + # real_components maps TL keys to (remote_path, actual_instance) tuples; + # for list components, actual_instance is a list of instances. + self.real_components: Dict[str, tuple] = {} + if not hasattr(self.cfg, "device") or self.cfg.device is None: + try: + self.cfg.device = str(next(self.original_model.parameters()).device) + except StopIteration: + self.cfg.device = "cpu" + set_original_components(self, self.adapter, self.__dict__["original_model"]) + self._initialize_hook_registry() + self._register_aliases() + self._register_all_aliases_recursive() + # Re-scan after alias registration so alias names (hook_resid_pre, …) + # join the registry alongside their canonical targets. Shared HookPoint + # instances, so no double-firing. + self._scan_existing_hooks(self, "") + self._setup_hook_compatibility() + # Backfill supported_hook_points = registry − non_fireable. Whitelist- + # semantic drivers (Inspect) declared their own non-empty set and skip. + if not self._driver.supported_hook_points: + self._driver.supported_hook_points = ( + frozenset(self._hook_registry) - self._driver.non_fireable_hook_points + ) + # Fail fast on a misshapen driver here, not at first capture. + validate_driver(self._driver, after_bridge_construction=True) + self.processor = None + + @classmethod + def boot_transformers( + cls, + model_name: str, + hf_config_overrides: Optional[dict] = None, + device: Optional[Union[str, torch.device]] = None, + dtype: torch.dtype = torch.float32, + tokenizer: Optional[Any] = None, + load_weights: bool = True, + trust_remote_code: bool = False, + model_class: Optional[type] = None, + hf_model: Optional[Any] = None, + device_map: Optional[Union[str, Dict[str, Union[str, int]]]] = None, + n_devices: Optional[int] = None, + max_memory: Optional[Dict[Union[str, int], str]] = None, + n_ctx: Optional[int] = None, + revision: Optional[str] = None, + checkpoint_index: Optional[int] = None, + checkpoint_value: Optional[int] = None, + ) -> "TransformerBridge": + """Boot a model from HuggingFace (alias for sources.transformers.boot). + + Returns raw HF weights by default — logits/activations match HF, *not* + legacy ``HookedTransformer`` (which folds LayerNorm + centers weights). + Call ``enable_compatibility_mode()`` on the result for HookedTransformer- + equivalent numerics. Generation, argmax, and CE loss are unaffected. + + Attention implementation is forced to ``"eager"`` so hooks can capture scores + and patterns. For an apples-to-apples HF comparison, load the HF model with + ``attn_implementation="eager"`` too; comparing against the default ``"sdpa"`` + shows ~1e-3 fp32 drift from kernel-level op reordering, not a bridge bug. + + Args: + model_name: The name of the model to load. + hf_config_overrides: Optional overrides applied to the HuggingFace config before model load. + device: The device to use. If None, will be determined automatically. Mutually exclusive + with ``device_map``. + dtype: The dtype to use for the model. + tokenizer: Optional pre-initialized tokenizer to use; if not provided one will be created. + load_weights: If False, load model without weights (on meta device) for config inspection only. + trust_remote_code: Whether to trust remote code for custom model architectures. + model_class: Optional HuggingFace model class to use instead of the default + auto-detected class (e.g., BertForNextSentencePrediction). + hf_model: Optional pre-loaded HuggingFace model to use instead of loading one. Useful + for models loaded with custom configurations (e.g., quantization via + BitsAndBytesConfig). When provided, load_weights is ignored. If the pre-loaded + model was built with a ``device_map``, ``cfg.device`` and ``cfg.n_devices`` are + derived from its ``hf_device_map`` automatically. + device_map: HuggingFace-style device map for multi-GPU inference. Pass ``"auto"``, + ``"balanced"``, ``"sequential"``, or an explicit ``{submodule_path: device}`` dict. + Mutually exclusive with ``device``. + n_devices: Convenience shortcut: split the model across this many CUDA devices. + Translated to a ``max_memory`` dict over devices 0..n_devices-1 and passed as + ``device_map`` to HF. Requires CUDA with at least this many visible devices. + max_memory: Optional per-device memory budget, passed through to HF's dispatcher. + Only used when ``device_map`` or ``n_devices`` is in effect. + n_ctx: Optional context length override. Writes to the appropriate HF config field + for this model automatically (callers don't need to know the field name). + Warns if larger than the model's default context length. + revision: Optional HF revision (branch, tag, or commit). Forwarded to the underlying + ``AutoConfig.from_pretrained`` and ``AutoModelForCausalLM.from_pretrained`` calls. + Mutually exclusive with ``checkpoint_index`` / ``checkpoint_value``. + checkpoint_index: Index into the available training checkpoints for the model family + (currently ``EleutherAI/pythia*`` and ``stanford-crfm/*``). Resolved to a revision + string via known per-family naming conventions. + checkpoint_value: Training step or token count of the desired checkpoint. Alternative + to ``checkpoint_index``; must match an entry in the family's checkpoint label list. + + Returns: + The bridge to the loaded model. + """ + from transformer_lens.model_bridge.sources.transformers import boot + + return boot( + model_name=model_name, + hf_config_overrides=hf_config_overrides, + device=device, + dtype=dtype, + tokenizer=tokenizer, + load_weights=load_weights, + trust_remote_code=trust_remote_code, + model_class=model_class, + hf_model=hf_model, + device_map=device_map, + n_devices=n_devices, + max_memory=max_memory, + n_ctx=n_ctx, + revision=revision, + checkpoint_index=checkpoint_index, + checkpoint_value=checkpoint_value, + ) + + @property + def original_model(self) -> nn.Module: + """The wrapped ``nn.Module``. Raises :class:`AttributeError` for + non-torch drivers (vLLM, Inspect) that don't expose a local module.""" + underlying = getattr(self._driver, "underlying_model", None) + if underlying is None: + raise AttributeError( + f"{type(self._driver).__name__} does not expose an nn.Module — " + "non-torch drivers (vLLM, Inspect) operate without a local module." + ) + return underlying + + @original_model.setter + def original_model(self, value: nn.Module) -> None: + """Used by weight-processing paths that move the model across devices.""" + self.__dict__["original_model"] = value + # Sync via the driver's public API; non-torch drivers don't implement it. + setter = getattr(self._driver, "set_underlying_model", None) + if callable(setter): + setter(value) + + def _set_processed_weight_attributes(self) -> None: + """Create 3D processed weight attributes for attention components. + + For each attention component, if it has 2D weights (q.weight, k.weight, v.weight), + reshape them to 3D format [n_heads, d_model, d_head] and set as: + - _processed_W_Q + - _processed_W_K + - _processed_W_V + - _processed_b_Q + - _processed_b_K + - _processed_b_V + + This allows property aliases (W_Q, W_K, W_V) to return 3D format for + HookedTransformer compatibility while keeping 2D format for calculations. + """ + + n_heads = self.cfg.n_heads + d_head = self.cfg.d_head + d_model = self.cfg.d_model + if not hasattr(self, "blocks"): + return + for block in self.blocks: + if "attn" not in block._modules: + continue + attn = block.attn + if not (hasattr(attn, "q") and hasattr(attn.q, "weight")): + continue + try: + w_q_2d = attn.q.weight.data + w_k_2d = attn.k.weight.data + w_v_2d = attn.v.weight.data + attn._processed_W_Q = einops.rearrange( + w_q_2d, "m (i h) -> i m h", i=n_heads, h=d_head + ) + attn._processed_W_K = einops.rearrange( + w_k_2d, "m (i h) -> i m h", i=n_heads, h=d_head + ) + attn._processed_W_V = einops.rearrange( + w_v_2d, "m (i h) -> i m h", i=n_heads, h=d_head + ) + if hasattr(attn.q, "bias") and attn.q.bias is not None: + b_q_2d = attn.q.bias.data + b_k_2d = attn.k.bias.data + b_v_2d = attn.v.bias.data + attn._processed_b_Q = einops.rearrange( + b_q_2d, "(i h) -> i h", i=n_heads, h=d_head + ) + attn._processed_b_K = einops.rearrange( + b_k_2d, "(i h) -> i h", i=n_heads, h=d_head + ) + attn._processed_b_V = einops.rearrange( + b_v_2d, "(i h) -> i h", i=n_heads, h=d_head + ) + if hasattr(attn, "o") and hasattr(attn.o, "weight"): + w_o_2d = attn.o.weight.data + w_o_transposed = w_o_2d.T + attn._processed_W_O = einops.rearrange( + w_o_transposed, "m (i h) -> i h m", i=n_heads, h=d_head + ) + if hasattr(attn.o, "bias") and attn.o.bias is not None: + attn._processed_b_O = attn.o.bias.data + except Exception: + pass + + def _register_all_aliases_recursive(self) -> None: + """Recursively register aliases on all bridge components. + + This walks through all components and calls _register_aliases() on each one. + Used after weight processing to ensure aliases point to processed weights. + """ + if hasattr(self, "_register_aliases"): + self._register_aliases() + for module in self.modules(): + if module is not self and hasattr(module, "_register_aliases"): + getattr(module, "_register_aliases")() + + def __setattr__(self, name: str, value: Any) -> None: + """Override setattr to track HookPoint objects dynamically.""" + super().__setattr__(name, value) + if isinstance(value, HookPoint): + value.name = name + self._hook_registry[name] = value + elif hasattr(value, "get_hooks") and callable(getattr(value, "get_hooks")): + component_hooks = value.get_hooks() + for hook_name, hook in component_hooks.items(): + full_name = f"{name}.{hook_name}" + hook.name = full_name + self._hook_registry[full_name] = hook + + def _scan_existing_hooks(self, module: nn.Module, prefix: str = "") -> None: + """Scan existing modules for hooks and add them to registry.""" + visited = set() + # Protect canonical HookPoint names from alias overwrites + named_hook_ids: set = set() + + def scan_module(mod: nn.Module, path: str = "") -> None: + obj_id = id(mod) + if obj_id in visited: + return + visited.add(obj_id) + if hasattr(mod, "get_hooks") and callable(getattr(mod, "get_hooks")): + component_hooks = mod.get_hooks() # type: ignore[operator] + if isinstance(component_hooks, dict): + hooks_dict = cast(Dict[str, HookPoint], component_hooks) + for hook_name, hook in hooks_dict.items(): + full_name = f"{path}.{hook_name}" if path else hook_name + hook_id = id(hook) + if hook_id not in named_hook_ids: + hook.name = full_name + named_hook_ids.add(hook_id) + self._hook_registry[full_name] = hook + for attr_name in dir(mod): + if attr_name.startswith("_"): + continue + if attr_name == "original_component" or attr_name == "original_model": + continue + if attr_name in [ + "OV", + "QK", + "W_V", + "W_O", + "W_Q", + "W_K", + "W_in", + "W_gate", + "W_out", + "b_V", + "b_O", + "b_Q", + "b_K", + "b_in", + "b_out", + ]: + continue + try: + attr = getattr(mod, attr_name) + except (AttributeError, NameError, RuntimeError, TypeError): + continue + name = f"{path}.{attr_name}" if path else attr_name + if isinstance(attr, HookPoint): + hook_id = id(attr) + if hook_id not in named_hook_ids: + attr.name = name + named_hook_ids.add(hook_id) + self._hook_registry[name] = attr + for child_name, child_module in mod.named_children(): + if ( + child_name == "original_component" + or child_name == "_original_component" + or child_name == "original_model" + ): + continue + child_path = f"{path}.{child_name}" if path else child_name + scan_module(child_module, child_path) + + scan_module(module, prefix) + + @property + def n_params_total(self) -> int: + """Total number of parameters in the model, including embeddings, biases, + and layer norm weights. + + Mirrors :attr:`HookedTransformer.n_params_total`. Use this when you want + the actual parameter count for memory budgeting, comparison with + HuggingFace's ``model.num_parameters()``, or alignment with reported + model sizes in papers (e.g. the Pythia suite). + + Returns: + int: ``sum(p.numel() for p in self.parameters())`` + """ + return sum(p.numel() for p in self.parameters()) + + def __getattr__(self, name: str) -> Any: + """Provide a clear error message for missing attributes.""" + # Re-invoke original_model's property so its descriptive AttributeError + # for non-torch drivers isn't shadowed by the __dict__ fallback below. + if name == "original_model": + prop = type(self).__dict__.get("original_model") + if isinstance(prop, property) and prop.fget is not None: + return prop.fget(self) + if name in self.__dict__: # type: ignore[arg-type] + return self.__dict__[name] + # Use __dict__ directly to avoid recursion + if "_modules" in self.__dict__ and name in self.__dict__["_modules"]: # type: ignore[arg-type] + return self.__dict__["_modules"][name] + if "original_model" in self.__dict__ and self.__dict__["original_model"] is not None: + try: + name_split = name.split(".") + if len(name_split) > 1: + current = getattr(self.__dict__["original_model"], name_split[0]) + for part in name_split[1:]: # type: ignore[operator] + current = getattr(current, part) + return current + else: + return getattr(self.__dict__["original_model"], name) + except AttributeError: + pass # type: ignore[operator,assignment] + raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") + + def __str__(self) -> str: + """Get a string representation of the bridge. + # type: ignore[operator] + Returns: + A string describing the bridge's components # type: ignore[operator] + """ + lines = ["TransformerBridge:"] + mapping = self.adapter.get_component_mapping() + lines.extend(self._format_component_mapping(mapping, indent=1)) + return "\n".join(lines) + + def enable_compatibility_mode( + self, + disable_warnings: bool = False, + no_processing: bool = False, + fold_ln: bool = True, + center_writing_weights: bool = True, + center_unembed: bool = True, + fold_value_biases: bool = True, + refactor_factored_attn_matrices: bool = False, + ) -> None: + """Apply HookedTransformer-equivalent weight processing and legacy hook compatibility. + + Defaults match HookedTransformer's load-time processing (fold_ln + weight + centering) — required for analyses that reason in HookedTransformer's + post-processed coordinate system: logit lens, direct logit attribution, + residual-stream norms. Also enables legacy hook/component name aliases. + + Args: + disable_warnings: Whether to disable warnings about legacy components/hooks + no_processing: Whether to disable ALL pre-processing steps of the model. + If True, overrides fold_ln, center_writing_weights, and center_unembed to False. + fold_ln: Whether to fold layer norm weights into the subsequent linear layers. + Default: True. Ignored if no_processing=True. + center_writing_weights: Whether to center the writing weights (W_out in attention and MLPs). + Default: True. Ignored if no_processing=True. + center_unembed: Whether to center the unembedding matrix. + Default: True. Ignored if no_processing=True. + fold_value_biases: Whether to fold value biases into output bias. + Default: True. Ignored if no_processing=True. + refactor_factored_attn_matrices: Whether to refactor factored attention matrices. + Default: False. Ignored if no_processing=True. + """ + from transformer_lens.utilities.bridge_components import ( + apply_fn_to_all_components, + ) + + self.compatibility_mode = True + + def set_compatibility_mode(component: Any) -> None: + """Set compatibility mode on a component.""" + component.compatibility_mode = True + component.disable_warnings = disable_warnings + + apply_fn_to_all_components(self, set_compatibility_mode) + self.clear_hook_registry() + try: + if not no_processing: + self.process_weights( + fold_ln=fold_ln, + center_writing_weights=center_writing_weights, + center_unembed=center_unembed, + fold_value_biases=fold_value_biases, + refactor_factored_attn_matrices=refactor_factored_attn_matrices, + ) + finally: + # Re-initialize hooks even on failure so bridge stays usable + self._initialize_hook_registry() + self._setup_hook_compatibility() + self._register_all_aliases_recursive() + + def _setup_hook_compatibility(self) -> None: + """Setup hook compatibility transformations to match HookedTransformer behavior. + + This method sets up hook conversions and wrappers that ensure Bridge hooks + have the same shapes and behavior as HookedTransformer hooks. This includes: + 1. hook_z reshaping from [batch, seq, d_model] to [batch, seq, n_heads, d_head] + 2. Wrapping HF attention forward to inject position embeddings/attention masks + 3. Architecture-specific setup (e.g., rotary embedding references) + + This is called during __init__ and should always be run, regardless of whether + compatibility mode or weight processing is enabled. + + Note: This method is idempotent - can be called multiple times safely. + """ + if hasattr(self.adapter, "setup_hook_compatibility"): + self.adapter.setup_hook_compatibility(self) + elif hasattr(self.adapter, "setup_no_processing_hooks"): + self.adapter.setup_no_processing_hooks(self) + blocks_to_process = [] + if hasattr(self, "blocks"): + blocks_to_process.extend(self.blocks) + if hasattr(self, "encoder_blocks"): + blocks_to_process.extend(self.encoder_blocks) + if hasattr(self, "decoder_blocks"): + blocks_to_process.extend(self.decoder_blocks) + for block in blocks_to_process: + for attn_name in ["attn", "self_attn", "cross_attn"]: + if hasattr(block, attn_name): + attn = getattr(block, attn_name) + if hasattr(attn, "setup_hook_compatibility"): + attn.setup_hook_compatibility() + elif hasattr(attn, "setup_no_processing_hooks"): + attn.setup_no_processing_hooks() + + def process_weights( + self, + verbose: bool = False, + fold_ln: bool = True, + center_writing_weights: bool = True, + center_unembed: bool = True, + fold_value_biases: bool = True, + refactor_factored_attn_matrices: bool = False, + ) -> None: + """Process weights directly using ProcessWeights and architecture adapter. + + This method applies weight processing transformations to improve model interpretability + without requiring a reference HookedTransformer model. Works with all architectures + supported by TransformerBridge, including GPT-OSS and other new models. + + Args: + verbose: If True, print detailed progress messages. Default: False + fold_ln: Fold LayerNorm weights/biases into subsequent layers. Default: True + center_writing_weights: Center weights that write to residual stream. Default: True + center_unembed: Center unembedding weights (translation invariant). Default: True + fold_value_biases: Fold value biases into output bias. Default: True + refactor_factored_attn_matrices: Experimental QK/OV factorization. Default: False + """ + from transformer_lens.weight_processing import ProcessWeights + + if verbose: + print(f"Processing weights for {self.cfg.model_name}...") + + # Soft capping (tanh) is not translation-invariant; centering would change output. + if center_unembed and getattr(self.cfg, "output_logits_soft_cap", -1.0) > 0.0: + import logging + + logging.warning( + "center_unembed=True is incompatible with logit softcapping " + "(output_logits_soft_cap=%.1f). Disabling center_unembed.", + self.cfg.output_logits_soft_cap, + ) + center_unembed = False + + if verbose: + print(" Extracting state dict from existing model...") + state_dict = self.state_dict() + adapter = self.adapter + + # Untie embed/unembed weights (GPT-2) so centering affects only unembed + embed_key = "embed.weight" + unembed_key = "unembed.weight" + + if embed_key in state_dict and unembed_key in state_dict: + # Check if they point to the same tensor (weight tying) + if state_dict[embed_key].data_ptr() == state_dict[unembed_key].data_ptr(): + if verbose: + print(" Breaking weight tying between embed and unembed in state dict...") + # Clone the unembed weight to break the tie + state_dict[unembed_key] = state_dict[unembed_key].clone() + + if adapter and hasattr(adapter, "preprocess_weights"): + adapter._fold_ln_requested = fold_ln # type: ignore[union-attr] + state_dict = adapter.preprocess_weights(state_dict) + + # Use unified ProcessWeights.process_weights() like HookedTransformer does. + # Float32 upcasting for precision is handled centrally in process_weights(). + if verbose: + print(" Processing weights (fold_ln, center_writing_weights, etc.)...") + state_dict = ProcessWeights.process_weights( + state_dict, + self.cfg, + fold_ln=fold_ln, + center_writing_weights=center_writing_weights, + center_unembed=center_unembed, + fold_value_biases=fold_value_biases, + refactor_factored_attn_matrices=refactor_factored_attn_matrices, + adapter=adapter, + ) + + # Normalize HF-prefix keys to TL format for weight routing + import re + + hf_to_tl_prefix = {} + for tl_name, (remote_path, _component) in self.real_components.items(): + if remote_path and remote_path != tl_name: + hf_to_tl_prefix[remote_path] = tl_name + + normalized_state_dict = {} + for key, value in state_dict.items(): + new_key = key + for hf_prefix, tl_prefix in hf_to_tl_prefix.items(): + if key.startswith(hf_prefix + "."): + suffix = key[len(hf_prefix) + 1 :] + new_key = f"{tl_prefix}.{suffix}" + break + normalized_state_dict[new_key] = value + state_dict = normalized_state_dict + + if verbose: + print(" Distributing weights to generalized components...") + ProcessWeights.distribute_weights_to_components( + state_dict=state_dict, + component_mapping=self.real_components, + ) + + def _calculate_loss(self, logits, tokens, loss_per_token=False): + """Calculate cross-entropy loss.""" + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = tokens[..., 1:].contiguous() + loss_fct = torch.nn.CrossEntropyLoss(reduction="none" if loss_per_token else "mean") + flat_logits = shift_logits.view(-1, shift_logits.size(-1)) + flat_labels = shift_labels.view(-1) + loss = loss_fct(flat_logits, flat_labels) + if loss_per_token: + return loss.view(shift_labels.shape) + else: + return loss + + def _extract_hf_weights(self): + """Extract weights from the original HuggingFace model.""" + hf_state_dict = self.state_dict() + for layer_idx in range(self.cfg.n_layers): + combined_qkv_key = f"transformer.h.{layer_idx}.attn.c_attn.weight" + combined_qkv_bias_key = f"transformer.h.{layer_idx}.attn.c_attn.bias" + if combined_qkv_key in hf_state_dict: + separate_keys_to_remove = [ + f"transformer.h.{layer_idx}.attn.q.weight", + f"transformer.h.{layer_idx}.attn.q.bias", + f"transformer.h.{layer_idx}.attn.k.weight", + f"transformer.h.{layer_idx}.attn.k.bias", + f"transformer.h.{layer_idx}.attn.v.weight", + f"transformer.h.{layer_idx}.attn.v.bias", + ] + for key_to_remove in separate_keys_to_remove: + if key_to_remove in hf_state_dict: + del hf_state_dict[key_to_remove] + return hf_state_dict + + def to_tokens( + self, + input: Union[str, List[str]], + prepend_bos: Optional[bool] = None, + padding_side: Optional[str] = None, + move_to_device: bool = True, + truncate: bool = True, + ) -> torch.Tensor: + """Converts a string to a tensor of tokens. + + See the class-level "Tokenization notes" for full ``prepend_bos`` + semantics, the ``default_prepend_bos`` / + ``tokenizer_prepends_bos`` interaction, and the whitespace- + sensitivity gotcha. **Pass ``prepend_bos=False`` whenever you're + tokenizing only part of a prompt.** + + Args: + input: The input to tokenize. + prepend_bos: Overrides ``self.cfg.default_prepend_bos``. Defaults + to ``None`` (use the cfg setting). Pass ``True`` or ``False`` + to override locally. + padding_side: Which side to pad on when tokenizing multiple + strings of different lengths. Defaults to the tokenizer's + ``padding_side``. + move_to_device: Whether to move the result to ``cfg.device``. + truncate: Whether to truncate inputs longer than ``cfg.n_ctx``. + + Returns: + Token tensor of shape ``[batch, pos]``. + """ + assert self.tokenizer is not None, "Cannot use to_tokens without a tokenizer" + if prepend_bos is None: + prepend_bos = getattr(self.cfg, "default_prepend_bos", True) + if padding_side is None: + padding_side = getattr(self.tokenizer, "padding_side", "right") + tokenizer_prepends_bos = getattr(self.cfg, "tokenizer_prepends_bos", True) + if prepend_bos and (not tokenizer_prepends_bos): + input = utils.get_input_with_manually_prepended_bos(self.tokenizer.bos_token, input) + if isinstance(input, str): + input = [input] + tokens = self.tokenizer( + input, + return_tensors="pt", + padding=True, + truncation=truncate, + max_length=self.cfg.n_ctx if truncate else None, + )["input_ids"] + # Strip auto-appended EOS tokens (e.g., OLMo) + if ( + getattr(self.cfg, "tokenizer_appends_eos", False) + and self.tokenizer.eos_token_id is not None + ): + # Remove trailing EOS, keep at least 1 token + while tokens.shape[-1] > 1 and (tokens[:, -1] == self.tokenizer.eos_token_id).all(): + tokens = tokens[:, :-1] + if not prepend_bos and tokenizer_prepends_bos: + tokens = utils.get_tokens_with_bos_removed(self.tokenizer, tokens) + if move_to_device: + tokens = tokens.to(self.cfg.device) + return tokens + + def to_string( + self, tokens: Union[List[int], torch.Tensor, np.ndarray] + ) -> Union[str, List[str]]: + """Convert tokens to string(s). + + Args: + tokens: Tokens to convert + + Returns: + Decoded string(s) + """ + if not isinstance(tokens, torch.Tensor): + tokens = torch.tensor(tokens) + if len(tokens.shape) == 2: + return self.tokenizer.batch_decode(tokens, clean_up_tokenization_spaces=False) + elif len(tokens.shape) <= 1: + return self.tokenizer.decode(tokens, clean_up_tokenization_spaces=False) + else: + raise ValueError(f"Invalid shape passed in: {tokens.shape}") + + def to_str_tokens( + self, + input: Union[str, torch.Tensor, np.ndarray, List], + prepend_bos: Optional[bool] = None, + padding_side: Optional[str] = None, + ) -> Union[List[str], List[List[str]]]: + """Map text or tokens to a list of tokens as strings. + + See the class-level "Tokenization notes" for full ``prepend_bos`` + semantics. **Pass ``prepend_bos=False`` whenever you're tokenizing + only part of a prompt.** When ``input`` is already a tensor or + array, ``prepend_bos`` and ``padding_side`` are ignored. + + Args: + input: A string, list of strings, or tensor/array of token IDs. + prepend_bos: Overrides ``self.cfg.default_prepend_bos``. Only + applies when ``input`` is a string. Defaults to ``None`` + (use the cfg setting). + padding_side: Which side to pad on. Only applies when ``input`` + is a string. + + Returns: + List of token strings. + """ + if isinstance(input, list): + return cast( + List[List[str]], + [self.to_str_tokens(item, prepend_bos, padding_side) for item in input], + ) + elif isinstance(input, str): + tokens = self.to_tokens(input, prepend_bos=prepend_bos, padding_side=padding_side)[0] + elif isinstance(input, torch.Tensor): + tokens = input.squeeze() + if tokens.dim() == 0: + tokens = tokens.unsqueeze(0) + assert ( + tokens.dim() == 1 + ), f"Invalid tokens input to to_str_tokens, has shape: {tokens.shape}" + elif isinstance(input, np.ndarray): + tokens_np = input.squeeze() + if tokens_np.ndim == 0: + tokens_np = np.expand_dims(tokens_np, axis=0) + assert ( + tokens_np.ndim == 1 + ), f"Invalid tokens input to to_str_tokens, has shape: {tokens_np.shape}" + tokens = torch.tensor(tokens_np) + else: + raise ValueError(f"Invalid input type to to_str_tokens: {type(input)}") + # v5 compat: wrap each token so batch_decode decodes them individually + tokens_list = [[int(t)] for t in tokens.tolist()] + str_tokens = self.tokenizer.batch_decode(tokens_list, clean_up_tokenization_spaces=False) + return str_tokens + + def to_single_token(self, string: str) -> int: + """Map a string that makes up a single token to the id for that token. + + Args: + string: The string to convert + + Returns: + Token ID + + Raises: + AssertionError: If string is not a single token + """ + token = self.to_tokens(string, prepend_bos=False).squeeze() + if token.numel() != 1: + raise AssertionError(f"Input string: {string} is not a single token!") + return int(token.item()) + + def get_token_position( + self, + single_token: Union[str, int], + input: Union[str, torch.Tensor], + mode="first", + prepend_bos: Optional[Union[bool, None]] = None, + padding_side: Optional[Union[Literal["left", "right"], None]] = None, + ): + """Get the position of a single_token in a string or sequence of tokens. + + Raises an error if the token is not present. + + When ``input`` is a string it's tokenized internally — see the + class-level "Tokenization notes" for ``prepend_bos`` semantics. + Off-by-one position errors usually mean ``prepend_bos`` is on + when it shouldn't be (or vice versa); pass ``prepend_bos=False`` + when ``input`` is a fragment of a larger prompt. + + Args: + single_token (Union[str, int]): The token to search for. Can + be a token index, or a string (but the string must correspond to a single token). + input (Union[str, torch.Tensor]): The sequence to + search in. Can be a string or a rank 1 tensor of tokens or a rank 2 tensor of tokens + with a dummy batch dimension. + mode (str, optional): If there are multiple matches, which match to return. Supports + "first" or "last". Defaults to "first". + prepend_bos (bool, optional): Overrides ``self.cfg.default_prepend_bos``. Only + applies when ``input`` is a string. Defaults to ``None`` (use the cfg setting). + padding_side (Union[Literal["left", "right"], None], optional): Specifies which + side to pad when tokenizing multiple strings of different lengths. + """ + if isinstance(input, str): + tokens = self.to_tokens(input, prepend_bos=prepend_bos, padding_side=padding_side) + else: + tokens = input + if len(tokens.shape) == 2: + assert ( + tokens.shape[0] == 1 + ), f"If tokens are rank two, they must have shape [1, seq_len], not {tokens.shape}" + tokens = tokens[0] + if isinstance(single_token, str): + single_token = self.to_single_token(single_token) + elif isinstance(single_token, torch.Tensor): + single_token = single_token.item() + indices = torch.arange(len(tokens), device=tokens.device)[tokens == single_token] + assert len(indices) > 0, "The token does not occur in the prompt" + if mode == "first": + return indices[0].item() + elif mode == "last": + return indices[-1].item() + else: + raise ValueError(f"mode must be 'first' or 'last', not {mode}") + + def to_single_str_token(self, int_token: int) -> str: + """Get the single token corresponding to an int in string form. + + Args: + int_token: The token ID + + Returns: + The token string + """ + assert isinstance(int_token, int) + token = self.to_str_tokens(torch.tensor([int_token])) + if isinstance(token, list) and len(token) == 1: + return str(token[0]) + raise AssertionError("Expected a single string token.") + + def blocks_with(self, submodule: str) -> List[Tuple[int, "GeneralizedComponent"]]: + """Return (index, block) pairs for blocks with the named bridged submodule. + + Checks _modules (not hasattr) so HF-internal attrs don't match. + Use instead of assuming blocks[0] is representative on hybrid models. + """ + if not hasattr(self, "blocks"): + return [] + return [(i, block) for i, block in enumerate(self.blocks) if submodule in block._modules] + + def stack_params_for( + self, submodule: str, attr_path: str, reshape_fn: Optional[Callable] = None + ) -> Tuple[List[int], torch.Tensor]: + """Stack a parameter across matching blocks only. Returns (layer_indices, tensor). + + Use for hybrid models where not all blocks have the submodule. + """ + matching = self.blocks_with(submodule) + if not matching: + raise ValueError( + f"No blocks have submodule '{submodule}'. " + f"Available submodules can be checked with blocks_with()." + ) + indices: List[int] = [] + weights: List[torch.Tensor] = [] + for idx, block in matching: + w = _resolve_attr_path(block, attr_path) + if reshape_fn is not None: + w = reshape_fn(w) + weights.append(w) + indices.append(idx) + return indices, torch.stack(weights, dim=0) + + def _stack_block_params( + self, attr_path: str, reshape_fn: Optional[Callable] = None + ) -> torch.Tensor: + """Stack a parameter across all blocks; falls back to matching-only on hybrids. + + On hybrid models, logs a warning about index mapping and returns only + blocks that have the submodule. First path segment is checked against + _modules; deeper segments resolve via getattr (intentional — W_Q etc. + are exposed via __getattr__ delegation). + """ + first_attr = attr_path.split(".")[0] + matching_blocks = [ + (i, block) for i, block in enumerate(self.blocks) if first_attr in block._modules + ] + + if len(matching_blocks) == 0: + raise AttributeError( + f"No blocks have submodule '{first_attr}'. " + f"Use bridge.blocks_with('{first_attr}') to check availability." + ) + + if len(matching_blocks) < len(self.blocks): + indices = [i for i, _ in matching_blocks] + logging.warning( + "Hybrid model: only %d/%d blocks have '%s'. Returning stacked tensor " + "for layers %s only. Tensor index i corresponds to original layer " + "indices[i], not layer i. For explicit index mapping, use " + "bridge.stack_params_for('%s', '%s').", + len(matching_blocks), + len(self.blocks), + first_attr, + indices, + first_attr, + attr_path, + ) + + weights: List[torch.Tensor] = [] + for _, block in matching_blocks: + w = _resolve_attr_path(block, attr_path) + if reshape_fn is not None: + w = reshape_fn(w) + weights.append(w) + # Under a device_map split, per-block tensors live on different devices. + # torch.stack requires a common device; gather onto cfg.device (the embedding / + # input device — a natural "home" for cross-layer reductions). + if getattr(self.cfg, "n_devices", 1) > 1 and weights and self.cfg.device is not None: + target_device = torch.device(self.cfg.device) + weights = [w.to(target_device) for w in weights] + return torch.stack(weights, dim=0) + + def _reshape_qkv(self, w: torch.Tensor) -> torch.Tensor: + """Reshape 2D [d_model, d_model] QKV weight to 3D [n_heads, d_model, d_head].""" + if w.shape == (self.cfg.d_model, self.cfg.d_model): + d_head = self.cfg.d_model // self.cfg.n_heads + return w.reshape(self.cfg.n_heads, self.cfg.d_model, d_head) + return w + + def _reshape_o(self, w: torch.Tensor) -> torch.Tensor: + """Reshape 2D [d_model, d_model] O weight to 3D [n_heads, d_head, d_model].""" + if w.shape == (self.cfg.d_model, self.cfg.d_model): + d_head = self.cfg.d_model // self.cfg.n_heads + return w.reshape(self.cfg.n_heads, d_head, self.cfg.d_model) + return w + + @property + def W_K(self) -> torch.Tensor: + """Stack the key weights across all layers.""" + return self._stack_block_params("attn.W_K", self._reshape_qkv) + + @property + def W_Q(self) -> torch.Tensor: + """Stack the query weights across all layers.""" + return self._stack_block_params("attn.W_Q", self._reshape_qkv) + + @property + def W_V(self) -> torch.Tensor: + """Stack the value weights across all layers.""" + return self._stack_block_params("attn.W_V", self._reshape_qkv) + + @property + def W_O(self) -> torch.Tensor: + """Stack the attn output weights across all layers.""" + return self._stack_block_params("attn.W_O", self._reshape_o) + + @property + def W_in(self) -> torch.Tensor: + """Stack the MLP input weights across all layers.""" + return self._stack_block_params("mlp.W_in") + + @property + def W_gate(self) -> Union[torch.Tensor, None]: + """Stack the MLP gate weights across all layers (gated MLPs only).""" + if getattr(self.cfg, "gated_mlp", False): + return self._stack_block_params("mlp.W_gate") + return None + + @property + def W_out(self) -> torch.Tensor: + """Stack the MLP output weights across all layers.""" + return self._stack_block_params("mlp.W_out") + + @property + def b_K(self) -> torch.Tensor: + """Stack the key biases across all layers.""" + return self._stack_block_params("attn.b_K") + + @property + def b_Q(self) -> torch.Tensor: + """Stack the query biases across all layers.""" + return self._stack_block_params("attn.b_Q") + + @property + def b_V(self) -> torch.Tensor: + """Stack the value biases across all layers.""" + return self._stack_block_params("attn.b_V") + + @property + def b_O(self) -> torch.Tensor: + """Stack the attn output biases across all layers.""" + return self._stack_block_params("attn.b_O") + + @property + def b_in(self) -> torch.Tensor: + """Stack the MLP input biases across all layers.""" + return self._stack_block_params("mlp.b_in") + + @property + def b_out(self) -> torch.Tensor: + """Stack the MLP output biases across all layers.""" + return self._stack_block_params("mlp.b_out") + + @property + def W_U(self) -> torch.Tensor: + """Unembedding matrix (d_model, d_vocab). Maps residual stream to logits.""" + return self.unembed.W_U + + @property + def b_U(self) -> torch.Tensor: + """Unembedding bias (d_vocab).""" + return self.unembed.b_U + + @property + def W_E(self) -> torch.Tensor: + """Token embedding matrix (d_vocab, d_model).""" + return self.embed.W_E + + @property + def QK(self): + """QK circuit. On hybrids, returns attn layers only (with warning). See QK_for_attn_layers().""" + return FactoredMatrix(self.W_Q, self.W_K.transpose(-2, -1)) + + @property + def OV(self): + """OV circuit. On hybrids, returns attn layers only (with warning). See OV_for_attn_layers().""" + return FactoredMatrix(self.W_V, self.W_O) + + def QK_for_attn_layers(self) -> Tuple[List[int], FactoredMatrix]: + """QK circuit for attention layers only. Returns (layer_indices, FactoredMatrix).""" + q_indices, W_Q = self.stack_params_for("attn", "attn.W_Q", self._reshape_qkv) + _, W_K = self.stack_params_for("attn", "attn.W_K", self._reshape_qkv) + return q_indices, FactoredMatrix(W_Q, W_K.transpose(-2, -1)) + + def OV_for_attn_layers(self) -> Tuple[List[int], FactoredMatrix]: + """OV circuit for attention layers only. Returns (layer_indices, FactoredMatrix).""" + v_indices, W_V = self.stack_params_for("attn", "attn.W_V", self._reshape_qkv) + _, W_O = self.stack_params_for("attn", "attn.W_O", self._reshape_o) + return v_indices, FactoredMatrix(W_V, W_O) + + # ------------------------------------------------------------------ + # Mechanistic interpretability analysis methods + # ------------------------------------------------------------------ + + def tokens_to_residual_directions( + self, + tokens: Union[str, int, torch.Tensor], + ) -> torch.Tensor: + """Map tokens to their unembedding vectors (residual stream directions). + + Returns the columns of W_U corresponding to the given tokens — i.e. the + directions in the residual stream that the model dots with to produce the + logit for each token. + + WARNING: If you use this without folding in LayerNorm (compatibility mode), + the results will be misleading because LN weights change the unembed map. + + Args: + tokens: A single token (str, int, or scalar tensor), a 1-D tensor of + token IDs, or a 2-D batch of token IDs. + + Returns: + Tensor of unembedding vectors with shape matching the input token shape + plus a trailing d_model dimension. + """ + if isinstance(tokens, torch.Tensor) and tokens.numel() > 1: + residual_directions = self.W_U[:, tokens] + residual_directions = einops.rearrange( + residual_directions, "d_model ... -> ... d_model" + ) + return residual_directions + else: + if isinstance(tokens, str): + token = self.to_single_token(tokens) + elif isinstance(tokens, int): + token = tokens + elif isinstance(tokens, torch.Tensor) and tokens.numel() == 1: + token = int(tokens.item()) + else: + raise ValueError(f"Invalid token type: {type(tokens)}") + residual_direction = self.W_U[:, token] + return residual_direction + + # Variant → attr paths for the output bias that feeds the residual stream. + _VARIANT_OUTPUT_BIAS_ATTRS: Dict[str, tuple] = { + "attn": ("b_O",), + "linear_attn": ("out_proj.bias",), + "mamba": ("out_proj.bias",), + "mixer": ("out_proj.bias",), + "ssm": ("out_proj.bias",), + } + + def _get_block_variant_bias(self, block: "GeneralizedComponent") -> Optional[torch.Tensor]: + """Return the output bias from this block's variant submodule, or None.""" + for name in VARIANT_SUBMODULE_NAMES: + if name not in block._modules: + continue + variant = block._modules[name] + for attr_path in self._VARIANT_OUTPUT_BIAS_ATTRS.get(name, ()): + obj = variant + try: + for attr in attr_path.split("."): + obj = getattr(obj, attr) + except AttributeError: + continue + if obj is not None and isinstance(obj, torch.Tensor): + return obj + return None + + def accumulated_bias( + self, + layer: int, + mlp_input: bool = False, + include_mlp_biases: bool = True, + ) -> torch.Tensor: + """Sum of variant + MLP output biases through the residual stream up to `layer`. + + Includes all layer types (attn, SSM, linear-attn). Set mlp_input=True + to include the variant bias of the target layer itself. + """ + accumulated = torch.zeros(self.cfg.d_model, device=self.cfg.device) + for i in range(layer): + block = self.blocks[i] + b_O = self._get_block_variant_bias(block) + if b_O is not None: + accumulated = accumulated + b_O.to(accumulated.device) + if include_mlp_biases and "mlp" in block._modules: + b_out = getattr(block.mlp, "b_out", None) + if b_out is not None: + accumulated = accumulated + b_out.to(accumulated.device) + if mlp_input: + assert layer < self.cfg.n_layers, "Cannot include attn_bias from beyond the final layer" + block = self.blocks[layer] + b_O = self._get_block_variant_bias(block) + if b_O is not None: + accumulated = accumulated + b_O.to(accumulated.device) + return accumulated + + def all_composition_scores(self, mode: str) -> CompositionScores: + """Composition scores for all attention head pairs. Returns CompositionScores. + + See https://transformer-circuits.pub/2021/framework/index.html + On hybrid models, only attention layers are included; layer_indices + maps tensor position i to original layer number. + """ + attn_blocks = self.blocks_with("attn") + if not attn_blocks: + raise ValueError("No attention layers found — cannot compute composition scores.") + + indices = [idx for idx, _ in attn_blocks] + blocks_list = [block for _, block in attn_blocks] + + def _stack(attr_path: str, reshape_fn: Optional[Callable] = None) -> torch.Tensor: + weights: List[torch.Tensor] = [] + for block in blocks_list: + w = _resolve_attr_path(block, attr_path) + if reshape_fn is not None: + w = reshape_fn(w) + weights.append(w) + # See _stack_block_params: gather per-block tensors onto cfg.device when split. + if getattr(self.cfg, "n_devices", 1) > 1 and weights and self.cfg.device is not None: + target_device = torch.device(self.cfg.device) + weights = [w.to(target_device) for w in weights] + return torch.stack(weights, dim=0) + + W_V = _stack("attn.W_V", self._reshape_qkv) + W_O = _stack("attn.W_O", self._reshape_o) + left = FactoredMatrix(W_V, W_O) + + if mode == "Q": + W_Q = _stack("attn.W_Q", self._reshape_qkv) + W_K = _stack("attn.W_K", self._reshape_qkv) + right = FactoredMatrix(W_Q, W_K.transpose(-2, -1)) + elif mode == "K": + W_Q = _stack("attn.W_Q", self._reshape_qkv) + W_K = _stack("attn.W_K", self._reshape_qkv) + right = FactoredMatrix(W_Q, W_K.transpose(-2, -1)).T + elif mode == "V": + right = left + else: + raise ValueError(f"mode must be one of ['Q', 'K', 'V'] not {mode}") + + scores = utils.composition_scores(left, right, broadcast_dims=True) + n_attn = len(indices) + idx_tensor = torch.arange(n_attn, device=self.cfg.device) + mask = idx_tensor[:, None, None, None] < idx_tensor[None, None, :, None] + scores = torch.where(mask, scores, torch.zeros_like(scores)) + + labels = [f"L{l}H{h}" for l in indices for h in range(self.cfg.n_heads)] + return CompositionScores(scores=scores, layer_indices=indices, head_labels=labels) + + def composition_layer_indices(self) -> List[int]: + """Original layer indices for attention layers (maps composition score positions).""" + return [idx for idx, _ in self.blocks_with("attn")] + + def block_hooks(self, layer_idx: int) -> List[str]: + """Sorted hook names available on block `layer_idx` (block-relative paths).""" + prefix = f"blocks.{layer_idx}." + return sorted(name[len(prefix) :] for name in self.hook_dict if name.startswith(prefix)) + + def block_submodules(self, layer_idx: int) -> List[str]: + """Return bridged submodule names on block `layer_idx`.""" + block = self.blocks[layer_idx] + return [name for name in block._modules if name not in _BLOCK_INTERNAL_MODULES] + + def layer_types(self) -> List[str]: + """Per-block type labels, e.g. ["attn+mlp", "ssm+mlp", ...]. Deterministic order.""" + types = [] + for block in self.blocks: + variants = [n for n in VARIANT_SUBMODULE_NAMES if n in block._modules] + universals = sorted( + n + for n in block._modules + if n not in _VARIANT_SUBMODULE_SET + and n not in _BLOCK_INTERNAL_MODULES + and not n.startswith(_NORM_PREFIXES) + ) + parts = variants + universals + types.append("+".join(parts) if parts else "unknown") + return types + + @property + def all_head_labels(self) -> list[str]: + """Human-readable labels for all attention heads, e.g. ['L0H0', 'L0H1', ...].""" + return [f"L{l}H{h}" for l in range(self.cfg.n_layers) for h in range(self.cfg.n_heads)] + + @property + def attn_head_labels(self) -> list[str]: + """Head labels for attention layers only — matches all_composition_scores() dims.""" + return [ + f"L{l}H{h}" for l in self.composition_layer_indices() for h in range(self.cfg.n_heads) + ] + + def parameters(self, recurse: bool = True) -> Iterator[nn.Parameter]: + """Returns parameters following standard PyTorch semantics. + + This method delegates to the underlying HuggingFace model's parameters(). + For TransformerLens-style parameter generator, use tl_parameters() instead. + + Args: + recurse: If True, yields parameters of this module and all submodules + + Returns: + Iterator of nn.Parameter objects + """ + return self.original_model.parameters(recurse=recurse) + + def named_parameters( + self, prefix: str = "", recurse: bool = True, remove_duplicate: bool = True + ) -> Iterator[tuple[str, nn.Parameter]]: + """Returns named parameters following standard PyTorch semantics. + + This method delegates to the underlying HuggingFace model's named_parameters(). + For TransformerLens-style generator, use tl_named_parameters() instead. + + Args: + prefix: Prefix to prepend to all parameter names + recurse: If True, yields parameters of this module and all submodules + remove_duplicate: If True, removes duplicate parameters + + Returns: + Iterator of (name, parameter) tuples + """ + return self.original_model.named_parameters(prefix, recurse, remove_duplicate) + + def tl_parameters(self) -> dict[str, torch.Tensor]: + """Returns TransformerLens-style parameter dictionary. + + Parameter names follow TransformerLens conventions (e.g., 'blocks.0.attn.W_Q') and may + include processed weights (non-leaf tensors). This format is expected by SVDInterpreter + among other analysis tools. + + Returns: + Dictionary mapping TransformerLens parameter names to tensors + + Example: + >>> bridge = TransformerBridge.boot_transformers("gpt2") + >>> tl_params = bridge.tl_parameters() + >>> W_Q = tl_params["blocks.0.attn.W_Q"] # Shape: [n_heads, d_model, d_head] + """ + return self.get_params() + + def tl_named_parameters(self) -> Iterator[tuple[str, torch.Tensor]]: + """Returns iterator of TransformerLens-style named parameters. + + This provides the same parameters as tl_parameters() but as an iterator + for consistency with PyTorch's named_parameters() API pattern. + + Returns: + Iterator of (name, tensor) tuples with TransformerLens naming conventions + + Example: + >>> bridge = TransformerBridge.boot_transformers("gpt2") + >>> for name, param in bridge.tl_named_parameters(): + ... if "attn.W_Q" in name: + ... print(f"{name}: {param.shape}") # doctest: +ELLIPSIS + blocks.0.attn.W_Q: torch.Size([12, 768, 64]) + ... + """ + return iter(self.get_params().items()) + + def forward( + self, + input: Union[str, List[str], torch.Tensor], + return_type: Optional[str] = "logits", + loss_per_token: bool = False, + prepend_bos: Optional[bool] = None, + padding_side: Optional[str] = None, + attention_mask: Optional[torch.Tensor] = None, + start_at_layer: Optional[int] = None, + stop_at_layer: Optional[int] = None, + pixel_values: Optional[torch.Tensor] = None, + input_values: Optional[torch.Tensor] = None, + **kwargs, + ) -> Any: + """Forward pass through the model. + + Args: + input: Input to the model + return_type: Type of output to return ('logits', 'loss', 'both', 'predictions', None) + loss_per_token: Whether to return loss per token + prepend_bos: Whether to prepend BOS token + padding_side: Which side to pad on + start_at_layer: Not implemented in TransformerBridge. The bridge delegates + to HuggingFace's model.forward() which owns the layer iteration loop, + making start_at_layer infeasible without monkey-patching HF internals + (fragile across HF versions) or exception-based layer skipping (corrupts + model state). Raises NotImplementedError if a non-None value is passed. + stop_at_layer: Layer to stop forward pass at + pixel_values: Optional image tensor for multimodal models (e.g., LLaVA, Gemma3). + The tensor is passed directly to the underlying HuggingFace model. + Only valid when cfg.is_multimodal is True. + input_values: Optional audio waveform tensor for audio models (e.g., HuBERT). + The tensor is passed directly to the underlying HuggingFace model. + Only valid when cfg.is_audio_model is True. + **kwargs: Additional arguments passed to model + + Returns: + Model output based on return_type + """ + + if start_at_layer is not None: + raise NotImplementedError( + "start_at_layer is not supported in TransformerBridge. " + "The bridge delegates to HuggingFace's model.forward() which controls " + "the layer iteration loop. See the TransformerBridge review plan for a " + "detailed analysis of implementation approaches and their tradeoffs." + ) + + # Set stop_at_layer flag on all blocks if requested + if stop_at_layer is not None and hasattr(self, "blocks"): + for block in self.blocks: + block._stop_at_layer_idx = stop_at_layer + + # Map HookedEncoderDecoder-style kwargs to HF-compatible names + if "decoder_input" in kwargs: + kwargs["decoder_input_ids"] = kwargs.pop("decoder_input") + if "one_zero_attention_mask" in kwargs: + if attention_mask is None: + attention_mask = kwargs.pop("one_zero_attention_mask") + else: + kwargs.pop("one_zero_attention_mask") + + # Detect batched list input that will need padding. For this case we force + # left-padding internally and auto-compute attention_mask + position_ids + # (unless the caller passed them explicitly) so pad tokens don't contaminate + # attention or position embeddings. + _is_batched_list = ( + isinstance(input, list) + and len(input) > 1 + and not getattr(self.cfg, "is_audio_model", False) + ) + + try: + if isinstance(input, (str, list)): + if getattr(self.cfg, "is_audio_model", False): + raise ValueError( + "Audio models require tensor input (raw waveform), not text. " + "Pass a torch.Tensor or use the input_values parameter." + ) + if _is_batched_list and padding_side is None: + # Force left-padding so real tokens are flush-right. + _orig_padding_side = self.tokenizer.padding_side + self.tokenizer.padding_side = "left" + try: + input_ids = self.to_tokens( + input, prepend_bos=prepend_bos, padding_side=padding_side + ) + finally: + self.tokenizer.padding_side = _orig_padding_side + else: + input_ids = self.to_tokens( + input, prepend_bos=prepend_bos, padding_side=padding_side + ) + else: + input_ids = input + # Promote 1D integer token tensors to 2D [batch=1, seq] to match + # HookedTransformer's contract. Float tensors (inputs_embeds, + # audio waveforms) are passed through unchanged. + if ( + isinstance(input_ids, torch.Tensor) + and input_ids.ndim == 1 + and not input_ids.is_floating_point() + ): + input_ids = input_ids.unsqueeze(0) + + # Detect inputs_embeds: if the tensor is floating point, it's pre-computed + # embeddings (e.g., from multimodal models) rather than token IDs. + _is_inputs_embeds = ( + isinstance(input_ids, torch.Tensor) and input_ids.is_floating_point() + ) + + # Auto-compute attention_mask + position_ids for batched list input + # when the caller didn't supply them. Matches HF generation convention. + if ( + _is_batched_list + and attention_mask is None + and self.tokenizer is not None + and self.tokenizer.pad_token_id is not None + and not _is_inputs_embeds + ): + _prev_side = self.tokenizer.padding_side + self.tokenizer.padding_side = "left" + try: + attention_mask = utils.get_attention_mask( + self.tokenizer, + input_ids, + prepend_bos=getattr(self.cfg, "default_prepend_bos", True), + ).to(self.cfg.device) + finally: + self.tokenizer.padding_side = _prev_side + if "position_ids" not in kwargs: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + kwargs["position_ids"] = position_ids + + if attention_mask is not None: + kwargs["attention_mask"] = attention_mask + if kwargs.pop("use_past_kv_cache", False) or kwargs.get("use_cache", False): + kwargs["use_cache"] = True + # Auto-generate decoder_input_ids for encoder-decoder models + if ( + "decoder_input_ids" not in kwargs + and hasattr(self.original_model, "config") + and getattr(self.original_model.config, "is_encoder_decoder", False) + ): + decoder_start_token_id = getattr( + self.original_model.config, "decoder_start_token_id", None + ) + if decoder_start_token_id is not None: + shifted = input_ids[:, :-1] + start_tokens = torch.full( + (input_ids.shape[0], 1), + decoder_start_token_id, + dtype=input_ids.dtype, + device=input_ids.device, + ) + kwargs["decoder_input_ids"] = torch.cat([start_tokens, shifted], dim=1) + else: + kwargs["decoder_input_ids"] = input_ids + + # Tell PosEmbedBridge to expand batch=1 position_ids to full batch. + if hasattr(self, "pos_embed"): + self.pos_embed._current_batch_size = input_ids.shape[0] + + # Handle pixel_values for multimodal models + if pixel_values is not None: + if not getattr(self.cfg, "is_multimodal", False): + raise ValueError( + "pixel_values can only be passed to multimodal models " + "(cfg.is_multimodal must be True)" + ) + kwargs["pixel_values"] = pixel_values + + # Handle input_values for audio models + if input_values is not None: + if not getattr(self.cfg, "is_audio_model", False): + raise ValueError( + "input_values can only be passed to audio models " + "(cfg.is_audio_model must be True)" + ) + kwargs["input_values"] = input_values + + # Audio models use input_values (waveform), not input_ids + if getattr(self.cfg, "is_audio_model", False): + if input_values is not None: + result = self._driver.forward(**kwargs) + elif isinstance(input, torch.Tensor): + kwargs["input_values"] = input + result = self._driver.forward(**kwargs) + else: + raise ValueError( + "Audio models require tensor input (raw waveform). " + "Pass a torch.Tensor or use input_values parameter." + ) + elif _is_inputs_embeds: + result = self._driver.forward(inputs_embeds=input_ids, **kwargs) + else: + # By keyword so kw-only ``input_ids`` drivers don't TypeError. + result = self._driver.forward(input_ids=input_ids, **kwargs) + output = result.raw_output + # No-op for HF (its hooks already fired); load-bearing for vLLM/Inspect. + if result.captured: + self._replay_captures(result.captured) + # Convert TensorLike to torch at the boundary; let weird shapes + # (audio/CTC dataclasses, tuple-of-tuples) pass through unchanged — + # downstream return_type branches catch them with specific errors. + logits = result.logits + if isinstance(logits, torch.Tensor): + pass + elif logits is not None and isinstance(logits, TensorLike): + logits = to_torch(logits) + # Stash only the cache object (not the full output) for generate(). + if getattr(self, "_capture_hf_cache", False): + self._last_hf_cache = getattr(output, "past_key_values", None) + return self._finalize_return( + return_type, + logits, + input_ids, + is_audio_model=getattr(self.cfg, "is_audio_model", False), + inputs_embeds_was_used=_is_inputs_embeds, + loss_per_token=loss_per_token, + ) + except StopAtLayerException as e: + # Execution stopped at the requested layer + return e.layer_output + finally: + # Clean up state that may be inconsistent after StopAtLayerException + if stop_at_layer is not None and hasattr(self, "blocks"): + # Reset the stop flag on all blocks + for block in self.blocks: + block._stop_at_layer_idx = None + + # Clear any stale KV cache — layers after the stop point didn't + # execute, so the cache is incomplete and would corrupt subsequent + # generate() calls that expect a full cache. + if hasattr(self, "_last_hf_cache"): + del self._last_hf_cache + + # loss_fn inherited from BridgeCore + + def _generate_tokens( + self, + current_tokens: torch.Tensor, + input_tokens: torch.Tensor, + batch_size: int, + *, + max_new_tokens: int, + do_sample: bool, + top_k: Optional[int], + top_p: Optional[float], + temperature: float, + freq_penalty: float, + repetition_penalty: float, + stop_at_eos: bool, + stop_tokens: List[int], + eos_token_for_padding: int, + finished_sequences: torch.Tensor, + use_past_kv_cache: bool, + use_stateful_cache: bool, + mamba_cache: Any, + mamba_conv_kernel: int, + is_encoder_decoder: bool, + _is_batched_list: bool, + _generate_from_embeds: bool, + encoder_input: Optional[torch.Tensor], + decoder_tokens: Optional[torch.Tensor], + generated_token_ids: Optional[List[torch.Tensor]], + pixel_values: Optional[torch.Tensor], + multimodal_kwargs: Dict[str, Any], + verbose: bool, + ) -> Generator[Tuple[torch.Tensor, torch.Tensor, bool], None, None]: + """Core generation loop. Yields (sampled_tokens, final_logits, all_finished) per step. + + Owns the forward pass, sampling, EOS handling, token accumulation, and + KV cache management. Callers are responsible for try/finally cleanup of + ``_capture_hf_cache``. + """ + _hf_kv_cache = None + + for gen_step_idx in tqdm.tqdm(range(max_new_tokens), disable=not verbose): + with torch.no_grad(): + if is_encoder_decoder: + logits = self( + encoder_input, + return_type="logits", + decoder_input=decoder_tokens, + ) + else: + forward_kwargs: Dict[str, Any] = {} + # Compute attention mask and position_ids for batched + # inputs with padding. + if ( + _is_batched_list + and self.tokenizer is not None + and self.tokenizer.pad_token_id is not None + ): + _prev_side = self.tokenizer.padding_side + self.tokenizer.padding_side = "left" + attn_mask = utils.get_attention_mask( + self.tokenizer, + current_tokens, + prepend_bos=getattr(self.cfg, "default_prepend_bos", True), + ).to(self.cfg.device) + self.tokenizer.padding_side = _prev_side + forward_kwargs["attention_mask"] = attn_mask + position_ids = attn_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attn_mask == 0, 1) + forward_kwargs["position_ids"] = position_ids + if gen_step_idx == 0: + if pixel_values is not None: + forward_kwargs["pixel_values"] = pixel_values + if multimodal_kwargs: + forward_kwargs.update(multimodal_kwargs) + if use_stateful_cache: + forward_kwargs["cache_params"] = mamba_cache + forward_kwargs["use_cache"] = True + if gen_step_idx == 0: + cache_position = torch.arange( + 0, mamba_conv_kernel, device=self.cfg.device + ) + forward_kwargs["cache_position"] = cache_position + logits = self( + current_tokens, + return_type="logits", + **forward_kwargs, + ) + else: + input_seq_pos = input_tokens.shape[1] + gen_step_idx - 1 + cache_position = torch.tensor([input_seq_pos], device=self.cfg.device) + forward_kwargs["cache_position"] = cache_position + if "position_ids" in forward_kwargs: + forward_kwargs["position_ids"] = forward_kwargs["position_ids"][ + :, -1: + ] + logits = self( + current_tokens[:, -1:], + return_type="logits", + **forward_kwargs, + ) + elif use_past_kv_cache: + forward_kwargs["use_cache"] = True + if _hf_kv_cache is not None: + forward_kwargs["past_key_values"] = _hf_kv_cache + if "position_ids" in forward_kwargs: + forward_kwargs["position_ids"] = forward_kwargs["position_ids"][ + :, -1: + ] + logits = self( + current_tokens[:, -1:], + return_type="logits", + **forward_kwargs, + ) + else: + logits = self( + current_tokens, + return_type="logits", + **forward_kwargs, + ) + else: + logits = self(current_tokens, return_type="logits", **forward_kwargs) + if use_past_kv_cache and hasattr(self, "_last_hf_cache"): + _hf_kv_cache = self._last_hf_cache or _hf_kv_cache + del self._last_hf_cache + final_logits = logits[:, -1, :] + + # Sample next token + penalty_tokens = ( + torch.stack(generated_token_ids, dim=1) + if _generate_from_embeds and generated_token_ids + else None + ) + if do_sample: + sampled_tokens = utils.sample_logits( + final_logits, + top_k=top_k, + top_p=top_p, + temperature=temperature, + freq_penalty=freq_penalty, + repetition_penalty=repetition_penalty, + tokens=penalty_tokens + if _generate_from_embeds + else (decoder_tokens if is_encoder_decoder else current_tokens), + ).to(self.cfg.device) + else: + sampled_tokens = utils.sample_logits( + final_logits, + temperature=0.0, + repetition_penalty=repetition_penalty, + tokens=penalty_tokens + if _generate_from_embeds + else (decoder_tokens if is_encoder_decoder else current_tokens), + ).to(self.cfg.device) + + # Handle EOS + if stop_at_eos: + sampled_tokens[finished_sequences] = eos_token_for_padding + finished_sequences.logical_or_( + torch.isin( + sampled_tokens.to(self.cfg.device), + torch.tensor(stop_tokens).to(self.cfg.device), + ) + ) + + # Update token sequences + if is_encoder_decoder: + assert decoder_tokens is not None + decoder_tokens = torch.cat([decoder_tokens, sampled_tokens.unsqueeze(1)], dim=1) + elif _generate_from_embeds: + assert generated_token_ids is not None + generated_token_ids.append(sampled_tokens) + embed_fn = self.original_model.get_input_embeddings() # type: ignore[operator] + assert embed_fn is not None + new_embed = embed_fn(sampled_tokens.unsqueeze(1)).to(current_tokens.dtype) + current_tokens = torch.cat([current_tokens, new_embed], dim=1) + else: + current_tokens = torch.cat([current_tokens, sampled_tokens.unsqueeze(1)], dim=1) + + all_finished = bool(stop_at_eos and finished_sequences.all().item()) + + yield sampled_tokens, final_logits, all_finished + + if all_finished: + return + + def generate( + self, + input: Union[str, List[str], torch.Tensor] = "", + max_new_tokens: int = 10, + stop_at_eos: bool = True, + eos_token_id: Optional[int] = None, + do_sample: bool = True, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + temperature: float = 1.0, + freq_penalty: float = 0.0, + repetition_penalty: float = 1.0, + use_past_kv_cache: bool = True, + prepend_bos: Optional[bool] = None, + padding_side: Optional[str] = None, + return_type: Optional[str] = "input", + verbose: bool = True, + output_logits: bool = False, + pixel_values: Optional[torch.Tensor] = None, + **multimodal_kwargs, + ) -> str | list[str] | torch.Tensor | Any: # Any for transformers.utils.ModelOutput + # Any: beartype forward ref limitation (beartype#546) + """Sample tokens from the model. + + Sample tokens from the model until the model outputs eos_token or max_new_tokens is reached. + This implementation is based on HookedTransformer.generate() to ensure consistent behavior. + + Args: + input: Text string, list of strings, or tensor of tokens + max_new_tokens: Maximum number of tokens to generate + stop_at_eos: If True, stop generating tokens when the model outputs eos_token + eos_token_id: The token ID to use for end of sentence + do_sample: If True, sample from the model's output distribution. Otherwise, use greedy search + top_k: Number of tokens to sample from. If None, sample from all tokens + top_p: Probability mass to sample from. If 1.0, sample from all tokens + temperature: Temperature for sampling. Higher values will make the model more random + freq_penalty: Frequency penalty for sampling - how much to penalise previous tokens + repetition_penalty: HuggingFace-style repetition penalty. Values > 1.0 discourage + repetition by dividing positive logits and multiplying negative logits for + previously seen tokens. Default 1.0 (no penalty). + use_past_kv_cache: If True, use KV caching for faster generation + prepend_bos: Accepted for API compatibility but not applied during generation. + The HF model expects tokens in its native format (tokenizer defaults). + Overriding BOS can silently degrade generation quality. + padding_side: Which side to pad when tokenizing multiple strings of different + lengths. For batched list inputs, left-padding is forced internally for + correct generation behavior. Defaults to None (tokenizer default). + return_type: The type of output to return - 'input', 'str', or 'tokens' + verbose: Not used in Bridge (kept for API compatibility) + output_logits: If True, return a ModelOutput with sequences and logits tuple + pixel_values: Optional image tensor for multimodal models. Only passed on the + first generation step (the vision encoder processes the image once, then + embeddings are part of the token sequence for subsequent steps). + + Returns: + Generated sequence as string, list of strings, or tensor depending on input type and return_type. + If output_logits=True, returns a ModelOutput-like object with 'sequences' and 'logits' attributes. + """ + # prepend_bos is intentionally not applied during generation. + # The HF model expects tokens in its native format. Overriding BOS can silently + # degrade quality. + if prepend_bos is not None: + import warnings + + warnings.warn( + "prepend_bos is ignored during TransformerBridge.generate(). " + "The HF model expects tokens with the tokenizer's default BOS handling. " + "To control BOS, tokenize with to_tokens(prepend_bos=...) and pass the " + "resulting tensor to generate().", + stacklevel=2, + ) + # padding_side is handled internally: for batched list inputs, left-padding + # is forced to ensure correct generation. See _is_batched_list logic below. + + # Stateful dispatch is decided after input parsing so we can fall back + # to hf_generate() for input types the stateful loop doesn't handle. + is_stateful_model = getattr(self.cfg, "is_stateful", False) + + _is_batched_list = isinstance(input, list) and len(input) > 1 + + _generate_from_embeds = False + if isinstance(input, str): + input_tokens = self.to_tokens(input, move_to_device=True, truncate=False) + input_type = "str" + elif isinstance(input, list): + # Force left-padding for batched generation so real tokens are + # flush-right and logits[:, -1, :] is always the last real token. + if _is_batched_list: + _orig_padding_side = self.tokenizer.padding_side + self.tokenizer.padding_side = "left" + input_tokens = self.to_tokens(input, move_to_device=True, truncate=False) + if _is_batched_list: + self.tokenizer.padding_side = _orig_padding_side + input_type = "list" + elif isinstance(input, torch.Tensor) and input.is_floating_point(): + # inputs_embeds: pre-computed embeddings (e.g., from multimodal models) + input_tokens = input.to(self.cfg.device) + input_type = "embeds" + _generate_from_embeds = True + else: + input_tokens = input.to(self.cfg.device) + input_type = "tokens" + + # Determine return type + if return_type == "input": + if input_type in ["str", "list"]: + return_type = "str" + elif input_type == "embeds": + return_type = "tokens" + else: + return_type = "tokens" + + batch_size = input_tokens.shape[0] + + # Setup EOS token handling + stop_tokens = [] + eos_token_for_padding = 0 + if stop_at_eos: + tokenizer_has_eos_token = ( + self.tokenizer is not None and self.tokenizer.eos_token_id is not None + ) + if eos_token_id is None: + assert ( + tokenizer_has_eos_token + ), "Must pass eos_token_id if stop_at_eos is True and tokenizer is None or has no eos_token_id" + assert self.tokenizer is not None + eos_token_id = self.tokenizer.eos_token_id + + if isinstance(eos_token_id, int): + stop_tokens = [eos_token_id] + eos_token_for_padding = eos_token_id + else: + stop_tokens = list(eos_token_id) + if tokenizer_has_eos_token: + assert self.tokenizer is not None + eos_token_for_padding = self.tokenizer.eos_token_id + else: + eos_token_for_padding = eos_token_id[0] + + # Track which sequences have finished + finished_sequences = torch.zeros(batch_size, dtype=torch.bool, device=self.cfg.device) + + # Optionally collect logits at each generation step for downstream tooling/tests + logits_seq_list: list[torch.Tensor] | None = [] if output_logits else None + + # Detect encoder-decoder models (T5, BART, etc.) + is_encoder_decoder = hasattr(self.original_model, "config") and getattr( + self.original_model.config, "is_encoder_decoder", False + ) + + # HF cache flows opaquely through the component chain via + # _reconstruct_attention() → _update_kv_cache() on each layer. + _hf_kv_cache = None + if use_past_kv_cache and is_encoder_decoder: + # Encoder-decoder models (T5, BART) don't support the opaque + # cache path — silently disable rather than crash, since + # use_past_kv_cache=True is the default. + use_past_kv_cache = False + + # SSMs (Mamba/Mamba-2) run through a dedicated cache path so hooks + # fire on every step. Unsupported input types fall back to hf_generate(). + use_stateful_cache = ( + is_stateful_model + and use_past_kv_cache + and not is_encoder_decoder + and not _generate_from_embeds + and pixel_values is None + and not multimodal_kwargs + ) + if is_stateful_model and not use_stateful_cache: + hf_kwargs: dict[str, Any] = { + "max_new_tokens": max_new_tokens, + "do_sample": do_sample, + "temperature": temperature, + } + if top_k is not None: + hf_kwargs["top_k"] = top_k + if top_p is not None: + hf_kwargs["top_p"] = top_p + if eos_token_id is not None: + hf_kwargs["eos_token_id"] = eos_token_id + return self.hf_generate(input, **hf_kwargs) + + # SSM cache is built once and mutated in place across forward calls. + # Adapter owns the cache-type choice; new SSMs just override + # create_stateful_cache(). + mamba_cache: Any = None + mamba_conv_kernel: int = 0 + if use_stateful_cache: + hf_model: Any = self.original_model + mamba_conv_kernel = int(getattr(hf_model.config, "conv_kernel", 4)) + cache_dtype = self.cfg.dtype or torch.float32 + mamba_cache = self.adapter.create_stateful_cache( + hf_model=hf_model, + batch_size=batch_size, + device=self.cfg.device, + dtype=cache_dtype, + ) + + if use_past_kv_cache and not use_stateful_cache: + self._capture_hf_cache = True # Signal forward() to stash cache + + # Generate tokens + current_tokens = input_tokens.clone() + # For inputs_embeds generation, also track generated token IDs for decoding + if _generate_from_embeds: + generated_token_ids: list[torch.Tensor] = [] + sampled_tokens_list = [] + + # For encoder-decoder models, keep encoder input fixed and grow decoder input + if is_encoder_decoder: + encoder_input = input_tokens.clone() + decoder_start_token_id = getattr( + self.original_model.config, "decoder_start_token_id", 0 + ) + decoder_tokens = torch.full( + (batch_size, 1), + decoder_start_token_id, + dtype=input_tokens.dtype, + device=self.cfg.device, + ) + + try: + for sampled_tokens, final_logits, all_finished in self._generate_tokens( + current_tokens, + input_tokens, + batch_size, + max_new_tokens=max_new_tokens, + do_sample=do_sample, + top_k=top_k, + top_p=top_p, + temperature=temperature, + freq_penalty=freq_penalty, + repetition_penalty=repetition_penalty, + stop_at_eos=stop_at_eos, + stop_tokens=stop_tokens, + eos_token_for_padding=eos_token_for_padding, + finished_sequences=finished_sequences, + use_past_kv_cache=use_past_kv_cache, + use_stateful_cache=use_stateful_cache, + mamba_cache=mamba_cache, + mamba_conv_kernel=mamba_conv_kernel, + is_encoder_decoder=is_encoder_decoder, + _is_batched_list=_is_batched_list, + _generate_from_embeds=_generate_from_embeds, + encoder_input=encoder_input if is_encoder_decoder else None, + decoder_tokens=decoder_tokens if is_encoder_decoder else None, + generated_token_ids=generated_token_ids if _generate_from_embeds else None, + pixel_values=pixel_values, + multimodal_kwargs=multimodal_kwargs if multimodal_kwargs else {}, + verbose=verbose, + ): + sampled_tokens_list.append(sampled_tokens.unsqueeze(1)) + if logits_seq_list is not None: + logits_seq_list.append(final_logits.clone()) + if all_finished: + break + finally: + self._capture_hf_cache = False + if hasattr(self, "_last_hf_cache"): + del self._last_hf_cache + + # Concatenate all sampled tokens + sampled_tokens = torch.cat(sampled_tokens_list, dim=1) + if is_encoder_decoder: + # Reconstruct full decoder sequence: start token + generated tokens + output_tokens = torch.cat([decoder_tokens[:, :1], sampled_tokens], dim=1) + elif _generate_from_embeds: + # For inputs_embeds, we only have the generated token IDs (no input token IDs) + output_tokens = sampled_tokens + else: + output_tokens = torch.cat([input_tokens, sampled_tokens], dim=1) + + # Return ModelOutput if output_logits was requested + if output_logits and logits_seq_list is not None: + from transformers.utils import ModelOutput # type: ignore + + def _logits_to_tuple(logits_list: list[torch.Tensor]) -> tuple[torch.Tensor, ...]: + assert logits_list is not None + # Convert list of [batch, vocab] tensors to tuple + return tuple(logits_list) + + try: + from transformers.generation.utils import GenerateDecoderOnlyOutput + + # Return a HF-compatible ModelOutput structure + # GenerateDecoderOnlyOutput expects: sequences, scores (optional), logits (optional) + return GenerateDecoderOnlyOutput( + sequences=cast(torch.LongTensor, output_tokens), + # HF's type hint says tuple[FloatTensor] but should be tuple[FloatTensor, ...] + # (variable-length tuple with one element per generated token) + logits=_logits_to_tuple(logits_seq_list), # type: ignore[arg-type] + ) + except (ImportError, AttributeError): + # Fallback if GenerateDecoderOnlyOutput not available in this transformers version + return ModelOutput( + sequences=output_tokens, + logits=_logits_to_tuple(logits_seq_list), + ) + + # Format output + if return_type == "str": + assert self.tokenizer is not None + if input_type == "str": + return self.tokenizer.decode(output_tokens[0], skip_special_tokens=True) + else: + decoded_texts = [ + self.tokenizer.decode(tokens, skip_special_tokens=True) + for tokens in output_tokens + ] + return decoded_texts[0] if len(decoded_texts) == 1 else decoded_texts + else: # return_type == "tokens" + return output_tokens + + @torch.no_grad() + def generate_stream( + self, + input: Union[str, List[str], torch.Tensor] = "", + max_new_tokens: int = 10, + max_tokens_per_yield: int = 25, + stop_at_eos: bool = True, + eos_token_id: Optional[int] = None, + do_sample: bool = True, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + temperature: float = 1.0, + freq_penalty: float = 0.0, + repetition_penalty: float = 1.0, + use_past_kv_cache: bool = True, + prepend_bos: Optional[bool] = None, + padding_side: Optional[str] = None, + return_type: Optional[str] = "input", + verbose: bool = True, + ) -> Generator[Union[torch.Tensor, str], None, None]: + """Stream tokens from the model as they are generated. + + Yields batches of tokens progressively during generation rather than + waiting for the entire sequence. Uses the same core loop as generate(). + + Args: + input: Text string, list of strings, or tensor of tokens. + max_new_tokens: Maximum number of tokens to generate. + max_tokens_per_yield: Yield accumulated tokens every this many steps. + stop_at_eos: If True, stop when eos_token is produced. + eos_token_id: Token ID(s) for end of sentence. Defaults to tokenizer's. + do_sample: If True, sample; otherwise greedy. + top_k: Top-k sampling. None means no filtering. + top_p: Nucleus sampling threshold. + temperature: Sampling temperature. + freq_penalty: Frequency penalty for previous tokens. + repetition_penalty: HF-style repetition penalty (>1.0 discourages repeats). + use_past_kv_cache: Use KV caching for faster generation. + prepend_bos: Not applied (API compatibility). See generate() docstring. + padding_side: Which side to pad for batched list inputs. Left-padding + is forced internally for batched generation. + return_type: 'input' (match input type), 'str', or 'tokens'. + verbose: Show progress bar. + + Yields: + Token tensors [batch, seq_len] or strings, accumulated up to + max_tokens_per_yield tokens between yields. First yield includes + the input tokens; subsequent yields contain only new tokens. + """ + if prepend_bos is not None: + warnings.warn( + "prepend_bos is ignored during TransformerBridge.generate_stream(). " + "The HF model expects tokens with the tokenizer's default BOS handling.", + stacklevel=2, + ) + + # --- Input parsing (mirrors generate()) --- + _is_batched_list = isinstance(input, list) and len(input) > 1 + + if isinstance(input, str): + input_tokens = self.to_tokens(input, move_to_device=True, truncate=False) + input_type = "str" + elif isinstance(input, list): + if _is_batched_list: + _orig_ps = self.tokenizer.padding_side + self.tokenizer.padding_side = "left" + try: + input_tokens = self.to_tokens(input, move_to_device=True, truncate=False) + finally: + if _is_batched_list: + self.tokenizer.padding_side = _orig_ps + input_type = "list" + else: + input_tokens = input.to(self.cfg.device) + input_type = "tokens" + + if return_type == "input": + return_type = "str" if input_type in ["str", "list"] else "tokens" + + batch_size = input_tokens.shape[0] + + # --- EOS setup --- + stop_tokens: List[int] = [] + eos_token_for_padding = 0 + if stop_at_eos: + tokenizer_has_eos_token = ( + self.tokenizer is not None and self.tokenizer.eos_token_id is not None + ) + if eos_token_id is None: + assert ( + tokenizer_has_eos_token + ), "Must pass eos_token_id if stop_at_eos is True and tokenizer is None or has no eos_token_id" + assert self.tokenizer is not None + eos_token_id = self.tokenizer.eos_token_id + if isinstance(eos_token_id, int): + stop_tokens = [eos_token_id] + eos_token_for_padding = eos_token_id + else: + stop_tokens = list(eos_token_id) + if tokenizer_has_eos_token: + assert self.tokenizer is not None + eos_token_for_padding = self.tokenizer.eos_token_id + else: + eos_token_for_padding = eos_token_id[0] + + finished_sequences = torch.zeros(batch_size, dtype=torch.bool, device=self.cfg.device) + + # --- Cache setup --- + if use_past_kv_cache: + self._capture_hf_cache = True + + current_tokens = input_tokens.clone() + + # --- Streaming loop --- + # All yields are token tensors [batch, seq_len]. Each yield contains + # only the newly generated tokens since the previous yield (the first + # yield additionally prepends the input tokens for context). + accumulated_tokens: Optional[torch.Tensor] = None + tokens_since_last_yield = 0 + + def _maybe_decode( + tokens: torch.Tensor, + ) -> Union[torch.Tensor, str]: + if return_type == "str": + assert self.tokenizer is not None + return self.tokenizer.decode(tokens[0], skip_special_tokens=True) + return tokens + + try: + for step_idx, (sampled_tokens, _, all_finished) in enumerate( + self._generate_tokens( + current_tokens, + input_tokens, + batch_size, + max_new_tokens=max_new_tokens, + do_sample=do_sample, + top_k=top_k, + top_p=top_p, + temperature=temperature, + freq_penalty=freq_penalty, + repetition_penalty=repetition_penalty, + stop_at_eos=stop_at_eos, + stop_tokens=stop_tokens, + eos_token_for_padding=eos_token_for_padding, + finished_sequences=finished_sequences, + use_past_kv_cache=use_past_kv_cache, + use_stateful_cache=False, + mamba_cache=None, + mamba_conv_kernel=0, + is_encoder_decoder=False, + _is_batched_list=_is_batched_list, + _generate_from_embeds=False, + encoder_input=None, + decoder_tokens=None, + generated_token_ids=None, + pixel_values=None, + multimodal_kwargs={}, + verbose=verbose, + ) + ): + new_tokens = sampled_tokens.unsqueeze(-1) + + if step_idx == 0: + accumulated_tokens = torch.cat([input_tokens, new_tokens], dim=-1) + tokens_since_last_yield = accumulated_tokens.shape[1] + else: + if accumulated_tokens is None: + accumulated_tokens = new_tokens + else: + accumulated_tokens = torch.cat([accumulated_tokens, new_tokens], dim=-1) + tokens_since_last_yield += 1 + + if tokens_since_last_yield >= max_tokens_per_yield: + yield _maybe_decode(accumulated_tokens) + tokens_since_last_yield = 0 + accumulated_tokens = None + + if all_finished: + if accumulated_tokens is not None: + yield _maybe_decode(accumulated_tokens) + break + + # Yield remainder after loop completes without break + if accumulated_tokens is not None: + yield _maybe_decode(accumulated_tokens) + finally: + self._capture_hf_cache = False + if hasattr(self, "_last_hf_cache"): + del self._last_hf_cache + + def hf_generate( + self, + input: str | list[str] | torch.Tensor = "", + max_new_tokens: int = 10, + stop_at_eos: bool = True, + eos_token_id: int | None = None, + do_sample: bool = True, + top_k: int | None = None, + top_p: float | None = None, + temperature: float = 1.0, + use_past_kv_cache: bool = True, + return_type: str | None = "input", + pixel_values: torch.Tensor | None = None, + **generation_kwargs, + ) -> str | list[str] | torch.Tensor | Any: # Any for HF ModelOutput types + # Any: beartype forward ref limitation (beartype#546) + """Generate text using the underlying HuggingFace model with full HF API support. + + This method provides direct access to HuggingFace's generation API, forwarding all + generation parameters (including output_scores, output_logits, output_attentions, + output_hidden_states) directly to the underlying HF model. Use this when you need + full HuggingFace generation features not supported by the standard generate() method. + + For standard generation compatible with HookedTransformer, use generate() instead. + + Args: + input: Text string, list of strings, or tensor of tokens + max_new_tokens: Maximum number of tokens to generate + stop_at_eos: If True, stop generating tokens when the model outputs eos_token + eos_token_id: The token ID to use for end of sentence + do_sample: If True, sample from the model's output distribution + top_k: Number of tokens to sample from + top_p: Probability mass to sample from + temperature: Temperature for sampling + use_past_kv_cache: If True, use KV caching for faster generation + return_type: The type of output to return - 'input', 'str', or 'tokens' + **generation_kwargs: Additional HuggingFace generation parameters including: + - output_scores: Return generation scores + - output_logits: Return generation logits + - output_attentions: Return attention weights + - output_hidden_states: Return hidden states + - return_dict_in_generate: Return ModelOutput object + - And any other HF generation parameters + + Returns: + Generated sequence as string, list of strings, tensor, or HF ModelOutput + depending on input type, return_type, and generation_kwargs. + + Example:: + + # Get full HF ModelOutput with logits and attentions + from transformer_lens import HookedTransformer + model = HookedTransformer.from_pretrained("tiny-stories-1M") + result = model.hf_generate( + "Hello world", + max_new_tokens=5, + output_logits=True, + output_attentions=True, + return_dict_in_generate=True + ) + print(result.sequences) # Generated tokens + print(result.logits) # Logits for each generation step + print(result.attentions) # Attention weights + """ + # Handle string input by tokenizing it + if isinstance(input, str): + inputs = self.tokenizer(input, return_tensors="pt", padding=False, truncation=False).to( + self.cfg.device + ) + input_ids = inputs["input_ids"] + input_type = "str" + elif isinstance(input, list): + inputs = self.tokenizer(input, return_tensors="pt", padding=True, truncation=False).to( + self.cfg.device + ) + input_ids = inputs["input_ids"] + input_type = "list" + else: + input_ids = input + if input_ids.device != self.cfg.device: + input_ids = input_ids.to(self.cfg.device) + input_type = "tokens" + + # Build generation_kwargs from explicit args and kwargs + generation_kwargs = dict(generation_kwargs) if generation_kwargs is not None else {} + generation_kwargs.update( + { + "max_new_tokens": max_new_tokens, + "do_sample": do_sample, + "temperature": temperature, + "pad_token_id": self.tokenizer.eos_token_id, + } + ) + + if top_k is not None: + generation_kwargs["top_k"] = top_k + if top_p is not None: + generation_kwargs["top_p"] = top_p + if eos_token_id is not None: + generation_kwargs["eos_token_id"] = eos_token_id + elif stop_at_eos and self.tokenizer.eos_token_id is not None: + generation_kwargs["eos_token_id"] = self.tokenizer.eos_token_id + + if pixel_values is not None: + generation_kwargs["pixel_values"] = pixel_values + + if use_past_kv_cache: + generation_kwargs["use_cache"] = True + + # HF dict flags that trigger ModelOutput returns + hf_dict_flags = ( + "output_scores", + "output_logits", + "output_attentions", + "output_hidden_states", + ) + + # If any HF-style output flags are provided, ensure return_dict_in_generate is set + any_flag_set = False + for f in hf_dict_flags: + if generation_kwargs.get(f) is not None: + generation_kwargs[f] = bool(generation_kwargs[f]) + any_flag_set = True + + if any_flag_set: + generation_kwargs.setdefault("return_dict_in_generate", True) + + # Generate using the original HuggingFace model + with torch.no_grad(): + outputs = self.original_model.generate(input_ids, **generation_kwargs) # type: ignore[operator] + + # Check if output is a ModelOutput + try: + from transformers.utils import ModelOutput # type: ignore + + is_model_output = isinstance(outputs, ModelOutput) + except Exception: + is_model_output = False + + # Return based on return_type and input format + if return_type == "input" or return_type is None: + if input_type == "str": + # Decode the full output back to string + if is_model_output and hasattr(outputs, "sequences"): + return self.tokenizer.decode(outputs.sequences[0], skip_special_tokens=True) + return self.tokenizer.decode(outputs[0], skip_special_tokens=True) + elif input_type == "list": + # Decode each sequence in the batch + if is_model_output and hasattr(outputs, "sequences"): + return [ + self.tokenizer.decode(seq, skip_special_tokens=True) + for seq in outputs.sequences + ] + return [self.tokenizer.decode(seq, skip_special_tokens=True) for seq in outputs] + else: + # Return the full token sequence including input + return outputs + elif return_type == "tokens": + return outputs + else: + # For other return types, default to the decoded text + if input_type == "str": + if is_model_output and hasattr(outputs, "sequences"): + return self.tokenizer.decode(outputs.sequences[0], skip_special_tokens=True) + return self.tokenizer.decode(outputs[0], skip_special_tokens=True) + elif input_type == "list": + if is_model_output and hasattr(outputs, "sequences"): + return [ + self.tokenizer.decode(seq, skip_special_tokens=True) + for seq in outputs.sequences + ] + return [self.tokenizer.decode(seq, skip_special_tokens=True) for seq in outputs] + else: + return outputs + + def prepare_multimodal_inputs( + self, + text: Union[str, List[str]], + images: Optional[Any] = None, + ) -> Dict[str, torch.Tensor]: + """Prepare multimodal inputs using the model's processor. + + Converts text and images into model-ready tensors (input_ids, pixel_values, + attention_mask, etc.) using the HuggingFace processor loaded during boot(). + + Args: + text: Text prompt(s), typically containing image placeholder tokens + (e.g., "" for LLaVA). + images: PIL Image or list of PIL Images to process. Pass None for + text-only inputs on a multimodal model. + + Returns: + Dictionary with 'input_ids', 'pixel_values', 'attention_mask', etc. + All tensors are moved to the model's device. + + Raises: + ValueError: If model is not multimodal or processor is not available. + """ + if not getattr(self.cfg, "is_multimodal", False): + raise ValueError( + "prepare_multimodal_inputs() requires a multimodal model " + "(cfg.is_multimodal must be True)" + ) + if self.processor is None: + raise ValueError( + "No processor available. Load model with boot_transformers() or " + "set bridge.processor = AutoProcessor.from_pretrained(...) manually." + ) + inputs = self.processor(text=text, images=images, return_tensors="pt") + return {k: v.to(self.cfg.device) if hasattr(v, "to") else v for k, v in inputs.items()} + + def to(self, *args, **kwargs) -> "TransformerBridge": + """Move model to device and/or change dtype. + + Args: + args: Positional arguments for nn.Module.to + kwargs: Keyword arguments for nn.Module.to + print_details: Whether to print details about device/dtype changes (default: True) + + Returns: + Self for chaining + """ + # Extract print_details if provided + print_details = kwargs.pop("print_details", True) + + # Handle both device and dtype changes + # torch.nn.Module.to() supports: to(device), to(dtype), to(device, dtype), + # to(device=...), to(dtype=...), to(device=..., dtype=...) + target_device, target_dtype = None, None + + if len(args) >= 1: + first_arg = args[0] + if isinstance(first_arg, (torch.device, str)): + target_device = first_arg + elif isinstance(first_arg, torch.dtype): + target_dtype = first_arg + if len(args) >= 2: + second_arg = args[1] + if isinstance(second_arg, torch.dtype): + target_dtype = second_arg + + # these override positional args + if "device" in kwargs: + target_device = kwargs["device"] + if "dtype" in kwargs: + target_dtype = kwargs["dtype"] + + # Moving a multi-device (device_map-dispatched) model to a single device would + # collapse the split and break accelerate's hook routing. Warn and drop the + # device move; still honor dtype changes. + if target_device is not None and getattr(self.cfg, "n_devices", 1) > 1: + warnings.warn( + f"TransformerBridge.to({target_device!r}) ignored: model is dispatched " + f"across {self.cfg.n_devices} devices via device_map. Reload with " + "device=... (and no device_map/n_devices) to move to a single device.", + stacklevel=2, + ) + target_device = None + + if target_device is not None: + move_to_and_update_config(self, target_device, print_details) + if target_dtype is not None: + move_to_and_update_config(self, target_dtype, print_details) + + # Move the original model with all original args/kwargs (with print_details removed). + # When we've nulled target_device for multi-GPU safety, strip device args so the + # underlying module isn't moved either. + if target_device is None and (len(args) > 0 or "device" in kwargs): + kwargs.pop("device", None) + # Filter positional args: drop devices/strings, keep dtypes. + args = tuple(a for a in args if not isinstance(a, (torch.device, str))) + self.original_model = self.original_model.to(*args, **kwargs) + return self + + def cuda(self, device: Optional[Union[int, torch.device]] = None) -> "TransformerBridge": + """Move model to CUDA. + + Args: + device: CUDA device + + Returns: + Self for chaining + """ + if isinstance(device, int): + return self.to(f"cuda:{device}") + elif device is None: + return self.to("cuda") + else: + return self.to(device) + + def cpu(self) -> "TransformerBridge": + """Move model to CPU. + + Returns: + Self for chaining + """ + return self.to(torch.device("cpu")) + + def mps(self) -> "TransformerBridge": + """Move model to MPS. + + Returns: + Self for chaining + """ + return self.to(torch.device("mps")) + + def set_use_attn_result(self, use_attn_result: bool): + """Toggle whether to explicitly calculate and expose the result for each attention head. + + Useful for interpretability but can easily burn through GPU memory. + """ + if use_attn_result: + self._validate_attention_fork_supported("use_attn_result") + self.cfg.use_attn_result = use_attn_result + self._propagate_attention_flag("use_attn_result", use_attn_result) + + def set_use_split_qkv_input(self, use_split_qkv_input: bool): + """Toggle independent residual copies for Q/K/V so each path can be patched alone. + + Mutually exclusive with `use_attn_in` — set that flag off first if it's on. + """ + if use_split_qkv_input: + if bool(getattr(self.cfg, "use_attn_in", False)): + raise ValueError( + "use_split_qkv_input and use_attn_in are mutually exclusive. " + "Call set_use_attn_in(False) before enabling use_split_qkv_input." + ) + self._validate_attention_fork_supported("use_split_qkv_input") + self.cfg.use_split_qkv_input = use_split_qkv_input + self._propagate_attention_flag("use_split_qkv_input", use_split_qkv_input) + + def set_use_attn_in(self, use_attn_in: bool): + """Toggle a single 4D residual copy feeding all three Q/K/V projections. + + Mutually exclusive with `use_split_qkv_input` — set that flag off first + if it's on. When on, `hook_attn_in` fires at + `[batch, pos, n_heads, d_model]`, enabling coarse-grained interventions + on the residual-stream copy shared across Q/K/V. + """ + if use_attn_in: + if bool(getattr(self.cfg, "use_split_qkv_input", False)): + raise ValueError( + "use_attn_in and use_split_qkv_input are mutually exclusive. " + "Call set_use_split_qkv_input(False) before enabling use_attn_in." + ) + self._validate_attention_fork_supported("use_attn_in") + self.cfg.use_attn_in = use_attn_in + self._propagate_attention_flag("use_attn_in", use_attn_in) + + def _propagate_attention_flag(self, flag_name: str, value: bool) -> None: + """Mirror `bridge.cfg.` onto every block's attention config. + + Some adapters (Llama family) deep-copy the block template during + `setup_blocks_bridge`, cloning the attention bridge's config along + with it. Others (Pythia, GPT-2) override `__deepcopy__` to share the + config. Setting the flag only on `self.cfg` silently misses the + cloned-config case. Propagating explicitly keeps both patterns + honest — a no-op when configs are shared, a correctness fix when + they aren't. + """ + if not hasattr(self, "blocks"): + return + for block in self.blocks: + attn = block._modules.get("attn") if hasattr(block, "_modules") else None + if attn is None: + continue + attn_cfg = getattr(attn, "config", None) + if attn_cfg is not None and attn_cfg is not self.cfg: + try: + setattr(attn_cfg, flag_name, value) + except Exception: + # Some cfg objects may be frozen/immutable. Skip silently — + # the block simply won't honor the flag, which is the + # same outcome as before this fix. + pass + + def _validate_attention_fork_supported(self, flag_name: str) -> None: + """Raise / warn if the model can't honor a fine-grained attention flag. + + The post-ln1 fork path lives on JointQKVAttentionBridge and + PositionEmbeddingsAttentionBridge. Plain AttentionBridge delegates to + HF and exposes no fork point; we raise rather than setting the flag + silently. For hybrid models (some attention layers, some not), we warn + and list which layers will honor the flag. + """ + # Deferred imports: tight circular dependency with bridge setup. + from transformer_lens.model_bridge.generalized_components.joint_qkv_attention import ( + JointQKVAttentionBridge, + ) + from transformer_lens.model_bridge.generalized_components.position_embeddings_attention import ( + PositionEmbeddingsAttentionBridge, + ) + + if not hasattr(self, "blocks"): + raise NotImplementedError( + f"{flag_name}: this bridge has no `blocks` attribute, so no " + "attention bridges to apply the flag to." + ) + supported_classes = (JointQKVAttentionBridge, PositionEmbeddingsAttentionBridge) + supporting_layers: list[int] = [] + attn_classes: set[str] = set() + total_with_attn = 0 + for idx, block in enumerate(self.blocks): + attn = block._modules.get("attn") if hasattr(block, "_modules") else None + if attn is None: + continue + total_with_attn += 1 + attn_classes.add(type(attn).__name__) + if isinstance(attn, supported_classes): + supporting_layers.append(idx) + if total_with_attn == 0: + raise NotImplementedError(f"{flag_name}: no attention bridges found on self.blocks.") + if not supporting_layers: + raise NotImplementedError( + f"{flag_name}: none of this model's attention bridges support " + "the fine-grained Q/K/V hook fork. Found attention classes: " + f"{sorted(attn_classes)}. Supported classes: " + f"{[c.__name__ for c in supported_classes]}. Plain " + "AttentionBridge delegates to HuggingFace and exposes no hook " + "point before the Q/K/V projection." + ) + if len(supporting_layers) < total_with_attn: + skipped = total_with_attn - len(supporting_layers) + warnings.warn( + f"{flag_name}: {skipped} of {total_with_attn} attention layers " + "use an attention-bridge class that cannot honor this flag " + f"(attention classes present: {sorted(attn_classes)}). " + f"The flag will affect layers: {supporting_layers}.", + stacklevel=3, + ) + + def _is_valid_bridge_path(self, hf_path: str) -> bool: + """Check if a HuggingFace path corresponds to a valid bridge component. + + This validates that the path follows the bridge component structure and doesn't + contain nested HuggingFace components that should have been wrapped. + + Args: + hf_path: HuggingFace path after removing _original_component + + Returns: + True if the path is valid, False if it contains nested HF components + """ + # Split the path into parts + parts = hf_path.split(".") + + # Get the component mapping for validation + component_mapping = self.adapter.component_mapping + if not component_mapping: + return True # If no mapping, accept all keys + + # Walk through the path and check if each level is a registered bridge component + # For example, transformer.h.0.mlp.in.weight should be valid + # but transformer.h.0.mlp.c_fc.weight should be invalid (c_fc is nested HF component) + + # Start from the root + current_component = None + idx = 0 + + # Find which top-level component this belongs to + for tl_name, component in component_mapping.items(): + if component.name and hf_path.startswith(component.name + "."): + current_component = component + # Skip past the HF prefix + remaining_path = hf_path[len(component.name) + 1 :] + parts = remaining_path.split(".") + idx = 0 + break + + if current_component is None: + return True # Path doesn't match any component, let it through + + # Special handling for blocks + if hasattr(current_component, "is_list_item") and current_component.is_list_item: + # Skip the layer index + if idx < len(parts) and parts[idx].isdigit(): + idx += 1 + + # Now validate the rest of the path against submodules + while idx < len(parts): + part = parts[idx] + + # If we hit 'weight' or 'bias', we're at a parameter - this is valid + if part in ("weight", "bias"): + return True + + # Check if this part is a registered submodule + if hasattr(current_component, "submodules") and current_component.submodules: + if part in current_component.submodules: + current_component = current_component.submodules[part] + idx += 1 + continue + else: + # This part is not a registered bridge component + # It's likely a nested HF component (like c_fc, c_proj, c_attn) + return False + else: + # No submodules to check, but not at a parameter yet + # Check if next is weight/bias + if idx + 1 < len(parts) and parts[idx + 1] in ("weight", "bias"): + return True + # Otherwise this is likely a nested HF component + return False + + idx += 1 + + return True + + def _normalize_bridge_key_to_hf(self, key: str) -> str: + """Normalize a key that uses bridge attribute names to use HF module names. + + PyTorch's state_dict uses the Python attribute names (e.g., 'ln1') + but the conversion logic expects HF module names (e.g., 'ln_1'). This + function only replaces non-nested component names, leaving bridge + subcomponents (like 'in', 'out', 'q', 'k', 'v') unchanged since they're + handled by the component structure. + + Args: + key: Key that may use bridge attribute names + + Returns: + Key with attribute names replaced by module names where needed + """ + component_mapping = self.adapter.component_mapping + if not component_mapping: + return key + + # Build a mapping of only the direct module attribute names to HF names + # We only care about top-level and block-level component names, NOT subcomponents + attr_to_hf = {} + + # Map top-level components + for tl_name, component in component_mapping.items(): + if component.name and tl_name != "blocks": + # Skip if TL name is already a suffix of the HF path (avoids doubling). + if tl_name != component.name and not component.name.endswith("." + tl_name): + attr_to_hf[tl_name] = component.name + + # Map block-level components (ln1, ln2, attn, mlp) + blocks_component = component_mapping.get("blocks") + if blocks_component and hasattr(blocks_component, "submodules"): + for tl_subname, subcomponent in blocks_component.submodules.items(): + if subcomponent.name: + # Only map if the names differ (e.g., ln1 -> ln_1, but attn -> attn) + if tl_subname != subcomponent.name: + attr_to_hf[tl_subname] = subcomponent.name + + # Replace only these specific attribute names in the key + # We need to be careful to only replace whole path components, not substrings + parts = key.split(".") + result_parts = [] + + for part in parts: + if part in attr_to_hf: + result_parts.append(attr_to_hf[part]) + else: + result_parts.append(part) + + return ".".join(result_parts) + + def state_dict(self, destination=None, prefix="", keep_vars=False): + """Get state dict with TransformerLens format keys. + + Converts HuggingFace format keys to TransformerLens format and filters out + _original_component references and nested HuggingFace components. + + This returns a clean state dict with only bridge component paths converted to TL format, + excluding nested HF components (like c_fc, c_proj, c_attn) that exist inside + original_component modules. + + Args: + destination: Optional dict to store state dict in + prefix: Optional prefix to add to all keys + keep_vars: Whether to keep variables as Variables instead of tensors + + Returns: + Dict containing the state dict with TransformerLens format keys + """ + if destination is not None: + raw_state_dict = self.original_model.state_dict( + destination=destination, prefix=prefix, keep_vars=keep_vars + ) + else: + raw_state_dict = self.original_model.state_dict(prefix=prefix, keep_vars=keep_vars) + + # Clean _original_component references and convert to TL format + # Also filter out nested HuggingFace components that are wrapped by bridge components + tl_state_dict = {} + + for key, value in raw_state_dict.items(): + # Skip _original_component keys + if key == "_original_component" or key.startswith("_original_component."): + continue + + # Remove all _original_component from the key + clean_key = key.replace("._original_component", "") + + # Check if this is a valid bridge path (not a nested HF component) + if not self._is_valid_bridge_path(clean_key): + continue + + # Normalize bridge component names to HF names for conversion + # (e.g., 'ln1' -> 'ln_1', 'mlp.in' -> 'mlp.c_fc') + hf_key = self._normalize_bridge_key_to_hf(clean_key) + + # Convert to TL format - this uses the adapter's component_mapping + tl_key = self.adapter.convert_hf_key_to_tl_key(hf_key) + + # Only add if we haven't seen this TL key yet (handles duplicates) + if tl_key not in tl_state_dict: + tl_state_dict[tl_key] = value + + return tl_state_dict + + def load_state_dict(self, state_dict, strict=True, assign=False): + """Load state dict into the model, handling both clean keys and original keys with _original_component references. + + Args: + state_dict: Dictionary containing a whole state of the module + strict: Whether to strictly enforce that the keys in state_dict match the keys returned by this module's state_dict() function + assign: Whether to assign items in the state dictionary to their corresponding keys in the module instead of copying them + + Returns: + NamedTuple with missing_keys and unexpected_keys fields + """ + current_state_dict = self.original_model.state_dict() + clean_to_actual = {} + actual_to_clean = {} + for actual_key in current_state_dict.keys(): + if actual_key != "_original_component": + clean_key = actual_key.replace("._original_component", "") + clean_to_actual[clean_key] = actual_key + actual_to_clean[actual_key] = clean_key + mapped_state_dict = {} + for input_key, value in state_dict.items(): + if input_key in current_state_dict: + mapped_state_dict[input_key] = value + else: + if input_key in clean_to_actual: + actual_key = clean_to_actual[input_key] + mapped_state_dict[actual_key] = value + else: + mapped_state_dict[input_key] = value + effective_strict = strict and len(mapped_state_dict) == len(current_state_dict) + return self.original_model.load_state_dict( + mapped_state_dict, strict=effective_strict, assign=assign + ) + + def get_params(self): + """Access to model parameters in the format expected by SVDInterpreter. + + For missing weights, returns zero tensors of appropriate shape instead of raising exceptions. + This ensures compatibility across different model architectures. + + Returns: + dict: Dictionary of parameter tensors with TransformerLens naming convention + + Raises: + ValueError: If configuration is inconsistent (e.g., cfg.n_layers != len(blocks)) + """ + return get_bridge_params(self) + + # NOTE: list_supported_models and check_model_support are attached to this class + # dynamically by transformer_lens.model_bridge.sources.transformers module. + # These are HuggingFace-specific methods that belong in the transformers source module. From 2e236da53144797f1a2c8f04d4c9ea707e76f101 Mon Sep 17 00:00:00 2001 From: Jonah Larson Date: Thu, 28 May 2026 08:53:52 -0500 Subject: [PATCH 02/87] vLLM Batches (#1338) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Refactored `transformers` so that bridge configuration code is source-agnostic * Initial breakup of Bridge into core and bridge only components. Setup remote driver in prep * Initial setup of the vllm source Cherry-picked from 98bbb835 on feature/vllm-integration. Resolved against dev-4.x's Phase A type split: - bridge.py: kept the shim (TransformerBridge content lives in transformer_bridge.py now). - sources/transformers.py: accepted the scaffold's package restructure (transformers.py → transformers/{__init__,helpers,source}.py + _hf_format.py). - sources/transformers/source.py: re-applied Phase A's explicit TransformersDriver construction in boot(). sources/vllm/source.py's boot_vllm currently sets bridge._forward_impl, which Phase A removed. The function is dead until the next chunk lifts it onto VLLMDriver + RemoteBridge — landing as-is so the scaffold tree is on dev-4.x for incremental Phase B work. Co-Authored-By: Claude Opus 4.7 (1M context) * Revision round 1 of the vLLM system * Full vLLM integration initial completion * jupiter patch * dynamo patching * pass through batch size * add max model length to remote bridge boot * Updating config construction to properly pass through to vLLM * Wiring TLWorkerExtension * Additional testing + diagnostic * Another spot check * Remove logging * Fix issue with logit generation on hook_out * Adjust to use token_ids directly instead of logprobs * Updating Step 5 and driver bridge cleanup * Add decoder layer for materializing the residual stream * Updating tests due to changes on `dev` * Fix pip install path * Testing ln_final divergence * documenting differences in ln_final * Format cleanup * Initial batching test * Add version print * Setup batching for vllm --------- Co-authored-by: Claude Opus 4.7 (1M context) --- demos/vLLM_Bridge_Integration_Test.ipynb | 326 +++++++++++++----- tests/unit/model_bridge/test_vllm_driver.py | 182 ++++++++++ .../unit/model_bridge/test_vllm_internals.py | 51 +++ .../test_vllm_worker_extension.py | 155 +++++++++ .../model_bridge/sources/vllm/driver.py | 123 +++++++ .../model_bridge/sources/vllm/internals.py | 48 +++ .../model_bridge/sources/vllm/plugin.py | 169 ++++++--- .../model_bridge/sources/vllm/source.py | 13 + .../sources/vllm/worker_extension.py | 78 ++++- 9 files changed, 1008 insertions(+), 137 deletions(-) create mode 100644 tests/unit/model_bridge/test_vllm_internals.py create mode 100644 tests/unit/model_bridge/test_vllm_worker_extension.py diff --git a/demos/vLLM_Bridge_Integration_Test.ipynb b/demos/vLLM_Bridge_Integration_Test.ipynb index e712db1c41..106ec40986 100644 --- a/demos/vLLM_Bridge_Integration_Test.ipynb +++ b/demos/vLLM_Bridge_Integration_Test.ipynb @@ -4,15 +4,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# vLLM Bridge \u2014 Integration Test\n", + "# vLLM Bridge — Integration Test\n", "\n", "End-to-end validation of `boot_vllm` on a real GPU. Runs the v4 Driver protocol's vLLM backend against `meta-llama/Llama-3.2-1B`, compares per-hook activations and next-token argmax against the HF transformers backend, and exercises the affine intervention path.\n", "\n", - "**Branch:** `dev-4.x`. **Hardware:** any CUDA GPU with \u226510 GB VRAM (free Colab T4 is sufficient).\n", + "**Branch:** `dev-4.x`. **Hardware:** any CUDA GPU with ≥10 GB VRAM (free Colab T4 is sufficient).\n", "\n", "## Scope: observation + mutation\n", "\n", - "This source extends past vllm-lens's observation-only scope. Each capture hook applies an affine transform (`output = output * scale + bias`, default identity) and returns the modified tensor, so interventions propagate to downstream layers. Step 6 below is the load-bearing verification that this works end-to-end under `torch.compile` + CUDA graphs \u2014 unit tests can't reach the compiled-graph path.\n", + "This source extends past vllm-lens's observation-only scope. Each capture hook applies an affine transform (`output = output * scale + bias`, default identity) and returns the modified tensor, so interventions propagate to downstream layers. Step 6 below is the load-bearing verification that this works end-to-end under `torch.compile` + CUDA graphs — unit tests can't reach the compiled-graph path.\n", "\n", "## What this validates\n", "1. `boot_vllm` returns a `RemoteBridge` end-to-end.\n", @@ -30,8 +30,8 @@ "source": [ "## Setup\n", "\n", - "1. **Runtime \u2192 Change runtime type \u2192 GPU** (T4 / L4 / A100 all work).\n", - "2. **Secrets \u2192 add `HF_TOKEN`** with a token that has access to `meta-llama/Llama-3.2-1B` (gated).\n", + "1. **Runtime → Change runtime type → GPU** (T4 / L4 / A100 all work).\n", + "2. **Secrets → add `HF_TOKEN`** with a token that has access to `meta-llama/Llama-3.2-1B` (gated).\n", "\n", "The environment cell below patches `sys.stdout.fileno` because ipykernel's captured stdout doesn't expose a real file descriptor and vLLM's worker init calls `fileno()`. Without the patch, Step 2 fails with `UnsupportedOperation: fileno`." ], @@ -42,14 +42,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "# Install vllm and TransformerLens @ feature/driver-system. ~3-5 minutes.\n", - "# vllm pinned to 0.20.2 \u2014 the version the internal-API walks in\n", - "# transformer_lens/model_bridge/sources/vllm/{plugin,internals}.py were validated against.\n", - "# vLLM rearranges its internal class paths every 4-6 weeks; re-validate before bumping.\n", - "%pip install -q \"vllm==0.20.2\"\n", - "%pip install -q git+https://github.com/TransformerLensOrg/TransformerLens.git@feature/driver-system" - ], + "source": "# Install vllm and TransformerLens @ feature/vllm-batched. ~3-5 minutes.\n# Branch must match this notebook: feature/vllm-batched has the hook-fire counter\n# (Step 7) and the batched capture surface (Step 10). feature/driver-system lacks\n# both, so collective_rpc raises NotImplementedError on tl_reset_counter / the\n# batched RPCs.\n# vllm pinned to 0.20.2 — the version the internal-API walks in\n# transformer_lens/model_bridge/sources/vllm/{plugin,internals}.py were validated against.\n# vLLM rearranges its internal class paths every 4-6 weeks; re-validate before bumping.\n%pip install -q \"vllm==0.20.2\"\n%pip install -q git+https://github.com/TransformerLensOrg/TransformerLens.git@feature/vllm-batched", "id": "4ab1eb60e6b1" }, { @@ -57,45 +50,14 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "import gc\n", - "import os\n", - "import sys\n", - "\n", - "import torch\n", - "\n", - "# HF_TOKEN comes from Colab secrets. Falls back to env var for non-Colab runs.\n", - "try:\n", - " from google.colab import userdata\n", - " os.environ.setdefault(\"HF_TOKEN\", userdata.get(\"HF_TOKEN\"))\n", - "except (ImportError, Exception):\n", - " pass\n", - "assert os.environ.get(\"HF_TOKEN\"), \"Set HF_TOKEN in Colab Secrets (gear icon, left sidebar).\"\n", - "\n", - "# Colab/Jupyter compatibility: ipykernel's stdout doesn't expose a fileno();\n", - "# vLLM's worker init calls sys.stdout.fileno() during parallel-state setup\n", - "# and crashes with UnsupportedOperation: fileno. Patch fileno to return the\n", - "# underlying process FDs (1, 2) \u2014 Colab writes back to those anyway.\n", - "if \"ipykernel\" in sys.modules:\n", - " sys.stdout.fileno = lambda: 1 # type: ignore[method-assign]\n", - " sys.stderr.fileno = lambda: 2 # type: ignore[method-assign]\n", - "\n", - "MODEL = \"meta-llama/Llama-3.2-1B\"\n", - "PROMPT = \"The quick brown fox jumps over the\"\n", - "DTYPE = torch.float16\n", - "torch.manual_seed(0)\n", - "\n", - "print(f\"CUDA available: {torch.cuda.is_available()}\")\n", - "print(f\"Device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU only \u2014 abort'}\")\n", - "assert torch.cuda.is_available(), \"GPU runtime required.\"" - ], + "source": "import gc\nimport os\nimport sys\n\nimport torch\n\n# HF_TOKEN comes from Colab secrets. Falls back to env var for non-Colab runs.\ntry:\n from google.colab import userdata\n os.environ.setdefault(\"HF_TOKEN\", userdata.get(\"HF_TOKEN\"))\nexcept (ImportError, Exception):\n pass\nassert os.environ.get(\"HF_TOKEN\"), \"Set HF_TOKEN in Colab Secrets (gear icon, left sidebar).\"\n\n# Colab/Jupyter compatibility: ipykernel's stdout doesn't expose a fileno();\n# vLLM's worker init calls sys.stdout.fileno() during parallel-state setup\n# and crashes with UnsupportedOperation: fileno. Patch fileno to return the\n# underlying process FDs (1, 2) — Colab writes back to those anyway.\nif \"ipykernel\" in sys.modules:\n sys.stdout.fileno = lambda: 1 # type: ignore[method-assign]\n sys.stderr.fileno = lambda: 2 # type: ignore[method-assign]\n\n# Read the installed vllm version from package metadata, NOT `import vllm` —\n# importing vllm loads its CUDA C extension (vllm._C), which is exactly what\n# fails with `libcudart.so.NN not found` when a newer wheel built for a CUDA\n# version Colab doesn't ship gets installed. Metadata read works regardless.\nfrom importlib.metadata import PackageNotFoundError\nfrom importlib.metadata import version as _pkg_version\n\n_PINNED_VLLM = \"0.20.2\"\ntry:\n _vllm_ver = _pkg_version(\"vllm\")\n print(f\"vllm version: {_vllm_ver}\")\n if _vllm_ver != _PINNED_VLLM:\n print(\n f\"⚠ expected vllm=={_PINNED_VLLM} (the version the capture plugin is validated \"\n f\"against); got {_vllm_ver}. Newer wheels target a CUDA the Colab image may not \"\n \"ship (→ libcudart.so error at boot) and may move vLLM internals the plugin walks. \"\n f\"Re-pin with %pip install -q 'vllm=={_PINNED_VLLM}' and restart the runtime.\"\n )\nexcept PackageNotFoundError:\n print(\"⚠ vllm is not installed — run the install cell above.\")\n\nMODEL = \"meta-llama/Llama-3.2-1B\"\nPROMPT = \"The quick brown fox jumps over the\"\nDTYPE = torch.float16\ntorch.manual_seed(0)\n\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nprint(f\"Device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU only — abort'}\")\nassert torch.cuda.is_available(), \"GPU runtime required.\"", "id": "1748b73ca8b0" }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Step 1 \u2014 HF reference\n", + "## Step 1 — HF reference\n", "\n", "Boot the transformers backend first, capture activations and argmax, then drop it so vLLM has the GPU to itself." ], @@ -132,7 +94,7 @@ "\n", "cache_hf_cpu = {name: t.detach().cpu().clone() for name, t in cache_hf.cache_dict.items()}\n", "next_token_hf = bridge_hf.tokenizer.decode([argmax_hf])\n", - "print(f\"HF argmax token id: {argmax_hf} \u2192 {next_token_hf!r}\")\n", + "print(f\"HF argmax token id: {argmax_hf} → {next_token_hf!r}\")\n", "print(f\"HF cache: {len(cache_hf_cpu)} entries (filtered to overlay's fireable set)\")\n", "\n", "# Move parameters to CPU before deletion to force release even if a reference\n", @@ -148,7 +110,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Step 2 \u2014 Boot vLLM bridge\n", + "## Step 2 — Boot vLLM bridge\n", "\n", "`boot_vllm` constructs the LLM, monkey-patches `Worker.load_model` pre-compile to install capture hooks, then wraps it in a `RemoteBridge`." ], @@ -163,7 +125,7 @@ "from transformer_lens.model_bridge.remote_bridge import RemoteBridge\n", "\n", "# max_model_len=2048 caps the KV cache reservation. Llama-3.2-1B's native\n", - "# context is 131072 (128k) \u2014 the default reservation is ~4 GiB and overshoots\n", + "# context is 131072 (128k) — the default reservation is ~4 GiB and overshoots\n", "# the free T4 budget. The test prompt is ~10 tokens, so 2048 is plenty.\n", "bridge = RemoteBridge.boot_vllm(\n", " MODEL,\n", @@ -182,9 +144,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Step 3 \u2014 Capture pipeline\n", + "## Step 3 — Capture pipeline\n", "\n", - "Run a single forward, populate the cache via `collective_rpc \u2192 tl_read_captures`, and sanity-check shapes." + "Run a single forward, populate the cache via `collective_rpc → tl_read_captures`, and sanity-check shapes." ], "id": "284538a2c556" }, @@ -199,7 +161,7 @@ "argmax_vllm = int(logits_vllm[0, -1].argmax().item())\n", "next_token_vllm = bridge.tokenizer.decode([argmax_vllm])\n", "\n", - "print(f\"vLLM argmax token id: {argmax_vllm} \u2192 {next_token_vllm!r}\")\n", + "print(f\"vLLM argmax token id: {argmax_vllm} → {next_token_vllm!r}\")\n", "print(f\"vLLM cache entries: {len(cache_vllm.cache_dict)}\")\n", "\n", "for name in sorted(bridge._driver.supported_hook_points)[:5]:\n", @@ -213,7 +175,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Step 4 \u2014 Greedy parity \n", + "## Step 4 — Greedy parity \n", "\n", "vLLM and HF must produce the same next-token argmax on the same prompt." ], @@ -226,9 +188,9 @@ "outputs": [], "source": [ "parity = argmax_vllm == argmax_hf\n", - "status = \"\u2705 PASS\" if parity else \"\u274c FAIL\"\n", - "print(f\"{status}: HF\u2192{argmax_hf} ({next_token_hf!r}) vs vLLM\u2192{argmax_vllm} ({next_token_vllm!r})\")\n", - "assert parity, \"Greedy parity failed \u2014 kernel divergence or overlay misconfiguration.\"" + "status = \"✅ PASS\" if parity else \"❌ FAIL\"\n", + "print(f\"{status}: HF→{argmax_hf} ({next_token_hf!r}) vs vLLM→{argmax_vllm} ({next_token_vllm!r})\")\n", + "assert parity, \"Greedy parity failed — kernel divergence or overlay misconfiguration.\"" ], "id": "a59824b0e3be" }, @@ -236,11 +198,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Step 5 \u2014 Per-hook L2 (acceptance gate)\n", + "## Step 5 — Per-hook L2 (acceptance gate)\n", "\n", "Target is relative L2 < 5e-3 in fp16 for every fireable hook. One hook remains exempted from the strict gate:\n", "\n", - "- **`ln_final.hook_normalized`** \u2014 vLLM's `model.norm` is invoked as part of the fused-residual norm kernel and the captured value scales ~2\u00d7 HF's. Open investigation (likely a residual-fusion semantic discrepancy at the model boundary rather than a real divergence; argmax + downstream parity work correctly).\n", + "- **`ln_final.hook_normalized`** — vLLM's `model.norm` is invoked as part of the fused-residual norm kernel and the captured value scales ~2× HF's. Open investigation (likely a residual-fusion semantic discrepancy at the model boundary rather than a real divergence; argmax + downstream parity work correctly).\n", "\n", "All other fireable hooks (including `blocks.{i}.hook_out`, which is now materialized from vLLM's `(mlp_delta, residual)` tuple) must be within target." ], @@ -272,21 +234,21 @@ " base = t_hf.norm().item() or 1.0\n", " rel = diff / base\n", " if rel < TARGET_REL_L2:\n", - " mark = \"\u2705\"\n", + " mark = \"✅\"\n", " note = \"\"\n", " elif name in _RESIDUAL_FUSION_DIVERGENT:\n", - " mark = \"\u26a0\"\n", + " mark = \"⚠\"\n", " note = \"residual-fusion (expected)\"\n", " else:\n", - " mark = \"\u274c\"\n", + " mark = \"❌\"\n", " note = \"\"\n", " rows.append((name, rel, mark, note))\n", "\n", "print(f\"{'hook':<40} {'rel L2':>10} status note\")\n", "print(\"-\" * 80)\n", "for name, rel, mark, note in rows:\n", - " rel_str = f\"{rel:.3e}\" if isinstance(rel, float) else \"\u2014\"\n", - " print(f\"{name:<40} {rel_str:>10} {mark or '\u2014':<6} {note}\")\n", + " rel_str = f\"{rel:.3e}\" if isinstance(rel, float) else \"—\"\n", + " print(f\"{name:<40} {rel_str:>10} {mark or '—':<6} {note}\")\n", "\n", "# Strict gate: only the non-residual-fusion hooks need to be within target.\n", "failed = [(name, rel) for name, rel, _, _ in rows\n", @@ -301,7 +263,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Step 6 \u2014 Intervention smoke (load-bearing)\n", + "## Step 6 — Intervention smoke (load-bearing)\n", "\n", "**This cell is the load-bearing verification for the vLLM source's mutation claim.** Unit tests only exercise the dispatch protocol (mocked LLM); they cannot prove that the affine math (`output = output * scale + bias`) traces correctly through `torch.compile` and propagates to downstream layers under CUDA-graph replay. That guarantee depends on this cell passing.\n", "\n", @@ -315,41 +277,102 @@ "metadata": {}, "outputs": [], "source": [ - "# Suppress (zero) the embedding output. Forward should behave very differently.\n", + "# (a) suppress: zero the embedding output. Multiplicative op (scale=0).\n", "logits_suppressed, cache_suppressed = bridge.run_with_cache(\n", " tokens,\n", " intervene={\"embed.hook_out\": {\"op\": \"suppress\"}},\n", ")\n", "argmax_suppressed = int(logits_suppressed[0, -1].argmax().item())\n", - "\n", - "# Embed cache should be all zeros after suppress.\n", "embed_norm = cache_suppressed[\"embed.hook_out\"].abs().max().item()\n", - "print(f\"Embed |max| after suppress: {embed_norm:.6f} (should be 0.0)\")\n", - "assert embed_norm == 0.0, \"Suppress did not zero embed.hook_out \u2014 intervention path broken.\"\n", - "\n", - "# Argmax should differ from the clean run.\n", - "argmax_shifted = argmax_suppressed != argmax_vllm\n", - "print(f\"Clean argmax: {argmax_vllm} Suppressed argmax: {argmax_suppressed} Shifted: {argmax_shifted}\")\n", + "print(f\"suppress: embed |max|={embed_norm:.6f} (→0) argmax {argmax_vllm} → {argmax_suppressed}\")\n", + "assert embed_norm == 0.0, \"suppress did not zero embed.hook_out\"\n", + "assert argmax_suppressed != argmax_vllm, \"suppress did not change the prediction\"\n", + "\n", + "# (b) set: force embed to a constant vector. Additive op (scale=0, bias=value).\n", + "d_model = bridge.cfg.d_model\n", + "set_value = [0.5] * d_model\n", + "logits_set, cache_set = bridge.run_with_cache(\n", + " tokens,\n", + " intervene={\"embed.hook_out\": {\"op\": \"set\", \"value\": set_value}},\n", + ")\n", + "set_max = cache_set[\"embed.hook_out\"].abs().max().item()\n", + "print(f\"set: embed |max|={set_max:.6f} (→0.5) argmax {argmax_vllm} → {int(logits_set[0, -1].argmax().item())}\")\n", + "assert abs(set_max - 0.5) < 1e-2, \"set did not write the constant value to embed.hook_out\"\n", "\n", - "# Verify the next forward (no intervene) reverts \u2014 interventions are not sticky.\n", + "# (c) reset: a clean forward must revert — interventions are not sticky.\n", "logits_revert, _ = bridge.run_with_cache(tokens)\n", "argmax_revert = int(logits_revert[0, -1].argmax().item())\n", - "print(f\"Revert argmax: {argmax_revert} matches clean: {argmax_revert == argmax_vllm}\")\n", - "assert argmax_revert == argmax_vllm, \"Intervention persisted across calls \u2014 reset path broken.\"" + "print(f\"reset: argmax → {argmax_revert} matches clean: {argmax_revert == argmax_vllm}\")\n", + "assert argmax_revert == argmax_vllm, \"intervention persisted across calls — reset path broken\"" ], "id": "8c94e6f538f4" }, + { + "cell_type": "markdown", + "id": "a21f3155a525", + "metadata": {}, + "source": [ + "## Step 7 — Hook fires exactly once\n", + "\n", + "A correctness gate for the capture mechanism: under `torch.compile` + CUDA graphs, each capture hook must fire exactly once per forward. A double-fire would silently overwrite the buffer (and with multi-token decode, write the wrong step's activation). The plugin increments a shared GPU counter on every hook fire; after one forward it should equal the number of installed hooks." + ] + }, + { + "cell_type": "code", + "id": "53ee0d1f31e7", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "bridge._driver._llm.collective_rpc(\"tl_reset_counter\")\n", + "bridge.run_with_cache(tokens)\n", + "fire_count = bridge._driver._llm.collective_rpc(\"tl_read_counter\")[0]\n", + "expected = len(bridge._driver.supported_hook_points)\n", + "print(f\"Hook fires: {fire_count} expected (one per hook): {expected}\")\n", + "assert fire_count == expected, (\n", + " f\"Expected {expected} fires (one per installed hook), got {fire_count} — \"\n", + " \"a hook fired more than once under compile, or some didn't fire.\"\n", + ")\n", + "print(\"✅ every capture hook fired exactly once.\")" + ] + }, + { + "cell_type": "markdown", + "id": "2795d60c55a9", + "metadata": {}, + "source": [ + "## Step 8 — Stream safety\n", + "\n", + "Capturing the same prompt repeatedly must be deterministic — back-to-back forwards through the compiled graph should produce bitwise-identical captures. Drift here would indicate a stale-buffer race or non-deterministic kernel path in the capture machinery." + ] + }, + { + "cell_type": "code", + "id": "8213e889960f", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "refs = []\n", + "for _ in range(5):\n", + " _, cache_i = bridge.run_with_cache(tokens)\n", + " refs.append(cache_i[\"embed.hook_out\"].detach().cpu().clone())\n", + "for i, c in enumerate(refs[1:], start=1):\n", + " assert torch.equal(refs[0], c), f\"Run {i} differs from run 0 — non-deterministic capture\"\n", + "print(\"✅ 5 back-to-back captures are bitwise identical.\")" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Step 7 \u2014 Lifetime\n", + "## Step 9 — Lifetime\n", "\n", "`bridge.close()` is responsible for releasing **our** resources:\n", "- detaches the per-Worker capture hooks via `tl_remove_hooks`\n", "- tears down vLLM's distributed environment (`destroy_distributed_environment`)\n", "\n", - "It cannot release **vLLM's** internal state in 0.20.2 \u2014 there's no `LLM.shutdown()` API. Model weights, KV cache pool, and Inductor compile cache stay resident in PyTorch's caching allocator until process exit. A clean Colab `Runtime \u2192 Restart session` is the only way to fully reclaim GPU memory.\n", + "It cannot release **vLLM's** internal state in 0.20.2 — there's no `LLM.shutdown()` API. Model weights, KV cache pool, and Inductor compile cache stay resident in PyTorch's caching allocator until process exit. A clean Colab `Runtime → Restart session` is the only way to fully reclaim GPU memory.\n", "\n", "What we *can* verify: `bridge.close()` ran without error and didn't leak more memory. The residual is informational, not a gate." ], @@ -371,31 +394,154 @@ "print(f\"GPU memory released: {released:.2f} GB\")\n", "print()\n", "if released > 0.05:\n", - " print(f\"\u2705 close() released ~{released:.2f} GB (hooks + capture buffers + distributed state).\")\n", + " print(f\"✅ close() released ~{released:.2f} GB (hooks + capture buffers + distributed state).\")\n", "else:\n", - " print(\"\u26a0 close() released < 50 MB \u2014 likely vLLM's model weights and KV cache pool\")\n", + " print(\"⚠ close() released < 50 MB — likely vLLM's model weights and KV cache pool\")\n", " print(\" staying resident. This is expected on vLLM 0.20.2; restart the runtime\")\n", " print(\" for a clean GPU.\")" ], "id": "51dbe16a57b2" }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Step 10 — Batched capture (`enable_batching=True`)\n\nThe default path above is compile-validated but single-prompt. The throughput\npath for SAE/probe data collection is `enable_batching=True`: `enforce_eager`\n(no torch.compile), per-request accumulation across chunked prefill, and\n`batch_size > 1`.\n\n> **⚠ Restart the runtime, and use a GPU bigger than a T4, before running this\n> section.** Two reasons:\n> 1. It boots a *second* vLLM engine, and vLLM 0.20.2 can't release the first\n> in-process (no shutdown API; weights + KV pool stay resident — see Step 9).\n> 2. The multi-prompt batched forward **deadlocked on a 16 GB T4** under memory\n> pressure; an L4/A100 ran 10a–10c clean. Larger GPUs (compute ≥ 8) also use\n> the FlashAttention backend, which exposes query offsets more directly than\n> the T4's FlashInfer fallback.\n>\n> **Runtime → Restart session**, switch to L4/A100, then run **only**: the\n> install cell, the Setup cell, and this Step 10 section (each Step 10 cell\n> re-imports what it needs — you do not need to re-run Steps 1–9).", + "id": "5ff55aa42ae6" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "# Self-contained: re-imports RemoteBridge so this section runs standalone after a\n# runtime restart (only the Setup cell needs to have run first, for MODEL/DTYPE).\nfrom transformer_lens.model_bridge.remote_bridge import RemoteBridge\n\nbridge_b = RemoteBridge.boot_vllm(\n MODEL,\n dtype=DTYPE,\n gpu_memory_utilization=0.5,\n max_model_len=2048,\n # Small cap so a long prompt below forces chunked prefill (multi-forward).\n max_num_batched_tokens=512,\n enable_batching=True,\n)\nprint(f\"Batched bridge booted (enforce_eager). Fireable hooks: {len(bridge_b._driver.supported_hook_points)}\")\nSAMPLE_HOOK = sorted(bridge_b._driver.supported_hook_points)[0]\nprint(f\"Sampling correctness on hook: {SAMPLE_HOOK!r}\")", + "id": "e81f834a7b60" + }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Summary\n", + "### 10a — Batch parity\n", "\n", - "If all asserts above passed, the v4 Driver-protocol vLLM backend is sound on this architecture:\n", + "Each row's real-token activations must equal the same prompt run alone. vLLM\n", + "computes each request independently under PagedAttention, so right-padding the\n", + "batch is a pure cache-assembly artifact — real tokens are padding-independent." + ], + "id": "5f6cae085def" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "PROMPTS = [\"The capital of France is\", \"The quick brown fox jumps over the\", \"Hello\"]\n", + "batch_tokens = [bridge_b.to_tokens(p)[0].tolist() for p in PROMPTS]\n", "\n", - "- `boot_vllm` returns `RemoteBridge` end-to-end.\n", - "- `collective_rpc \u2192 tl_read_captures` populates the cache.\n", - "- Greedy argmax matches HF (greedy parity).\n", - "- Per-hook L2 < 5e-3 vs HF across every fireable hook (hook-fidelity gate).\n", - "- Affine interventions (suppress / scale / add / set) apply per-forward and reset cleanly.\n", - "- GPU lifetime is well-behaved.\n", + "_, cache_batch = bridge_b.run_with_cache(batch_tokens)\n", + "print(f\"batched cache '{SAMPLE_HOOK}' shape: {tuple(cache_batch[SAMPLE_HOOK].shape)} (batch, max_seq, d_model)\")\n", "\n", - "Next: extend to other architectures (Qwen / Mistral / Gemma) via the `DecoderOnlyOverlay`, or add `clamp` to the intervention vocabulary." + "TARGET_REL_L2 = 5e-3\n", + "for k, ids in enumerate(batch_tokens):\n", + " L = len(ids)\n", + " _, cache_single = bridge_b.run_with_cache([ids])\n", + " bt = cache_batch[SAMPLE_HOOK][k, :L].float()\n", + " st = cache_single[SAMPLE_HOOK][0, :L].float()\n", + " rel = (bt - st).norm().item() / (st.norm().item() or 1.0)\n", + " print(f\" row {k} ({L:>2} tok): rel L2 = {rel:.3e}\")\n", + " assert rel < TARGET_REL_L2, f\"row {k} batched != single (rel {rel:.3e})\"\n", + " # Pad positions past the real length must be zero.\n", + " assert torch.equal(cache_batch[SAMPLE_HOOK][k, L:], torch.zeros_like(cache_batch[SAMPLE_HOOK][k, L:]))\n", + "print(\"✅ batched rows match single-prompt runs; pad positions zeroed.\")" ], + "id": "0ea345b5b8fc" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 10b — Chunked-prefill accumulation\n", + "\n", + "A prompt longer than `max_num_batched_tokens` splits across forwards. The eager\n", + "hook accumulates per-request slices and `torch.cat`s them — a single-buffer\n", + "overwrite (the compiled design) would keep only the last chunk. Assert the\n", + "captured sequence length equals the full prompt length." + ], + "id": "808190b7edfc" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "base_ids = bridge_b.to_tokens(\"The quick brown fox jumps over the lazy dog. \")[0].tolist()\n", + "long_ids = (base_ids * 60)[:600] # > max_num_batched_tokens=512 → forces chunking\n", + "print(f\"prompt length {len(long_ids)} tok > max_num_batched_tokens=512\")\n", + "\n", + "_, cache_long = bridge_b.run_with_cache([long_ids])\n", + "seq = cache_long[SAMPLE_HOOK].shape[1]\n", + "print(f\"captured seq length: {seq}\")\n", + "assert seq == len(long_ids), f\"chunked prefill lost tokens: captured {seq} of {len(long_ids)} — accumulation broken\"\n", + "assert torch.isfinite(cache_long[SAMPLE_HOOK]).all(), \"non-finite values in accumulated capture\"\n", + "print(\"✅ chunked-prefill accumulation reconstructs the full sequence.\")" + ], + "id": "c5ede5236742" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 10c — Batched interventions (global)\n", + "\n", + "Interventions in batched mode are global across the batch. A `suppress` on the\n", + "embedding shifts every row's prediction; a clean batch reverts." + ], + "id": "97f65eadc7ad" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "logits_clean, _ = bridge_b.run_with_cache(batch_tokens)\n", + "clean_argmax = [int(logits_clean[k, len(ids) - 1].argmax().item()) for k, ids in enumerate(batch_tokens)]\n", + "\n", + "logits_supp, cache_supp = bridge_b.run_with_cache(\n", + " batch_tokens, intervene={\"embed.hook_out\": {\"op\": \"suppress\"}}\n", + ")\n", + "supp_argmax = [int(logits_supp[k, len(ids) - 1].argmax().item()) for k, ids in enumerate(batch_tokens)]\n", + "embed_max = cache_supp[\"embed.hook_out\"].abs().max().item()\n", + "print(f\"clean argmax: {clean_argmax}\")\n", + "print(f\"suppress argmax: {supp_argmax} embed |max|={embed_max:.6f} (→0)\")\n", + "assert embed_max == 0.0, \"suppress did not zero embed across the batch\"\n", + "assert supp_argmax != clean_argmax, \"suppress did not shift any row's prediction\"\n", + "\n", + "logits_revert, _ = bridge_b.run_with_cache(batch_tokens)\n", + "revert_argmax = [int(logits_revert[k, len(ids) - 1].argmax().item()) for k, ids in enumerate(batch_tokens)]\n", + "print(f\"revert argmax: {revert_argmax} matches clean: {revert_argmax == clean_argmax}\")\n", + "assert revert_argmax == clean_argmax, \"intervention persisted across batched calls\"\n", + "print(\"✅ batched interventions apply globally and reset cleanly.\")" + ], + "id": "b58d8362010e" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bridge_b.close()\n", + "del bridge_b, cache_batch, cache_long, logits_clean, logits_supp, cache_supp, logits_revert\n", + "gc.collect(); torch.cuda.empty_cache()\n", + "print(f\"GPU memory after batched-bridge close: {torch.cuda.memory_allocated() / 1e9:.2f} GB\")" + ], + "id": "aac4cf7b621d" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Summary\n\nIf all asserts above passed, the v4 Driver-protocol vLLM backend is sound on this architecture:\n\n- `boot_vllm` returns `RemoteBridge` end-to-end.\n- `collective_rpc → tl_read_captures` populates the cache.\n- Greedy argmax matches HF (greedy parity).\n- Per-hook L2 < 5e-3 vs HF across every fireable hook (hook-fidelity gate; `ln_final.hook_normalized` exempted — see overlay docstring).\n- Affine interventions (suppress / scale / add / set) apply per-forward and reset cleanly.\n- Each hook fires exactly once per forward under compile (no double-fire / silent overwrite).\n- Back-to-back captures are deterministic (stream-safe).\n- GPU lifetime behaves as expected for vLLM 0.20.x (full release needs a runtime restart).\n- **Batched mode** (`enable_batching=True`, Step 10): `batch_size > 1` parity vs single-prompt runs, chunked-prefill accumulation reconstructs full sequences, and global interventions apply + reset across the batch.\n\nKnown divergences (documented, not bugs): `ln_final.hook_normalized` post-weight vs pre-weight, Gemma `embed.hook_out` scaling. Both reconciled by conversions noted in `sources.vllm.overlays.decoder_only`.\n\nOut of scope: multi-token generation (`max_new_tokens > 1`), per-request (non-global) batched interventions, tensor/pipeline parallelism.", "id": "fedb17c9a539" } ], @@ -417,4 +563,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/tests/unit/model_bridge/test_vllm_driver.py b/tests/unit/model_bridge/test_vllm_driver.py index 8a46a63938..964a761116 100644 --- a/tests/unit/model_bridge/test_vllm_driver.py +++ b/tests/unit/model_bridge/test_vllm_driver.py @@ -243,6 +243,188 @@ def test_forward_rejects_prompt_exceeding_buffer(self): _driver(captures={}, max_num_batched_tokens=4).forward(torch.tensor([[1, 2, 3, 4, 5]])) +def _batched_request_output(request_id, generated_token=None, top_logprobs=None): + """RequestOutput-shaped mock carrying a request_id for the accumulator join.""" + completion = MagicMock() + completion.token_ids = [generated_token] if generated_token is not None else [] + completion.logprobs = ( + [{tid: MagicMock(logprob=lp) for tid, lp in top_logprobs.items()}] + if top_logprobs is not None + else [] + ) + ro = MagicMock() + ro.request_id = request_id + ro.outputs = [completion] + return ro + + +def _batched_driver(*, outputs, captures_by_req, hf_config=None) -> VLLMDriver: + """Batched-mode VLLMDriver. ``outputs`` is the llm.generate return (in submission + order, each carrying .request_id); ``captures_by_req`` is the + tl_read_batched_captures payload keyed by req_id.""" + llm = MagicMock() + llm.generate = MagicMock(return_value=outputs) + # collective_rpc("tl_read_batched_captures")[0] is the only indexed call; the + # reset/set calls ignore the return value. + llm.collective_rpc = MagicMock(return_value=[captures_by_req]) + return VLLMDriver( + llm=llm, + adapter=_adapter(), + tokenizer=None, + overlay=_overlay(), + hf_config=hf_config or _hf_config(), + max_num_batched_tokens=2048, + enable_batching=True, + ) + + +class TestNormalizeInputIdsBatched: + """_normalize_input_ids_batched accepts tensors, flat lists, and ragged lists.""" + + def test_1d_tensor_is_single_prompt(self): + assert VLLMDriver._normalize_input_ids_batched(torch.tensor([1, 2, 3])) == [[1, 2, 3]] + + def test_2d_tensor_is_per_row(self): + assert VLLMDriver._normalize_input_ids_batched(torch.tensor([[1, 2], [3, 4]])) == [ + [1, 2], + [3, 4], + ] + + def test_flat_list_is_single_prompt(self): + assert VLLMDriver._normalize_input_ids_batched([1, 2, 3]) == [[1, 2, 3]] + + def test_ragged_list_of_lists_preserved(self): + assert VLLMDriver._normalize_input_ids_batched([[1, 2, 3], [4, 5]]) == [[1, 2, 3], [4, 5]] + + +class TestBatchedForward: + """Batched path: req_id join, right-padded assembly, per-row logit synthesis.""" + + def test_req_id_join_not_positional(self): + """Accumulator keys are out-of-order req_ids; join via outputs[k].request_id.""" + pytest.importorskip("vllm") + outputs = [ + _batched_request_output("req-A", top_logprobs={7: 2.0}), + _batched_request_output("req-B", top_logprobs={3: 2.0}), + ] + # Captures keyed by req_id, deliberately reverse insertion order. + captures_by_req = { + "req-B": {"embed.hook_out": torch.full((2, 4), 2.0)}, + "req-A": {"embed.hook_out": torch.full((3, 4), 1.0)}, + } + result = _batched_driver(outputs=outputs, captures_by_req=captures_by_req).forward( + [[1, 2, 3], [4, 5]] + ) + emb = result.captured["embed.hook_out"] + assert tuple(emb.shape) == (2, 3, 4) # (batch, max_seq, width) + # Row 0 = req-A (3 real tokens, value 1.0); row 1 = req-B (2 real, value 2.0). + assert torch.equal(emb[0, :3], torch.full((3, 4), 1.0)) + assert torch.equal(emb[1, :2], torch.full((2, 4), 2.0)) + + def test_join_strips_internal_req_id_suffix(self): + """vLLM keys the worker accumulator by f'{public}-{hash}'; RequestOutput + carries only the public id. Join must match exact-or-prefix.""" + pytest.importorskip("vllm") + outputs = [ + _batched_request_output("10", top_logprobs={7: 2.0}), + _batched_request_output("11", top_logprobs={3: 2.0}), + ] + # Worker keys carry the engine-internal "-" suffix. + captures_by_req = { + "10-83c3532c": {"embed.hook_out": torch.full((3, 4), 1.0)}, + "11-a1b2c3d4": {"embed.hook_out": torch.full((2, 4), 2.0)}, + } + result = _batched_driver(outputs=outputs, captures_by_req=captures_by_req).forward( + [[1, 2, 3], [4, 5]] + ) + emb = result.captured["embed.hook_out"] + assert tuple(emb.shape) == (2, 3, 4) + assert torch.equal(emb[0, :3], torch.full((3, 4), 1.0)) + assert torch.equal(emb[1, :2], torch.full((2, 4), 2.0)) + + def test_join_prefix_does_not_collide_on_numeric_ids(self): + """Public '1' must not match worker key '10-...'; the '-' delimiter guards it.""" + pytest.importorskip("vllm") + outputs = [ + _batched_request_output("1", top_logprobs={7: 2.0}), + _batched_request_output("10", top_logprobs={3: 2.0}), + ] + captures_by_req = { + "1-aaaaaaaa": {"embed.hook_out": torch.full((2, 4), 1.0)}, + "10-bbbbbbbb": {"embed.hook_out": torch.full((3, 4), 9.0)}, + } + result = _batched_driver(outputs=outputs, captures_by_req=captures_by_req).forward( + [[1, 2], [1, 2, 3]] + ) + emb = result.captured["embed.hook_out"] + assert torch.equal(emb[0, :2], torch.full((2, 4), 1.0)) # req "1" + assert torch.equal(emb[1, :3], torch.full((3, 4), 9.0)) # req "10" + + def test_shorter_rows_right_padded_with_zeros(self): + pytest.importorskip("vllm") + outputs = [ + _batched_request_output("r0", top_logprobs={1: 1.0}), + _batched_request_output("r1", top_logprobs={1: 1.0}), + ] + captures_by_req = { + "r0": {"embed.hook_out": torch.ones(3, 4)}, + "r1": {"embed.hook_out": torch.ones(1, 4)}, + } + result = _batched_driver(outputs=outputs, captures_by_req=captures_by_req).forward( + [[1, 2, 3], [9]] + ) + emb = result.captured["embed.hook_out"] + # Row 1 has 1 real token; positions 1,2 are zero pad. + assert torch.equal(emb[1, 1:], torch.zeros(2, 4)) + + def test_logits_at_per_row_last_token(self): + """Next-token logits land at prompt_len-1 per row, never -1 (a pad row).""" + pytest.importorskip("vllm") + outputs = [ + _batched_request_output("r0", top_logprobs={7: 5.0}), + _batched_request_output("r1", top_logprobs={3: 5.0}), + ] + captures_by_req = { + "r0": {"embed.hook_out": torch.ones(3, 4)}, + "r1": {"embed.hook_out": torch.ones(1, 4)}, + } + result = _batched_driver(outputs=outputs, captures_by_req=captures_by_req).forward( + [[1, 2, 3], [9]] + ) + logits = result.logits + assert logits is not None and tuple(logits.shape) == (2, 3, 16) + # Row 0 last token at pos 2; row 1 last token at pos 0 (not 2, which is pad). + assert int(logits[0, 2].argmax().item()) == 7 + assert int(logits[1, 0].argmax().item()) == 3 + assert torch.isinf(logits[1, 2]).all() # pad position stays -inf + + def test_resets_accumulator_before_generate(self): + """tl_reset_accumulators must fire so prior-forward chunks don't leak.""" + pytest.importorskip("vllm") + driver = _batched_driver( + outputs=[_batched_request_output("r0", top_logprobs={1: 1.0})], + captures_by_req={"r0": {"embed.hook_out": torch.ones(2, 4)}}, + ) + driver.forward([[1, 2]]) + methods = [c.args[0] for c in driver._llm.collective_rpc.call_args_list] + assert "tl_reset_accumulators" in methods + assert "tl_set_batched_interventions" in methods + + def test_batched_intervention_spec_pushed(self): + pytest.importorskip("vllm") + driver = _batched_driver( + outputs=[_batched_request_output("r0", top_logprobs={1: 1.0})], + captures_by_req={"r0": {"embed.hook_out": torch.ones(2, 4)}}, + ) + driver.forward([[1, 2]], intervene={"embed.hook_out": {"op": "suppress"}}) + calls = [c.args for c in driver._llm.collective_rpc.call_args_list] + assert any( + a[0] == "tl_set_batched_interventions" + and a[1] == ({"embed.hook_out": {"op": "suppress"}},) + for a in calls + ) + + class TestVLLMDriverThroughBridge: """End-to-end via RemoteBridge — drivers' captures flow into the HookPoint tree.""" diff --git a/tests/unit/model_bridge/test_vllm_internals.py b/tests/unit/model_bridge/test_vllm_internals.py new file mode 100644 index 0000000000..cbca9315e8 --- /dev/null +++ b/tests/unit/model_bridge/test_vllm_internals.py @@ -0,0 +1,51 @@ +"""Unit tests for the vLLM internals chokepoint. + +``segment_by_request``'s primary path reads ``model_runner.query_start_loc`` and +needs no vLLM install (backend-agnostic). The attn-metadata fallback imports +vLLM's forward context, so it's exercised on GPU via the Colab notebook instead. +""" +from __future__ import annotations + +from types import SimpleNamespace + +import numpy as np +import torch + +from transformer_lens.model_bridge.sources.vllm.internals import segment_by_request + + +def _model_runner(query_start_loc, req_ids): + return SimpleNamespace( + query_start_loc=query_start_loc, + input_batch=SimpleNamespace(req_ids=req_ids), + ) + + +class TestSegmentByRequest: + """Per-request query offsets come from model_runner.query_start_loc.""" + + def test_plain_tensor_sliced_to_num_reqs_plus_one(self): + # The buffer is padded past the active batch; only the first n+1 entries + # are this step's offsets ([0, len0, len0+len1, ...]). + mr = _model_runner(torch.tensor([0, 6, 14, 16, 99, 99]), ["21", "22", "23"]) + offsets, req_ids = segment_by_request(mr) + assert req_ids == ["21", "22", "23"] + assert offsets.tolist() == [0, 6, 14, 16] + + def test_cpu_gpu_buffer_via_cpu_accessor(self): + """vLLM's CpuGpuBuffer exposes the data tensor as ``.cpu``.""" + buf = SimpleNamespace(cpu=torch.tensor([0, 3, 5, 100])) + offsets, _ = segment_by_request(_model_runner(buf, ["a", "b"])) + assert offsets.tolist() == [0, 3, 5] + + def test_cpu_gpu_buffer_via_np_accessor(self): + """Some buffers surface a numpy view as ``.np``.""" + buf = SimpleNamespace(np=np.array([0, 2, 7, 9, 50])) + offsets, _ = segment_by_request(_model_runner(buf, ["a", "b", "c"])) + assert offsets.tolist() == [0, 2, 7, 9] + + def test_single_request(self): + mr = _model_runner(torch.tensor([0, 5]), ["only"]) + offsets, req_ids = segment_by_request(mr) + assert offsets.tolist() == [0, 5] + assert req_ids == ["only"] diff --git a/tests/unit/model_bridge/test_vllm_worker_extension.py b/tests/unit/model_bridge/test_vllm_worker_extension.py new file mode 100644 index 0000000000..adf120e4d1 --- /dev/null +++ b/tests/unit/model_bridge/test_vllm_worker_extension.py @@ -0,0 +1,155 @@ +"""Unit tests for TLWorkerExtension — the post-compile read/write RPC surface. + +These methods are pure torch + Python (no vLLM install needed). Hook installation +and the in-compile counter increment live in plugin.py and can only be exercised +on a real GPU via demos/vLLM_Bridge_Integration_Test.ipynb. +""" +from __future__ import annotations + +import torch + +from transformer_lens.model_bridge.sources.vllm.worker_extension import ( + TLWorkerExtension, + _apply_intervention, + _apply_op, +) + + +class TestFireCounter: + """tl_reset_counter / tl_read_counter operate on the shared GPU counter.""" + + def test_read_counter_reflects_value(self): + ext = TLWorkerExtension() + ext._tl_fire_counter = torch.tensor([7], dtype=torch.int64) + assert ext.tl_read_counter() == 7 + + def test_reset_counter_zeros(self): + ext = TLWorkerExtension() + ext._tl_fire_counter = torch.tensor([42], dtype=torch.int64) + ext.tl_reset_counter() + assert ext.tl_read_counter() == 0 + + def test_missing_counter_reads_zero(self): + """Before patched_load_model runs, the attribute doesn't exist yet.""" + ext = TLWorkerExtension() + assert ext.tl_read_counter() == 0 + ext.tl_reset_counter() # no-op, must not raise + + +class TestApplyIntervention: + """_apply_intervention translates spec dicts to affine buffer writes.""" + + def _buffers(self, width=4): + return torch.ones(width), torch.zeros(width) + + def test_suppress_zeros_both(self): + scale, bias = self._buffers() + _apply_intervention(scale, bias, {"op": "suppress"}) + assert torch.equal(scale, torch.zeros(4)) + assert torch.equal(bias, torch.zeros(4)) + + def test_scale_sets_factor(self): + scale, bias = self._buffers() + _apply_intervention(scale, bias, {"op": "scale", "factor": 0.5}) + assert torch.equal(scale, torch.full((4,), 0.5)) + assert torch.equal(bias, torch.zeros(4)) + + def test_add_scalar_broadcasts(self): + scale, bias = self._buffers() + _apply_intervention(scale, bias, {"op": "add", "value": 0.5}) + assert torch.equal(scale, torch.ones(4)) + assert torch.equal(bias, torch.full((4,), 0.5)) + + def test_add_vector_elementwise(self): + scale, bias = self._buffers() + vec = [1.0, 2.0, 3.0, 4.0] + _apply_intervention(scale, bias, {"op": "add", "value": vec}) + assert torch.equal(bias, torch.tensor(vec)) + + def test_set_zeros_scale(self): + scale, bias = self._buffers() + _apply_intervention(scale, bias, {"op": "set", "value": 2.0}) + assert torch.equal(scale, torch.zeros(4)) + assert torch.equal(bias, torch.full((4,), 2.0)) + + def test_value_shape_mismatch_raises(self): + import pytest + + scale, bias = self._buffers(width=4) + with pytest.raises(ValueError, match="must be a scalar or shape"): + _apply_intervention(scale, bias, {"op": "add", "value": [1.0, 2.0]}) + + +class TestApplyOp: + """_apply_op is the eager batched path's tensor-level intervention.""" + + def _t(self): + return torch.ones(3, 4) + + def test_suppress_zeros(self): + assert torch.equal(_apply_op(self._t(), {"op": "suppress"}), torch.zeros(3, 4)) + + def test_scale_multiplies(self): + assert torch.equal( + _apply_op(self._t(), {"op": "scale", "factor": 0.5}), torch.full((3, 4), 0.5) + ) + + def test_add_scalar_broadcasts(self): + assert torch.equal( + _apply_op(self._t(), {"op": "add", "value": 1.0}), torch.full((3, 4), 2.0) + ) + + def test_add_vector_elementwise(self): + out = _apply_op(self._t(), {"op": "add", "value": [0.0, 1.0, 2.0, 3.0]}) + assert torch.equal(out[0], torch.tensor([1.0, 2.0, 3.0, 4.0])) + + def test_set_replaces(self): + assert torch.equal( + _apply_op(self._t(), {"op": "set", "value": 9.0}), torch.full((3, 4), 9.0) + ) + + def test_unsupported_op_raises(self): + import pytest + + with pytest.raises(ValueError, match="Unsupported intervention op"): + _apply_op(self._t(), {"op": "clamp", "value": 1.0}) + + def test_value_shape_mismatch_raises(self): + import pytest + + with pytest.raises(ValueError, match="must be a scalar or shape"): + _apply_op(self._t(), {"op": "add", "value": [1.0, 2.0]}) + + +class TestBatchedAccumulators: + """tl_reset_accumulators / tl_read_batched_captures / tl_set_batched_interventions.""" + + def test_read_concatenates_chunks_in_order(self): + ext = TLWorkerExtension() + ext._tl_accum = { + ("req-A", "embed.hook_out"): [torch.ones(2, 4), torch.full((1, 4), 2.0)], + ("req-B", "embed.hook_out"): [torch.full((3, 4), 5.0)], + } + out = ext.tl_read_batched_captures() + # req-A's two chunks cat to (3, 4); chunk order is token order. + assert tuple(out["req-A"]["embed.hook_out"].shape) == (3, 4) + assert torch.equal(out["req-A"]["embed.hook_out"][2], torch.full((4,), 2.0)) + assert tuple(out["req-B"]["embed.hook_out"].shape) == (3, 4) + + def test_reset_clears_accumulator(self): + ext = TLWorkerExtension() + ext._tl_accum = {("r", "h"): [torch.ones(1, 4)]} + ext.tl_reset_accumulators() + assert ext.tl_read_batched_captures() == {} + + def test_set_batched_interventions_validates_op(self): + import pytest + + ext = TLWorkerExtension() + with pytest.raises(ValueError, match="Unsupported intervention op"): + ext.tl_set_batched_interventions({"h": {"op": "clamp"}}) + + def test_set_batched_interventions_stores_specs(self): + ext = TLWorkerExtension() + ext.tl_set_batched_interventions({"h": {"op": "suppress"}}) + assert ext._tl_intervention_specs == {"h": {"op": "suppress"}} diff --git a/transformer_lens/model_bridge/sources/vllm/driver.py b/transformer_lens/model_bridge/sources/vllm/driver.py index e6733c3ef1..a9fa167121 100644 --- a/transformer_lens/model_bridge/sources/vllm/driver.py +++ b/transformer_lens/model_bridge/sources/vllm/driver.py @@ -30,10 +30,12 @@ def __init__( overlay: Any, hf_config: Any, max_num_batched_tokens: int, + enable_batching: bool = False, ) -> None: super().__init__(adapter.cfg, tokenizer) self._llm = llm self._max_num_batched_tokens = max_num_batched_tokens + self._enable_batching = enable_batching self.supported_hook_points = frozenset(overlay.capture_specs(hf_config).keys()) @@ -65,6 +67,9 @@ def forward( ) intervene_specs = self._validate_interventions(intervene or {}) + if self._enable_batching: + return self._forward_batched(input_ids, intervene_specs, return_logits) + ids_list = self._normalize_input_ids(input_ids) if len(ids_list) > self._max_num_batched_tokens: # Worker buffers silently clamp on overflow — fail loud here instead. @@ -104,6 +109,111 @@ def forward( return ForwardResult(logits=logits, captured=captured, raw_output=outputs[0]) + def _forward_batched( + self, + input_ids: TensorLike, + intervene_specs: dict, + return_logits: bool, + ) -> ForwardResult: + """Eager batched path: per-request capture, right-padded to (B, S, W). + + No per-prompt length gate — chunked prefill accumulates long prompts + across forwards. Interventions are global across the batch. + """ + from vllm import SamplingParams + from vllm.inputs import TokensPrompt + + prompts_ids = self._normalize_input_ids_batched(input_ids) + prompt_lens = [len(ids) for ids in prompts_ids] + + # Reset accumulators so prior-forward chunks don't leak into the cat. + self._llm.collective_rpc("tl_reset_accumulators") + self._llm.collective_rpc("tl_reset_counter") + self._llm.collective_rpc("tl_set_batched_interventions", args=(intervene_specs,)) + + d_vocab = self.bridge_config.d_vocab + outputs = self._llm.generate( + prompts=[TokensPrompt(prompt_token_ids=ids) for ids in prompts_ids], + sampling_params=SamplingParams( + max_tokens=1, + temperature=0.0, + logprobs=d_vocab if return_logits else None, + ), + ) + + # Keyed by req_id (no guaranteed order) — _assemble_padded joins to slot + # k via outputs[k].request_id, not by position. + worker_captures = self._llm.collective_rpc("tl_read_batched_captures")[0] + captured = self._assemble_padded(outputs, worker_captures, prompt_lens) + + logits: torch.Tensor | None = None + if return_logits: + logits = self._synthesize_logits_batched(outputs, prompt_lens, d_vocab) + + return ForwardResult(logits=logits, captured=captured, raw_output=outputs) + + @staticmethod + def _assemble_padded( + outputs: list, + worker_captures: Mapping[str, Mapping[str, torch.Tensor]], + prompt_lens: list[int], + ) -> dict[str, torch.Tensor]: + """Stack per-request captures into right-padded ``(batch, max_seq, width)``. + + Pad is a cache-assembly artifact only: vLLM computes each request + independently, so real-token activations don't depend on the padding. + """ + batch = len(outputs) + max_seq = max(prompt_lens) if prompt_lens else 0 + # Worker keys are engine-internal req_ids ("10-83c3532c"); RequestOutput + # carries only the public id ("10"). Join exact-or-prefix; the "-" keeps + # "1" from matching "10-...". + worker_keys = list(worker_captures.keys()) + + def _captures_for(public_rid: str) -> Mapping[str, torch.Tensor]: + if public_rid in worker_captures: + return worker_captures[public_rid] + matches = [k for k in worker_keys if k.startswith(f"{public_rid}-")] + return worker_captures[matches[0]] if len(matches) == 1 else {} + + per_slot = [_captures_for(o.request_id) for o in outputs] + + hook_names: set[str] = set() + for caps in per_slot: + hook_names |= set(caps.keys()) + + assembled: dict[str, torch.Tensor] = {} + for name in hook_names: + sample = next(caps[name] for caps in per_slot if name in caps) + buf = torch.zeros(batch, max_seq, sample.shape[-1], dtype=sample.dtype) + for k, caps in enumerate(per_slot): + t = caps.get(name) + if t is not None: + buf[k, : t.shape[0]] = t + assembled[name] = buf + return assembled + + @staticmethod + def _synthesize_logits_batched( + outputs: list, prompt_lens: list[int], d_vocab: int + ) -> torch.Tensor: + """Build ``(batch, max_seq, d_vocab)`` logits; next-token dist at each row's + ``prompt_lens[k] - 1``, never ``-1`` (a pad position for shorter prompts).""" + batch = len(outputs) + max_seq = max(prompt_lens) if prompt_lens else 0 + logits = torch.full((batch, max_seq, d_vocab), float("-inf"), dtype=torch.float16) + for k, request_output in enumerate(outputs): + gen = request_output.outputs[0] if request_output.outputs else None + if gen is None: + continue + pos = prompt_lens[k] - 1 + if gen.logprobs: + for token_id, lp_obj in gen.logprobs[0].items(): + logits[k, pos, int(token_id)] = float(lp_obj.logprob) + elif gen.token_ids: + logits[k, pos, int(gen.token_ids[0])] = 0.0 + return logits + def close(self) -> None: # Detach hooks before dropping the LLM so they don't stay registered on # worker modules for the life of the process (long-running notebooks). @@ -205,3 +315,16 @@ def _normalize_input_ids(input_ids: Any) -> list: raise NotImplementedError("VLLMDriver supports batch_size=1 only.") ids_list = ids_list[0] return ids_list + + @staticmethod + def _normalize_input_ids_batched(input_ids: Any) -> list[list[int]]: + """Coerce to ``list[list[int]]`` (one per prompt); accepts 1-D/2-D tensor, + flat list (single prompt), or ragged list-of-lists.""" + if isinstance(input_ids, torch.Tensor): + if input_ids.dim() == 1: + return [input_ids.tolist()] + return [row.tolist() for row in input_ids] + seq = list(input_ids) + if seq and isinstance(seq[0], (list, tuple)): + return [list(row) for row in seq] + return [list(seq)] diff --git a/transformer_lens/model_bridge/sources/vllm/internals.py b/transformer_lens/model_bridge/sources/vllm/internals.py index bdabf5455a..185a5a9a4b 100644 --- a/transformer_lens/model_bridge/sources/vllm/internals.py +++ b/transformer_lens/model_bridge/sources/vllm/internals.py @@ -22,3 +22,51 @@ def extract_hf_config(llm: Any) -> Any: "Could not locate hf_config under llm.llm_engine.model_config. " "vLLM may have moved it; update extract_hf_config() to match." ) from e + + +# Cumulative per-request query offsets: FlashAttention/Triton name them +# query_start_loc, FlashInfer names them qo_indptr. Request i = rows i:i+1. +_QUERY_OFFSET_ATTRS = ("query_start_loc", "qo_indptr") + + +def _to_cpu_offsets(buf: Any) -> Any: + """Coerce a CpuGpuBuffer (``.cpu``/``.np``/``.gpu``) or tensor to a CPU tensor.""" + import torch + + for accessor in ("cpu", "np", "gpu"): + data = getattr(buf, accessor, None) + if data is not None and hasattr(data, "__len__"): + return torch.as_tensor(data) + return torch.as_tensor(buf) if hasattr(buf, "shape") else None + + +def segment_by_request(model_runner: Any) -> Any: + """Return ``(query_offsets_cpu, req_ids)``; request i = rows offsets[i]:offsets[i+1]. + + Only valid inside a forward. ``req_ids`` is row order, NOT submission order — + join on it. Reads ``model_runner.query_start_loc`` (backend-agnostic; the + runner builds it before any attention backend, whereas FlashInfer buries its + offsets in an opaque C++ wrapper), falling back to attn metadata for backends + that surface them directly. ``(None, req_ids)`` ⇒ caller single-slices. + """ + req_ids = list(model_runner.input_batch.req_ids) + n = len(req_ids) + + qsl = getattr(model_runner, "query_start_loc", None) + if qsl is not None: + offsets = _to_cpu_offsets(qsl) + if offsets is not None and len(offsets) >= n + 1: # buffer is padded to max batch + return offsets[: n + 1].detach().cpu(), req_ids + + from vllm.forward_context import get_forward_context + + attn_metadata = get_forward_context().attn_metadata + if isinstance(attn_metadata, list): # dual-batch-overlap returns a list of dicts + attn_metadata = attn_metadata[0] + if isinstance(attn_metadata, dict): + for meta in attn_metadata.values(): + for attr in _QUERY_OFFSET_ATTRS: + off = getattr(meta, attr, None) + if off is not None: + return off.detach().cpu(), req_ids + return None, req_ids diff --git a/transformer_lens/model_bridge/sources/vllm/plugin.py b/transformer_lens/model_bridge/sources/vllm/plugin.py index 02ddf9a586..b3b0dd4e2b 100644 --- a/transformer_lens/model_bridge/sources/vllm/plugin.py +++ b/transformer_lens/model_bridge/sources/vllm/plugin.py @@ -4,21 +4,24 @@ load and before ``compile_or_warm_up_model`` — the only window where hooks make it into the compiled FX graph (PyTorch #117758). -Hook body invariants under ``torch.compile``: in-place writes to a -pre-allocated GPU tensor; no ``.cpu()`` (illegal during CUDA-graph capture); -SymInt-indexed slicing only (Python ``.shape`` access forces specialization). - -Interventions ride the same hook. Each hook applies an affine transform -``output = output * scale_buf + bias_buf`` before capturing. Defaults are -``scale=ones`` / ``bias=zeros`` (identity). The driver swaps buffer contents -between forwards via ``tl_set_interventions`` — the FX graph references the -buffers, so swaps take effect on the next dispatch without recompiling. - -Memory cost: the affine transform allocates a transient output-shape tensor -per hook per forward, even in identity mode. Peak forward memory is roughly -1.5× the prior capture-only design — the caching allocator reuses the slot -but the peak pressure rises. Branching the hook to skip the affine in -identity mode would defeat the swap-via-buffer trick and break the FX graph. +Two hook flavors, selected by ``configure(enable_batching=...)``: + +* Compiled (default): in-place writes to a pre-allocated GPU tensor; no + ``.cpu()`` (illegal during CUDA-graph capture); SymInt-indexed slicing only + (Python ``.shape`` access forces specialization). Single prompt. + Interventions ride the same hook as an affine transform + ``output = output * scale_buf + bias_buf`` (defaults identity). The driver + swaps buffer contents between forwards via ``tl_set_interventions`` — the FX + graph references the buffers, so swaps take effect without recompiling. + Memory cost: the affine allocates a transient output-shape tensor per hook + per forward, even at identity — peak forward memory ~1.5× capture-only. + Branching to skip the affine would defeat the swap trick and break the graph. + +* Batched (``enable_batching=True``, runs ``enforce_eager``): the hook reads + per-request token boundaries via ``segment_by_request`` (only valid inside a + forward, untraceable under compile — hence eager), slices each request's rows + to CPU, and appends to per-(req_id, hook) accumulators across chunked-prefill + forwards. Interventions apply directly to the tensor via ``_apply_op``. """ from __future__ import annotations @@ -27,10 +30,11 @@ import torch -# Matches dot-paths like "model.layers.0", "model.layers.15" — vLLM's decoder -# layers, which return the fused-residual (mlp_delta, residual) 2-tuple. Hooks -# on these need to materialize the sum so the capture matches HF's -# blocks.{i}.hook_out semantics (full residual stream). +from .internals import segment_by_request +from .worker_extension import _apply_op + +# Decoder layers return the fused-residual (mlp_delta, residual) 2-tuple, so +# their hooks materialize the sum (see _make_capture_hook); other modules don't. _DECODER_LAYER_PATH = re.compile(r"^model\.layers\.\d+$") # Transient signal driver → worker during LLM construction. Per-Worker buffers @@ -44,11 +48,13 @@ def configure( capture_specs: Dict[str, Tuple[str, int]], max_num_batched_tokens: int, dtype: torch.dtype, + enable_batching: bool = False, ) -> None: - """Set capture specs, buffer length, and dtype before ``LLM(...)``.""" + """Set capture specs, buffer length, dtype, and hook flavor before ``LLM(...)``.""" _config["capture_specs"] = capture_specs _config["max_num_batched_tokens"] = max_num_batched_tokens _config["dtype"] = dtype + _config["enable_batching"] = enable_batching def register() -> None: @@ -77,6 +83,7 @@ def patched_load_model(self): return # not a TL-driven LLM; no hooks to install max_n = _config["max_num_batched_tokens"] dtype = _config["dtype"] + enable_batching = _config.get("enable_batching", False) device = next(self.model_runner.model.parameters()).device # Detach prior handles before reassigning — vLLM doesn't double-load @@ -87,25 +94,46 @@ def patched_load_model(self): self._tl_scale_buffers = {} self._tl_bias_buffers = {} self._tl_hook_handles = [] + # Batched-mode per-(req_id, hook) accumulators + global spec dict. + self._tl_accum = {} + self._tl_intervention_specs = {} + # Shared counter — surfaces hook double-fire under compile via tl_read_counter. + self._tl_fire_counter = torch.zeros(1, device=device, dtype=torch.int64) for canonical_name, (dot_path, width) in specs.items(): target = self.model_runner.model for seg in dot_path.split("."): target = target[int(seg)] if seg.isdigit() else getattr(target, seg) - capture_buf = torch.zeros(max_n, width, device=device, dtype=dtype) - # Affine identity at install. Driver swaps via tl_set_interventions - # to enable suppress/scale/add/set ops between forwards. - scale_buf = torch.ones(width, device=device, dtype=dtype) - bias_buf = torch.zeros(width, device=device, dtype=dtype) - self._tl_buffers[canonical_name] = capture_buf - self._tl_scale_buffers[canonical_name] = scale_buf - self._tl_bias_buffers[canonical_name] = bias_buf # Decoder layers return vLLM's (mlp_delta, residual) tuple; the # hook materializes their sum so the capture semantically matches # HF's full residual stream. Other modules use the default path. materialize = bool(_DECODER_LAYER_PATH.match(dot_path)) - handle = target.register_forward_hook( - _make_capture_hook(capture_buf, scale_buf, bias_buf, materialize=materialize) - ) + if enable_batching: + handle = target.register_forward_hook( + _make_batched_hook( + self, + canonical_name, + self._tl_fire_counter, + materialize=materialize, + ) + ) + else: + capture_buf = torch.zeros(max_n, width, device=device, dtype=dtype) + # Affine identity at install. Driver swaps via tl_set_interventions + # to enable suppress/scale/add/set ops between forwards. + scale_buf = torch.ones(width, device=device, dtype=dtype) + bias_buf = torch.zeros(width, device=device, dtype=dtype) + self._tl_buffers[canonical_name] = capture_buf + self._tl_scale_buffers[canonical_name] = scale_buf + self._tl_bias_buffers[canonical_name] = bias_buf + handle = target.register_forward_hook( + _make_capture_hook( + capture_buf, + scale_buf, + bias_buf, + self._tl_fire_counter, + materialize=materialize, + ) + ) self._tl_hook_handles.append(handle) Worker.load_model = patched_load_model @@ -116,6 +144,7 @@ def _make_capture_hook( capture_buf: torch.Tensor, scale_buf: torch.Tensor, bias_buf: torch.Tensor, + fire_counter: torch.Tensor, *, materialize: bool = False, ): @@ -127,10 +156,13 @@ def _make_capture_hook( blocks.{i}.hook_out semantics) and return ``(modified - residual, residual)`` so the next layer's input_layernorm sees the same fused sum. Mutations propagate through both the capture and the downstream graph. + + ``fire_counter`` is incremented per call for the fire-once check. """ @torch.no_grad() def hook(_module, _inputs, output): + fire_counter.add_(1) if materialize and isinstance(output, tuple) and len(output) == 2: hidden, residual = output if isinstance(hidden, torch.Tensor) and isinstance(residual, torch.Tensor): @@ -138,10 +170,8 @@ def hook(_module, _inputs, output): modified = t * scale_buf + bias_buf n = t.shape[0] capture_buf.narrow(0, 0, n).copy_(modified) - # ``(modified - residual) + residual`` reconstructs ``modified`` in - # the next layer's fused input_layernorm. Identity case is exact; - # for interventions, fp16 precision loss is bounded by one ulp at - # the residual stream's magnitude — small relative to mutation effect. + # Reconstructs ``modified`` in the next layer's fused norm: exact at + # identity, bounded fp16 error under intervention. return (modified - residual, residual) tuple_tail: tuple = () @@ -155,14 +185,71 @@ def hook(_module, _inputs, output): # Affine transform; default scale=1 / bias=0 means identity. Driver # swaps buffer contents to enable interventions. modified = t * scale_buf + bias_buf - # ``narrow`` produces an explicit-shape view ``(n, width)`` that Dynamo - # traces correctly under dynamic shapes; the ``[:n]`` getitem form gets - # erased under fake-tensor tracing and ``copy_`` then sees the full - # buffer as destination, raising "expand s72 -> max_n". + # narrow() keeps the dynamic shape; [:n] gets erased under fake-tensor + # tracing and copy_ then sees the full buffer ("expand s72 -> max_n"). n = t.shape[0] capture_buf.narrow(0, 0, n).copy_(modified) - # Preserve the input wrapping — a 1-tuple input must come back as a 1-tuple - # (``tuple_tail`` is falsy in that case, so don't gate on it). + # Gate on isinstance, not truthy tuple_tail — a 1-tuple has an empty tail. + if isinstance(output, tuple): + return (modified,) + tuple_tail + return modified + + return hook + + +def _make_batched_hook( + worker: Any, + canonical_name: str, + fire_counter: torch.Tensor, + *, + materialize: bool = False, +): + """Eager hook: intervene, then append each request's rows to ``worker._tl_accum``. + + Chunked prefill fires this once per chunk; appends are token-order for the + later cat. ``materialize`` mirrors the compiled hook (fused-residual sum). + """ + + @torch.no_grad() + def hook(_module, _inputs, output): + fire_counter.add_(1) + residual = None + if materialize and isinstance(output, tuple) and len(output) == 2: + hidden, residual = output + if isinstance(hidden, torch.Tensor) and isinstance(residual, torch.Tensor): + t = hidden + residual + else: + return None + tuple_tail: tuple = () + elif isinstance(output, tuple): + t = output[0] + tuple_tail = output[1:] + else: + t = output + tuple_tail = () + if not isinstance(t, torch.Tensor): + return None + + modified = t + for spec in getattr(worker, "_tl_intervention_specs", {}).values(): + modified = _apply_op(modified, spec) + + qsl, req_ids = segment_by_request(worker.model_runner) + accum: Dict[tuple, list] = worker._tl_accum + if qsl is None: + # No per-request boundaries available — treat the batch as one request. + req_id = req_ids[0] if req_ids else "0" + accum.setdefault((req_id, canonical_name), []).append(modified.detach().cpu()) + else: + for i, req_id in enumerate(req_ids): + start, end = int(qsl[i]), int(qsl[i + 1]) + if end <= start: + continue + chunk = modified[start:end].detach().cpu() + accum.setdefault((req_id, canonical_name), []).append(chunk) + + if residual is not None: + return (modified - residual, residual) if isinstance(output, tuple): return (modified,) + tuple_tail return modified diff --git a/transformer_lens/model_bridge/sources/vllm/source.py b/transformer_lens/model_bridge/sources/vllm/source.py index e5b7c083fe..cfafc5d63d 100644 --- a/transformer_lens/model_bridge/sources/vllm/source.py +++ b/transformer_lens/model_bridge/sources/vllm/source.py @@ -38,6 +38,7 @@ def boot_vllm( gpu_memory_utilization: float = 0.5, max_model_len: Optional[int] = None, max_num_batched_tokens: int = 2048, + enable_batching: bool = False, **vllm_kwargs: Any, ) -> RemoteBridge: """Boot a model via vLLM and wrap it in a :class:`RemoteBridge` via :class:`VLLMDriver`. @@ -73,6 +74,11 @@ def boot_vllm( model's native context (e.g. 131072 for Llama-3.2-1B) — easily 4+ GiB even on a 1B model. Pass an explicit ``max_model_len`` (e.g. ``2048`` for typical mech-interp prompts) to keep the budget on smaller GPUs. + + ``enable_batching`` switches to the eager batched path (``enforce_eager``, + ``batch_size > 1``) — the throughput path for SAE/probe data collection. + Default ``False`` keeps the compile-validated single-prompt path. Batched + caches are right-padded with zeros to the longest sequence. """ _reject_locked_overrides(vllm_kwargs) @@ -90,6 +96,7 @@ def boot_vllm( capture_specs=overlay.capture_specs(hf_config_preview), max_num_batched_tokens=max_num_batched_tokens, dtype=resolved_dtype, + enable_batching=enable_batching, ) plugin.register() @@ -109,6 +116,10 @@ def boot_vllm( from vllm import LLM + # Batched capture reads query_start_loc from the forward context, untraceable + # under torch.compile — so the batched path must run eager. + eager_kwargs: Dict[str, Any] = {"enforce_eager": True} if enable_batching else {} + llm = LLM( model=model_name, gpu_memory_utilization=gpu_memory_utilization, @@ -128,6 +139,7 @@ def boot_vllm( # too small for mech-interp). One ~512 KB buffer per call; negligible. max_logprobs=hf_config_preview.vocab_size, dtype=str(resolved_dtype).replace("torch.", "") if dtype is not None else "auto", + **eager_kwargs, **_LOCKED_KWARGS, **vllm_kwargs, ) @@ -157,6 +169,7 @@ def boot_vllm( overlay=overlay, hf_config=hf_config, max_num_batched_tokens=max_num_batched_tokens, + enable_batching=enable_batching, ) bridge = RemoteBridge(adapter=adapter, tokenizer=tokenizer, driver=driver) _log_hook_summary(model_name, architecture, driver) diff --git a/transformer_lens/model_bridge/sources/vllm/worker_extension.py b/transformer_lens/model_bridge/sources/vllm/worker_extension.py index 70a98ad39a..17a8dc1f84 100644 --- a/transformer_lens/model_bridge/sources/vllm/worker_extension.py +++ b/transformer_lens/model_bridge/sources/vllm/worker_extension.py @@ -1,11 +1,18 @@ -"""Worker extension exposed to collective_rpc for capture-buffer reads and -intervention-buffer writes. +"""Worker extension exposed to collective_rpc for capture reads and +intervention writes. Hook *installation* lives in :mod:`plugin` (must happen pre-compile). This class -only exposes the post-compile read/write surface. Buffers are per-Worker -(``self._tl_buffers`` / ``self._tl_scale_buffers`` / ``self._tl_bias_buffers``) -so concurrent ``boot_vllm`` calls don't collide. All methods prefixed ``tl_`` -to avoid colliding with vLLM ``Worker`` attributes. +only exposes the read/write surface. State is per-Worker so concurrent +``boot_vllm`` calls don't collide. All methods prefixed ``tl_`` to avoid +colliding with vLLM ``Worker`` attributes. + +Two capture modes, selected at boot: + * Compiled (default): per-hook GPU buffers + affine scale/bias swap. Single + prompt. ``tl_read_captures`` / ``tl_set_interventions``. + * Batched (``enable_batching=True``, eager): per-(req_id, hook) CPU + accumulators filled in the hook via query_start_loc segmentation; arbitrary + batch + chunked prefill. ``tl_read_batched_captures`` / + ``tl_set_batched_interventions`` / ``tl_reset_accumulators``. """ from __future__ import annotations @@ -23,6 +30,10 @@ class TLWorkerExtension: _tl_buffers: Dict[str, torch.Tensor] _tl_scale_buffers: Dict[str, torch.Tensor] _tl_bias_buffers: Dict[str, torch.Tensor] + _tl_fire_counter: torch.Tensor + # Batched-mode state (eager). + _tl_accum: Dict[tuple, List[torch.Tensor]] + _tl_intervention_specs: Dict[str, Dict[str, Any]] def tl_read_captures(self, prompt_lens: List[int]) -> Dict[str, torch.Tensor]: """Slice each capture buffer back to ``sum(prompt_lens)`` rows; CPU copies. @@ -54,6 +65,39 @@ def tl_set_interventions(self, specs: Dict[str, Dict[str, Any]]) -> None: raise KeyError(f"Unknown hook for intervention: {hook_name!r}") _apply_intervention(scale_bufs[hook_name], bias_bufs[hook_name], spec) + def tl_reset_counter(self) -> None: + """Zero the shared hook-fire counter before a forward.""" + counter = getattr(self, "_tl_fire_counter", None) + if counter is not None: + counter.zero_() + + def tl_read_counter(self) -> int: + """Total hook fires since the last reset.""" + counter = getattr(self, "_tl_fire_counter", None) + return int(counter.item()) if counter is not None else 0 + + def tl_reset_accumulators(self) -> None: + """Clear capture chunks before each generate, else prior chunks leak into the cat.""" + self._tl_accum = {} + + def tl_read_batched_captures(self) -> Dict[str, Dict[str, torch.Tensor]]: + """Cat per-request chunks into ``{req_id: {hook: (seq, width)}}`` (chunks are token-order).""" + accum: Dict[tuple, List[torch.Tensor]] = getattr(self, "_tl_accum", {}) + out: Dict[str, Dict[str, torch.Tensor]] = {} + for (req_id, name), chunks in accum.items(): + out.setdefault(req_id, {})[name] = torch.cat(chunks, dim=0) + return out + + def tl_set_batched_interventions(self, specs: Dict[str, Dict[str, Any]]) -> None: + """Store the global spec dict the eager hook reads; ``{}`` clears.""" + for spec in specs.values(): + op = spec.get("op") + if op not in SUPPORTED_OPS: + raise ValueError( + f"Unsupported intervention op: {op!r}. Supported: {sorted(SUPPORTED_OPS)}" + ) + self._tl_intervention_specs = dict(specs) + def tl_remove_hooks(self) -> None: """Detach all capture hooks and drop buffer references. Idempotent.""" for handle in getattr(self, "_tl_hook_handles", []): @@ -62,6 +106,28 @@ def tl_remove_hooks(self) -> None: self._tl_buffers = {} self._tl_scale_buffers = {} self._tl_bias_buffers = {} + self._tl_accum = {} + self._tl_intervention_specs = {} + + +def _apply_op(t: torch.Tensor, spec: Dict[str, Any]) -> torch.Tensor: + """Apply a spec to a tensor in-line (eager path; no GPU buffer to swap).""" + op = spec.get("op") + if op not in SUPPORTED_OPS: + raise ValueError(f"Unsupported intervention op: {op!r}. Supported: {sorted(SUPPORTED_OPS)}") + if op == "suppress": + return torch.zeros_like(t) + if op == "scale": + return t * float(spec["factor"]) + value = torch.as_tensor(spec["value"], device=t.device, dtype=t.dtype) + if value.ndim != 0 and value.shape != (t.shape[-1],): + raise ValueError( + f"Intervention 'value' must be a scalar or shape {(t.shape[-1],)}; " + f"got shape {tuple(value.shape)}" + ) + if op == "add": + return t + value + return torch.zeros_like(t) + value # set def _apply_intervention( From d8b471e534a3732678bb04442398318bedd35bf2 Mon Sep 17 00:00:00 2001 From: Jonah Larson Date: Thu, 28 May 2026 12:17:37 -0500 Subject: [PATCH 03/87] vLLM Driver Bugs (#1343) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Refactored `transformers` so that bridge configuration code is source-agnostic * Initial breakup of Bridge into core and bridge only components. Setup remote driver in prep * Initial setup of the vllm source Cherry-picked from 98bbb835 on feature/vllm-integration. Resolved against dev-4.x's Phase A type split: - bridge.py: kept the shim (TransformerBridge content lives in transformer_bridge.py now). - sources/transformers.py: accepted the scaffold's package restructure (transformers.py → transformers/{__init__,helpers,source}.py + _hf_format.py). - sources/transformers/source.py: re-applied Phase A's explicit TransformersDriver construction in boot(). sources/vllm/source.py's boot_vllm currently sets bridge._forward_impl, which Phase A removed. The function is dead until the next chunk lifts it onto VLLMDriver + RemoteBridge — landing as-is so the scaffold tree is on dev-4.x for incremental Phase B work. Co-Authored-By: Claude Opus 4.7 (1M context) * Revision round 1 of the vLLM system * Full vLLM integration initial completion * jupiter patch * dynamo patching * pass through batch size * add max model length to remote bridge boot * Updating config construction to properly pass through to vLLM * Wiring TLWorkerExtension * Additional testing + diagnostic * Another spot check * Remove logging * Fix issue with logit generation on hook_out * Adjust to use token_ids directly instead of logprobs * Updating Step 5 and driver bridge cleanup * Add decoder layer for materializing the residual stream * Updating tests due to changes on `dev` * Fix pip install path * Testing ln_final divergence * documenting differences in ln_final * Format cleanup * Initial batching test * Add version print * Setup batching for vllm * Additional documentation and coverage * CI Failure cleanup --------- Co-authored-by: Claude Opus 4.7 (1M context) --- tests/unit/model_bridge/test_vllm_driver.py | 80 +++++++++++++++++++ tests/unit/model_bridge/test_vllm_plugin.py | 56 +++++++++++++ .../test_vllm_worker_extension.py | 55 +++++++++++++ .../model_bridge/remote_bridge.py | 72 ++++++++++++++++- .../model_bridge/sources/_driver_base.py | 5 ++ .../model_bridge/sources/vllm/driver.py | 68 +++++++++++----- .../sources/vllm/overlays/decoder_only.py | 10 +-- .../model_bridge/sources/vllm/plugin.py | 5 +- .../model_bridge/sources/vllm/source.py | 7 ++ .../sources/vllm/worker_extension.py | 42 ++++++++-- 10 files changed, 366 insertions(+), 34 deletions(-) create mode 100644 tests/unit/model_bridge/test_vllm_plugin.py diff --git a/tests/unit/model_bridge/test_vllm_driver.py b/tests/unit/model_bridge/test_vllm_driver.py index 964a761116..f676a005d7 100644 --- a/tests/unit/model_bridge/test_vllm_driver.py +++ b/tests/unit/model_bridge/test_vllm_driver.py @@ -133,6 +133,31 @@ def test_non_fireable_expanded_per_layer(self): ) +class TestVLLMDriverConfig: + """Boot-time config: logprobs request must match boot's max_logprobs.""" + + def test_n_logprobs_uses_vocab_size(self): + # hf_config.vocab_size (16) is the request count, matching boot's + # max_logprobs — not bridge_config.d_vocab, which may be padded larger. + assert _driver()._n_logprobs == 16 + + +class TestVLLMDriverGetParam: + """get_param fetches a named worker tensor via collective_rpc.""" + + def test_returns_rpc_result(self): + driver = _driver(captures={}) + driver._llm.collective_rpc = MagicMock(return_value=[torch.ones(4)]) + out = driver.get_param("model.norm.weight") + assert torch.equal(out, torch.ones(4)) + assert driver._llm.collective_rpc.call_args.args[0] == "tl_get_param" + + def test_returns_none_when_closed(self): + driver = _driver(captures={}) + driver._llm = None + assert driver.get_param("model.norm.weight") is None + + class TestVLLMDriverForward: """forward dispatches via llm.generate and surfaces captures via ForwardResult.""" @@ -398,6 +423,23 @@ def test_logits_at_per_row_last_token(self): assert int(logits[1, 0].argmax().item()) == 3 assert torch.isinf(logits[1, 2]).all() # pad position stays -inf + def test_assemble_padded_raises_on_missing_join(self): + """A request with no matching worker key must raise, not zero-fill silently.""" + outputs = [SimpleNamespace(request_id="99")] + worker_captures = {"10-83c3532c": {"embed.hook_out": torch.ones(2, 4)}} + with pytest.raises(RuntimeError, match="Cannot join request '99'"): + VLLMDriver._assemble_padded(outputs, worker_captures, [2]) + + def test_assemble_padded_raises_on_ambiguous_join(self): + """Two worker keys sharing the public-id prefix is ambiguous — raise.""" + outputs = [SimpleNamespace(request_id="1")] + worker_captures = { + "1-aaaaaaaa": {"embed.hook_out": torch.ones(2, 4)}, + "1-bbbbbbbb": {"embed.hook_out": torch.ones(2, 4)}, + } + with pytest.raises(RuntimeError, match="found 2 key"): + VLLMDriver._assemble_padded(outputs, worker_captures, [2]) + def test_resets_accumulator_before_generate(self): """tl_reset_accumulators must fire so prior-forward chunks don't leak.""" pytest.importorskip("vllm") @@ -438,3 +480,41 @@ def test_bridge_replays_vllm_captures(self): bridge.forward(torch.tensor([[1, 2, 3]])) assert len(fired) == 1 assert tuple(fired[0].shape) == (1, 3, 4) + + def test_forward_rejects_loss_return_types(self): + """Synthesized logits are last-position-only; loss/both would be nan, so refuse. + + The guard is the first thing forward() does, so this needs no vllm install. + """ + bridge = RemoteBridge(adapter=_adapter(), tokenizer=None, driver=_driver(captures={})) + for rt in ("loss", "both"): + with pytest.raises(NotImplementedError, match="return_type"): + bridge.forward(torch.tensor([[1, 2, 3]]), return_type=rt) + + def test_run_with_hooks_rejects_bwd_hooks(self): + """No backward pass on a remote driver — bwd_hooks raise before any forward.""" + bridge = RemoteBridge(adapter=_adapter(), tokenizer=None, driver=_driver(captures={})) + with pytest.raises(NotImplementedError, match="backward"): + bridge.run_with_hooks( + torch.tensor([[1, 2, 3]]), + bwd_hooks=[("embed.hook_out", lambda a, hook: a)], + ) + + def test_context_manager_closes_engine(self): + """`with bridge:` releases the engine on exit (close() nulls the LLM).""" + driver = _driver(captures={}) + bridge = RemoteBridge(adapter=_adapter(), tokenizer=None, driver=driver) + with bridge as entered: + assert entered is bridge + assert driver._llm is None # close() ran + + def test_run_with_hooks_warns_fwd_hooks_are_read_only(self): + """A mutating fwd_hook is a no-op on a remote driver — warn loudly.""" + pytest.importorskip("vllm") + driver = _driver(captures={"embed.hook_out": torch.randn(3, 4)}) + bridge = RemoteBridge(adapter=_adapter(), tokenizer=None, driver=driver) + with pytest.warns(UserWarning, match="read-only"): + bridge.run_with_hooks( + torch.tensor([[1, 2, 3]]), + fwd_hooks=[("embed.hook_out", lambda a, hook: a)], + ) diff --git a/tests/unit/model_bridge/test_vllm_plugin.py b/tests/unit/model_bridge/test_vllm_plugin.py new file mode 100644 index 0000000000..4de4383589 --- /dev/null +++ b/tests/unit/model_bridge/test_vllm_plugin.py @@ -0,0 +1,56 @@ +"""Unit tests for the eager batched capture hook (plugin._make_batched_hook). + +The hook closure is pure torch + the segmentation helper, so it runs without a +vLLM install or GPU: a mock worker carries ``query_start_loc`` (segment_by_request's +backend-agnostic source) and the per-(req_id, hook) accumulator. +""" +from __future__ import annotations + +from types import SimpleNamespace + +import torch + +from transformer_lens.model_bridge.sources.vllm.plugin import _make_batched_hook + + +def _worker(specs): + """Single-request worker mock: query_start_loc=[0, 2], one req id.""" + return SimpleNamespace( + _tl_accum={}, + _tl_intervention_specs=specs, + model_runner=SimpleNamespace( + query_start_loc=torch.tensor([0, 2]), + input_batch=SimpleNamespace(req_ids=["r0"]), + ), + ) + + +class TestBatchedHookInterventionTargeting: + """A batched hook applies ONLY the spec keyed to its own hook name.""" + + def test_non_targeted_hook_is_unmodified(self): + # Spec targets embed; a hook belonging to blocks.0.hook_out must NOT apply it. + worker = _worker({"embed.hook_out": {"op": "suppress"}}) + counter = torch.zeros(1, dtype=torch.int64) + hook = _make_batched_hook(worker, "blocks.0.hook_out", counter) + + out = hook(None, None, torch.ones(2, 4)) + assert torch.equal(out, torch.ones(2, 4)), "spec leaked onto a non-targeted hook" + assert torch.equal(worker._tl_accum[("r0", "blocks.0.hook_out")][0], torch.ones(2, 4)) + + def test_targeted_hook_is_modified(self): + worker = _worker({"embed.hook_out": {"op": "suppress"}}) + counter = torch.zeros(1, dtype=torch.int64) + hook = _make_batched_hook(worker, "embed.hook_out", counter) + + out = hook(None, None, torch.ones(2, 4)) + assert torch.equal(out, torch.zeros(2, 4)), "suppress did not apply to its target" + assert torch.equal(worker._tl_accum[("r0", "embed.hook_out")][0], torch.zeros(2, 4)) + + def test_no_spec_leaves_output_unchanged(self): + worker = _worker({}) + counter = torch.zeros(1, dtype=torch.int64) + hook = _make_batched_hook(worker, "embed.hook_out", counter) + + out = hook(None, None, torch.ones(2, 4) * 3) + assert torch.equal(out, torch.ones(2, 4) * 3) diff --git a/tests/unit/model_bridge/test_vllm_worker_extension.py b/tests/unit/model_bridge/test_vllm_worker_extension.py index adf120e4d1..5d06ca40a6 100644 --- a/tests/unit/model_bridge/test_vllm_worker_extension.py +++ b/tests/unit/model_bridge/test_vllm_worker_extension.py @@ -6,6 +6,8 @@ """ from __future__ import annotations +from types import SimpleNamespace + import torch from transformer_lens.model_bridge.sources.vllm.worker_extension import ( @@ -15,6 +17,59 @@ ) +class TestGetParam: + """tl_get_param resolves a dotted path to a CPU tensor clone, or None.""" + + def test_reads_named_tensor(self): + ext = TLWorkerExtension() + weight = torch.ones(4) + ext.model_runner = SimpleNamespace( # type: ignore[attr-defined] + model=SimpleNamespace(norm=SimpleNamespace(weight=weight)) + ) + out = ext.tl_get_param("norm.weight") + assert torch.equal(out, weight) + assert out is not weight # cloned, not a live reference + + def test_missing_path_returns_none(self): + ext = TLWorkerExtension() + ext.model_runner = SimpleNamespace(model=SimpleNamespace()) # type: ignore[attr-defined] + assert ext.tl_get_param("norm.weight") is None + + def test_non_tensor_target_returns_none(self): + ext = TLWorkerExtension() + ext.model_runner = SimpleNamespace( # type: ignore[attr-defined] + model=SimpleNamespace(norm=SimpleNamespace()) + ) + assert ext.tl_get_param("norm") is None + + +class TestReadCapturesFiltering: + """names restricts the GPU→CPU read; None reads all.""" + + def _ext(self): + ext = TLWorkerExtension() + ext._tl_buffers = {"a": torch.ones(3, 4), "b": torch.zeros(3, 4)} + return ext + + def test_names_filters(self): + out = self._ext().tl_read_captures([2], names=["a"]) + assert set(out) == {"a"} + assert tuple(out["a"].shape) == (2, 4) + + def test_none_reads_all(self): + out = self._ext().tl_read_captures([2]) + assert set(out) == {"a", "b"} + + def test_batched_names_filters(self): + ext = TLWorkerExtension() + ext._tl_accum = { + ("r", "a"): [torch.ones(2, 4)], + ("r", "b"): [torch.zeros(2, 4)], + } + out = ext.tl_read_batched_captures(names=["a"]) + assert set(out["r"]) == {"a"} + + class TestFireCounter: """tl_reset_counter / tl_read_counter operate on the shared GPU counter.""" diff --git a/transformer_lens/model_bridge/remote_bridge.py b/transformer_lens/model_bridge/remote_bridge.py index b41fa10352..0c8cc5b25d 100644 --- a/transformer_lens/model_bridge/remote_bridge.py +++ b/transformer_lens/model_bridge/remote_bridge.py @@ -1,7 +1,8 @@ """Non-torch bridge: vLLM workers, Inspect remote providers.""" from __future__ import annotations -from typing import Any +import warnings +from typing import Any, Callable, List, Optional, Tuple, Union from transformer_lens.hook_points import HookIntrospectionMixin, HookPoint from transformer_lens.model_bridge.bridge_core import BridgeCore @@ -68,6 +69,16 @@ def forward( **kwargs: Any, ) -> Any: """Tokenize → driver.forward → replay captures → finalize per return_type.""" + if return_type in ("loss", "both") and not getattr( + self._driver, "provides_sequence_logits", True + ): + # Final-position-only logits ⇒ loss over the -inf earlier positions is nan. + raise NotImplementedError( + f"RemoteBridge does not support return_type={return_type!r} on this driver: " + "it provides next-token logits for the final position only, so loss over " + "earlier positions is undefined. Use return_type='logits' and read " + "logits[..., -1, :] (or the per-row last token in batched mode)." + ) if isinstance(input, str): assert self.tokenizer is not None, "Tokenizer must be set for string input." tokens = self.tokenizer.encode(input, return_tensors="pt") @@ -93,8 +104,67 @@ def forward( loss_per_token=loss_per_token, ) + def run_with_hooks( + self, + input: Any, + fwd_hooks: List[Tuple[Union[str, Callable], Callable]] = [], + bwd_hooks: List[Tuple[Union[str, Callable], Callable]] = [], + reset_hooks_end: bool = True, + clear_contexts: bool = False, + return_type: Optional[str] = "logits", + names_filter: Optional[Union[str, List[str], Callable[[str], bool]]] = None, + stop_at_layer: Optional[int] = None, + remove_batch_dim: bool = False, + **kwargs: Any, + ) -> Any: + """Run with hooks. Remote fwd_hooks fire post-forward on captured + activations (read-only) — they can't alter the computation, so warn; use + ``intervene=`` specs to mutate. bwd_hooks are unsupported (no backward).""" + if bwd_hooks: + raise NotImplementedError( + "RemoteBridge has no backward pass; bwd_hooks are unsupported." + ) + if fwd_hooks: + warnings.warn( + "RemoteBridge fwd_hooks fire on already-captured activations (read-only): " + "a hook that returns a modified tensor does NOT change the forward " + "computation or logits — the return is discarded. To intervene on the " + "computation, pass intervene={hook_name: {'op': ...}} to " + "forward()/run_with_cache().", + UserWarning, + stacklevel=2, + ) + return super().run_with_hooks( + input, + fwd_hooks=fwd_hooks, + bwd_hooks=bwd_hooks, + reset_hooks_end=reset_hooks_end, + clear_contexts=clear_contexts, + return_type=return_type, + names_filter=names_filter, + stop_at_layer=stop_at_layer, + remove_batch_dim=remove_batch_dim, + **kwargs, + ) + def to_tokens(self, text: Any, *args: Any, **kwargs: Any) -> Any: """Tokenize via ``self.tokenizer``. BOS/padding handling lives on :class:`TransformerBridge`.""" assert self.tokenizer is not None, "Tokenizer must be set." return self.tokenizer.encode(text, return_tensors="pt") + + def __enter__(self) -> "RemoteBridge": + """Use as a context manager so the engine is released on exit: + ``with RemoteBridge.boot_vllm(...) as bridge: ...``.""" + return self + + def __exit__(self, *exc: Any) -> None: + self.close() + + def __del__(self) -> None: + # Best-effort safety net for notebooks that drop the bridge without + # close() — repeated boot_vllm would otherwise OOM. close() is idempotent. + try: + self.close() + except Exception: + pass diff --git a/transformer_lens/model_bridge/sources/_driver_base.py b/transformer_lens/model_bridge/sources/_driver_base.py index 175d3fee24..ac3be638ff 100644 --- a/transformer_lens/model_bridge/sources/_driver_base.py +++ b/transformer_lens/model_bridge/sources/_driver_base.py @@ -29,6 +29,11 @@ class DriverBase(ABC): # Subclasses override with the capability strings they actually serve. _supported_features: frozenset[str] = frozenset() + # True if forward() returns logits for every position, so loss is computable. + # Drivers that synthesize only the final position set this False; the bridge + # then refuses return_type=loss/both rather than return nan. + provides_sequence_logits: bool = True + def __init__( self, bridge_config: TransformerBridgeConfig, diff --git a/transformer_lens/model_bridge/sources/vllm/driver.py b/transformer_lens/model_bridge/sources/vllm/driver.py index a9fa167121..cf231c3cc7 100644 --- a/transformer_lens/model_bridge/sources/vllm/driver.py +++ b/transformer_lens/model_bridge/sources/vllm/driver.py @@ -1,6 +1,7 @@ """vLLM Driver: forward dispatches via ``llm.generate``; captures via ``collective_rpc``.""" from __future__ import annotations +import gc import logging from typing import Any, Mapping @@ -21,6 +22,8 @@ class VLLMDriver(DriverBase): # vLLM owns the model in a worker — no torch surface (parameters/state_dict/grads). _supported_features = frozenset() + # Logits synthesized for the final position only (sampler bypass). + provides_sequence_logits = False def __init__( self, @@ -36,6 +39,9 @@ def __init__( self._llm = llm self._max_num_batched_tokens = max_num_batched_tokens self._enable_batching = enable_batching + # Logprobs per forward = real vocab (boot's max_logprobs). d_vocab can be + # padded larger, which vLLM would reject; the logits tensor stays d_vocab. + self._n_logprobs = int(getattr(hf_config, "vocab_size", self.bridge_config.d_vocab)) self.supported_hook_points = frozenset(overlay.capture_specs(hf_config).keys()) @@ -67,8 +73,12 @@ def forward( ) intervene_specs = self._validate_interventions(intervene or {}) + # Restrict the GPU→CPU read to these hooks (None = all). run_with_cache + # doesn't derive this from names_filter yet — only explicit forward(capture=). + names = list(capture) or None + if self._enable_batching: - return self._forward_batched(input_ids, intervene_specs, return_logits) + return self._forward_batched(input_ids, intervene_specs, return_logits, names) ids_list = self._normalize_input_ids(input_ids) if len(ids_list) > self._max_num_batched_tokens: @@ -85,21 +95,19 @@ def forward( # Push intervention state (possibly empty) before generate — this also # resets stale interventions from prior forwards. self._llm.collective_rpc("tl_set_interventions", args=(intervene_specs,)) - # Request full-vocab logprobs so the driver can populate position -1 of - # the synthesized logits with the real next-token distribution. vLLM's - # ``max_logprobs`` was set to d_vocab at boot to make this legal. + # Full-vocab logprobs → position -1 of the synthesized logits (see _n_logprobs). outputs = self._llm.generate( prompts=[TokensPrompt(prompt_token_ids=ids_list)], sampling_params=SamplingParams( max_tokens=int(max_new_tokens), temperature=0.0, - logprobs=self.bridge_config.d_vocab if return_logits else None, + logprobs=self._n_logprobs if return_logits else None, ), ) n_tokens = len(ids_list) # collective_rpc returns one result per worker; single-rank, so [0]. - worker_captures = self._llm.collective_rpc("tl_read_captures", args=([n_tokens],))[0] + worker_captures = self._llm.collective_rpc("tl_read_captures", args=([n_tokens], names))[0] # Add batch dim: vLLM hands back (n_tokens, width); bridge expects (1, n_tokens, width). captured = {name: t.unsqueeze(0) for name, t in worker_captures.items()} @@ -114,11 +122,13 @@ def _forward_batched( input_ids: TensorLike, intervene_specs: dict, return_logits: bool, + names: list[str] | None = None, ) -> ForwardResult: """Eager batched path: per-request capture, right-padded to (B, S, W). No per-prompt length gate — chunked prefill accumulates long prompts - across forwards. Interventions are global across the batch. + across forwards. Interventions are global across the batch. ``names`` + restricts the returned hooks (``None`` = all). """ from vllm import SamplingParams from vllm.inputs import TokensPrompt @@ -137,13 +147,13 @@ def _forward_batched( sampling_params=SamplingParams( max_tokens=1, temperature=0.0, - logprobs=d_vocab if return_logits else None, + logprobs=self._n_logprobs if return_logits else None, ), ) # Keyed by req_id (no guaranteed order) — _assemble_padded joins to slot # k via outputs[k].request_id, not by position. - worker_captures = self._llm.collective_rpc("tl_read_batched_captures")[0] + worker_captures = self._llm.collective_rpc("tl_read_batched_captures", args=(names,))[0] captured = self._assemble_padded(outputs, worker_captures, prompt_lens) logits: torch.Tensor | None = None @@ -174,7 +184,16 @@ def _captures_for(public_rid: str) -> Mapping[str, torch.Tensor]: if public_rid in worker_captures: return worker_captures[public_rid] matches = [k for k in worker_keys if k.startswith(f"{public_rid}-")] - return worker_captures[matches[0]] if len(matches) == 1 else {} + # Raise, never silently zero-fill the row: a missing or ambiguous join + # is indistinguishable from a genuine zero activation, which is silent + # data loss on the collection path. + if len(matches) != 1: + raise RuntimeError( + f"Cannot join request {public_rid!r} to worker captures: found " + f"{len(matches)} key(s) in {worker_keys}. Expected exactly one " + f"(exact or '{public_rid}-')." + ) + return worker_captures[matches[0]] per_slot = [_captures_for(o.request_id) for o in outputs] @@ -214,6 +233,14 @@ def _synthesize_logits_batched( logits[k, pos, int(gen.token_ids[0])] = 0.0 return logits + def get_param(self, dotted_name: str) -> torch.Tensor | None: + """Fetch a named worker tensor (e.g. ``model.norm.weight``) for conversions + the bridge can't otherwise do (ln_final post→pre-weight; see the overlay). + None if closed or the path is missing.""" + if self._llm is None: + return None + return self._llm.collective_rpc("tl_get_param", args=(dotted_name,))[0] + def close(self) -> None: # Detach hooks before dropping the LLM so they don't stay registered on # worker modules for the life of the process (long-running notebooks). @@ -229,9 +256,7 @@ def close(self) -> None: # vLLM 0.20.2 has no LLM.shutdown() — model weights and KV cache stay # resident until process exit unless we explicitly tear down the # distributed environment vLLM set up at construction. Both calls are - # best-effort: they're no-ops if there's no distributed state. After - # this, the caller still needs gc.collect() + torch.cuda.empty_cache() - # to drop PyTorch's caching allocator entries. + # best-effort: they're no-ops if there's no distributed state. try: from vllm.distributed.parallel_state import ( destroy_distributed_environment, @@ -242,17 +267,22 @@ def close(self) -> None: destroy_distributed_environment() except Exception as e: log.debug("vLLM distributed teardown failed during close(): %s", e) + # Free the caching allocator's blocks here so the caller doesn't have to. + gc.collect() + try: + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception as e: + log.debug("torch.cuda.empty_cache failed during close(): %s", e) @staticmethod def _synthesize_logits(request_output: Any, n_tokens: int, d_vocab: int) -> torch.Tensor: """Build a (1, n_tokens, d_vocab) logits-like tensor from vLLM's sampler output. - vLLM's lm_head bypass means our hook never fires; the sampler returns - full-vocab logprobs (we set ``max_logprobs=d_vocab`` at boot to allow this). - Position -1 — the input's last token, = next-token prediction — is populated - from those logprobs. Earlier positions stay at ``-inf`` so any argmax there - is loud rather than silently misleading; populating them would need - ``prompt_logprobs`` requested per call (much more expensive). + Values are **log-probs**, not raw logits (vLLM returns log_softmax): fine + for argmax/next-token, wrong for absolute scale (temperature, logit-lens). + lm_head is bypassed so only position -1 is filled (next-token); earlier + positions stay -inf — populating them needs prompt_logprobs per call. """ logits = torch.full((1, n_tokens, d_vocab), float("-inf"), dtype=torch.float16) gen = request_output.outputs[0] if request_output.outputs else None diff --git a/transformer_lens/model_bridge/sources/vllm/overlays/decoder_only.py b/transformer_lens/model_bridge/sources/vllm/overlays/decoder_only.py index fbed07c8c4..0ce38efb37 100644 --- a/transformer_lens/model_bridge/sources/vllm/overlays/decoder_only.py +++ b/transformer_lens/model_bridge/sources/vllm/overlays/decoder_only.py @@ -15,11 +15,11 @@ separately (fused-residual). The plugin's hook materializes the sum so the captured value matches HF's "post-MLP residual stream". - ``ln_final.hook_normalized``: vLLM exposes ``x * rsqrt(var+eps) * weight``; - HF/HT exposes the pre-weight value. Divide the vLLM capture by - ``model.norm.weight`` (or ``1 + weight`` for Gemma) for HT-equivalent output. - -Non-decoder-only architectures (Mamba SSM, T5 encoder-decoder, BERT, MoE -per-expert) break the convention and would need their own overlays. + HF/HT exposes the pre-weight value. They are NOT auto-converted — the cache + carries vLLM's post-weight value under this name, so a direct diff against + ``boot_transformers`` will mismatch here. To convert, fetch the weight via + ``bridge._driver.get_param("model.norm.weight")`` and divide the capture by it + (or by ``1 + weight`` for Gemma). """ from __future__ import annotations diff --git a/transformer_lens/model_bridge/sources/vllm/plugin.py b/transformer_lens/model_bridge/sources/vllm/plugin.py index b3b0dd4e2b..d2ee620b9b 100644 --- a/transformer_lens/model_bridge/sources/vllm/plugin.py +++ b/transformer_lens/model_bridge/sources/vllm/plugin.py @@ -230,8 +230,11 @@ def hook(_module, _inputs, output): if not isinstance(t, torch.Tensor): return None + # Only this hook's spec — keyed by name like the compiled per-hook buffers. + # Iterating all specs would apply every intervention to every hook. modified = t - for spec in getattr(worker, "_tl_intervention_specs", {}).values(): + spec = getattr(worker, "_tl_intervention_specs", {}).get(canonical_name) + if spec is not None: modified = _apply_op(modified, spec) qsl, req_ids = segment_by_request(worker.model_runner) diff --git a/transformer_lens/model_bridge/sources/vllm/source.py b/transformer_lens/model_bridge/sources/vllm/source.py index cfafc5d63d..bb6d03d501 100644 --- a/transformer_lens/model_bridge/sources/vllm/source.py +++ b/transformer_lens/model_bridge/sources/vllm/source.py @@ -62,6 +62,13 @@ def boot_vllm( :mod:`transformer_lens.model_bridge.sources.vllm.overlays.decoder_only` for which hooks diverge and the conversion to apply for HT-equivalent values. + **Returned logits are log-probs, not raw logits.** vLLM bypasses ``lm_head``; + the driver fills the final position from the sampler's ``log_softmax`` output. + Correct for argmax / next-token, but absolute scale is off — temperature + scaling and logit-lens magnitude analysis will be wrong. Only the final + position is populated (earlier positions are ``-inf``); ``return_type`` in + ``{"loss", "both"}`` is rejected for that reason. + GPU memory cost: each capture buffer is ``max_num_batched_tokens × width`` at the model's dtype. For Llama-3.2-1B at fp16 with ``max_num_batched_tokens=2048``, the unembed buffer alone is ~525 MB (2048 × 128256 × 2 bytes); residual-stream diff --git a/transformer_lens/model_bridge/sources/vllm/worker_extension.py b/transformer_lens/model_bridge/sources/vllm/worker_extension.py index 17a8dc1f84..a462e826fd 100644 --- a/transformer_lens/model_bridge/sources/vllm/worker_extension.py +++ b/transformer_lens/model_bridge/sources/vllm/worker_extension.py @@ -16,7 +16,7 @@ """ from __future__ import annotations -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import torch @@ -35,15 +35,19 @@ class TLWorkerExtension: _tl_accum: Dict[tuple, List[torch.Tensor]] _tl_intervention_specs: Dict[str, Dict[str, Any]] - def tl_read_captures(self, prompt_lens: List[int]) -> Dict[str, torch.Tensor]: - """Slice each capture buffer back to ``sum(prompt_lens)`` rows; CPU copies. + def tl_read_captures( + self, prompt_lens: List[int], names: Optional[List[str]] = None + ) -> Dict[str, torch.Tensor]: + """Slice each capture buffer to ``sum(prompt_lens)`` rows; CPU copies. - Caller (VLLMDriver.forward) gates ``sum(prompt_lens) <= max_num_batched_tokens`` - before the RPC, so ``total`` is always within buffer bounds. + ``names`` restricts the read (``None`` = all) — this is the only GPU→CPU + crossing, so it's where a names_filtered run saves bandwidth. Caller gates + ``sum(prompt_lens) <= max_num_batched_tokens``, so ``total`` is in bounds. """ total = sum(prompt_lens) buffers: Dict[str, torch.Tensor] = getattr(self, "_tl_buffers", {}) - return {name: buf[:total].detach().cpu().clone() for name, buf in buffers.items()} + wanted = buffers.keys() if names is None else [n for n in names if n in buffers] + return {name: buffers[name][:total].detach().cpu().clone() for name in wanted} def tl_set_interventions(self, specs: Dict[str, Dict[str, Any]]) -> None: """Reset all affine buffers to identity, then apply each spec. @@ -65,6 +69,19 @@ def tl_set_interventions(self, specs: Dict[str, Dict[str, Any]]) -> None: raise KeyError(f"Unknown hook for intervention: {hook_name!r}") _apply_intervention(scale_bufs[hook_name], bias_bufs[hook_name], spec) + def tl_get_param(self, dotted_name: str) -> Optional[torch.Tensor]: + """Read a named model tensor (e.g. ``model.norm.weight``) as a CPU clone. + + ``None`` if the path doesn't resolve to a tensor. The bridge has no general + weight surface, so this is how callers reach e.g. the ln_final weight. + """ + target = getattr(self, "model_runner").model + for seg in dotted_name.split("."): + target = target[int(seg)] if seg.isdigit() else getattr(target, seg, None) + if target is None: + return None + return target.detach().cpu().clone() if isinstance(target, torch.Tensor) else None + def tl_reset_counter(self) -> None: """Zero the shared hook-fire counter before a forward.""" counter = getattr(self, "_tl_fire_counter", None) @@ -80,11 +97,20 @@ def tl_reset_accumulators(self) -> None: """Clear capture chunks before each generate, else prior chunks leak into the cat.""" self._tl_accum = {} - def tl_read_batched_captures(self) -> Dict[str, Dict[str, torch.Tensor]]: - """Cat per-request chunks into ``{req_id: {hook: (seq, width)}}`` (chunks are token-order).""" + def tl_read_batched_captures( + self, names: Optional[List[str]] = None + ) -> Dict[str, Dict[str, torch.Tensor]]: + """Cat per-request chunks into ``{req_id: {hook: (seq, width)}}`` (token-order). + + ``names`` restricts to those hooks (``None`` = all). Note the per-chunk + GPU→CPU copy already happened in the hook, so this only saves the cat. + """ accum: Dict[tuple, List[torch.Tensor]] = getattr(self, "_tl_accum", {}) + nameset = None if names is None else set(names) out: Dict[str, Dict[str, torch.Tensor]] = {} for (req_id, name), chunks in accum.items(): + if nameset is not None and name not in nameset: + continue out.setdefault(req_id, {})[name] = torch.cat(chunks, dim=0) return out From 59a1a97819a24e903ebc8587aa102cd7a482ba33 Mon Sep 17 00:00:00 2001 From: jlarson4 Date: Fri, 29 May 2026 18:05:55 -0500 Subject: [PATCH 04/87] Pass through bridge changes --- transformer_lens/model_bridge/bridge_core.py | 78 +++++----- .../model_bridge/transformer_bridge.py | 137 ++++++++++++++++-- 2 files changed, 164 insertions(+), 51 deletions(-) diff --git a/transformer_lens/model_bridge/bridge_core.py b/transformer_lens/model_bridge/bridge_core.py index 301432dbfe..96f30b94a5 100644 --- a/transformer_lens/model_bridge/bridge_core.py +++ b/transformer_lens/model_bridge/bridge_core.py @@ -421,6 +421,20 @@ def add_hook( else: raise AttributeError(f"Hook point '{hook_name}' not found on component") + def add_perma_hook( + self, + name: Union[str, Callable[[str], bool]], + hook_fn: Callable, + dir: Literal["fwd", "bwd"] = "fwd", + ) -> None: + """Add a permanent hook that survives ``reset_hooks()`` calls. + + Convenience wrapper for ``add_hook(..., is_permanent=True)``. To remove, + call ``reset_hooks(including_permanent=True)`` or remove from the + underlying ``HookPoint`` directly. + """ + self.add_hook(name, hook_fn, dir=dir, is_permanent=True) + def reset_hooks(self, clear_contexts: bool = True) -> None: """Remove all hooks. Registry is canonical; nn.Module children walked additively.""" for hp in self._hook_registry.values(): @@ -455,7 +469,7 @@ def hooks( @contextmanager def _hooks_context() -> Iterator["BridgeCore"]: - added_hooks: List[Tuple[HookPoint, str]] = [] + added_hooks: List[Tuple[HookPoint, Literal["fwd", "bwd"]]] = [] def add_hook_to_point( hook_point: HookPoint, @@ -471,7 +485,7 @@ def add_hook_to_point( hook_point.add_hook(hook_fn, dir=dir, alias_names=alias_names_list) else: hook_point.add_hook(hook_fn, dir=dir) - added_hooks.append((hook_point, name)) + added_hooks.append((hook_point, dir)) def apply_hooks(hook_list: List[Tuple[Any, Callable]], is_fwd: bool) -> None: direction: Literal["fwd", "bwd"] = "fwd" if is_fwd else "bwd" @@ -504,8 +518,8 @@ def apply_hooks(hook_list: List[Tuple[Any, Callable]], is_fwd: bool) -> None: yield self finally: if reset_hooks_end: - for hook_point, _ in added_hooks: - hook_point.remove_hooks() + for hook_point, direction in added_hooks: + hook_point.remove_hooks(dir=direction) return _hooks_context() @@ -530,7 +544,7 @@ def run_with_hooks( (KV cache cleaned up on stop). ``remove_batch_dim`` squeezes/unsqueezes the batch dim around hook callbacks (batch_size==1 only). """ - added_hooks: List[Tuple[HookPoint, str]] = [] + added_hooks: List[Tuple[HookPoint, Literal["fwd", "bwd"]]] = [] effective_stop_layer = None if stop_at_layer is not None and hasattr(self, "blocks"): if stop_at_layer < 0: @@ -559,7 +573,7 @@ def add_hook_to_point( hook_point.add_hook(hook_fn, dir=dir, alias_names=alias_names_list) else: hook_point.add_hook(hook_fn, dir=dir) - added_hooks.append((hook_point, name)) + added_hooks.append((hook_point, dir)) if stop_at_layer is not None and hasattr(self, "blocks"): if stop_at_layer < 0: @@ -629,8 +643,8 @@ def wrapped_hook_fn(tensor, hook, _orig_fn=original_hook_fn): return output finally: if reset_hooks_end: - for hook_point, _ in added_hooks: - hook_point.remove_hooks() + for hook_point, direction in added_hooks: + hook_point.remove_hooks(dir=direction) # ---- high-level execution: run_with_cache ---- @@ -668,6 +682,8 @@ def run_with_cache( """Run the model and cache activations. Returns ``(output, cache)``. ``stop_at_layer`` raises :class:`StopAtLayerException` to stop early. + ``device`` offloads cached activations (matches ``ActivationCache.to``); the + model and inputs stay where the caller put them. """ aliases = build_alias_to_canonical_map(self.hook_dict) @@ -776,37 +792,19 @@ def stop_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: hook_dict[block_hook_name].add_hook(stop_hook) hooks.append((hook_dict[block_hook_name], block_hook_name)) filtered_kwargs = kwargs.copy() - if cache_device is not None: - if getattr(self.cfg, "n_devices", 1) > 1: - # Moving a dispatched model to a single device collapses accelerate's - # split and breaks its routing hooks. The cache will stay spread across - # the per-layer devices; callers can .to(cache_device) on cache entries - # after the fact if they need a single-device cache. - warnings.warn( - f"run_with_cache(device={cache_device!r}) ignored: model is dispatched " - f"across {self.cfg.n_devices} devices via device_map. Cached activations " - "will remain on their per-layer devices.", - stacklevel=2, - ) - else: - try: - # original_model / its setter exist on TransformerBridge; on - # non-torch bridges the property raises AttributeError. - underlying = getattr(self, "original_model") - setattr(self, "original_model", underlying.to(cache_device)) - except AttributeError: - # Non-torch driver: cache_device doesn't apply to the model. - warnings.warn( - f"run_with_cache(device={cache_device!r}) ignored: driver does not " - "expose a local model. Cached activations will stay on driver-managed " - "devices.", - stacklevel=2, - ) - if processed_args and isinstance(processed_args[0], torch.Tensor): - processed_args = [processed_args[0].to(cache_device)] + list(processed_args[1:]) - for key, value in filtered_kwargs.items(): - if isinstance(value, torch.Tensor): - filtered_kwargs[key] = value.to(cache_device) + # ``cache_device`` is honored by ``make_cache_hook`` above (``tensor.detach().to(cache_device)``); + # the model and inputs stay where the caller put them, matching ``ActivationCache.to``. + if cache_device is not None and getattr(self.cfg, "n_devices", 1) > 1: + # Moving a dispatched model to a single device collapses accelerate's + # split and breaks its routing hooks. The cache will stay spread across + # the per-layer devices; callers can .to(cache_device) on cache entries + # after the fact if they need a single-device cache. + warnings.warn( + f"run_with_cache(device={cache_device!r}) ignored: model is dispatched " + f"across {self.cfg.n_devices} devices via device_map. Cached activations " + "will remain on their per-layer devices.", + stacklevel=2, + ) try: if "output_attentions" not in filtered_kwargs: filtered_kwargs["output_attentions"] = True @@ -827,7 +825,7 @@ def stop_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: raise e finally: for hp, _ in hooks: - hp.remove_hooks() + hp.remove_hooks(dir="fwd") if self.compatibility_mode == True: reverse_aliases = {} for old_name, new_name in aliases.items(): diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index ae3830a911..138c8d7d75 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -28,6 +28,7 @@ from torch import nn from transformer_lens import utilities as utils +from transformer_lens.ActivationCache import ActivationCache from transformer_lens.FactoredMatrix import FactoredMatrix from transformer_lens.hook_points import HookIntrospectionMixin, HookPoint from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter @@ -504,6 +505,14 @@ def enable_compatibility_mode( post-processed coordinate system: logit lens, direct logit attribution, residual-stream norms. Also enables legacy hook/component name aliases. + Hook semantic parity (issue #1317): ``hook_q_input``, ``hook_k_input``, + ``hook_v_input``, ``hook_attn_in``, and ``hook_mlp_in`` fire on the + pre-norm residual. Carve-outs: post-norm architectures (OLMo 2, + BERT-style) read the post-attention residual instead, and MLA blocks + (DeepSeek V2/V3/R1) do not expose the split-qkv aliases. ``hook_mlp_in`` + is gated on ``cfg.use_hook_mlp_in``; toggle it via + :py:meth:`set_use_hook_mlp_in`. + Args: disable_warnings: Whether to disable warnings about legacy components/hooks no_processing: Whether to disable ALL pre-processing steps of the model. @@ -532,6 +541,11 @@ def set_compatibility_mode(component: Any) -> None: apply_fn_to_all_components(self, set_compatibility_mode) self.clear_hook_registry() + # Drop pre-ln capture handles from any prior call so they don't accumulate. + if hasattr(self, "blocks"): + for block in self.blocks: + if hasattr(block, "_teardown_pre_ln_capture"): + block._teardown_pre_ln_capture() try: if not no_processing: self.process_weights( @@ -1739,10 +1753,30 @@ def _generate_tokens( forward_kwargs["use_cache"] = True if _hf_kv_cache is not None: forward_kwargs["past_key_values"] = _hf_kv_cache + # HF v5 + macOS-arm64 NaNs when these are inferred + # from cache state alone. Mirror HF generate(): pass + # both an (batch, total_len) attention_mask and a + # (batch, 1) position_ids for the new token. + batch_size = current_tokens.shape[0] + total_len = current_tokens.shape[1] + device = current_tokens.device + if "attention_mask" not in forward_kwargs: + forward_kwargs["attention_mask"] = torch.ones( + (batch_size, total_len), + dtype=torch.long, + device=device, + ) if "position_ids" in forward_kwargs: forward_kwargs["position_ids"] = forward_kwargs["position_ids"][ :, -1: ] + else: + forward_kwargs["position_ids"] = torch.full( + (batch_size, 1), + total_len - 1, + dtype=torch.long, + device=device, + ) logits = self( current_tokens[:, -1:], return_type="logits", @@ -1838,9 +1872,14 @@ def generate( return_type: Optional[str] = "input", verbose: bool = True, output_logits: bool = False, + return_cache: bool = False, + names_filter: Optional[Union[str, List[str], Callable[[str], bool]]] = None, + device: Optional[Union[str, torch.device]] = None, pixel_values: Optional[torch.Tensor] = None, **multimodal_kwargs, - ) -> str | list[str] | torch.Tensor | Any: # Any for transformers.utils.ModelOutput + ) -> ( + str | list[str] | torch.Tensor | Any | tuple[Any, ActivationCache] + ): # Any for transformers.utils.ModelOutput # Any: beartype forward ref limitation (beartype#546) """Sample tokens from the model. @@ -1870,6 +1909,18 @@ def generate( return_type: The type of output to return - 'input', 'str', or 'tokens' verbose: Not used in Bridge (kept for API compatibility) output_logits: If True, return a ModelOutput with sequences and logits tuple + return_cache: If True, also return an ActivationCache for the full prompt + + generated sequence, identical to ``run_with_cache(output)``, and the call + returns an ``(output, cache)`` tuple. Implemented as one extra clean forward + over the output, so the cache includes every hook point (attention patterns + included). Supported only for single-sequence, decoder-only text generation; + encoder-decoder, SSM, multimodal, batched, and inputs_embeds inputs raise + NotImplementedError. The cache spans prompt + max_new_tokens and can be large, + use ``names_filter`` to scope it and/or ``device`` to offload it. + names_filter: Passed to ``run_with_cache`` when ``return_cache=True``; restricts + which activations are cached (str, list of str, or callable). + device: Passed through when ``return_cache=True`` to offload the cached tensors + to this device (e.g. "cpu") to save accelerator memory. pixel_values: Optional image tensor for multimodal models. Only passed on the first generation step (the vision encoder processes the image once, then embeddings are part of the token sequence for subsequent steps). @@ -1877,6 +1928,13 @@ def generate( Returns: Generated sequence as string, list of strings, or tensor depending on input type and return_type. If output_logits=True, returns a ModelOutput-like object with 'sequences' and 'logits' attributes. + If return_cache=True, returns an ``(output, ActivationCache)`` tuple where ``output`` is the + value that would otherwise be returned and the cache equals ``run_with_cache(output)``. + + Example: + ``out, cache = model.generate(prompt, max_new_tokens=20, return_cache=True)`` returns a + normal ActivationCache over the full prompt + generated sequence (equivalent to + ``run_with_cache(out)``). """ # prepend_bos is intentionally not applied during generation. # The HF model expects tokens in its native format. Overriding BOS can silently @@ -1970,6 +2028,37 @@ def generate( self.original_model.config, "is_encoder_decoder", False ) + # return_cache recomputes run_with_cache on the generated output (see issue #697). + # That is well-defined only for single-sequence, decoder-only text generation, so + # reject the paths whose cache would be wrong/undefined, with a clear pointer to the + # run_with_cache workaround. Fail fast here, before any generation work. + if return_cache: + if is_encoder_decoder: + raise NotImplementedError( + "generate(return_cache=True) is not supported for encoder-decoder " + "models yet. Run run_with_cache on the generated output instead." + ) + if is_stateful_model: + raise NotImplementedError( + "generate(return_cache=True) is not supported for stateful/SSM models " + "(e.g. Mamba); they do not expose standard transformer hook points." + ) + if pixel_values is not None or multimodal_kwargs: + raise NotImplementedError( + "generate(return_cache=True) is not supported for multimodal generation " + "yet. Run run_with_cache on the generated output instead." + ) + if _generate_from_embeds: + raise NotImplementedError( + "generate(return_cache=True) requires token input, not inputs_embeds." + ) + if batch_size > 1: + raise NotImplementedError( + "generate(return_cache=True) is not supported for batched/multi-prompt " + "generation yet. Pass a single prompt, or run run_with_cache on each " + "output sequence." + ) + # HF cache flows opaquely through the component chain via # _reconstruct_attention() → _update_kv_cache() on each layer. _hf_kv_cache = None @@ -2093,7 +2182,8 @@ def generate( else: output_tokens = torch.cat([input_tokens, sampled_tokens], dim=1) - # Return ModelOutput if output_logits was requested + # Build the formatted output (shape unchanged: ModelOutput / str / list[str] / tokens). + result: Any if output_logits and logits_seq_list is not None: from transformers.utils import ModelOutput # type: ignore @@ -2105,9 +2195,9 @@ def _logits_to_tuple(logits_list: list[torch.Tensor]) -> tuple[torch.Tensor, ... try: from transformers.generation.utils import GenerateDecoderOnlyOutput - # Return a HF-compatible ModelOutput structure + # HF-compatible ModelOutput structure. # GenerateDecoderOnlyOutput expects: sequences, scores (optional), logits (optional) - return GenerateDecoderOnlyOutput( + result = GenerateDecoderOnlyOutput( sequences=cast(torch.LongTensor, output_tokens), # HF's type hint says tuple[FloatTensor] but should be tuple[FloatTensor, ...] # (variable-length tuple with one element per generated token) @@ -2115,24 +2205,32 @@ def _logits_to_tuple(logits_list: list[torch.Tensor]) -> tuple[torch.Tensor, ... ) except (ImportError, AttributeError): # Fallback if GenerateDecoderOnlyOutput not available in this transformers version - return ModelOutput( + result = ModelOutput( sequences=output_tokens, logits=_logits_to_tuple(logits_seq_list), ) - - # Format output - if return_type == "str": + elif return_type == "str": assert self.tokenizer is not None if input_type == "str": - return self.tokenizer.decode(output_tokens[0], skip_special_tokens=True) + result = self.tokenizer.decode(output_tokens[0], skip_special_tokens=True) else: decoded_texts = [ self.tokenizer.decode(tokens, skip_special_tokens=True) for tokens in output_tokens ] - return decoded_texts[0] if len(decoded_texts) == 1 else decoded_texts + result = decoded_texts[0] if len(decoded_texts) == 1 else decoded_texts else: # return_type == "tokens" - return output_tokens + result = output_tokens + + if not return_cache: + return result + + # return_cache: recompute one clean forward over the full generated sequence so the + # cache is identical to run_with_cache(output_tokens) - all hook points, including + # attention patterns. The guards above restrict this to single-sequence, decoder-only + # text generation (see issue #697). + _, cache = self.run_with_cache(output_tokens, names_filter=names_filter, device=device) + return result, cache @torch.no_grad() def generate_stream( @@ -2673,6 +2771,23 @@ def set_use_attn_in(self, use_attn_in: bool): self.cfg.use_attn_in = use_attn_in self._propagate_attention_flag("use_attn_in", use_attn_in) + def set_use_hook_mlp_in(self, use_hook_mlp_in: bool) -> None: + """Toggle the pre-ln2 ``hook_mlp_in`` HookPoint, matching legacy semantics. + + See :py:meth:`HookedTransformer.set_use_hook_mlp_in`. + """ + self.cfg.use_hook_mlp_in = use_hook_mlp_in + if not hasattr(self, "blocks"): + return + for block in self.blocks: + block_cfg = getattr(block, "config", None) + if block_cfg is not None and block_cfg is not self.cfg: + try: + block_cfg.use_hook_mlp_in = use_hook_mlp_in + except Exception: + pass + block._use_hook_mlp_in = use_hook_mlp_in + def _propagate_attention_flag(self, flag_name: str, value: bool) -> None: """Mirror `bridge.cfg.` onto every block's attention config. From 91ea21dedc0ddf31333eefbfe3fddfda54761037 Mon Sep 17 00:00:00 2001 From: Jonah Larson Date: Mon, 8 Jun 2026 16:32:09 -0500 Subject: [PATCH 05/87] Inspect Driver (#1367) * INitial Inspect Driver setup * Additional improvements to inspect * Cleaning up inspect driver flaws * Adding additional coverage to ensure verify interventions & CUDA support * Improving architecture verification process * extended inspect tests to cover more model types * Clean up leaking rand change * Fix inspect memoize bug * Additional inspect improvements * Initial setup of inspect vllm provider * Additional inspect driver feature tightening * migrating dev 4.x changes * format checks * docstring cleanup * Fixing CI issues --- demos/Inspect_Bridge_Demo.ipynb | 3115 +++++++++++++++++ demos/Inspect_vLLM_Provider_Demo.ipynb | 360 ++ pyproject.toml | 11 + scripts/inspect_parity_report.py | 157 + tests/acceptance/model_bridge/conftest.py | 9 +- .../model_bridge/test_inspect_provider.py | 637 ++++ .../unit/model_bridge/test_inspect_driver.py | 438 +++ .../test_inspect_vllm_provider.py | 564 +++ .../model_bridge/remote_bridge.py | 52 +- .../model_bridge/sources/inspect/__init__.py | 25 + .../sources/inspect/_provider_base.py | 201 ++ .../model_bridge/sources/inspect/conftest.py | 19 + .../model_bridge/sources/inspect/driver.py | 182 + .../model_bridge/sources/inspect/eval.py | 118 + .../model_bridge/sources/inspect/hooks.py | 86 + .../sources/inspect/intervention.py | 60 + .../model_bridge/sources/inspect/profiles.py | 134 + .../model_bridge/sources/inspect/source.py | 151 + .../sources/inspect/transformers_provider.py | 522 +++ .../sources/inspect/vllm_provider.py | 439 +++ .../model_bridge/sources/inspect/wire.py | 70 + .../model_bridge/sources/vllm/driver.py | 4 + .../model_bridge/sources/vllm/plugin.py | 38 +- .../sources/vllm/worker_extension.py | 14 + uv.lock | 1023 ++++-- 25 files changed, 8219 insertions(+), 210 deletions(-) create mode 100644 demos/Inspect_Bridge_Demo.ipynb create mode 100644 demos/Inspect_vLLM_Provider_Demo.ipynb create mode 100644 scripts/inspect_parity_report.py create mode 100644 tests/acceptance/model_bridge/test_inspect_provider.py create mode 100644 tests/unit/model_bridge/test_inspect_driver.py create mode 100644 tests/unit/model_bridge/test_inspect_vllm_provider.py create mode 100644 transformer_lens/model_bridge/sources/inspect/__init__.py create mode 100644 transformer_lens/model_bridge/sources/inspect/_provider_base.py create mode 100644 transformer_lens/model_bridge/sources/inspect/conftest.py create mode 100644 transformer_lens/model_bridge/sources/inspect/driver.py create mode 100644 transformer_lens/model_bridge/sources/inspect/eval.py create mode 100644 transformer_lens/model_bridge/sources/inspect/hooks.py create mode 100644 transformer_lens/model_bridge/sources/inspect/intervention.py create mode 100644 transformer_lens/model_bridge/sources/inspect/profiles.py create mode 100644 transformer_lens/model_bridge/sources/inspect/source.py create mode 100644 transformer_lens/model_bridge/sources/inspect/transformers_provider.py create mode 100644 transformer_lens/model_bridge/sources/inspect/vllm_provider.py create mode 100644 transformer_lens/model_bridge/sources/inspect/wire.py diff --git a/demos/Inspect_Bridge_Demo.ipynb b/demos/Inspect_Bridge_Demo.ipynb new file mode 100644 index 0000000000..802106b236 --- /dev/null +++ b/demos/Inspect_Bridge_Demo.ipynb @@ -0,0 +1,3115 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "b7e48565", + "metadata": {}, + "source": [ + "# Inspect Driver — Demo\n", + "\n", + "Turn a model served through [`inspect_ai`](https://inspect.aisi.org.uk) into a\n", + "TransformerLens `HookedTransformer`: `run_with_cache`, named hook points, and\n", + "interventions all work over the Inspect boundary.\n", + "\n", + "`boot_inspect` boots the model behind an `inspect_ai` provider and wraps it in a\n", + "bridge with the standard TL hook contract.\n", + "\n", + "**Scope (v1):**\n", + "- Captures residual / attention / MLP outputs under their TransformerBridge names\n", + " (the same canonical names the vLLM driver uses):\n", + " `blocks.{i}.hook_in` (resid_pre) / `blocks.{i}.ln2.hook_in` (resid_mid) /\n", + " `blocks.{i}.hook_out` (resid_post), `blocks.{i}.attn.hook_out`,\n", + " `blocks.{i}.mlp.hook_out`. Head-split hooks (`hook_q/k/v/z`, `hook_pattern`)\n", + " aren't available — use `boot_transformers()`.\n", + "- Which of those a given model actually serves is decided by a per-model structural\n", + " self-check: `resid_mid` is gated on parallel / norm-variant architectures, and\n", + " `attn_out`/`mlp_out` when their submodule isn't locatable. gpt2 (here) serves all five.\n", + "- Returns full-sequence logits, so `return_type=\"loss\"` works too.\n", + "\n", + "Runs on CPU; no GPU required." + ] + }, + { + "cell_type": "markdown", + "id": "91511cda", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Install TransformerLens with the `inspect` extra (pulls in `inspect_ai`):\n", + "\n", + "```bash\n", + "pip install \"transformer_lens[inspect]\"\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "1b0c92d4", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-29T18:38:08.301264Z", + "iopub.status.busy": "2026-05-29T18:38:08.301205Z", + "iopub.status.idle": "2026-05-29T18:38:11.378083Z", + "shell.execute_reply": "2026-05-29T18:38:11.377648Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import warnings\n", + "warnings.filterwarnings(\"ignore\")\n", + "import torch\n", + "\n", + "from transformer_lens.model_bridge.remote_bridge import RemoteBridge\n", + "from transformer_lens.model_bridge.transformer_bridge import TransformerBridge\n", + "\n", + "MODEL = \"gpt2\"\n", + "PROMPT = \"The quick brown fox\"\n", + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "id": "f5f730f7", + "metadata": {}, + "source": [ + "## Step 1 — Boot the model through Inspect\n", + "\n", + "`boot_inspect` resolves the architecture/config (no weights loaded TL-side), boots\n", + "the model behind an `inspect_ai` provider, and wraps it in a `RemoteBridge` with\n", + "the standard hook contract." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "5ebf4efd", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-29T18:38:11.379416Z", + "iopub.status.busy": "2026-05-29T18:38:11.379269Z", + "iopub.status.idle": "2026-05-29T18:38:13.454236Z", + "shell.execute_reply": "2026-05-29T18:38:13.453801Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "43666d4ba2304c08baff0f8809084935", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading weights: 0%| | 0/148 [00:00` is also a normal `inspect_ai` model, so it plugs into the eval\n", + "pipeline — not just the bridge capture path above. The next cells show: **(6)** eval-native\n", + "generation with logprobs + usage, **(7)** capturing activations alongside a scored eval\n", + "(full tensors to a side-artifact + a reduction in `samples_df`), **(8)** per-turn capture\n", + "across a rollout, and **(9)** tool-aware generation." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "709a4c77", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-29T18:38:14.255424Z", + "iopub.status.busy": "2026-05-29T18:38:14.255357Z", + "iopub.status.idle": "2026-05-29T18:38:15.661882Z", + "shell.execute_reply": "2026-05-29T18:38:15.661407Z" + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7d0b2131829348a9a5c2309f34a71914", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading weights: 0%| | 0/148 [00:00 6 tokens\n", + "1st-token top-3: [(' the', -2.47), (' now', -3.04), (' a', -3.08)]\n" + ] + } + ], + "source": [ + "# (6) Eval-native generation: tl_bridge is a normal Inspect model — real multi-token\n", + "# generation with token logprobs + usage, read straight from the eval log.\n", + "import tempfile\n", + "\n", + "from inspect_ai import Task\n", + "from inspect_ai import eval as inspect_eval\n", + "from inspect_ai.dataset import Sample\n", + "from inspect_ai.solver import generate\n", + "\n", + "run = tempfile.mkdtemp()\n", + "gen_log = inspect_eval(\n", + " Task(dataset=[Sample(input=\"The capital of France is\")], solver=generate()),\n", + " model=\"tl_bridge/gpt2\",\n", + " max_tokens=6,\n", + " logprobs=True,\n", + " top_logprobs=3,\n", + " log_dir=f\"{run}/gen\",\n", + " display=\"none\",\n", + ")[0]\n", + "out = gen_log.samples[0].output\n", + "print(\"completion:\", repr(out.completion))\n", + "print(\"usage:\", out.usage.input_tokens, \"->\", out.usage.output_tokens, \"tokens\")\n", + "top3 = out.choices[0].logprobs.content[0].top_logprobs\n", + "print(\"1st-token top-3:\", [(t.token, round(t.logprob, 2)) for t in top3])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "0c84386e", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-29T18:38:15.662903Z", + "iopub.status.busy": "2026-05-29T18:38:15.662841Z", + "iopub.status.idle": "2026-05-29T18:38:16.703794Z", + "shell.execute_reply": "2026-05-29T18:38:16.703347Z" + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "510fe4459254404599829fa646a82d57", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading weights: 0%| | 0/148 [00:00{\"name\": \"add\", \"arguments\": {\"a\": 2, \"b\": 2}}')\nprint(\"parsed tool calls:\", [(c.function, c.arguments) for c in calls])" + }, + { + "cell_type": "markdown", + "id": "81fc52b5", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "- `boot_inspect(\"gpt2\")` returns a bridge with the standard TL hook contract.\n", + "- Residual / attention / MLP activations cross the Inspect boundary and match\n", + " `boot_transformers` to fp tolerance (exact next-token argmax) — conversion to torch\n", + " happens at the bridge.\n", + "- Full-sequence logits, so `return_type=\"loss\"` matches `boot_transformers` too.\n", + "- Full affine interventions (suppress/scale/add/set) apply and revert.\n", + "- `tl_bridge/` is also a normal Inspect model: eval-native generation with\n", + " logprobs + usage, `capture_activations` alongside a scored eval (→ `.npz` + `samples_df`),\n", + " per-turn capture across rollouts (`turn_activations`), and tool-aware generation.\n", + "\n", + "**Out of v1 scope:** head-split hooks (`hook_q/k/v/z`, `hook_pattern`), `embed`/`ln_final`\n", + "(convention), batch > 1, and live agent loops with tool *execution* (need a tool-capable\n", + "model). For those, use `boot_transformers()`." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "transformer-lens", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "039b65ce93724f1f8d988cf38f812441": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "049b91662c18485ba8d310a6ba5cdf75": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0dbf433c50ea4822a4aa778b76433fa9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_ceb7407ac8634c0797ce74cf2d1ec273", + "placeholder": "​", + "style": "IPY_MODEL_7657d3c31e644ecb8a61731ef81074c0", + "tabbable": null, + "tooltip": null, + "value": "Loading weights: 100%" + } + }, + "0f4ddd3fb96b426d9357af7a32f68f51": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_8b0e6dd40b724f09938ee1cf7fcd3920", + "placeholder": "​", + "style": "IPY_MODEL_aafb576cbcdb487080bfef599d9b5280", + "tabbable": null, + "tooltip": null, + "value": " 148/148 [00:00<00:00, 16029.46it/s]" + } + }, + "1508e54470b24d8bb040b7098446a600": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "154464f730194e2d977650ac26fac364": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_3a97799075e945189635b381c195df50", + "max": 148, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_b7e243057b82482389dc8c14f4d0c043", + "tabbable": null, + "tooltip": null, + "value": 148 + } + }, + "15d832fda0484f748d26c8bcc51cc6e0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "163bb672010f41e98f33c1842acf6ff7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1684be8213734d4ab7d7fb8502594275": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_c9c81b8b998145f9acf7396fc90d2488", + "placeholder": "​", + "style": "IPY_MODEL_80dd59d42a0f4fc08d6b40f8592c8646", + "tabbable": null, + "tooltip": null, + "value": "Loading weights: 100%" + } + }, + "1801dd8dc1494d15b5ded6765683ab96": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1ce90842f0114f1b83e2ed5fbf47f10d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_b42913c5fc2a472dabf6c1eddb6e5238", + "max": 148, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_ec038eccf291410ba0946f1c54c41e1c", + "tabbable": null, + "tooltip": null, + "value": 148 + } + }, + "1efa41221d824dc7b89757a4108a1ecc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2b141b703af44686a7a6faaaeecbc159": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2b9f283ea6364020b34f1908660f436f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2bc55ce15643487aba52c306a247a16f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_1efa41221d824dc7b89757a4108a1ecc", + "placeholder": "​", + "style": "IPY_MODEL_6297f7fdb27a47e49096fe1b2574f50e", + "tabbable": null, + "tooltip": null, + "value": " 148/148 [00:00<00:00, 25208.41it/s]" + } + }, + "2d94897c4ee5473b8ac65ce7df738e8c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "33456edc9b424dbcb23e399792d73723": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "345c6f94a03c480bb78841d1e5ebcd5f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "3596e9d35b23407e91e8a8bb2487b181": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3a97799075e945189635b381c195df50": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3bbb9853ab904247abd3387149795bcc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3f56b4c85fc3475ca10948bb4a77b81d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_15d832fda0484f748d26c8bcc51cc6e0", + "placeholder": "​", + "style": "IPY_MODEL_445917f730124d3284680ce2eefa0f52", + "tabbable": null, + "tooltip": null, + "value": " 148/148 [00:00<00:00, 22674.40it/s]" + } + }, + "445917f730124d3284680ce2eefa0f52": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "44ab299ad94d420187fb4f5a4de80d1b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_049b91662c18485ba8d310a6ba5cdf75", + "placeholder": "​", + "style": "IPY_MODEL_b641905d12aa45b2893211db21672211", + "tabbable": null, + "tooltip": null, + "value": "Loading weights: 100%" + } + }, + "485253be7f644fdba502cc068131dac9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "5a94b8f0ea244a8890c7e8858ca48605": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_a6d9da81e3bc4c839a140fd980a8bcea", + "placeholder": "​", + "style": "IPY_MODEL_6f39418339e44a539746fc0abecc417d", + "tabbable": null, + "tooltip": null, + "value": "Loading weights: 100%" + } + }, + "5b22465d443b45cb88bccb0b357d262f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_3596e9d35b23407e91e8a8bb2487b181", + "placeholder": "​", + "style": "IPY_MODEL_345c6f94a03c480bb78841d1e5ebcd5f", + "tabbable": null, + "tooltip": null, + "value": "Loading weights: 100%" + } + }, + "5ca604a82a194495abf4871c413115b2": { + "model_module": "@jupyter-widgets/output", + "model_module_version": "1.0.0", + "model_name": "OutputModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/output", + "_model_module_version": "1.0.0", + "_model_name": "OutputModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/output", + "_view_module_version": "1.0.0", + "_view_name": "OutputView", + "layout": "IPY_MODEL_648d530cb09047b2a043334c6178a26c", + "msg_id": "", + "outputs": [], + "tabbable": null, + "tooltip": null + } + }, + "610c42e408bb4b2199290e50ef2f8dbf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_2d94897c4ee5473b8ac65ce7df738e8c", + "max": 148, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_662f8455d13e43cc872b686ac37655e2", + "tabbable": null, + "tooltip": null, + "value": 148 + } + }, + "6297f7fdb27a47e49096fe1b2574f50e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "648d530cb09047b2a043334c6178a26c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "65d313562b014e4191d6d50f7d213500": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_0dbf433c50ea4822a4aa778b76433fa9", + "IPY_MODEL_610c42e408bb4b2199290e50ef2f8dbf", + "IPY_MODEL_7c89ec1b6a744f17be6a01e863dcd97e" + ], + "layout": "IPY_MODEL_ae00be5c86244a43b1587fb4cb389230", + "tabbable": null, + "tooltip": null + } + }, + "662f8455d13e43cc872b686ac37655e2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6f39418339e44a539746fc0abecc417d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "710de54c091346caa50464ed332222ec": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_8ef8501982bc46aaac18a218eaca75e2", + "max": 148, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_da2064a89aca48678c9d4b2611a5c8af", + "tabbable": null, + "tooltip": null, + "value": 148 + } + }, + "7492f0d02ce94737933f78a4479b6054": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_039b65ce93724f1f8d988cf38f812441", + "max": 148, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_86ab9534cefa40fbbc899d89ab66ffbb", + "tabbable": null, + "tooltip": null, + "value": 148 + } + }, + "7657d3c31e644ecb8a61731ef81074c0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "7c89ec1b6a744f17be6a01e863dcd97e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_d5c90e6f5ec2475eb6308d3c0b95a5a6", + "placeholder": "​", + "style": "IPY_MODEL_d30e782b7a62432983c77aeeb699f393", + "tabbable": null, + "tooltip": null, + "value": " 148/148 [00:00<00:00, 24612.70it/s]" + } + }, + "80dd59d42a0f4fc08d6b40f8592c8646": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "86ab9534cefa40fbbc899d89ab66ffbb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "87651219336b48eab027e004737206ca": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a9fa4413a9ff4afb875ef05e22a15d2c", + "IPY_MODEL_154464f730194e2d977650ac26fac364", + "IPY_MODEL_3f56b4c85fc3475ca10948bb4a77b81d" + ], + "layout": "IPY_MODEL_94f4afcfa29f4bf19daf21626d9ae7b3", + "tabbable": null, + "tooltip": null + } + }, + "8a6b53647a824d96a1f562ab72537508": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_e73be1310dfd49d19cce7b39dc932c0e", + "max": 148, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_93c1eaa4ecd241ff9bbdbec12989301b", + "tabbable": null, + "tooltip": null, + "value": 148 + } + }, + "8b0e6dd40b724f09938ee1cf7fcd3920": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8ec58ea9c4e8471da2e9678eee3ec5ae": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "8ef8501982bc46aaac18a218eaca75e2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8fe059ca8d95476d9c5248b0ce15cb8b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9396484a42eb4e93b67d06c45eb033f1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_2b141b703af44686a7a6faaaeecbc159", + "placeholder": "​", + "style": "IPY_MODEL_8ec58ea9c4e8471da2e9678eee3ec5ae", + "tabbable": null, + "tooltip": null, + "value": " 148/148 [00:00<00:00, 22705.91it/s]" + } + }, + "93c1eaa4ecd241ff9bbdbec12989301b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "94f4afcfa29f4bf19daf21626d9ae7b3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a6d9da81e3bc4c839a140fd980a8bcea": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a9fa4413a9ff4afb875ef05e22a15d2c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_f383a3567190447aaf32d702e8b6643f", + "placeholder": "​", + "style": "IPY_MODEL_485253be7f644fdba502cc068131dac9", + "tabbable": null, + "tooltip": null, + "value": "Loading weights: 100%" + } + }, + "aafb576cbcdb487080bfef599d9b5280": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "ae00be5c86244a43b1587fb4cb389230": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ae58b21474564d61a500970e2de702a9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_5a94b8f0ea244a8890c7e8858ca48605", + "IPY_MODEL_7492f0d02ce94737933f78a4479b6054", + "IPY_MODEL_2bc55ce15643487aba52c306a247a16f" + ], + "layout": "IPY_MODEL_8fe059ca8d95476d9c5248b0ce15cb8b", + "tabbable": null, + "tooltip": null + } + }, + "b42913c5fc2a472dabf6c1eddb6e5238": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b641905d12aa45b2893211db21672211": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "b7e243057b82482389dc8c14f4d0c043": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "bab5623c801e40e1bd2c3d72c525f74f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1684be8213734d4ab7d7fb8502594275", + "IPY_MODEL_8a6b53647a824d96a1f562ab72537508", + "IPY_MODEL_f4375282f79e4799ab414b4c52931f20" + ], + "layout": "IPY_MODEL_163bb672010f41e98f33c1842acf6ff7", + "tabbable": null, + "tooltip": null + } + }, + "bf4577111eeb43c1824c68837ae0d68c": { + "model_module": "@jupyter-widgets/output", + "model_module_version": "1.0.0", + "model_name": "OutputModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/output", + "_model_module_version": "1.0.0", + "_model_name": "OutputModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/output", + "_view_module_version": "1.0.0", + "_view_name": "OutputView", + "layout": "IPY_MODEL_3bbb9853ab904247abd3387149795bcc", + "msg_id": "", + "outputs": [], + "tabbable": null, + "tooltip": null + } + }, + "c5490c3fbd58414ba8aac8aeae090fc5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c9c81b8b998145f9acf7396fc90d2488": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ceb7407ac8634c0797ce74cf2d1ec273": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d30e782b7a62432983c77aeeb699f393": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "d51337139f614c86913d3ddbdded26c8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_44ab299ad94d420187fb4f5a4de80d1b", + "IPY_MODEL_1ce90842f0114f1b83e2ed5fbf47f10d", + "IPY_MODEL_0f4ddd3fb96b426d9357af7a32f68f51" + ], + "layout": "IPY_MODEL_1508e54470b24d8bb040b7098446a600", + "tabbable": null, + "tooltip": null + } + }, + "d5c90e6f5ec2475eb6308d3c0b95a5a6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "da2064a89aca48678c9d4b2611a5c8af": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "e73be1310dfd49d19cce7b39dc932c0e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e7d7ea997bff4327a957a6d26fadcecd": { + "model_module": "@jupyter-widgets/output", + "model_module_version": "1.0.0", + "model_name": "OutputModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/output", + "_model_module_version": "1.0.0", + "_model_name": "OutputModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/output", + "_view_module_version": "1.0.0", + "_view_name": "OutputView", + "layout": "IPY_MODEL_1801dd8dc1494d15b5ded6765683ab96", + "msg_id": "", + "outputs": [], + "tabbable": null, + "tooltip": null + } + }, + "ec038eccf291410ba0946f1c54c41e1c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "f383a3567190447aaf32d702e8b6643f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f4375282f79e4799ab414b4c52931f20": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_2b9f283ea6364020b34f1908660f436f", + "placeholder": "​", + "style": "IPY_MODEL_33456edc9b424dbcb23e399792d73723", + "tabbable": null, + "tooltip": null, + "value": " 148/148 [00:00<00:00, 24092.10it/s]" + } + }, + "fb1929f170da4f1cae3741651e9f1305": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_5b22465d443b45cb88bccb0b357d262f", + "IPY_MODEL_710de54c091346caa50464ed332222ec", + "IPY_MODEL_9396484a42eb4e93b67d06c45eb033f1" + ], + "layout": "IPY_MODEL_c5490c3fbd58414ba8aac8aeae090fc5", + "tabbable": null, + "tooltip": null + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/demos/Inspect_vLLM_Provider_Demo.ipynb b/demos/Inspect_vLLM_Provider_Demo.ipynb new file mode 100644 index 0000000000..75ca627cc2 --- /dev/null +++ b/demos/Inspect_vLLM_Provider_Demo.ipynb @@ -0,0 +1,360 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Inspect Driver — vLLM Provider Demo\n", + "\n", + "Same surface as the HF-backed `Inspect_Bridge_Demo`, but routes through `tl_bridge_vllm`\n", + "— vLLM under the hood instead of `AutoModelForCausalLM`. The point is dataset-scale\n", + "evals: vLLM's PagedAttention + continuous batching lets parallel `inspect_eval` samples\n", + "stream through a single resident model, where the HF provider serializes them.\n", + "\n", + "**Boundaries served (vLLM-specific):**\n", + "- ✅ `blocks.{i}.hook_out` (resid_post)\n", + "- ✅ `blocks.{i}.attn.hook_out` (attn_out)\n", + "- ✅ `blocks.{i}.mlp.hook_out` (mlp_out)\n", + "- ⛔ `blocks.{i}.hook_in` (resid_pre) — vLLM fuses the block-input read into the next\n", + " layer; no Python-visible hook point.\n", + "- ⛔ `blocks.{i}.ln2.hook_in` (resid_mid) — derived from resid_pre + attn_out.\n", + "- (Always non-fireable) head-split `hook_q/k/v/z`, `hook_pattern`, `embed`, `ln_final`,\n", + " `unembed`.\n", + "\n", + "Use `boot_inspect(provider=\"tl_bridge\")` for the gated boundaries (small models, CPU,\n", + "exact parity). Use `boot_inspect(provider=\"tl_bridge_vllm\")` for throughput on the\n", + "served boundaries.\n", + "\n", + "**Prerequisites:** an NVIDIA GPU with enough memory for the chosen model in fp16 +\n", + "vLLM's KV cache (≈4–8 GB for `meta-llama/Llama-3.2-1B`). vLLM is GPU-only; this\n", + "notebook will not run on CPU." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Install TransformerLens with both `inspect` and vLLM:\n", + "\n", + "```bash\n", + "pip install \"transformer_lens[inspect]\" vllm\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "warnings.filterwarnings(\"ignore\")\n", + "import torch\n", + "\n", + "assert torch.cuda.is_available(), \"vLLM is GPU-only; this notebook needs CUDA.\"\n", + "\n", + "from transformer_lens.model_bridge.remote_bridge import RemoteBridge\n", + "from transformer_lens.model_bridge.transformer_bridge import TransformerBridge\n", + "\n", + "MODEL = \"meta-llama/Llama-3.2-1B\"\n", + "PROMPT = \"The quick brown fox\"\n", + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1 — Boot the model through Inspect (vLLM provider)\n", + "\n", + "`boot_inspect(..., provider=\"tl_bridge_vllm\")` constructs the same `RemoteBridge` you\n", + "get from the HF provider; the swap is invisible to downstream code. The provider's\n", + "structural self-check enumerates which boundary kinds vLLM can serve; the bridge profile\n", + "is filtered to that set, and any gated boundary surfaces as a UserWarning.\n", + "\n", + "First boot takes ~30–60s (vLLM loads weights, profiles KV cache, captures CUDA graphs)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "bridge = RemoteBridge.boot_inspect(MODEL, provider=\"tl_bridge_vllm\")\nprint(\"bridge:\", type(bridge).__name__)\n# UserWarning surfaces on boot when boundaries are gated; print here for visibility.\nprint(\"capability note:\", bridge._driver._model.api.capability_note())" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2 — Listing available hooks\n", + "\n", + "`bridge.hook_dict` is provider-agnostic. The fireable set vs the HF provider differs by\n", + "the gated kinds (resid_pre, resid_mid); everything else is identical." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"hook_dict entries:\", len(bridge.hook_dict))\n", + "fireable = sorted(bridge._driver.supported_hook_points)\n", + "print(\"fireable (native):\", len(fireable))\n", + "print(\"layer 0:\", [h for h in fireable if h.startswith(\"blocks.0.\")])\n", + "print(\"non-fireable (use boot_transformers):\", len(bridge._driver.non_fireable_hook_points))\n", + "# resid_pre / resid_mid live in non-fireable for this provider:\n", + "print(\"layer 0 non-fireable:\", [h for h in sorted(bridge._driver.non_fireable_hook_points)\n", + " if h.startswith(\"blocks.0.\")])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3 — `run_with_cache` over the Inspect+vLLM boundary\n", + "\n", + "Activations cross the Inspect boundary as the same wire envelope the HF provider emits,\n", + "get converted to torch at the bridge boundary, and surface as a normal `ActivationCache`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tokens = bridge.to_tokens(PROMPT)\n", + "logits, cache = bridge.run_with_cache(tokens)\n", + "print(\"logits:\", tuple(logits.shape))\n", + "for hk in [\"blocks.0.hook_out\", \"blocks.0.attn.hook_out\", \"blocks.0.mlp.hook_out\"]:\n", + " print(f\"{hk}: shape={tuple(cache[hk].shape)} dtype={cache[hk].dtype}\")\n", + "nxt = int(logits[0, -1].argmax())\n", + "print(\"next-token:\", nxt, repr(bridge.tokenizer.decode([nxt])))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4 — Parity vs `boot_transformers`\n", + "\n", + "vLLM runs the same weights as HF, but in fp16 with fused kernels (PagedAttention,\n", + "QKVParallelLinear). The residual stream matches to **fp16 tolerance** — looser than the\n", + "HF provider's exact match, tight enough that next-token argmax agrees on the served\n", + "boundaries." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "hf = TransformerBridge.boot_transformers(MODEL, dtype=torch.float16)\n", + "toks = hf.to_tokens(PROMPT) # shared, BOS-prepended\n", + "\n", + "hf_logits, hf_cache = hf.run_with_cache(toks)\n", + "i_logits, i_cache = bridge.run_with_cache(toks)\n", + "\n", + "print(\"argmax match:\", int(hf_logits[0, -1].argmax()) == int(i_logits[0, -1].argmax()))\n", + "# resid_post / attn_out / mlp_out across a sampling of layers — vLLM's served kinds.\n", + "for hk in [\"blocks.0.hook_out\", \"blocks.0.attn.hook_out\", \"blocks.0.mlp.hook_out\",\n", + " \"blocks.8.hook_out\", \"blocks.15.hook_out\"]:\n", + " a, b = hf_cache[hk].float(), i_cache[hk].float()\n", + " rel = (a - b).norm() / (a.norm() + 1e-6)\n", + " print(f\"{hk}: rel_L2={rel.item():.2e} maxdiff={(a - b).abs().max().item():.2e}\")\n", + "del hf # free HF before vLLM keeps going" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5 — Interventions\n", + "\n", + "Full affine vocabulary (`suppress` / `scale` / `add` / `set`) flows through to the vLLM\n", + "worker via the per-hook scale+bias buffer swap. Same surface as the HF provider — the\n", + "intervention dict is identical." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "clean = int(bridge.forward(toks)[0, -1].argmax())\n", + "\n", + "hk = \"blocks.0.hook_out\" # resid_post\n", + "supp_logits, supp_cache = bridge.run_with_cache(\n", + " toks, intervene={hk: {\"op\": \"suppress\"}}\n", + ")\n", + "suppressed = int(supp_logits[0, -1].argmax())\n", + "print(f\"{hk} |max| after suppress:\", supp_cache[hk].abs().max().item())\n", + "print(f\"argmax: clean={clean} suppressed={suppressed} changed={clean != suppressed}\")\n", + "\n", + "revert = int(bridge.forward(toks)[0, -1].argmax())\n", + "print(f\"revert={revert} matches clean={revert == clean}\")\n", + "\n", + "# Tear down vLLM before the eval-pipeline section boots its own. vLLM has no\n", + "# LLM.shutdown(); the driver's close() destroys the distributed env so the next\n", + "# Inspect-launched LLM(...) starts clean.\n", + "bridge.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Inspect-native workflows\n", + "\n", + "`tl_bridge_vllm/` is also a normal `inspect_ai` model — the eval pipeline boots\n", + "vLLM, runs samples through it (parallel, batched), and reads activations off\n", + "`ModelOutput.metadata`. The next cells show the same four patterns as the HF demo:\n", + "**(6)** eval-native generation with logprobs + usage, **(7)** capture_activations\n", + "alongside a scored eval, **(8)** per-turn capture across a rollout, **(9)** tool-aware\n", + "generation. The helpers are provider-agnostic (they read the same wire envelope), so the\n", + "cells are essentially the HF cells with the model string swapped." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# (6) Eval-native generation: tl_bridge_vllm is a normal Inspect model — real multi-token\n", + "# generation with token logprobs + usage from vLLM's sampler.\n", + "import tempfile\n", + "\n", + "from inspect_ai import Task\n", + "from inspect_ai import eval as inspect_eval\n", + "from inspect_ai.dataset import Sample\n", + "from inspect_ai.solver import generate\n", + "\n", + "run = tempfile.mkdtemp()\n", + "gen_log = inspect_eval(\n", + " Task(dataset=[Sample(input=\"The capital of France is\")], solver=generate()),\n", + " model=f\"tl_bridge_vllm/{MODEL}\",\n", + " max_tokens=6,\n", + " logprobs=True,\n", + " top_logprobs=3,\n", + " log_dir=f\"{run}/gen\",\n", + " display=\"none\",\n", + ")[0]\n", + "out = gen_log.samples[0].output\n", + "print(\"completion:\", repr(out.completion))\n", + "print(\"usage:\", out.usage.input_tokens, \"->\", out.usage.output_tokens, \"tokens\")\n", + "top3 = out.choices[0].logprobs.content[0].top_logprobs\n", + "print(\"1st-token top-3:\", [(t.token, round(t.logprob, 2)) for t in top3])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# (7) Capture activations alongside a scored eval. Same helpers as the HF demo — they\n", + "# read metadata['activations'] which both providers emit in the same wire envelope.\n", + "import pathlib\n", + "\n", + "from inspect_ai.analysis import SampleSummary, samples_df\n", + "from inspect_ai.scorer import includes\n", + "\n", + "from transformer_lens.model_bridge.sources.inspect import activations_column, capture_activations\n", + "\n", + "cap_task = Task(\n", + " dataset=[\n", + " Sample(input=\"The capital of France is\", target=\"Paris\"),\n", + " Sample(input=\"The opposite of hot is\", target=\"cold\"),\n", + " ],\n", + " solver=[capture_activations([\"blocks.8.hook_out\"], output_dir=f\"{run}/acts\"), generate()],\n", + " scorer=includes(),\n", + ")\n", + "cap_log = inspect_eval(\n", + " cap_task, model=f\"tl_bridge_vllm/{MODEL}\", max_tokens=4, log_dir=f\"{run}/cap\", display=\"none\"\n", + ")[0]\n", + "print(\"status:\", cap_log.status, \"| npz:\", [p.name for p in pathlib.Path(f\"{run}/acts\").glob(\"*.npz\")])\n", + "df = samples_df(f\"{run}/cap\", columns=[*SampleSummary, activations_column()])\n", + "print(df[[\"id\", \"tl_activations\"]].to_string(index=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# (8) Per-turn capture during a multi-token generate. vLLM's compiled hooks overwrite\n", + "# buffer row 0 on each decode step, so the provider takes a single-token snapshot of the\n", + "# prompt activations BEFORE the eval generate (extra ~prompt prefill — small relative to\n", + "# typical multi-token completions).\n", + "from transformer_lens.model_bridge.sources.inspect import turn_activations\n", + "\n", + "turn_log = inspect_eval(\n", + " Task(dataset=[Sample(input=\"The capital of France is\")], solver=generate()),\n", + " model=f\"tl_bridge_vllm/{MODEL}\",\n", + " model_args={\"capture\": [\"blocks.8.hook_out\"]},\n", + " max_tokens=4,\n", + " log_dir=f\"{run}/turns\",\n", + " display=\"none\",\n", + ")[0]\n", + "turns = turn_activations(turn_log.samples[0])\n", + "print(\"model turns captured:\", len(turns))\n", + "print(\"turn 0:\", {k: tuple(v.shape) for k, v in turns[0].items()})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# (9) Tool-aware generation. Same parser as the HF provider — re-exported from the\n", + "# vLLM provider for symmetry. For a tool-capable instruct model the provider renders\n", + "# tool schemas into the chat template; here we just show the parser:\n", + "from transformer_lens.model_bridge.sources.inspect.vllm_provider import _parse_tool_calls\n", + "\n", + "calls = _parse_tool_calls('{\"name\": \"add\", \"arguments\": {\"a\": 2, \"b\": 2}}')\n", + "print(\"parsed tool calls:\", [(c.function, c.arguments) for c in calls])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "- `boot_inspect(MODEL, provider=\"tl_bridge_vllm\")` returns the same `RemoteBridge` as the\n", + " HF provider, backed by vLLM. The structural self-check gates `resid_pre`/`resid_mid`\n", + " (no Python-visible block-input hook under vLLM's fused execution).\n", + "- Residual / attn / mlp boundaries cross the Inspect boundary as the same wire envelope\n", + " the HF provider emits, match `boot_transformers` to fp16 tolerance (next-token argmax\n", + " agrees on served kinds), and convert to torch at the bridge.\n", + "- Full affine interventions (suppress / scale / add / set) flow through the per-hook\n", + " scale+bias buffer swap on the worker.\n", + "- `tl_bridge_vllm/` is also a normal Inspect model: eval-native generation,\n", + " `capture_activations`, per-turn capture, tool-call parsing — same helpers, same wire\n", + " format. Trade vs HF: gives up `resid_pre`/`resid_mid` and exact parity; gains vLLM's\n", + " throughput (PagedAttention + continuous batching) for dataset-scale evals." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index a98ab985c4..1f2c9a8569 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,12 +39,22 @@ # chardet<6 works around a `requests<=2.32` compatibility-check warning that fires # whenever chardet>=6 is installed. Remove the pin when psf/requests bumps the cap. evals=["lm-eval>=0.4", "chardet<6"] + inspect=["inspect_ai>=0.3"] lit=["lit-nlp>=1.3"] [project.scripts] build-docs="docs.make_docs:build_docs" docs-hot-reload="docs.make_docs:docs_hot_reload" + # Registers our HF-backed activation-extraction provider with inspect_ai so + # get_model("tl_bridge/") resolves it once the package is installed. + # (Named tl_bridge, not transformer_lens — inspect_ai ships a built-in by that name.) + # tl_bridge_vllm is the vLLM-backed sibling (lazy vllm import in __init__ so + # the module imports cleanly without vllm installed). + [project.entry-points.inspect_ai] + tl_bridge="transformer_lens.model_bridge.sources.inspect.transformers_provider" + tl_bridge_vllm="transformer_lens.model_bridge.sources.inspect.vllm_provider" + [dependency-groups] demo=["gradio>=4.0.0", "orjson>=3.11.7,<4.0"] dev=[ @@ -104,6 +114,7 @@ ] markers=[ "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "inspect: requires inspect_ai (deselect with '-m \"not inspect\"')", ] pythonpath=["."] testpaths=["tests", "transformer_lens"] # Only test these directories diff --git a/scripts/inspect_parity_report.py b/scripts/inspect_parity_report.py new file mode 100644 index 0000000000..4d643f0c47 --- /dev/null +++ b/scripts/inspect_parity_report.py @@ -0,0 +1,157 @@ +"""Empirical parity report: boot_inspect vs boot_transformers per architecture. + +For each model, compares (at the first and last layer) the boundaries the provider's +structural self-check OFFERS — gated ones (e.g. resid_mid on parallel/norm-variant archs) +are correctly withheld and reported, not compared. Validates that self-check against real +models: everything offered must match boot_transformers; it is not a per-PR CI job. + +Run: uv run python scripts/inspect_parity_report.py +Override the model list with TL_PARITY_MODELS="id1,id2,...". +""" +from __future__ import annotations + +import gc +import os +import sys +import warnings + +warnings.filterwarnings("ignore") + +import torch + +# Broad coverage across adapter families: real small/gated checkpoints (HF token from +# .env) where available, tiny-random elsewhere. Structure (not weights) drives parity, +# so tiny-random is fine for classification; bad/404 ids land in "couldn't run". +DEFAULT_MODELS = [ + # --- sequential post-norm (GPT2 family) --- + "sshleifer/tiny-gpt2", # GPT2 + "gpt2", # GPT2 + "bigcode/gpt_bigcode-santacoder", # GPTBigCode + "roneneldan/TinyStories-33M", # GPTNeo + # --- sequential pre-norm (Llama family) --- + "meta-llama/Llama-3.2-1B", # Llama (gated) + "mistralai/Mistral-7B-v0.1", # Mistral + "Qwen/Qwen2.5-0.5B", # Qwen2 + "Qwen/Qwen3-0.6B", # Qwen3 + "microsoft/Phi-3-mini-4k-instruct", # Phi3 + "allenai/OLMo-2-0425-1B", # Olmo2 + "ibm-granite/granite-3.0-2b-base", # Granite + "stabilityai/stablelm-2-1_6b", # StableLm + "HuggingFaceTB/SmolLM2-135M", # Llama + # --- norm-variant (extra pre/post-FF norms → resid_mid gated by identity check) --- + "google/gemma-2-2b", # Gemma2 (gated) + "google/gemma-3-1b-pt", # Gemma3 (gated) + "ibm-granite/granite-3.0-2b-base", # Granite — residual-multiplier; even attn/mlp diverge + # --- parallel residual (attn + mlp both read resid_pre → resid_mid gated by causal) --- + "EleutherAI/pythia-160m", # GPTNeoX + "Salesforce/codegen-350M-mono", # CodeGen + "microsoft/phi-1_5", # Phi + "tiiuae/falcon-rw-1b", # Falcon (attn attr self_attention → attn_out gated) + # --- other / non-standard composition --- + "facebook/opt-125m", # OPT (fc1/fc2 → mlp_out gated) + "bigscience/bloom-560m", # Bloom + # --- tiny-random fallbacks for families lacking a small real checkpoint --- + "hf-internal-testing/tiny-random-GPTJForCausalLM", # GPTJ (parallel) + "trl-internal-testing/tiny-CohereForCausalLM", # Cohere (tiny) + "hf-internal-testing/tiny-random-MixtralForCausalLM", # Mixtral (MoE block → mlp_out gated) +] + +PROMPT = "The quick brown fox" +ATOL, RTOL = 1e-3, 1e-3 + + +# Boundary kind -> TransformerBridge-native hook suffix. +KIND_SUFFIX = { + "resid_pre": "hook_in", + "resid_mid": "ln2.hook_in", + "resid_post": "hook_out", + "attn_out": "attn.hook_out", + "mlp_out": "mlp.hook_out", +} + + +def verify(model_id: str) -> dict: + from transformer_lens.model_bridge.remote_bridge import RemoteBridge + from transformer_lens.model_bridge.transformer_bridge import TransformerBridge + + result: dict = {"model": model_id, "arch": "?", "status": "", "detail": ""} + hf = inspect = None + try: + # Force matched fp32 + (provider) eager attention so the comparison isolates the + # boundary mapping, not dtype/attn-impl differences. + hf = TransformerBridge.boot_transformers(model_id, device="cpu", dtype=torch.float32) + result["arch"] = getattr(hf.cfg, "architecture", "?") + n_layers = int(hf.cfg.n_layers) + toks = hf.to_tokens(PROMPT) + + inspect = RemoteBridge.boot_inspect(model_id, dtype=torch.float32) + # Validate the structural self-check: only the boundaries the provider OFFERS must + # match; ones it gated (e.g. resid_mid on parallel archs) are correctly withheld. + offered = inspect._driver.supported_hook_points + kinds_offered = {k for k, suf in KIND_SUFFIX.items() if f"blocks.0.{suf}" in offered} + gated = sorted(set(KIND_SUFFIX) - kinds_offered) + + _, hf_cache = hf.run_with_cache(toks) + _, i_cache = inspect.run_with_cache(toks) + + worst = 0.0 + mism = [] + for i in sorted({0, n_layers - 1}): + for kind in kinds_offered: + hk = f"blocks.{i}.{KIND_SUFFIX[kind]}" + if hk not in i_cache or hk not in hf_cache: + mism.append(f"{hk} missing") + continue + a, b = hf_cache[hk].float(), i_cache[hk].float() + if a.shape != b.shape: + mism.append(f"{hk} shape {tuple(a.shape)}!={tuple(b.shape)}") + continue + d = (a - b).abs().max().item() + worst = max(worst, d) + if not torch.allclose(a, b, atol=ATOL, rtol=RTOL): + mism.append(f"{hk} maxdiff={d:.2e}") + gated_str = ",".join(gated) if gated else "none" + if mism: + result["status"] = "FAIL" + result["detail"] = f"gated={gated_str}; " + "; ".join(mism[:3]) + else: + result["status"] = "PASS" + result["detail"] = f"maxdiff={worst:.2e} (L={n_layers}, gated={gated_str})" + except Exception as e: # download/gating/locate failures are not parity failures + result["status"] = "SKIP" + result["detail"] = f"{type(e).__name__}: {str(e).splitlines()[0][:120]}" + finally: + for bridge in (inspect, hf): + try: + if bridge is not None: + bridge.close() + except Exception: + pass + del hf, inspect + gc.collect() + return result + + +def main() -> None: + ids = os.environ.get("TL_PARITY_MODELS") + models = [m.strip() for m in ids.split(",")] if ids else DEFAULT_MODELS + rows = [] + for m in models: + r = verify(m) + rows.append(r) + print(f"[{r['status']:4}] {r['arch']:32} {r['model']:48} {r['detail']}", flush=True) + + print("\n================ PARITY REPORT CARD ================") + for status in ("PASS", "FAIL", "SKIP"): + sel = [r for r in rows if r["status"] == status] + print(f"\n{status} ({len(sel)}):") + for r in sel: + print(f" {r['arch']:32} {r['model']:48} {r['detail']}") + passed = sorted({r["arch"] for r in rows if r["status"] == "PASS"}) + print("\nPARITY-VERIFIED ARCHITECTURES (measured PASS):") + print(" " + ", ".join(passed)) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/tests/acceptance/model_bridge/conftest.py b/tests/acceptance/model_bridge/conftest.py index 870b38a7a3..505bcb20e3 100644 --- a/tests/acceptance/model_bridge/conftest.py +++ b/tests/acceptance/model_bridge/conftest.py @@ -1,7 +1,12 @@ """Session fixtures for model_bridge acceptance tests. -transformer_lens imports stay inside fixture bodies — jaxtyping's pytest_configure -hook must install before the package is first imported. +Session-scoped fixtures avoid redundant model loads across test files. +All models used here must be in the CI cache (see .github/workflows/checks.yml). + +Imports of ``transformer_lens`` are deferred into the fixtures: this conftest is +loaded during pytest's initial conftest collection (before plugins' pytest_configure), +and a module-level TL import would trip jaxtyping's "package already imported" guard +when a single test file under this directory is run in isolation. """ import pytest diff --git a/tests/acceptance/model_bridge/test_inspect_provider.py b/tests/acceptance/model_bridge/test_inspect_provider.py new file mode 100644 index 0000000000..6bc51e7259 --- /dev/null +++ b/tests/acceptance/model_bridge/test_inspect_provider.py @@ -0,0 +1,637 @@ +"""Acceptance gate for the Inspect driver: gpt2 through our HF provider must match +``boot_transformers`` to fp tolerance (same backend + dtype + eager attention), with +exact next-token argmax, and interventions must take effect. + +Gated on ``inspect_ai`` being installed (the ``inspect`` extra); runs on CPU. +""" +from __future__ import annotations + +import pytest +import torch + +try: + import inspect_ai # noqa: F401 + + INSPECT_AVAILABLE = True +except ImportError: + INSPECT_AVAILABLE = False + +pytestmark = [ + pytest.mark.inspect, + pytest.mark.skipif( + not INSPECT_AVAILABLE, reason="inspect_ai not installed (pip install '.[inspect]')" + ), +] + +MODEL = "gpt2" +PROMPT = "The quick brown fox" +# A spread across boundary kinds and layers — all must match boot_transformers to fp +# tolerance. TransformerBridge-native names (the bridge cache carries these too). +PARITY_HOOKS = [ + "blocks.0.hook_in", # resid_pre + "blocks.0.attn.hook_out", # attn_out + "blocks.0.ln2.hook_in", # resid_mid + "blocks.0.mlp.hook_out", # mlp_out + "blocks.0.hook_out", # resid_post + "blocks.6.attn.hook_out", + "blocks.11.hook_out", +] + +# Boundary kind -> TransformerBridge-native hook suffix. +KIND_SUFFIX = { + "resid_pre": "hook_in", + "resid_mid": "ln2.hook_in", + "resid_post": "hook_out", + "attn_out": "attn.hook_out", + "mlp_out": "mlp.hook_out", +} + +# Token-free tiny-random checkpoints, one per structural-check code path: standard +# sequential (nothing gated), parallel-residual (resid_mid gated by the causal probe), +# post-norm (resid_mid gated by the linear-identity probe). Pins cross-family behavior +# so a detector/load-path regression on non-gpt2 archs fails CI instead of shipping silently. +STRUCTURAL_FAMILIES = [ + ("hf-internal-testing/tiny-random-LlamaForCausalLM", frozenset()), + ("hf-internal-testing/tiny-random-GPTJForCausalLM", frozenset({"resid_mid"})), + ("hf-internal-testing/tiny-random-Gemma2ForCausalLM", frozenset({"resid_mid"})), +] + + +@pytest.fixture(scope="module") +def hf_bridge(): + from transformer_lens.model_bridge.transformer_bridge import TransformerBridge + + return TransformerBridge.boot_transformers(MODEL) + + +@pytest.fixture(scope="module") +def inspect_bridge(): + from transformer_lens.model_bridge.remote_bridge import RemoteBridge + + bridge = RemoteBridge.boot_inspect(MODEL) + yield bridge + bridge.close() + + +@pytest.fixture(scope="module") +def tokens(hf_bridge): + # Shared token ids so the only variable under test is the backend path, not + # tokenization (to_tokens BOS parity is covered by test_string_input_parity). + return hf_bridge.to_tokens(PROMPT) + + +class TestInspectParity: + def test_argmax_matches_boot_transformers(self, hf_bridge, inspect_bridge, tokens): + hf_logits = hf_bridge.forward(tokens) + i_logits = inspect_bridge.forward(tokens) + assert int(hf_logits[0, -1].argmax()) == int(i_logits[0, -1].argmax()) + + def test_string_input_parity(self, hf_bridge, inspect_bridge): + # Same STRING (not shared tokens): exercises RemoteBridge.to_tokens BOS handling. + # A bare encode (no BOS) would diverge from boot_transformers here. + hf_argmax = int(hf_bridge.forward(PROMPT)[0, -1].argmax()) + i_argmax = int(inspect_bridge.forward(PROMPT)[0, -1].argmax()) + assert hf_argmax == i_argmax + + def test_hooks_match(self, hf_bridge, inspect_bridge, tokens): + _, hf_cache = hf_bridge.run_with_cache(tokens) + _, i_cache = inspect_bridge.run_with_cache(tokens) + for hook in PARITY_HOOKS: + a, b = hf_cache[hook].float(), i_cache[hook].float() + assert a.shape == b.shape + assert torch.allclose( + a, b, atol=1e-3, rtol=1e-3 + ), f"{hook} diverges: max {(a - b).abs().max().item():.2e}" + + def test_loss_matches(self, hf_bridge, inspect_bridge, tokens): + # Full-sequence logits ⇒ loss is computable and matches boot_transformers. + hf_loss = hf_bridge.forward(tokens, return_type="loss") + i_loss = inspect_bridge.forward(tokens, return_type="loss") + assert torch.allclose(hf_loss, i_loss, atol=1e-3) + + +# Each intervenable boundary kind at layer 0, with whether suppress flips the top token +# (mlp_out shifts the logits but not the argmax at layer 0). resid_pre exercises the +# forward_pre_hook path; the others the forward_hook path. +INTERVENE_HOOKS = [ + ("blocks.0.hook_in", True), # resid_pre + ("blocks.0.attn.hook_out", True), # attn_out + ("blocks.0.mlp.hook_out", False), # mlp_out + ("blocks.0.hook_out", True), # resid_post +] + + +class TestInspectInterventions: + @pytest.mark.parametrize("hook,flips_argmax", INTERVENE_HOOKS) + def test_suppress_applies_at_each_boundary(self, inspect_bridge, tokens, hook, flips_argmax): + clean_logits = inspect_bridge.forward(tokens) + clean_argmax = int(clean_logits[0, -1].argmax()) + + supp_logits, supp_cache = inspect_bridge.run_with_cache( + tokens, intervene={hook: {"op": "suppress"}} + ) + assert supp_cache[hook].abs().max().item() == 0.0 # capture reflects the intervention + assert not torch.allclose(supp_logits, clean_logits) # and it propagated to the logits + if flips_argmax: + assert int(supp_logits[0, -1].argmax()) != clean_argmax + + def test_intervention_reverts(self, inspect_bridge, tokens): + clean_argmax = int(inspect_bridge.forward(tokens)[0, -1].argmax()) + inspect_bridge.forward(tokens, intervene={"blocks.0.hook_out": {"op": "suppress"}}) + assert int(inspect_bridge.forward(tokens)[0, -1].argmax()) == clean_argmax # not sticky + + +class TestPreHookKwargs: + """The resid_pre pre-hook runs with_kwargs=True so it modifies the right tensor + whether hidden_states arrives positionally (args[0]) or as a kwarg.""" + + CALL = "test_call" + + def _hook(self): + from transformer_lens.model_bridge.sources.inspect.transformers_provider import ( + _pre_hook, + ) + + return _pre_hook( + layer=0, want_capture=True, spec={"op": "suppress"}, raw={}, call_id=self.CALL + ) + + def _scope(self): + from transformer_lens.model_bridge.sources.inspect.transformers_provider import ( + _current_call_id, + ) + + return _current_call_id.set(self.CALL) + + def test_positional_hidden_states(self): + from transformer_lens.model_bridge.sources.inspect.transformers_provider import ( + _current_call_id, + ) + + hidden = torch.ones(1, 2, 4) + token = self._scope() + try: + new_args, new_kwargs = self._hook()(None, (hidden, "mask"), {}) + finally: + _current_call_id.reset(token) + assert torch.allclose(new_args[0], torch.zeros_like(hidden)) and new_args[1] == "mask" + assert new_kwargs == {} + + def test_kwarg_hidden_states(self): + from transformer_lens.model_bridge.sources.inspect.transformers_provider import ( + _current_call_id, + ) + + hidden = torch.ones(1, 2, 4) + token = self._scope() + try: + new_args, new_kwargs = self._hook()( + None, (), {"hidden_states": hidden, "use_cache": True} + ) + finally: + _current_call_id.reset(token) + assert new_args == () + assert torch.allclose(new_kwargs["hidden_states"], torch.zeros_like(hidden)) # suppressed + assert new_kwargs["use_cache"] is True # other kwargs preserved + + +class TestStructuralProbe: + """The boot-time structural self-check: resid_mid is offered only when attn feeds mlp. + Toy modules (no download) exercise the causal probe for sequential vs parallel blocks.""" + + def _toy_model(self, parallel: bool = False, resid_scale: float = 1.0): + import torch.nn as nn + + class Sub(nn.Module): + def __init__(self, k): + super().__init__() + self.lin = nn.Linear(4, 4) + self.k = k + + def forward(self, x): + return self.lin(x) * self.k + + class Block(nn.Module): + def __init__(self): + super().__init__() + self.self_attn = Sub(0.5) + self.mlp = Sub(0.1) + + def forward(self, x): + a = self.self_attn(x) + m = self.mlp(x) if parallel else self.mlp(x + a) # parallel reads block input + # resid_scale != 1 mimics post-norm/residual-multiplier archs (Gemma2/OLMo2/ + # Granite): outputs don't add linearly, so resid_pre + attn_out is wrong. + return x + resid_scale * a + resid_scale * m + + class Model(nn.Module): + def __init__(self): + super().__init__() + self.embed = nn.Embedding(16, 4) + self.layers = nn.ModuleList([Block()]) + + def forward(self, ids): + x = self.embed(ids) + for b in self.layers: + x = b(x) + return x + + torch.manual_seed(0) + return Model() + + def test_sequential_offers_resid_mid(self): + from transformer_lens.model_bridge.sources.inspect.transformers_provider import ( + _detect_capabilities, + ) + + m = self._toy_model() # standard sequential, linear residual + kinds, note = _detect_capabilities(m, m.layers) + assert "resid_mid" in kinds + assert note == "" # nothing gated + + def test_parallel_gates_resid_mid(self): + from transformer_lens.model_bridge.sources.inspect.transformers_provider import ( + _detect_capabilities, + ) + + m = self._toy_model(parallel=True) # mlp reads block input, not attn output + kinds, note = _detect_capabilities(m, m.layers) + assert "resid_mid" not in kinds + assert {"resid_pre", "resid_post", "attn_out", "mlp_out"} <= kinds # rest still served + assert "resid_mid" in note # note explains the gate + + def test_nonlinear_residual_gates_resid_mid(self): + from transformer_lens.model_bridge.sources.inspect.transformers_provider import ( + _detect_capabilities, + ) + + # Sequential (attn feeds mlp) but outputs are scaled before the residual add, so + # resid_post != resid_pre + attn_out + mlp_out — resid_mid must still be gated. + m = self._toy_model(resid_scale=2.0) + kinds, note = _detect_capabilities(m, m.layers) + assert "resid_mid" not in kinds + assert {"resid_pre", "resid_post", "attn_out", "mlp_out"} <= kinds + assert "resid_mid" in note + + def test_probe_leaves_global_rng_untouched(self): + # The probe's attn perturbation must use a local generator — booting a model + # should never reset the caller's torch RNG. + from transformer_lens.model_bridge.sources.inspect.transformers_provider import ( + _detect_capabilities, + ) + + m = self._toy_model() + before = torch.get_rng_state() + _detect_capabilities(m, m.layers) + assert torch.equal(torch.get_rng_state(), before) + + +class TestStructuralCheckAcrossFamilies: + """Real tiny-random models, one per detector code path, run in CI (no token, seconds). + Locks both the gating decision and offered-boundary parity vs boot_transformers, so a + structural-check or load-path regression on non-gpt2 architectures fails here.""" + + @pytest.mark.parametrize("model_id,expected_gated", STRUCTURAL_FAMILIES) + def test_gating_and_parity(self, model_id, expected_gated): + from transformer_lens.model_bridge.remote_bridge import RemoteBridge + from transformer_lens.model_bridge.transformer_bridge import TransformerBridge + + # Matched fp32 + eager attention (boot_inspect's defaults) so the only variable + # is the boundary mapping. + hf = TransformerBridge.boot_transformers(model_id, device="cpu", dtype=torch.float32) + inspect = RemoteBridge.boot_inspect(model_id, dtype=torch.float32) + try: + supported = inspect._driver.supported_hook_points + offered = {k for k, suf in KIND_SUFFIX.items() if f"blocks.0.{suf}" in supported} + gated = frozenset(set(KIND_SUFFIX) - offered) + assert gated == expected_gated, f"{model_id}: gated {sorted(gated)}" + # gated kinds are reported non-fireable, not silently dropped + for kind in expected_gated: + assert f"blocks.0.{KIND_SUFFIX[kind]}" in inspect._driver.non_fireable_hook_points + + n_layers = int(hf.cfg.n_layers) + toks = hf.to_tokens(PROMPT) + _, hf_cache = hf.run_with_cache(toks) + _, i_cache = inspect.run_with_cache(toks) + assert int(hf.forward(toks)[0, -1].argmax()) == int( + inspect.forward(toks)[0, -1].argmax() + ) + for layer in sorted({0, n_layers - 1}): + for kind in offered: + hk = f"blocks.{layer}.{KIND_SUFFIX[kind]}" + a, b = hf_cache[hk].float(), i_cache[hk].float() + assert torch.allclose( + a, b, atol=1e-3, rtol=1e-3 + ), f"{model_id} {hk} diverges: {(a - b).abs().max().item():.2e}" + finally: + inspect.close() + + +class TestRebootSemantics: + """inspect_ai memoizes get_model by name; boot_inspect passes memoize=False so each + boot honors its own args and independently owns (and frees) its model.""" + + def test_reboot_is_fresh_and_honors_dtype(self): + from transformer_lens.model_bridge.remote_bridge import RemoteBridge + + mid = "hf-internal-testing/tiny-random-LlamaForCausalLM" + b1 = RemoteBridge.boot_inspect(mid, dtype=torch.float32) + b2 = RemoteBridge.boot_inspect(mid, dtype=torch.float16) + try: + # Distinct provider objects — not inspect_ai's memoized singleton. + assert b1._driver._model is not b2._driver._model + # Each provider loaded at the dtype its boot requested (not a stale cache). + assert next(b1._driver._model.api._hf.parameters()).dtype == torch.float32 + assert next(b2._driver._model.api._hf.parameters()).dtype == torch.float16 + finally: + b1.close() + b2.close() + + +class TestEvalNativeGeneration: + """Beyond TL-driven capture, the provider works as a normal Inspect model: real + multi-token generation from chat input, with logprobs + usage, so an Inspect eval runs.""" + + def _api(self): + from inspect_ai.model import get_model + + from transformer_lens.model_bridge.sources.inspect import ( # noqa: F401 register + transformers_provider, + ) + + return get_model("tl_bridge/gpt2", memoize=False).api + + def test_multi_token_generation_with_logprobs(self): + import asyncio + + from inspect_ai.model import ChatMessageUser, GenerateConfig + + out = asyncio.run( + self._api().generate( + [ChatMessageUser(content="The capital of France is")], + None, + None, + GenerateConfig(max_tokens=5, logprobs=True, top_logprobs=3), + ) + ) + choice = out.choices[0] + assert choice.message.text # non-empty completion + assert choice.stop_reason in ("max_tokens", "stop") + assert len(choice.logprobs.content) == 5 # one logprob per generated token + assert len(choice.logprobs.content[0].top_logprobs) == 3 + assert out.usage.output_tokens == 5 + assert out.usage.total_tokens == out.usage.input_tokens + 5 + + def test_eval_runs_end_to_end(self, tmp_path): + from inspect_ai import Task + from inspect_ai import eval as inspect_eval + from inspect_ai.dataset import Sample + from inspect_ai.scorer import includes + from inspect_ai.solver import generate + + from transformer_lens.model_bridge.sources.inspect import ( # noqa: F401 register + transformers_provider, + ) + + task = Task( + dataset=[Sample(input="The capital of France is", target="Paris")], + solver=generate(), + scorer=includes(), + ) + logs = inspect_eval( + task, model="tl_bridge/gpt2", max_tokens=8, log_dir=str(tmp_path), display="none" + ) + assert logs[0].status == "success" + assert logs[0].samples and logs[0].samples[0].output.completion + + +class TestCaptureInEval: + """The capture_activations solver: full activations to a per-sample side artifact + a + reduction in the store, surfaced by samples_df to correlate features with scores.""" + + def test_artifact_store_and_samples_df(self, tmp_path): + import json + import pathlib + + import numpy as np + from inspect_ai import Task + from inspect_ai import eval as inspect_eval + from inspect_ai.analysis import SampleSummary, samples_df + from inspect_ai.dataset import Sample + from inspect_ai.solver import generate + + from transformer_lens.model_bridge.sources.inspect import ( # noqa: F401 register + transformers_provider, + ) + from transformer_lens.model_bridge.sources.inspect.eval import ( + activations_column, + capture_activations, + ) + + acts, logs = str(tmp_path / "acts"), str(tmp_path / "logs") + task = Task( + dataset=[Sample(input="The capital of France is", target="Paris")], + solver=[capture_activations(["blocks.6.hook_out"], output_dir=acts), generate()], + ) + log = inspect_eval( + task, model="tl_bridge/gpt2", max_tokens=4, log_dir=logs, display="none" + )[0] + assert log.status == "success" + + # full activations side-artifact, keyed by hook name + npz = list(pathlib.Path(acts).glob("*.npz")) + assert len(npz) == 1 + assert np.load(npz[0])["blocks.6.hook_out"].shape[-1] == 768 + + # reduction in the store, queryable via samples_df + df = samples_df(logs, columns=[*SampleSummary, activations_column()]) + reduction = df["tl_activations"].iloc[0] + reduction = json.loads(reduction) if isinstance(reduction, str) else reduction + assert reduction["blocks.6.hook_out"]["shape"][-1] == 768 + + def test_rejects_unknown_hook(self): + from transformer_lens.model_bridge.sources.inspect.eval import ( + capture_activations, + ) + + with pytest.raises(ValueError, match="not a fireable hook"): + capture_activations(["blocks.0.not_a_hook"]) + + +class TestAgenticToolCapture: + """Honor tools (render into the template or raise clearly), best-effort tool-call + parsing, and per-turn activation capture across a rollout.""" + + def test_tool_call_parsing(self): + from transformer_lens.model_bridge.sources.inspect.transformers_provider import ( + _parse_tool_calls, + ) + + block = _parse_tool_calls( + 'ok {"name": "add", "arguments": {"a": 1}}' + ) + assert block and block[0].function == "add" and block[0].arguments == {"a": 1} + bare = _parse_tool_calls('{"name": "f", "arguments": {}}') + assert bare and bare[0].function == "f" + assert _parse_tool_calls("no tool calls in this text") is None + + def test_tools_without_chat_template_raise(self): + import asyncio + + from inspect_ai.model import ChatMessageUser, GenerateConfig, get_model + from inspect_ai.tool import ToolInfo, ToolParams + + from transformer_lens.model_bridge.sources.inspect import ( # noqa: F401 register + transformers_provider, + ) + + api = get_model("tl_bridge/gpt2", memoize=False).api + tool = ToolInfo(name="add", description="add two ints", parameters=ToolParams()) + with pytest.raises(NotImplementedError, match="tool-aware chat template"): + asyncio.run( + api.generate( + [ChatMessageUser(content="2+2?")], [tool], None, GenerateConfig(max_tokens=3) + ) + ) + + def test_per_turn_capture_collected_from_transcript(self, tmp_path): + from inspect_ai import Task + from inspect_ai import eval as inspect_eval + from inspect_ai.dataset import Sample + from inspect_ai.solver import generate + + from transformer_lens.model_bridge.sources.inspect import ( # noqa: F401 register + transformers_provider, + ) + from transformer_lens.model_bridge.sources.inspect.eval import turn_activations + + task = Task( + dataset=[Sample(input="The capital of France is", target="Paris")], solver=generate() + ) + log = inspect_eval( + task, + model="tl_bridge/gpt2", + model_args={"capture": ["blocks.6.hook_out"]}, + max_tokens=4, + log_dir=str(tmp_path), + display="none", + )[0] + assert log.status == "success" + # each model turn's activations are recoverable from the transcript + turns = turn_activations(log.samples[0]) + assert len(turns) >= 1 + assert turns[0]["blocks.6.hook_out"].shape[-1] == 768 + + +class TestEvalPathStructuralGating: + """Both eval entry points (``model_args={"capture": [...]}`` and the solver's + ``extra_args`` route through ``_generate_capture``) must enforce the same structural + gate as the driver path — pinned on GPT-J, where resid_mid is gated.""" + + PARALLEL = "hf-internal-testing/tiny-random-GPTJForCausalLM" + GATED_HOOK = "blocks.0.ln2.hook_in" # resid_mid + + def test_model_args_capture_rejects_gated_kind(self): + from inspect_ai.model import get_model + + from transformer_lens.model_bridge.sources.inspect import ( # noqa: F401 register + transformers_provider, + ) + + with pytest.raises(ValueError, match="resid_mid"): + get_model(f"tl_bridge/{self.PARALLEL}", memoize=False, capture=[self.GATED_HOOK]) + + def test_extra_args_capture_rejects_gated_kind(self): + import asyncio + + from inspect_ai.model import ChatMessageUser, GenerateConfig, get_model + + from transformer_lens.model_bridge.sources.inspect import ( # noqa: F401 register + transformers_provider, + ) + + api = get_model(f"tl_bridge/{self.PARALLEL}", memoize=False).api + with pytest.raises(ValueError, match="resid_mid"): + asyncio.run( + api.generate( + [ChatMessageUser(content="hi")], + None, + None, + GenerateConfig(extra_body={"extra_args": {"capture": ["0:resid_mid"]}}), + ) + ) + + def test_model_args_unresolvable_hook_raises(self): + # model_args["capture"] must validate like the solver does (was silently dropping). + from inspect_ai.model import get_model + + from transformer_lens.model_bridge.sources.inspect import ( # noqa: F401 register + transformers_provider, + ) + + with pytest.raises(ValueError, match="not a fireable hook"): + get_model("tl_bridge/gpt2", memoize=False, capture=["blocks.0.not_a_hook"]) + + +class TestProviderReviewFixes: + """Targeted regressions for the post-merge review: validation parity, output_dir + absolute-resolve, ContextVar hook isolation (concurrency-safe), and per-turn capture + without the extra prompt forward.""" + + def test_solver_output_dir_resolved_absolute_at_construction(self, tmp_path, monkeypatch): + # A multi-eval run from different CWDs must not scatter artifacts. + + from transformer_lens.model_bridge.sources.inspect.eval import ( + capture_activations, + ) + + monkeypatch.chdir(tmp_path) + # The solver factory captures output_dir in its closure; resolve at construction. + cap = capture_activations(["blocks.0.hook_out"], output_dir="tl_acts") + # Move CWD; the path the solver writes to must still resolve to tmp_path/tl_acts. + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + monkeypatch.chdir(elsewhere) + # Closure inspection: the wrapped solve fn closes over the absolute output_dir. + absolute = str(tmp_path / "tl_acts") + assert any(absolute == c.cell_contents for c in (cap.__closure__ or ())) + + def test_hooks_isolated_by_contextvar(self): + # The hook only fires for the call whose call_id matches the contextvar — so two + # concurrent inspect_eval samples (each in their own contextvar copy) don't write + # into each other's raw dicts. + from transformer_lens.model_bridge.sources.inspect.transformers_provider import ( + _current_call_id, + _out_hook, + ) + + raw: dict = {} + hook = _out_hook( + layer=0, kind="resid_post", want_capture=True, spec=None, raw=raw, call_id="MINE" + ) + + # Default contextvar ("") doesn't match -> hook is a no-op. + hook(None, None, torch.ones(1, 3, 4)) + assert raw == {} + + # Set the contextvar to a DIFFERENT call's id -> still skip (concurrent peer). + token = _current_call_id.set("OTHER") + try: + hook(None, None, torch.ones(1, 3, 4)) + assert raw == {} + finally: + _current_call_id.reset(token) + + # Set the contextvar to our id -> hook fires. + token = _current_call_id.set("MINE") + try: + hook(None, None, torch.ones(1, 3, 4)) + assert (0, "resid_post") in raw + + # First-write-wins: a second call with different data doesn't overwrite (so + # generate's decode-step forwards don't clobber the prompt-forward capture). + first = raw[(0, "resid_post")].copy() + hook(None, None, torch.zeros(1, 3, 4)) + assert (raw[(0, "resid_post")] == first).all() + finally: + _current_call_id.reset(token) diff --git a/tests/unit/model_bridge/test_inspect_driver.py b/tests/unit/model_bridge/test_inspect_driver.py new file mode 100644 index 0000000000..b4b7894459 --- /dev/null +++ b/tests/unit/model_bridge/test_inspect_driver.py @@ -0,0 +1,438 @@ +"""Unit tests for InspectDriver — mocks the inspect_ai Model so no provider runs. + +The driver is the torch-free consumer side; these tests exercise it with a fake +async model returning a wire-encoded ModelOutput. Real end-to-end parity (gpt2 +through our provider) lives in tests/acceptance/model_bridge/test_inspect_provider.py. +""" +from __future__ import annotations + +import ast +import pathlib +from types import SimpleNamespace + +import numpy as np +import pytest +import torch + +# The driver lazily imports inspect_ai inside ``_generate``; without the ``inspect`` extra +# the import fires at first call and tests fail with ModuleNotFoundError. Skip-collect here. +pytest.importorskip("inspect_ai") + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.driver_protocol import ( + Driver, + ForwardResult, + validate_driver, +) +from transformer_lens.model_bridge.remote_bridge import RemoteBridge +from transformer_lens.model_bridge.sources.inspect import ( + hooks, + intervention, + profiles, + wire, +) +from transformer_lens.model_bridge.sources.inspect.driver import InspectDriver + +N_LAYERS, D_MODEL, D_VOCAB = 2, 4, 16 + + +def _cfg() -> TransformerBridgeConfig: + return TransformerBridgeConfig( + d_model=D_MODEL, + d_head=2, + n_layers=N_LAYERS, + n_ctx=8, + n_heads=2, + d_vocab=D_VOCAB, + d_mlp=8, + architecture="GPT2LMHeadModel", + ) + + +def _adapter() -> ArchitectureAdapter: + adapter = ArchitectureAdapter(_cfg()) + adapter.component_mapping = {} + return adapter + + +def _fake_model(*, captures=None, logits=None, raises=None): + """Stand-in inspect_ai Model. ``captures``: {wire_key: (seq,d)}; ``logits``: (seq,vocab).""" + metadata: dict = {} + if captures is not None: + metadata["activations"] = wire.encode_activations(captures) + if logits is not None: + metadata["tl_logits"] = wire.encode_array(logits) + output = SimpleNamespace(metadata=metadata) + + model = SimpleNamespace() + + async def generate(_input, config=None): + if raises is not None: + raise raises + return output + + model.generate = generate + return model + + +def _driver(model=None) -> InspectDriver: + return InspectDriver(model=model or _fake_model(), adapter=_adapter(), tokenizer=None) + + +class TestProtocolConformance: + def test_is_driver_and_validates(self): + driver = _driver() + assert isinstance(driver, Driver) + validate_driver(driver) + + def test_no_torch_capabilities(self): + driver = _driver() + for feature in ("parameters", "state_dict", "gradients", "weight_access"): + assert driver.supports(feature) is False + + def test_provides_sequence_logits_true(self): + # Provider returns full-sequence logits, so loss/both are available. + assert _driver().provides_sequence_logits is True + + def test_tl_bridge_profile_can_disable_sequence_logits(self): + # The vLLM provider routes through TLBridgeProfile with provides_sequence_logits=False + # (set in source.py from api.provides_sequence_logits) so RemoteBridge.forward rejects + # loss/both — synthesized logits only cover the gen position; earlier positions are -inf. + driver = InspectDriver( + _fake_model(), + _adapter(), + tokenizer=None, + profile=profiles.TLBridgeProfile(provides_sequence_logits=False), + ) + assert driver.provides_sequence_logits is False + + +class TestHookSets: + def test_full_residual_attn_mlp_set(self): + supported = _driver().supported_hook_points + assert len(supported) == 5 * N_LAYERS # one canonical name per boundary, no aliases + for name in [ + "blocks.0.hook_in", # resid_pre + "blocks.0.ln2.hook_in", # resid_mid + "blocks.0.hook_out", # resid_post + "blocks.0.attn.hook_out", # attn_out + "blocks.0.mlp.hook_out", # mlp_out + ]: + assert name in supported + assert "blocks.0.hook_attn_out" not in supported # HookedTransformer alias not duplicated + + def test_nonfireable_disjoint_and_includes_headsplit(self): + driver = _driver() + assert driver.supported_hook_points.isdisjoint(driver.non_fireable_hook_points) + assert "blocks.0.attn.hook_pattern" in driver.non_fireable_hook_points + assert "ln_final.hook_normalized" in driver.non_fireable_hook_points + + def test_vllm_lens_profile_narrows_to_residual(self): + driver = InspectDriver( + _fake_model(), _adapter(), tokenizer=None, profile=profiles.VLLMLensProfile() + ) + assert driver.supported_hook_points == frozenset( + f"blocks.{i}.hook_out" for i in range(N_LAYERS) + ) + assert driver.provides_sequence_logits is False # no full logits via that path + # attn/mlp hooks our provider could do are non-fireable for the residual-only peer. + assert "blocks.0.attn.hook_out" in driver.non_fireable_hook_points + + +class TestKindGating: + """The provider's structural self-check feeds a kind set through to the driver.""" + + def test_supported_hook_points_filters_kinds(self): + only = hooks.supported_hook_points(2, kinds={"resid_pre", "resid_post"}) + assert only == frozenset( + f"blocks.{i}.{suffix}" for i in range(2) for suffix in ("hook_in", "hook_out") + ) + + def test_profile_restricts_to_detected_kinds(self): + # resid_mid gated (e.g. parallel-residual arch) → ln2.hook_in absent, rest present. + prof = profiles.TLBridgeProfile( + supported_kinds={"resid_pre", "resid_post", "attn_out", "mlp_out"} + ) + names = prof.supported_hooks(N_LAYERS) + assert "blocks.0.ln2.hook_in" not in names + assert "blocks.0.attn.hook_out" in names + assert len(names) == 4 * N_LAYERS + + def test_driver_moves_gated_kind_to_nonfireable(self): + prof = profiles.TLBridgeProfile( + supported_kinds={"resid_pre", "resid_post", "attn_out", "mlp_out"} + ) + driver = InspectDriver(_fake_model(), _adapter(), tokenizer=None, profile=prof) + assert "blocks.0.ln2.hook_in" not in driver.supported_hook_points + assert "blocks.0.ln2.hook_in" in driver.non_fireable_hook_points + + +class TestNormalizeInputIds: + def test_1d_tensor(self): + assert InspectDriver._normalize_input_ids(torch.tensor([1, 2, 3])) == [1, 2, 3] + + def test_2d_single_row(self): + assert InspectDriver._normalize_input_ids(torch.tensor([[1, 2, 3]])) == [1, 2, 3] + + def test_batch_gt_one_raises(self): + with pytest.raises(NotImplementedError, match="batch_size=1"): + InspectDriver._normalize_input_ids(torch.tensor([[1, 2], [3, 4]])) + + def test_cuda_like_tensor_moved_to_cpu(self): + # np.asarray can't read CUDA memory; the torch-free driver must duck-type + # .detach().cpu() first. Simulate: __array__ raises (as a CUDA tensor would), + # cpu() yields a real ndarray. + class FakeCuda: + def __init__(self, data): + self._data = data + self.moved = False + + def detach(self): + return self + + def cpu(self): + self.moved = True + return np.asarray(self._data) + + def __array__(self, *a, **k): + raise TypeError("can't convert cuda:0 device tensor to numpy") + + fake = FakeCuda([[1, 2, 3]]) + assert InspectDriver._normalize_input_ids(fake) == [1, 2, 3] + assert fake.moved + + +class TestForward: + def test_assembles_named_captures_and_full_logits(self): + caps = { + "0:resid_post": np.ones((3, D_MODEL), np.float32), + "0:attn_out": np.full((3, D_MODEL), 2.0, np.float32), + } + logits = np.zeros((3, D_VOCAB), np.float32) + logits[-1, 7] = 5.0 + driver = _driver(_fake_model(captures=caps, logits=logits)) + + result = driver.forward( + torch.tensor([[1, 2, 3]]), + capture=("blocks.0.hook_out", "blocks.0.attn.hook_out"), + ) + assert isinstance(result, ForwardResult) + assert tuple(result.captured["blocks.0.hook_out"].shape) == (1, 3, D_MODEL) # resid_post + assert tuple(result.captured["blocks.0.attn.hook_out"].shape) == (1, 3, D_MODEL) # attn_out + assert tuple(result.logits.shape) == (1, 3, D_VOCAB) # full sequence + assert int(result.logits[0, -1].argmax()) == 7 + + def test_empty_capture_captures_nothing(self): + # capture=() means "logits only" — the driver requests no activations. + driver = _driver( + _fake_model( + captures={"0:resid_post": np.ones((2, D_MODEL), np.float32)}, + logits=np.zeros((2, D_VOCAB), np.float32), + ) + ) + result = driver.forward(torch.tensor([[1, 2]])) + assert result.captured == {} + assert result.logits is not None + + def test_missing_hook_warns(self): + # Provider returns no activation for a requested+supported hook → warn, key absent. + driver = _driver(_fake_model(captures={}, logits=np.zeros((2, D_VOCAB), np.float32))) + with pytest.warns(UserWarning, match="no activation"): + result = driver.forward(torch.tensor([[1, 2]]), capture=("blocks.0.hook_out",)) + assert "blocks.0.hook_out" not in result.captured + + def test_resid_mid_consumed_under_its_name(self): + # The provider derives resid_mid (= resid_pre + attn_out) and sends it under + # its wire key; the driver surfaces it as blocks.{i}.ln2.hook_in. + caps = {"0:resid_mid": np.full((2, D_MODEL), 4.0, np.float32)} + driver = _driver(_fake_model(captures=caps, logits=np.zeros((2, D_VOCAB), np.float32))) + result = driver.forward(torch.tensor([[1, 2]]), capture=("blocks.0.ln2.hook_in",)) + assert np.allclose(result.captured["blocks.0.ln2.hook_in"][0], 4.0) + + def test_provider_error_propagates(self): + driver = _driver(_fake_model(raises=RuntimeError("boom"))) + with pytest.raises(RuntimeError, match="boom"): + driver.forward(torch.tensor([[1, 2, 3]])) + + def test_rejects_max_new_tokens_gt_one(self): + with pytest.raises(NotImplementedError, match="max_new_tokens=1"): + _driver().forward(torch.tensor([[1, 2]]), max_new_tokens=2) + + def test_closed_driver_raises(self): + driver = _driver() + driver.close() + with pytest.raises(RuntimeError, match="closed"): + driver.forward(torch.tensor([[1, 2]])) + + +class TestAsyncWrapper: + def test_reuses_one_loop_thread(self): + caps = {"0:resid_post": np.ones((1, D_MODEL), np.float32)} + driver = _driver(_fake_model(captures=caps, logits=np.zeros((1, D_VOCAB), np.float32))) + driver.forward(torch.tensor([[1]])) + first = driver._loop_thread + driver.forward(torch.tensor([[1]])) + assert driver._loop_thread is first and first.is_alive() + + def test_works_inside_running_loop(self): + """The Jupyter case: a loop already running on the calling thread.""" + import asyncio + + caps = {"0:resid_post": np.ones((1, D_MODEL), np.float32)} + driver = _driver(_fake_model(captures=caps, logits=np.zeros((1, D_VOCAB), np.float32))) + + async def run(): + return driver.forward(torch.tensor([[1]])) + + assert isinstance(asyncio.run(run()), ForwardResult) + + +class TestWire: + def test_array_round_trip(self): + arr = np.arange(12, dtype=np.float32).reshape(3, 4) + assert np.array_equal(wire.decode_array(wire.encode_array(arr)), arr) + + def test_flat_activations_round_trip(self): + caps = { + "0:resid_post": np.ones((2, 4), np.float32), + "1:attn_out": np.full((2, 4), 3.0, np.float32), + } + meta = {"activations": wire.encode_activations(caps)} + out = wire.decode_activations(meta, ["0:resid_post", "1:attn_out"]) + assert np.array_equal(out["1:attn_out"], caps["1:attn_out"]) + + def test_vllm_lens_residual_stream_fallback(self): + # A peer provider's nested residual_stream decodes for resid_post keys. + meta = { + "activations": { + "residual_stream": {"3": wire.encode_array(np.ones((2, 4), np.float32))} + } + } + out = wire.decode_activations(meta, ["3:resid_post"]) + assert tuple(out["3:resid_post"].shape) == (2, 4) + + +class TestHooksRegistry: + def test_resolve(self): + assert hooks.resolve("blocks.2.attn.hook_out") == (2, "attn_out") + assert hooks.resolve("blocks.2.mlp.hook_out") == (2, "mlp_out") + assert hooks.resolve("blocks.0.ln2.hook_in") == (0, "resid_mid") + assert hooks.resolve("blocks.0.hook_in") == (0, "resid_pre") + assert hooks.resolve("blocks.0.hook_out") == (0, "resid_post") + assert hooks.resolve("embed.hook_out") is None + assert ( + hooks.resolve("blocks.2.hook_attn_out") is None + ) # HookedTransformer alias not exposed + + +class TestInterventionTranslation: + def _supported(self): + return hooks.supported_hook_points(N_LAYERS) + + def test_op_translates_to_wire_key(self): + out = intervention.build_interventions( + {"blocks.1.hook_out": {"op": "suppress"}}, self._supported() + ) + assert out == {"1:resid_post": {"op": "suppress"}} + + def test_attn_out_intervention(self): + out = intervention.build_interventions( + {"blocks.0.attn.hook_out": {"op": "scale", "factor": 0.5}}, self._supported() + ) + assert out == {"0:attn_out": {"op": "scale", "factor": 0.5}} + + def test_resid_mid_is_capture_only(self): + with pytest.raises(ValueError, match="capture-only"): + intervention.build_interventions( + {"blocks.0.ln2.hook_in": {"op": "suppress"}}, self._supported() + ) + + def test_callable_rejected(self): + with pytest.raises(NotImplementedError, match="specs"): + intervention.build_interventions({"blocks.0.hook_out": lambda a: a}, self._supported()) + + def test_unknown_hook_rejected(self): + with pytest.raises(ValueError, match="not in supported_hook_points"): + intervention.build_interventions( + {"blocks.9.hook_out": {"op": "suppress"}}, self._supported() + ) + + def test_bad_op_rejected(self): + with pytest.raises(ValueError, match="Unsupported intervention op"): + intervention.build_interventions( + {"blocks.0.hook_out": {"op": "clamp"}}, self._supported() + ) + + +class TestVLLMLensProfile: + """The vllm-lens request/response codec (live path unverified; logic is testable).""" + + def test_build_request_uses_output_residual_stream(self): + tokenizer = SimpleNamespace(decode=lambda ids: "hello world") + prompt, extra = profiles.VLLMLensProfile().build_request( + [1, 2, 3], ["0:resid_post", "1:resid_post"], [], True, tokenizer + ) + assert prompt == "hello world" # detokenized — vllm-lens re-tokenizes + assert extra == {"output_residual_stream": [0, 1]} + + def test_decode_logits_one_hot_from_completion(self): + tokenizer = SimpleNamespace(encode=lambda text: [9]) + output = SimpleNamespace(completion="x", metadata={}) + logits = profiles.VLLMLensProfile().decode_logits(output, 3, D_VOCAB, tokenizer) + assert tuple(logits.shape) == (1, 3, D_VOCAB) + assert int(logits[0, -1].argmax()) == 9 + + def test_non_additive_intervention_rejected_without_vllm_lens(self): + # Validation happens before the lazy vllm_lens import, so this works uninstalled. + with pytest.raises(NotImplementedError, match="additive steering"): + profiles.VLLMLensProfile().translate_interventions( + {"blocks.0.hook_out": {"op": "suppress"}}, + frozenset({"blocks.0.hook_out"}), + ) + + def test_no_interventions_needs_no_vllm_lens(self): + assert profiles.VLLMLensProfile().translate_interventions({}, frozenset()) == [] + + +class TestThroughBridge: + def test_replays_as_torch(self): + caps = {"0:resid_post": np.ones((3, D_MODEL), np.float32)} + driver = _driver(_fake_model(captures=caps, logits=np.zeros((3, D_VOCAB), np.float32))) + bridge = RemoteBridge(adapter=_adapter(), tokenizer=None, driver=driver) + + fired: list = [] + bridge.add_hook("blocks.0.hook_out", lambda act, hook: fired.append(act)) + bridge.forward(torch.tensor([[1, 2, 3]])) + assert len(fired) == 1 and isinstance(fired[0], torch.Tensor) # numpy→torch at the bridge + assert tuple(fired[0].shape) == (1, 3, D_MODEL) + + def test_loss_now_supported(self): + # Full-sequence logits ⇒ the bridge computes loss (no longer rejected). + logits = np.zeros((3, D_VOCAB), np.float32) + driver = _driver(_fake_model(captures={}, logits=logits)) + bridge = RemoteBridge(adapter=_adapter(), tokenizer=None, driver=driver) + loss = bridge.forward(torch.tensor([[1, 2, 3]]), return_type="loss") + assert isinstance(loss, torch.Tensor) and loss.ndim == 0 + + def test_context_manager_closes(self): + driver = _driver() + bridge = RemoteBridge(adapter=_adapter(), tokenizer=None, driver=driver) + with bridge: + pass + assert driver._model is None + + +def test_driver_imports_no_torch(): + """The acceptance gate: the driver file must not import torch (data-only boundary).""" + import transformer_lens.model_bridge.sources.inspect.driver as drv + + tree = ast.parse(pathlib.Path(drv.__file__).read_text()) + offenders: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + offenders += [a.name for a in node.names if a.name.split(".")[0] == "torch"] + elif isinstance(node, ast.ImportFrom): + if (node.module or "").split(".")[0] == "torch": + offenders.append(node.module or "") + assert not offenders, f"driver.py must be torch-free; found torch imports: {offenders}" diff --git a/tests/unit/model_bridge/test_inspect_vllm_provider.py b/tests/unit/model_bridge/test_inspect_vllm_provider.py new file mode 100644 index 0000000000..921c8e6fd8 --- /dev/null +++ b/tests/unit/model_bridge/test_inspect_vllm_provider.py @@ -0,0 +1,564 @@ +"""Unit tests for the vLLM-backed Inspect provider (``tl_bridge_vllm``) — mocks vLLM so +no vllm install is needed. Live end-to-end verification (real vLLM, real GPU) is in the +Colab walkthrough; the mocked tests pin the construction, generate→ModelOutput conversion, +and the validation surface (locked kwargs, capture path, model_args["capture"]).""" +from __future__ import annotations + +import asyncio +import sys +import types +from types import SimpleNamespace +from typing import Any +from unittest.mock import MagicMock + +import pytest + +# Tests deferred-import ``inspect_ai.model`` inside method bodies; provider construction +# in the fixtures also pulls inspect_ai transitively. Skip when the ``inspect`` extra is +# absent rather than fail each test with ModuleNotFoundError. +pytest.importorskip("inspect_ai") + + +def _install_vllm_mocks(monkeypatch, llm_class: Any) -> None: + """Install fake ``vllm``/``vllm.inputs``/``vllm.distributed.parallel_state`` modules + in ``sys.modules`` so the provider's lazy ``from vllm import …`` resolves to mocks.""" + vllm = types.ModuleType("vllm") + vllm.LLM = llm_class + vllm.SamplingParams = MagicMock(name="SamplingParams") + vllm_inputs = types.ModuleType("vllm.inputs") + vllm_inputs.TokensPrompt = MagicMock(name="TokensPrompt") + vllm.inputs = vllm_inputs + vllm_dist = types.ModuleType("vllm.distributed") + vllm_ps = types.ModuleType("vllm.distributed.parallel_state") + vllm_ps.destroy_model_parallel = MagicMock(name="destroy_model_parallel") + vllm_ps.destroy_distributed_environment = MagicMock(name="destroy_distributed_environment") + vllm_dist.parallel_state = vllm_ps + monkeypatch.setitem(sys.modules, "vllm", vllm) + monkeypatch.setitem(sys.modules, "vllm.inputs", vllm_inputs) + monkeypatch.setitem(sys.modules, "vllm.distributed", vllm_dist) + monkeypatch.setitem(sys.modules, "vllm.distributed.parallel_state", vllm_ps) + + +def _fake_tokenizer(monkeypatch) -> Any: + """A fake ``AutoTokenizer.from_pretrained`` returning a tokenizer with a trivial + chat-template-free tokenize/decode (just maps ints↔single-char strings).""" + tok = SimpleNamespace(chat_template=None, pad_token_id=0, eos_token_id=1) + + def encode(text: str) -> list[int]: + return [ord(c) % 50 for c in str(text)[:32]] + + tok.__call__ = lambda text, **kw: SimpleNamespace(input_ids=encode(text)) # type: ignore[attr-defined] + callable_tok = MagicMock(wraps=tok) + callable_tok.side_effect = lambda text, **kw: SimpleNamespace(input_ids=encode(text)) + callable_tok.decode = lambda ids, **kw: "".join(chr(int(i)) for i in ids) + callable_tok.chat_template = None + callable_tok.pad_token_id = 0 + callable_tok.eos_token_id = 1 + + from transformers import AutoTokenizer + + monkeypatch.setattr(AutoTokenizer, "from_pretrained", lambda *a, **kw: callable_tok) + return callable_tok + + +def _fake_hf_config(n_layers: int = 2, d_model: int = 4, vocab_size: int = 128) -> Any: + """Minimal HF config the overlay + provider reach into (vocab_size, + num_hidden_layers, hidden_size, architectures, torch_dtype).""" + return SimpleNamespace( + architectures=["LlamaForCausalLM"], + hidden_size=d_model, + num_hidden_layers=n_layers, + vocab_size=vocab_size, + torch_dtype="float16", + ) + + +def _patch_construction(monkeypatch, hf_config: Any) -> None: + """Patch AutoConfig + plugin.configure/register + extract_hf_config so __init__ runs + without vllm / real worker processes.""" + from transformers import AutoConfig + + from transformer_lens.model_bridge.sources.vllm import internals, plugin + + monkeypatch.setattr(AutoConfig, "from_pretrained", lambda *a, **kw: hf_config) + monkeypatch.setattr(plugin, "configure", lambda **kw: None) + monkeypatch.setattr(plugin, "register", lambda: None) + monkeypatch.setattr(internals, "extract_hf_config", lambda llm: hf_config) + + +def _make_request_output( + new_token_ids: list[int], finish_reason: str = "length", logprobs: Any = None +) -> Any: + """Fake vLLM ``RequestOutput`` exposing ``.outputs[0].{token_ids, finish_reason, logprobs}``.""" + inner = SimpleNamespace( + token_ids=list(new_token_ids), finish_reason=finish_reason, logprobs=logprobs + ) + return SimpleNamespace(outputs=[inner], prompt_token_ids=[]) + + +def _make_provider(monkeypatch, llm_instance: Any, hf_config: Any | None = None): + """Construct the provider with mocked vllm.LLM + fake tokenizer + patched AutoConfig / + plugin / extract_hf_config; device='cpu' so intermediate tensors don't require CUDA.""" + _fake_tokenizer(monkeypatch) + hf_config = hf_config or _fake_hf_config() + _patch_construction(monkeypatch, hf_config) + llm_class = MagicMock(return_value=llm_instance, name="LLM") + _install_vllm_mocks(monkeypatch, llm_class) + from transformer_lens.model_bridge.sources.inspect.vllm_provider import ( + TransformerLensVLLMModelAPI, + ) + + return TransformerLensVLLMModelAPI("any/model", device="cpu"), llm_class + + +class TestModuleSafety: + def test_module_imports_without_vllm(self): + # Confirm the lazy-vllm pattern: importing the module loads no vllm symbols. + assert "vllm" not in sys.modules + from transformer_lens.model_bridge.sources.inspect import ( # noqa: F401 + vllm_provider, + ) + + assert "vllm" not in sys.modules + + def test_modelapi_registered_as_tl_bridge_vllm(self): + from transformer_lens.model_bridge.sources.inspect.vllm_provider import ( + PROVIDER_NAME, + ) + + assert PROVIDER_NAME == "tl_bridge_vllm" + + +class TestConstruction: + def test_rejects_locked_kwarg_override(self, monkeypatch): + _fake_tokenizer(monkeypatch) + _patch_construction(monkeypatch, _fake_hf_config()) + _install_vllm_mocks(monkeypatch, llm_class=MagicMock(name="LLM")) + from transformer_lens.model_bridge.sources.inspect.vllm_provider import ( + TransformerLensVLLMModelAPI, + ) + + with pytest.raises(ValueError, match="tensor_parallel_size"): + TransformerLensVLLMModelAPI("any/model", vllm_kwargs={"tensor_parallel_size": 2}) + + def test_gated_capture_kind_raises(self, monkeypatch): + # vLLM's overlay serves resid_post/attn_out/mlp_out; resid_pre and resid_mid are gated. + provider_factory_args: dict[str, Any] = {"capture": ["blocks.0.hook_in"]} # resid_pre + _fake_tokenizer(monkeypatch) + _patch_construction(monkeypatch, _fake_hf_config()) + _install_vllm_mocks(monkeypatch, llm_class=MagicMock(name="LLM")) + from transformer_lens.model_bridge.sources.inspect.vllm_provider import ( + TransformerLensVLLMModelAPI, + ) + + with pytest.raises(ValueError, match="resid_pre|gated"): + TransformerLensVLLMModelAPI("any/model", device="cpu", **provider_factory_args) + + def test_served_capture_kind_accepted(self, monkeypatch): + # resid_post (blocks.{i}.hook_out) IS served by the vLLM overlay — must not raise. + _fake_tokenizer(monkeypatch) + _patch_construction(monkeypatch, _fake_hf_config()) + _install_vllm_mocks(monkeypatch, llm_class=MagicMock(name="LLM")) + from transformer_lens.model_bridge.sources.inspect.vllm_provider import ( + TransformerLensVLLMModelAPI, + ) + + api = TransformerLensVLLMModelAPI("any/model", device="cpu", capture=["blocks.0.hook_out"]) + assert "0:resid_post" in api._eval_capture + + def test_passes_locked_kwargs_to_llm(self, monkeypatch): + provider, llm_class = _make_provider(monkeypatch, MagicMock()) + kwargs = llm_class.call_args.kwargs + assert kwargs["tensor_parallel_size"] == 1 + assert kwargs["skip_tokenizer_init"] is True + assert kwargs["disable_log_stats"] is True + # Inc 2: capture wiring requires worker_extension_cls + full-vocab logprobs. + assert kwargs["worker_extension_cls"].endswith("TLWorkerExtension") + assert kwargs["max_logprobs"] == 128 # _fake_hf_config().vocab_size + assert kwargs["max_num_batched_tokens"] == 2048 # default + + def test_supported_kinds_match_overlay(self, monkeypatch): + provider, _ = _make_provider(monkeypatch, MagicMock()) + assert provider.supported_kinds() == frozenset({"resid_post", "attn_out", "mlp_out"}) + assert "resid_pre" in provider.capability_note() + + def test_provides_sequence_logits_is_false(self, monkeypatch): + # vLLM's sampler bypasses lm_head; _synthesize_logits only populates the gen + # position, so RemoteBridge.forward(return_type='loss'|'both') would NaN. The + # class attr is the signal source.py reads to set TLBridgeProfile's flag. + provider, _ = _make_provider(monkeypatch, MagicMock()) + assert provider.provides_sequence_logits is False + + +class TestGenerateEval: + def _config(self, **overrides): + from inspect_ai.model import GenerateConfig + + return GenerateConfig(**overrides) + + def _user_msg(self, content="hi"): + from inspect_ai.model import ChatMessageUser + + return [ChatMessageUser(content=content)] + + def test_generate_eval_basic_completion_and_usage(self, monkeypatch): + llm = MagicMock() + llm.generate.return_value = [_make_request_output([65, 66, 67], finish_reason="length")] + provider, _ = _make_provider(monkeypatch, llm) + out = asyncio.run( + provider.generate(self._user_msg("xy"), None, None, self._config(max_tokens=5)) + ) + choice = out.choices[0] + assert choice.message.text == "ABC" + assert choice.stop_reason == "max_tokens" # vLLM 'length' → inspect 'max_tokens' + assert out.usage.output_tokens == 3 + assert out.usage.total_tokens == out.usage.input_tokens + 3 + + def test_generate_eval_resets_interventions_first(self, monkeypatch): + # Worker intervention buffers are persistent across calls; a prior capture-path + # forward(intervene=...) would leak into the eval generate without an explicit + # identity reset. Contract: tl_set_interventions({}) fires before llm.generate. + llm = MagicMock() + llm.generate.return_value = [_make_request_output([65], finish_reason="length")] + provider, _ = _make_provider(monkeypatch, llm) + asyncio.run(provider.generate(self._user_msg(), None, None, self._config(max_tokens=1))) + # First rpc call must be the identity reset (no _eval_capture configured ⇒ exactly one). + rpc_calls = llm.collective_rpc.call_args_list + assert rpc_calls, "eval path must call tl_set_interventions before generate" + assert rpc_calls[0].args[0] == "tl_set_interventions" + assert rpc_calls[0].kwargs["args"] == ({},) + + def test_generate_eval_finish_stop_maps_to_stop(self, monkeypatch): + llm = MagicMock() + llm.generate.return_value = [_make_request_output([65, 1], finish_reason="stop")] + provider, _ = _make_provider(monkeypatch, llm) + out = asyncio.run( + provider.generate(self._user_msg(), None, None, self._config(max_tokens=4)) + ) + assert out.choices[0].stop_reason == "stop" + + def test_generate_eval_with_logprobs(self, monkeypatch): + vllm_lp = SimpleNamespace(logprob=-0.5, rank=1, decoded_token="A") + vllm_alt = SimpleNamespace(logprob=-2.0, rank=2, decoded_token="B") + step = {65: vllm_lp, 66: vllm_alt} + llm = MagicMock() + llm.generate.return_value = [ + _make_request_output([65], finish_reason="length", logprobs=[step]) + ] + provider, _ = _make_provider(monkeypatch, llm) + out = asyncio.run( + provider.generate( + self._user_msg(), + None, + None, + self._config(max_tokens=1, logprobs=True, top_logprobs=2), + ) + ) + lp = out.choices[0].logprobs + assert lp is not None and len(lp.content) == 1 + assert lp.content[0].logprob == -0.5 + assert len(lp.content[0].top_logprobs) == 2 + + def test_generate_eval_passes_temperature_top_p(self, monkeypatch): + llm = MagicMock() + llm.generate.return_value = [_make_request_output([65], finish_reason="length")] + provider, _ = _make_provider(monkeypatch, llm) + asyncio.run( + provider.generate( + self._user_msg(), + None, + None, + self._config(max_tokens=1, temperature=0.7, top_p=0.9, seed=42), + ) + ) + # SamplingParams was instantiated; confirm the sampling kwargs flowed through. + from transformer_lens.model_bridge.sources.inspect import ( # noqa: F401 + vllm_provider as _vp, + ) + + sp_cls = sys.modules["vllm"].SamplingParams + sp_kwargs = sp_cls.call_args.kwargs + assert sp_kwargs["temperature"] == 0.7 + assert sp_kwargs["top_p"] == 0.9 + assert sp_kwargs["seed"] == 42 + + +class TestCapturePath: + """TL-driven single-forward capture: extra_args carries input_ids/capture/interventions; + provider pushes specs → generates → reads captures via collective_rpc.""" + + def _config_with(self, extra_args: dict[str, Any]): + from inspect_ai.model import GenerateConfig + + return GenerateConfig(extra_body={"extra_args": extra_args}) + + def _make_capture_provider(self, monkeypatch, *, captures=None, next_token: int = 65): + import torch + + vllm_lp = SimpleNamespace(logprob=-0.1, rank=1, decoded_token="A") + request_output = SimpleNamespace( + outputs=[ + SimpleNamespace( + token_ids=[next_token], + logprobs=[{next_token: vllm_lp}], + finish_reason="stop", + ) + ], + prompt_token_ids=[], + ) + llm = MagicMock() + llm.generate.return_value = [request_output] + captures = ( + captures + if captures is not None + else {"blocks.0.hook_out": torch.zeros(4, 4, dtype=torch.float32)} + ) + + def rpc(method, args=()): + return [captures] if method == "tl_read_captures" else [None] + + llm.collective_rpc.side_effect = rpc + provider, llm_class = _make_provider(monkeypatch, llm) + return provider, llm, llm_class + + def test_capture_pushes_interventions_and_reads(self, monkeypatch): + provider, llm, _ = self._make_capture_provider(monkeypatch) + asyncio.run( + provider.generate( + [], + None, + None, + self._config_with( + { + "input_ids": [10, 20, 30, 40], + "capture": ["0:resid_post"], + "interventions": {"0:resid_post": {"op": "suppress"}}, + } + ), + ) + ) + # rpc order: set_interventions (resets stale + applies new) → reset_capture_flags + # (opens the first-write-wins gate for the prefill below) → read. + methods = [c.args[0] for c in llm.collective_rpc.call_args_list] + assert methods == ["tl_set_interventions", "tl_reset_capture_flags", "tl_read_captures"] + # Specs translated wire key → TL hook name (worker is keyed by hook name). + set_call = llm.collective_rpc.call_args_list[0] + assert set_call.kwargs["args"] == ({"blocks.0.hook_out": {"op": "suppress"}},) + # Read takes [prompt_lens] + hook names so the worker slices the GPU buffer. + read_call = llm.collective_rpc.call_args_list[2] + assert read_call.kwargs["args"] == ([4], ["blocks.0.hook_out"]) + + def test_capture_returns_wire_format_activations(self, monkeypatch): + import torch + + captures = {"blocks.0.hook_out": torch.full((3, 4), 0.5, dtype=torch.float32)} + provider, _, _ = self._make_capture_provider(monkeypatch, captures=captures) + out = asyncio.run( + provider.generate( + [], + None, + None, + self._config_with({"input_ids": [1, 2, 3], "capture": ["0:resid_post"]}), + ) + ) + assert "0:resid_post" in out.metadata["activations"] + # Wire envelope is {"data": b64, "dtype": str, "shape": list} — driver decodes verbatim. + entry = out.metadata["activations"]["0:resid_post"] + assert entry["shape"] == [3, 4] and entry["dtype"] == "float32" + + def test_capture_synthesizes_logits_when_requested(self, monkeypatch): + provider, _, _ = self._make_capture_provider(monkeypatch) + out = asyncio.run( + provider.generate( + [], + None, + None, + self._config_with( + { + "input_ids": [10, 20, 30], + "capture": ["0:resid_post"], + "return_logits": True, + } + ), + ) + ) + # logits shape (n_tokens, d_vocab); last position holds vLLM's logprobs at the gen step. + entry = out.metadata["tl_logits"] + assert entry["shape"] == [3, 128] + + def test_capture_skips_logits_when_disabled(self, monkeypatch): + provider, _, _ = self._make_capture_provider(monkeypatch) + out = asyncio.run( + provider.generate( + [], + None, + None, + self._config_with( + { + "input_ids": [10, 20, 30], + "capture": ["0:resid_post"], + "return_logits": False, + } + ), + ) + ) + assert "tl_logits" not in out.metadata + + def test_capture_gated_kind_raises(self, monkeypatch): + # resid_pre (blocks.{i}.hook_in) is gated by vLLM's fused execution. + provider, _, _ = self._make_capture_provider(monkeypatch) + with pytest.raises(ValueError, match="resid_pre|gated"): + asyncio.run( + provider.generate( + [], + None, + None, + self._config_with({"input_ids": [1, 2], "capture": ["0:resid_pre"]}), + ) + ) + + def test_capture_unparseable_wire_key_raises(self, monkeypatch): + # Kind is served, but the layer prefix is non-numeric — name_from_wire_key returns None. + provider, _, _ = self._make_capture_provider(monkeypatch) + with pytest.raises(ValueError, match="unrecognised wire keys"): + asyncio.run( + provider.generate( + [], + None, + None, + self._config_with({"input_ids": [1, 2], "capture": ["abc:resid_post"]}), + ) + ) + + def test_intervention_unknown_wire_key_raises(self, monkeypatch): + provider, _, _ = self._make_capture_provider(monkeypatch) + with pytest.raises(ValueError, match="not a fireable hook"): + asyncio.run( + provider.generate( + [], + None, + None, + self._config_with( + { + "input_ids": [1, 2], + "capture": ["0:resid_post"], + "interventions": {"abc:resid_post": {"op": "suppress"}}, + } + ), + ) + ) + + def test_prompt_exceeds_max_batched_tokens_raises(self, monkeypatch): + provider, _, _ = self._make_capture_provider(monkeypatch) + with pytest.raises(ValueError, match="max_num_batched_tokens"): + asyncio.run( + provider.generate( + [], + None, + None, + self._config_with( + {"input_ids": list(range(3000)), "capture": ["0:resid_post"]} + ), + ) + ) + + def test_capture_completion_decodes_generated_token(self, monkeypatch): + # The ModelOutput choice's text must be the decoded next token (parity with HF provider). + provider, _, _ = self._make_capture_provider(monkeypatch, next_token=72) + out = asyncio.run( + provider.generate( + [], + None, + None, + self._config_with({"input_ids": [1, 2], "capture": ["0:resid_post"]}), + ) + ) + # The fake tokenizer decodes by joining chr(id) — 72 → 'H'. + assert out.choices[0].message.text == "H" + + +class TestPerTurnCapture: + """``model_args['capture']`` triggers a snapshot-via-single-token-forward before every + eval generate (vLLM's decode steps overwrite buffer row 0, so the HF first-write-wins + trick can't apply); the snapshot lands in ``ModelOutput.metadata`` for agent rollouts.""" + + def _user_msg(self, content: str = "hi"): + from inspect_ai.model import ChatMessageUser + + return [ChatMessageUser(content=content)] + + def _make_provider_with_capture(self, monkeypatch, capture: list[str]): + import torch + + # llm.generate runs twice per eval (snapshot then real generate); the same fake + # output works for both. collective_rpc returns canned captures for tl_read_captures. + llm = MagicMock() + llm.generate.return_value = [_make_request_output([65, 66], finish_reason="length")] + captures = {"blocks.0.hook_out": torch.zeros(4, 4, dtype=torch.float32)} + + def rpc(method, args=()): + return [captures] if method == "tl_read_captures" else [None] + + llm.collective_rpc.side_effect = rpc + + _fake_tokenizer(monkeypatch) + _patch_construction(monkeypatch, _fake_hf_config()) + llm_class = MagicMock(return_value=llm, name="LLM") + _install_vllm_mocks(monkeypatch, llm_class) + from transformer_lens.model_bridge.sources.inspect.vllm_provider import ( + TransformerLensVLLMModelAPI, + ) + + return TransformerLensVLLMModelAPI("any/model", device="cpu", capture=capture), llm + + def test_eval_capture_during_generate_no_extra_forward(self, monkeypatch): + from inspect_ai.model import GenerateConfig + + provider, llm = self._make_provider_with_capture(monkeypatch, ["blocks.0.hook_out"]) + out = asyncio.run( + provider.generate(self._user_msg(), None, None, GenerateConfig(max_tokens=4)) + ) + # rpc order: leak-guard reset → open capture gates → read after the eval generate. + # Prefill of the eval generate captures (first-write-wins); decode steps self-copy. + methods = [c.args[0] for c in llm.collective_rpc.call_args_list] + assert methods == [ + "tl_set_interventions", + "tl_reset_capture_flags", + "tl_read_captures", + ] + leak_guard = llm.collective_rpc.call_args_list[0] + assert leak_guard.kwargs["args"] == ({},) + # Single llm.generate now — the per-turn capture rides the same forward (was 2 before). + assert llm.generate.call_count == 1 + # Capture lands in metadata as the same wire format the InspectDriver consumes. + assert out.metadata is not None and "0:resid_post" in out.metadata["activations"] + + def test_no_capture_only_leak_guard_rpc(self, monkeypatch): + from inspect_ai.model import GenerateConfig + + llm = MagicMock() + llm.generate.return_value = [_make_request_output([65], finish_reason="stop")] + provider, _ = _make_provider(monkeypatch, llm) + out = asyncio.run( + provider.generate(self._user_msg(), None, None, GenerateConfig(max_tokens=1)) + ) + # No per-turn capture ⇒ no snapshot, no metadata. The only rpc is the identity + # reset guarding against stale interventions leaking from a prior capture-path call. + assert out.metadata is None + rpc_calls = llm.collective_rpc.call_args_list + assert len(rpc_calls) == 1 + assert rpc_calls[0].args[0] == "tl_set_interventions" + assert rpc_calls[0].kwargs["args"] == ({},) + + def test_eval_capture_emits_eval_completion_too(self, monkeypatch): + # The eval still returns the real eval generate's completion + usage — snapshot is + # an additive metadata side channel, not a substitute for the eval output. + from inspect_ai.model import GenerateConfig + + provider, _ = self._make_provider_with_capture(monkeypatch, ["blocks.0.hook_out"]) + out = asyncio.run( + provider.generate(self._user_msg(), None, None, GenerateConfig(max_tokens=4)) + ) + # _make_request_output([65, 66]) ⇒ "AB" via fake tokenizer. + assert out.choices[0].message.text == "AB" + assert out.usage.output_tokens == 2 diff --git a/transformer_lens/model_bridge/remote_bridge.py b/transformer_lens/model_bridge/remote_bridge.py index 0c8cc5b25d..a5c4ad9a05 100644 --- a/transformer_lens/model_bridge/remote_bridge.py +++ b/transformer_lens/model_bridge/remote_bridge.py @@ -57,6 +57,15 @@ def boot_vllm(*args: Any, **kwargs: Any) -> "RemoteBridge": return _boot_vllm(*args, **kwargs) + @staticmethod + def boot_inspect(*args: Any, **kwargs: Any) -> "RemoteBridge": + """Boot a model via an inspect_ai provider. Returns a RemoteBridge wrapping + an InspectDriver. Lazy import keeps remote_bridge inspect-agnostic. See + :func:`sources.inspect.boot_inspect` for kwargs.""" + from .sources.inspect import boot_inspect as _boot_inspect + + return _boot_inspect(*args, **kwargs) + def _scan_existing_hooks(self, module: Any, prefix: str = "") -> None: """No-op: registry built from driver declarations in __init__.""" @@ -80,12 +89,17 @@ def forward( "logits[..., -1, :] (or the per-row last token in batched mode)." ) if isinstance(input, str): - assert self.tokenizer is not None, "Tokenizer must be set for string input." - tokens = self.tokenizer.encode(input, return_tensors="pt") - kwargs["input_ids"] = tokens + kwargs["input_ids"] = self.to_tokens(input) # BOS-aware, matches boot_transformers elif input is not None: kwargs["input_ids"] = input + # Only request hooks with a registered handler, so a plain forward(tokens) + # ships logits alone instead of the full residual decomposition every call. + if "capture" not in kwargs: + kwargs["capture"] = tuple( + name for name, hp in self._hook_registry.items() if hp.fwd_hooks + ) + result: ForwardResult = self._driver.forward(**kwargs) if result.captured: self._replay_captures(result.captured) @@ -147,11 +161,35 @@ def run_with_hooks( **kwargs, ) - def to_tokens(self, text: Any, *args: Any, **kwargs: Any) -> Any: - """Tokenize via ``self.tokenizer``. BOS/padding handling lives on - :class:`TransformerBridge`.""" + def to_tokens(self, input: Any, prepend_bos: bool | None = None, truncate: bool = True) -> Any: + """Tokenize a string with the same BOS handling as ``TransformerBridge``. + + Mirrors ``cfg.default_prepend_bos`` / ``tokenizer_prepends_bos`` so + ``boot_inspect(m).run_with_cache("text")`` matches ``boot_transformers(m)`` + on the same string — a bare ``encode`` (no BOS) would silently diverge. + """ + from transformer_lens import utils + assert self.tokenizer is not None, "Tokenizer must be set." - return self.tokenizer.encode(text, return_tensors="pt") + if prepend_bos is None: + prepend_bos = getattr(self.cfg, "default_prepend_bos", True) + tokenizer_prepends_bos = getattr(self.cfg, "tokenizer_prepends_bos", True) + if prepend_bos and not tokenizer_prepends_bos: + input = utils.get_input_with_manually_prepended_bos(self.tokenizer.bos_token, input) + if isinstance(input, str): + input = [input] + # A single sequence never needs padding; only pad when batching (which also + # avoids requiring a pad_token on tokenizers that lack one, e.g. raw gpt2). + tokens = self.tokenizer( + input, + return_tensors="pt", + padding=len(input) > 1, + truncation=truncate, + max_length=self.cfg.n_ctx if truncate else None, + )["input_ids"] + if not prepend_bos and tokenizer_prepends_bos: + tokens = utils.get_tokens_with_bos_removed(self.tokenizer, tokens) + return tokens def __enter__(self) -> "RemoteBridge": """Use as a context manager so the engine is released on exit: diff --git a/transformer_lens/model_bridge/sources/inspect/__init__.py b/transformer_lens/model_bridge/sources/inspect/__init__.py new file mode 100644 index 0000000000..a31d5920b5 --- /dev/null +++ b/transformer_lens/model_bridge/sources/inspect/__init__.py @@ -0,0 +1,25 @@ +"""Inspect driver: turn an ``inspect_ai``-served model into a TransformerLens bridge. + +We ship our own HF-backed provider (``provider.py``) and a torch-free consumer +(``driver.py``). + +Maintainer note: this package is named ``inspect``, shadowing the stdlib module. +Only ever import the stdlib ``inspect`` via an absolute import from outside this +package; never write a bare ``import inspect`` inside these modules. +""" +from __future__ import annotations + +from typing import Any + +from .source import boot_inspect + +__all__ = ["activations_column", "boot_inspect", "capture_activations", "turn_activations"] + + +def __getattr__(name: str) -> Any: + # Lazy so importing the package stays inspect_ai-free; eval.py imports inspect_ai. + if name in ("capture_activations", "activations_column", "turn_activations"): + from . import eval as _eval + + return getattr(_eval, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/transformer_lens/model_bridge/sources/inspect/_provider_base.py b/transformer_lens/model_bridge/sources/inspect/_provider_base.py new file mode 100644 index 0000000000..6248bcbafe --- /dev/null +++ b/transformer_lens/model_bridge/sources/inspect/_provider_base.py @@ -0,0 +1,201 @@ +"""Shared base for ``tl_bridge``-style Inspect providers (HF, vLLM). + +Subclasses load their model + tokenizer, run their structural self-check, and implement +backend-specific ``_generate_capture`` / ``_generate_eval`` — this base owns the +``generate()`` dispatch, message-to-ids rendering (chat template + tools), logprob +construction, and validation of the per-turn ``capture=[...]`` config. + +Subclass contract: +- Set ``self._tokenizer`` (HF-style ``AutoTokenizer``), ``self._device`` (str), + ``self._kinds`` (frozenset of served boundary kinds), ``self._capability_note`` (str + explaining any gating), and ``self._eval_capture`` (dict ``{wire_key: hook_name}``, + built via :meth:`_parse_eval_capture`) before any ``generate()`` call. +- Implement ``_generate_capture(input, extra_args, config)`` (TL-driven single forward) and + ``_generate_eval(input, config, tools)`` (multi-token chat generation). +""" +from __future__ import annotations + +import json +import re +import uuid +from typing import Any, Mapping + +import torch +from inspect_ai.model import GenerateConfig, Logprob, ModelAPI, TopLogprob +from inspect_ai.tool import ToolCall + +from . import hooks + + +def _message_text(message: Any) -> str: + """Text of an Inspect chat message — its ``.text`` (handles multimodal content), + falling back to string content, else ''.""" + text = getattr(message, "text", None) + if isinstance(text, str): + return text + content = getattr(message, "content", None) + return content if isinstance(content, str) else "" + + +def _tool_schema(tool: Any) -> dict[str, Any]: + """Inspect ``ToolInfo`` → OpenAI-style function schema for ``apply_chat_template``.""" + params = tool.parameters + params = params.model_dump() if hasattr(params, "model_dump") else dict(params) + return { + "type": "function", + "function": {"name": tool.name, "description": tool.description, "parameters": params}, + } + + +# Tool-call blocks emitted by common instruct templates (Qwen/Hermes-style); the bare-JSON +# fallback covers models that emit a single {"name", "arguments"} object. +_TOOL_CALL_BLOCK = re.compile(r"\s*(\{.*?\})\s*", re.S) + + +def _parse_tool_calls(text: str) -> list[ToolCall] | None: + """Best-effort parse of tool calls from a completion. Model-specific formats vary; this + handles ``{json}`` blocks and a single bare ``{name, arguments}``.""" + blocks = _TOOL_CALL_BLOCK.findall(text) + if not blocks: + match = re.search(r"\{.*\}", text, re.S) + blocks = [match.group(0)] if match else [] + calls = [] + for block in blocks: + try: + obj = json.loads(block) + except (ValueError, TypeError): + continue + name = obj.get("name") if isinstance(obj, dict) else None + if not isinstance(name, str): + continue + args = obj.get("arguments") or obj.get("parameters") or {} + calls.append( + ToolCall( + id=uuid.uuid4().hex[:8], + function=name, + arguments=args if isinstance(args, dict) else {}, + ) + ) + return calls or None + + +def _require_served(kind: str, served: frozenset[str], note: str, context: str) -> None: + """Raise if ``kind`` was gated by the structural self-check — without this the eval path + would silently return a derivation (e.g. ``resid_mid``) the driver path excludes.""" + if kind not in served: + raise ValueError( + f"{context} requests kind {kind!r} which this model gated. {note} " + f"Served kinds: {sorted(served)}." + ) + + +class _InspectModelAPIBase(ModelAPI): + """Shared Inspect ModelAPI scaffolding for ``tl_bridge``-style providers. + + Subclasses populate ``self._tokenizer``/``_device``/``_kinds``/``_capability_note``/ + ``_eval_capture`` in ``__init__`` and implement the backend-specific generate paths. + """ + + # Attributes set by subclass __init__ before any generate() call (declared here so the + # shared helpers' type-checking sees them; not initialized to avoid masking bugs). + _tokenizer: Any + _device: Any + _kinds: frozenset + _capability_note: str + _eval_capture: dict[str, str] + + # Class-level capability flag, read by source.py → TLBridgeProfile → InspectDriver → + # RemoteBridge.forward to gate return_type ∈ {loss, both}. Must be set by every + # subclass: True iff every position 0..n-1 of metadata['tl_logits'] holds real values + # (not -inf padding). HF does a true forward → full logits; vLLM's sampler bypasses + # lm_head and only the generated position has logprobs, so vLLM=False. + provides_sequence_logits: bool + + # --- subclass contract ------------------------------------------------------------- + + def _generate_capture( + self, input: Any, extra_args: Mapping[str, Any], config: GenerateConfig + ) -> Any: + """TL-driven single-forward capture (residual/attn/mlp boundaries + full logits).""" + raise NotImplementedError + + def _generate_eval(self, input: Any, config: GenerateConfig, tools: Any) -> Any: + """Plain Inspect generation: chat input → multi-token completion + Logprobs + + ModelUsage (+ parsed tool calls when ``tools`` is non-empty + per-turn capture).""" + raise NotImplementedError + + # --- shared API -------------------------------------------------------------------- + + def supported_kinds(self) -> frozenset: + """Boundary kinds this model is structurally able to serve.""" + return self._kinds + + def capability_note(self) -> str: + """Human-readable reason for any gated boundary, or '' if all are served.""" + return self._capability_note + + async def generate(self, input, tools, tool_choice, config): # type: ignore[override] + # Two callers: the TL driver (extra_args carries input_ids/capture/interventions — + # single-forward activation capture) and a plain Inspect eval (chat messages, real + # multi-token generation). Branch on whether a TL request is present. + extra_args: Mapping[str, Any] = (config.extra_body or {}).get("extra_args", {}) + if extra_args.get("input_ids") is not None or extra_args.get("capture"): + return self._generate_capture(input, extra_args, config) + return self._generate_eval(input, config, tools or []) + + # --- shared helpers ---------------------------------------------------------------- + + def _parse_eval_capture(self, model_args: dict[str, Any]) -> dict[str, str]: + """Validate ``model_args["capture"]`` against the structural self-check (same + protection as the driver path) and key by wire key. Returns ``{wire_key: name}``.""" + eval_capture: dict[str, str] = {} + for name in model_args.pop("capture", None) or []: + resolved = hooks.resolve(name) + if resolved is None: + raise ValueError(f"capture={name!r} is not a fireable hook name.") + _require_served(resolved[1], self._kinds, self._capability_note, f"capture={name!r}") + eval_capture[hooks.wire_key(*resolved)] = name + return eval_capture + + def _messages_to_ids(self, input: Any, tools: Any = ()) -> Any: + """Render Inspect chat messages (+ any ``tools``) to input ids — chat template + when the tokenizer has one, else newline-joined message text (e.g. gpt2).""" + if isinstance(input, str): + messages = [{"role": "user", "content": input}] + else: + messages = [ + {"role": getattr(m, "role", "user"), "content": _message_text(m)} for m in input + ] + template = getattr(self._tokenizer, "chat_template", None) + if len(tools) and not template: + raise NotImplementedError( + f"tl_bridge: tool use needs a tool-aware chat template; {self.model_name} has " + "none. Serve a tool-capable instruct model for agentic evals." + ) + if template: + kwargs: dict[str, Any] = {"add_generation_prompt": True} + if len(tools): + kwargs["tools"] = [_tool_schema(t) for t in tools] + token_ids = self._tokenizer.apply_chat_template(messages, **kwargs) + else: + token_ids = self._tokenizer("\n".join(m["content"] for m in messages)).input_ids + return torch.tensor([list(token_ids)], device=self._device) + + def _logprob_entry(self, token_id: int, step_logits: Any, top_n: Any) -> Logprob: + """One token's log-prob (+ top-k alternatives) from a position's logits.""" + lp = torch.log_softmax(step_logits.float(), dim=-1) + top = [] + if top_n: + vals, idx = lp.topk(int(top_n)) + top = [ + TopLogprob( + token=str(self._tokenizer.decode([int(i)])), logprob=float(v), bytes=None + ) + for v, i in zip(vals.tolist(), idx.tolist()) + ] + return Logprob( + token=str(self._tokenizer.decode([int(token_id)])), + logprob=float(lp[int(token_id)]), + bytes=None, + top_logprobs=top, + ) diff --git a/transformer_lens/model_bridge/sources/inspect/conftest.py b/transformer_lens/model_bridge/sources/inspect/conftest.py new file mode 100644 index 0000000000..baa9985ec0 --- /dev/null +++ b/transformer_lens/model_bridge/sources/inspect/conftest.py @@ -0,0 +1,19 @@ +"""Skip doctest-modules collection of inspect-provider files when ``inspect_ai`` is not +installed (it's an optional extra). Each file does a top-level ``from inspect_ai.model +import …`` for the ``ModelAPI`` base + helper types — guarding that import per-file would +bury the dependency contract; one collect-time skip here keeps the providers' source clean +and lets ``make docstring-test`` run cleanly without the ``inspect`` extra.""" +from __future__ import annotations + +from importlib import util as _importlib_util + +collect_ignore_glob = ( + [] + if _importlib_util.find_spec("inspect_ai") is not None + else [ + "_provider_base.py", + "eval.py", + "transformers_provider.py", + "vllm_provider.py", + ] +) diff --git a/transformer_lens/model_bridge/sources/inspect/driver.py b/transformer_lens/model_bridge/sources/inspect/driver.py new file mode 100644 index 0000000000..f30240cf96 --- /dev/null +++ b/transformer_lens/model_bridge/sources/inspect/driver.py @@ -0,0 +1,182 @@ +"""InspectDriver — torch-free consumer of an ``inspect_ai`` provider's output. + +Talks to a provider through the inspect_ai ``ModelOutput`` envelope. Everything +provider-specific — the request schema, which hooks are served, full vs last-token +logits, intervention translation — lives in a :mod:`profiles` Profile; the driver just +drives it. Stays numpy-only (``to_torch`` runs at the bridge boundary), so this file +imports zero torch symbols (enforced by a unit test). ``inspect_ai`` is imported lazily. +""" +from __future__ import annotations + +import asyncio +import logging +import threading +import warnings +from typing import Any, Mapping + +import numpy as np + +from transformer_lens.model_bridge.driver_protocol import ( + ForwardResult, + Intervention, + TensorLike, +) +from transformer_lens.model_bridge.sources._driver_base import DriverBase + +from . import hooks, wire +from .profiles import TLBridgeProfile + + +class InspectDriver(DriverBase): + """Driver wrapping an ``inspect_ai`` model; capture + interventions via a Profile.""" + + # Remote provider — no torch weight/grad surface. + _supported_features = frozenset() + + def __init__(self, model: Any, adapter: Any, tokenizer: Any, profile: Any = None) -> None: + super().__init__(adapter.cfg, tokenizer) + self._model = model + self._profile = profile if profile is not None else TLBridgeProfile() + self._n_layers = int(self.bridge_config.n_layers) + self._d_vocab = int(self.bridge_config.d_vocab) + # Provider-specific: loss/both allowed only if the provider returns full logits. + self.provides_sequence_logits = self._profile.provides_sequence_logits + self.supported_hook_points = self._profile.supported_hooks(self._n_layers) + full = hooks.supported_hook_points(self._n_layers) + self.non_fireable_hook_points = hooks.nonfireable_hook_points(self._n_layers) | ( + full - self.supported_hook_points + ) + # Background event loop, created lazily on first forward. + self._loop: asyncio.AbstractEventLoop | None = None + self._loop_thread: threading.Thread | None = None + self._warned_missing: set[str] = set() # hooks we've already warned were absent + + def forward( + self, + input_ids: TensorLike | None = None, + *, + capture: tuple[str, ...] = (), + intervene: Mapping[str, Intervention] | None = None, + max_new_tokens: int = 1, + return_logits: bool = True, + **kwargs: Any, + ) -> ForwardResult: + if self._model is None: + raise RuntimeError("InspectDriver is closed.") + if input_ids is None: + raise ValueError("InspectDriver requires input_ids") + if int(max_new_tokens) != 1: + raise NotImplementedError( + "InspectDriver supports max_new_tokens=1 only (single-forward capture)." + ) + ids = self._normalize_input_ids(input_ids) + # capture is authoritative: the bridge passes exactly the hooks with handlers, + # so () means "capture nothing" (logits only), not "capture everything". + names = list(capture) + wire_keys = self._wire_keys(names) + interventions = self._profile.translate_interventions( + intervene or {}, self.supported_hook_points + ) + prompt, extra_args = self._profile.build_request( + ids, wire_keys, interventions, return_logits, self.tokenizer + ) + + output = self._run_coro(self._generate(prompt, extra_args)) + + captured = self._assemble_captures(output, names) + logits = ( + self._profile.decode_logits(output, len(ids), self._d_vocab, self.tokenizer) + if return_logits + else None + ) + return ForwardResult(logits=logits, captured=captured, raw_output=output) + + async def _generate(self, prompt: Any, extra_args: dict[str, Any]) -> Any: + from inspect_ai.model import GenerateConfig + + config = GenerateConfig( + temperature=0.0, max_tokens=1, extra_body={"extra_args": extra_args} + ) + return await self._model.generate(prompt, config=config) + + def _assemble_captures(self, output: Any, names: list[str]) -> dict[str, np.ndarray]: + """Decode the requested boundaries → ``{hook_name: (1, seq, d_model)}``; names the + provider didn't return are skipped (and warned once).""" + metadata = getattr(output, "metadata", None) or {} + decoded = wire.decode_activations(metadata, self._wire_keys(names)) + captured: dict[str, np.ndarray] = {} + missing: list[str] = [] + for name in names: + resolved = hooks.resolve(name) + if resolved is None: + continue + arr = decoded.get(hooks.wire_key(*resolved)) + if arr is None: + missing.append(name) + continue + captured[name] = arr[np.newaxis, ...] if arr.ndim == 2 else arr + self._warn_missing(missing) + return captured + + def _warn_missing(self, missing: list[str]) -> None: + """Warn once per hook the provider was asked for but didn't return — else its + cache entry is silently absent and surfaces only as a later KeyError.""" + new = [name for name in missing if name not in self._warned_missing] + if new: + self._warned_missing.update(new) + warnings.warn( + f"InspectDriver: provider returned no activation for {sorted(new)} " + "(requested and in supported_hook_points); those cache keys will be absent.", + UserWarning, + stacklevel=2, + ) + + def close(self) -> None: + log = logging.getLogger("transformer_lens.inspect") + if self._loop is not None: + try: + self._loop.call_soon_threadsafe(self._loop.stop) + if self._loop_thread is not None: + self._loop_thread.join(timeout=5) + self._loop.close() + except Exception as e: + log.debug("event-loop teardown failed during close(): %s", e) + self._loop = None + self._loop_thread = None + self._model = None # drop the provider reference; the server owns its own lifecycle + + # ---- helpers ---- + + def _ensure_loop(self) -> asyncio.AbstractEventLoop: + """A private loop on a daemon thread — works even when the caller is already + inside a running loop (Jupyter), unlike asyncio.run().""" + if self._loop is None: + self._loop = asyncio.new_event_loop() + self._loop_thread = threading.Thread( + target=self._loop.run_forever, daemon=True, name="inspect-driver-loop" + ) + self._loop_thread.start() + return self._loop + + def _run_coro(self, coro: Any) -> Any: + future = asyncio.run_coroutine_threadsafe(coro, self._ensure_loop()) + return future.result() # blocks the sync caller; re-raises provider errors + + def _wire_keys(self, names: list[str]) -> list[str]: + """Unique ``:`` keys for the requested hook names (aliases collapse).""" + keys = {hooks.wire_key(*r) for r in (hooks.resolve(n) for n in names) if r is not None} + return sorted(keys) + + @staticmethod + def _normalize_input_ids(input_ids: Any) -> list[int]: + """Coerce to a flat list[int] (batch_size=1 only); numpy/list/tensor, no torch import.""" + # Duck-type a torch tensor onto CPU first — np.asarray can't read CUDA memory, + # and a torch-free driver can't import torch to .cpu() it. + if hasattr(input_ids, "detach") and hasattr(input_ids, "cpu"): + input_ids = input_ids.detach().cpu() + arr = np.asarray(input_ids) + if arr.ndim == 2: + if arr.shape[0] != 1: + raise NotImplementedError("InspectDriver supports batch_size=1 only.") + arr = arr[0] + return [int(x) for x in arr.tolist()] diff --git a/transformer_lens/model_bridge/sources/inspect/eval.py b/transformer_lens/model_bridge/sources/inspect/eval.py new file mode 100644 index 0000000000..eebad35931 --- /dev/null +++ b/transformer_lens/model_bridge/sources/inspect/eval.py @@ -0,0 +1,118 @@ +"""Inspect solver for capturing TransformerLens activations during an eval. + +``capture_activations([...])`` is a solver you add to a Task's solver chain to harvest +activations alongside a behavioral eval (with a ``tl_bridge``-served model). Full +activations go to a per-sample side artifact (``output_dir/.npz``) for +probing/SAE; a compact ``reduce(...)`` summary lands in the sample store so +``inspect_ai.analysis.samples_df`` can correlate activation features with scores. + +Imports ``inspect_ai`` at module load (like ``provider.py``); the package ``__init__`` +exposes ``capture_activations`` lazily so importing the package stays inspect_ai-free. +""" +from __future__ import annotations + +import os +from typing import Any, Callable, Mapping, Optional, Sequence + +import numpy as np +from inspect_ai.model import GenerateConfig, get_model +from inspect_ai.solver import Generate, Solver, TaskState, solver +from inspect_ai.util import store + +from . import hooks, wire + + +def _default_reduce(activations: Mapping[str, np.ndarray]) -> dict[str, Any]: + """Per-hook L2 norm + shape — small, JSON-able, queryable in samples_df.""" + return { + name: {"l2": float(np.linalg.norm(arr)), "shape": list(arr.shape)} + for name, arr in activations.items() + } + + +@solver +def capture_activations( + capture: Sequence[str], + output_dir: str = "tl_activations", + reduce: Optional[Callable[[Mapping[str, np.ndarray]], dict[str, Any]]] = None, + store_key: str = "tl_activations", +) -> Solver: + """Capture ``capture`` hooks for each sample's current messages. + + Writes full activations to ``output_dir/.npz`` and a ``reduce(...)`` summary + (default: per-hook L2 + shape) to the sample store under ``store_key`` (+ ``_path``). + Requires a ``tl_bridge``-served model; raises if the model returns no activations. + Place before ``generate()`` to capture the prompt, after it to include the completion. + """ + # Resolve at construction so a multi-eval run from different CWDs doesn't scatter + # artifacts (the solver runs later, possibly under a different working directory). + output_dir = os.path.abspath(output_dir) + reduce_fn = reduce or _default_reduce + # TL hook names → provider wire keys (and back, to key the saved arrays by hook name). + name_by_wire = {} + for name in capture: + resolved = hooks.resolve(name) + if resolved is None: + raise ValueError(f"capture_activations: {name!r} is not a fireable hook name.") + name_by_wire[hooks.wire_key(*resolved)] = name + wire_keys = list(name_by_wire) + + async def solve(state: TaskState, generate: Generate) -> TaskState: + # get_model() (no args) is the eval's active model; state.model is just its name. + output = await get_model().generate( + state.messages, + config=GenerateConfig( + extra_body={"extra_args": {"capture": wire_keys, "return_logits": False}} + ), + ) + decoded = wire.decode_activations(getattr(output, "metadata", None), wire_keys) + if not decoded: + raise RuntimeError( + "capture_activations got no activations back — the eval model must be a " + "tl_bridge provider (e.g. model='tl_bridge/gpt2')." + ) + activations = {name_by_wire[wk]: arr for wk, arr in decoded.items()} + os.makedirs(output_dir, exist_ok=True) + path = os.path.join(output_dir, f"{state.sample_id}.npz") + np.savez_compressed(path, **activations) + store().set(store_key, reduce_fn(activations)) + store().set(f"{store_key}_path", path) + return state + + return solve + + +def turn_activations(sample: Any) -> list[dict[str, np.ndarray]]: + """Per-turn activations from an eval sample's model events, for a provider booted with + ``capture=[...]`` (e.g. ``model_args={"capture": [...]}``). Returns one + ``{hook_name: (1, seq, d)}`` dict per model generation, in turn order — the activations + of an agentic/multi-turn rollout. + """ + turns = [] + for event in getattr(sample, "events", []) or []: + metadata = getattr(getattr(event, "output", None), "metadata", None) + if not metadata or "activations" not in metadata: + continue + decoded = wire.decode_activations(metadata, list(metadata["activations"])) + named = { + name: arr[np.newaxis, ...] if arr.ndim == 2 else arr + for wk, arr in decoded.items() + if (name := hooks.name_from_wire_key(wk)) is not None + } + if named: + turns.append(named) + return turns + + +def activations_column(store_key: str = "tl_activations", name: Optional[str] = None) -> Any: + """A ``samples_df`` column surfacing :func:`capture_activations`'s reduction, for + correlating activation features with scores:: + + df = samples_df(logs, columns=[*SampleSummary, activations_column()]) + """ + from inspect_ai.analysis import SampleColumn + + return SampleColumn(name or store_key, path=lambda s: s.store.get(store_key), full=True) + + +__all__ = ["activations_column", "capture_activations", "turn_activations"] diff --git a/transformer_lens/model_bridge/sources/inspect/hooks.py b/transformer_lens/model_bridge/sources/inspect/hooks.py new file mode 100644 index 0000000000..ed287bca0a --- /dev/null +++ b/transformer_lens/model_bridge/sources/inspect/hooks.py @@ -0,0 +1,86 @@ +"""Canonical hook names ↔ (layer, kind) for the Inspect HF provider. + +Torch-free; shared by the provider (capture/intervene) and the driver (supported set, +decode). Covers the ``d_model``-shaped decoder-layer boundaries; head-split hooks +(q/k/v/z, pattern), ``embed``, and ``ln_final`` (fold-LN convention) are non-fireable. + +Names are TransformerBridge-native (``blocks.{i}.hook_out``, ``.attn.hook_out``, ...), +not the HookedTransformer aliases. A bridge cache carries both with identical values, +so parity vs ``boot_transformers`` still resolves. + +Which boundaries are actually fireable is decided per-model by the provider's structural +self-check, not a hand-kept architecture list: it locates attn/mlp and probes whether the +``resid_pre + attn_out`` derivation holds, gating ``resid_mid`` otherwise. +``supported_hook_points(n_layers, kinds=...)`` filters to that detected set. +""" +from __future__ import annotations + +import re +from typing import Iterable, Optional + +ALL_KINDS = frozenset({"resid_pre", "resid_mid", "resid_post", "attn_out", "mlp_out"}) + +# One canonical TransformerBridge name per boundary (no aliases — avoids duplicate +# HookPoints/cache entries). resid_mid (ln2.hook_in) is derived (resid_pre + +# attn_out), so it's capture-only. +_KIND_NAMES = { + "resid_pre": "blocks.{i}.hook_in", + "resid_mid": "blocks.{i}.ln2.hook_in", + "resid_post": "blocks.{i}.hook_out", + "attn_out": "blocks.{i}.attn.hook_out", + "mlp_out": "blocks.{i}.mlp.hook_out", +} +INTERVENEABLE_KINDS = frozenset({"resid_pre", "attn_out", "mlp_out", "resid_post"}) + +_SUFFIX_TO_KIND = { + "hook_in": "resid_pre", + "ln2.hook_in": "resid_mid", + "hook_out": "resid_post", + "attn.hook_out": "attn_out", + "mlp.hook_out": "mlp_out", +} +_BLOCK = re.compile(r"^blocks\.(\d+)\.(.+)$") + + +def supported_hook_points(n_layers: int, kinds: Optional[Iterable[str]] = None) -> frozenset[str]: + """Fireable hook names across all layers. ``kinds=None`` means all boundaries; + pass the provider's detected kinds to gate (e.g. drop ``resid_mid`` for parallel).""" + selected = _KIND_NAMES if kinds is None else {k: _KIND_NAMES[k] for k in kinds} + return frozenset(name.format(i=i) for i in range(n_layers) for name in selected.values()) + + +def nonfireable_hook_points(n_layers: int) -> frozenset[str]: + """Hooks the residual/attn/mlp provider can't fire (head-split, embed, ln_final).""" + names = ["embed.hook_out", "ln_final.hook_normalized", "unembed.hook_out"] + for i in range(n_layers): + names += [ + f"blocks.{i}.attn.hook_pattern", + f"blocks.{i}.attn.hook_attn_scores", + f"blocks.{i}.attn.hook_q", + f"blocks.{i}.attn.hook_k", + f"blocks.{i}.attn.hook_v", + f"blocks.{i}.attn.hook_z", + ] + return frozenset(names) + + +def resolve(name: str) -> tuple[int, str] | None: + """Canonical hook name → ``(layer, kind)``, or ``None`` if not a fireable hook.""" + match = _BLOCK.match(name) + if match is None: + return None + kind = _SUFFIX_TO_KIND.get(match.group(2)) + return (int(match.group(1)), kind) if kind is not None else None + + +def wire_key(layer: int, kind: str) -> str: + """Stable key for one captured boundary in the activation payload.""" + return f"{layer}:{kind}" + + +def name_from_wire_key(key: str) -> str | None: + """Inverse of ``wire_key`` ∘ ``resolve``: ``":"`` → the canonical hook + name, or ``None`` if the kind is unknown.""" + layer, _, kind = key.partition(":") + template = _KIND_NAMES.get(kind) + return template.format(i=int(layer)) if template and layer.isdigit() else None diff --git a/transformer_lens/model_bridge/sources/inspect/intervention.py b/transformer_lens/model_bridge/sources/inspect/intervention.py new file mode 100644 index 0000000000..fcda9bc802 --- /dev/null +++ b/transformer_lens/model_bridge/sources/inspect/intervention.py @@ -0,0 +1,60 @@ +"""Validate intervention specs and key them by ``:`` for the provider. + +Our HF provider applies interventions as forward-hook affine ops at the residual/ +attn/mlp boundaries, so the full vocabulary works. ``resid_mid`` is derived +(capture-only), so intervening on it is rejected. +""" +from __future__ import annotations + +from typing import Any, Mapping + +from . import hooks + +# suppress (→0), scale (factor), add (value), set (value). +SUPPORTED_OPS = frozenset({"suppress", "scale", "add", "set"}) + + +def build_interventions( + intervene: Mapping[str, Any], + supported_hook_points: frozenset[str], +) -> dict[str, dict[str, Any]]: + """Validate specs and return ``{wire_key: spec}`` for the provider to apply. + + Rejects callables (remote drivers take specs, not callbacks), bad ops, unknown or + unsupported hooks, and the capture-only ``resid_mid``. + """ + out: dict[str, dict[str, Any]] = {} + for hook_name, spec in intervene.items(): + if callable(spec): + raise NotImplementedError( + "InspectDriver requires intervention specs (dict), not callables. " + "Supported ops: suppress, scale (factor: float), add/set (value: scalar or " + "width-shaped)." + ) + if not isinstance(spec, Mapping) or "op" not in spec: + raise ValueError( + f"Intervention spec for {hook_name!r} must be a dict with 'op' key; got {spec!r}" + ) + op = spec["op"] + if op not in SUPPORTED_OPS: + raise ValueError( + f"Unsupported intervention op {op!r} for {hook_name!r}. " + f"Supported: {sorted(SUPPORTED_OPS)}." + ) + if hook_name not in supported_hook_points: + raise ValueError(f"Cannot intervene on {hook_name!r}: not in supported_hook_points.") + resolved = hooks.resolve(hook_name) + if resolved is None or resolved[1] not in hooks.INTERVENEABLE_KINDS: + raise ValueError( + f"Cannot intervene on {hook_name!r}: capture-only " + f"(intervene on resid_pre/attn_out/mlp_out/resid_post instead)." + ) + if op == "scale" and "factor" not in spec: + raise ValueError(f"Intervention {hook_name!r}: op='scale' requires 'factor' (float).") + if op in ("add", "set") and "value" not in spec: + raise ValueError( + f"Intervention {hook_name!r}: op={op!r} requires 'value' (scalar or width-shaped)." + ) + layer, kind = resolved + out[hooks.wire_key(layer, kind)] = dict(spec) + return out diff --git a/transformer_lens/model_bridge/sources/inspect/profiles.py b/transformer_lens/model_bridge/sources/inspect/profiles.py new file mode 100644 index 0000000000..20222a529c --- /dev/null +++ b/transformer_lens/model_bridge/sources/inspect/profiles.py @@ -0,0 +1,134 @@ +"""Per-provider request/response codecs for the Inspect driver. + +The driver speaks one internal shape; each provider speaks its own. A Profile +encapsulates everything provider-specific: which hooks it serves, whether it +returns full-sequence logits, how to phrase the ``generate`` request (prompt + +``extra_args``), how to translate interventions, and how to read logits back. +Torch-free (numpy only) so the driver stays torch-free. + +``tl_bridge`` is our own HF provider (``provider.py``). ``vllm-lens`` is the +third-party provider. **The vllm-lens codec is written from its documented API and +is NOT verified against a live provider** (CI has none — it needs their GPU-served +provider). It's isolated here so this is the single place to fix once validated. +""" +from __future__ import annotations + +from typing import Any, Mapping + +import numpy as np + +from . import hooks, intervention, wire + + +class TLBridgeProfile: + """Codec for our own ``tl_bridge`` provider: residual/attn/mlp hooks, full-seq logits. + + ``supported_kinds`` is the provider's structurally-detected boundary set (e.g. + ``resid_mid`` dropped for parallel/norm-variant archs); ``None`` exposes all boundaries. + """ + + # Class-level default — overridden per-instance via __init__ for backends (vLLM) that + # only populate the gen position; lets RemoteBridge.forward reject loss/both there. + provides_sequence_logits = True + + def __init__(self, supported_kinds: Any = None, provides_sequence_logits: bool = True) -> None: + self._kinds = supported_kinds + self.provides_sequence_logits = provides_sequence_logits + + def supported_hooks(self, n_layers: int) -> frozenset[str]: + return hooks.supported_hook_points(n_layers, self._kinds) + + def translate_interventions(self, intervene, supported): + return intervention.build_interventions(intervene, supported) # {wire_key: spec} + + def build_request(self, ids, wire_keys, interventions, return_logits, tokenizer): + extra: dict[str, Any] = { + "input_ids": ids, + "capture": wire_keys, + "return_logits": return_logits, + } + if interventions: + extra["interventions"] = interventions + return "", extra # our provider reads input_ids from extra_args; prompt unused + + def decode_logits(self, output, n_tokens, d_vocab, tokenizer): + entry = (getattr(output, "metadata", None) or {}).get("tl_logits") + if entry is not None: + return wire.decode_array(entry)[np.newaxis, ...] # full (1, seq, d_vocab) + return np.full((1, n_tokens, d_vocab), -np.inf, dtype=np.float32) + + +class VLLMLensProfile: + """Codec for the third-party vllm-lens provider. + + Residual-stream-only, additive-steering-only, and last-token logits synthesized + one-hot from the generated token (argmax-only — vllm-lens doesn't hand back full + logits through this path). Prompt is the detokenized text, so vllm-lens + re-tokenizes it: activations reflect that re-tokenization, which may differ from + the exact ids. UNVERIFIED against a live provider. + """ + + provides_sequence_logits = False + + def supported_hooks(self, n_layers: int) -> frozenset[str]: + return frozenset(f"blocks.{i}.hook_out" for i in range(n_layers)) + + def translate_interventions(self, intervene: Mapping[str, Any], supported) -> list: + """op='add' with a width-shaped vector → vllm-lens SteeringVector; others raise. + + Validates before importing ``vllm_lens`` so non-additive ops (and the + no-intervention case) don't require the package installed. + """ + if not intervene: + return [] + steering_cls: Any = None # imported lazily once a valid additive spec is seen + vectors = [] + for name, spec in intervene.items(): + if callable(spec): + raise NotImplementedError("vllm-lens requires intervention specs, not callables.") + if not isinstance(spec, Mapping) or spec.get("op") != "add": + raise NotImplementedError( + f"vllm-lens supports only additive steering (op='add' with a width vector); " + f"got {spec!r} for {name!r}. Use boot_transformers() for suppress/scale/set." + ) + if name not in supported: + raise ValueError(f"Cannot intervene on {name!r}: not in supported_hook_points.") + resolved = hooks.resolve(name) + assert resolved is not None # supported ⇒ resolvable + if steering_cls is None: + from vllm_lens import SteeringVector + + steering_cls = SteeringVector + vectors.append( + steering_cls( + activations=np.asarray(spec["value"], dtype=np.float32), + layer_indices=[resolved[0]], + scale=float(spec.get("scale", 1.0)), + norm_match=bool(spec.get("norm_match", True)), + ) + ) + return vectors + + def build_request(self, ids, wire_keys, interventions, return_logits, tokenizer): + layers = sorted({int(key.split(":")[0]) for key in wire_keys}) + extra: dict[str, Any] = {"output_residual_stream": layers} + if interventions: + extra["apply_steering_vectors"] = interventions + prompt = tokenizer.decode(ids) if tokenizer is not None else "" + return prompt, extra + + def decode_logits(self, output, n_tokens, d_vocab, tokenizer): + # vllm-lens returns no full logits here; one-hot the generated token (argmax only). + logits = np.full((1, n_tokens, d_vocab), -np.inf, dtype=np.float32) + text = getattr(output, "completion", "") or "" + ids = tokenizer.encode(text) if (tokenizer is not None and text) else [] + if len(ids): + logits[0, -1, int(ids[0])] = 0.0 + return logits + + +def for_provider(provider: str) -> Any: + """Pick the codec for a provider name (the part before ``/`` in get_model).""" + if provider.startswith("vllm-lens"): + return VLLMLensProfile() + return TLBridgeProfile() diff --git a/transformer_lens/model_bridge/sources/inspect/source.py b/transformer_lens/model_bridge/sources/inspect/source.py new file mode 100644 index 0000000000..b23f565c83 --- /dev/null +++ b/transformer_lens/model_bridge/sources/inspect/source.py @@ -0,0 +1,151 @@ +"""``boot_inspect`` — wrap an ``inspect_ai`` provider in a RemoteBridge via InspectDriver.""" +from __future__ import annotations + +import logging +import warnings +from typing import Any, Optional + +import torch + +from transformer_lens.factories.architecture_adapter_factory import ( + ArchitectureAdapterFactory, +) +from transformer_lens.model_bridge.remote_bridge import RemoteBridge +from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_config_from_hf, + detect_tokenizer_bos_eos, +) +from transformer_lens.model_bridge.sources._hf_format import setup_tokenizer +from transformer_lens.utilities.hf_utils import get_hf_token + +from . import profiles +from .driver import InspectDriver + +# Providers that expose the structural self-check + capture wire format the InspectDriver +# consumes. boot_inspect queries supported_kinds on these; others route via for_provider. +_TL_BRIDGE_PROVIDERS = {"tl_bridge", "tl_bridge_vllm"} + + +def boot_inspect( + model_name: str, + tokenizer: Optional[Any] = None, + dtype: Optional[torch.dtype] = None, + provider: str = "tl_bridge", + **inspect_kwargs: Any, +) -> RemoteBridge: + """Boot a model via an ``inspect_ai`` provider and wrap it in a :class:`RemoteBridge`. + + The driver is provider-agnostic: ``provider`` defaults to our own HF-backed + ``tl_bridge`` provider (residual/attn/mlp capture + full affine interventions + + full-sequence logits); ``"vllm-lens"`` targets a running vllm-lens vLLM provider + (residual-only, additive-steering-only) — wire-aligned with its documented format, + but not yet verified against a live provider. + + Fireable hooks (``tl_bridge``, TransformerBridge-native names): ``blocks.{i}.hook_in`` + (resid_pre) / ``ln2.hook_in`` (resid_mid) / + ``hook_out`` (resid_post) / ``attn.hook_out`` / ``mlp.hook_out``. The provider runs + a structural self-check per model and gates any boundary it can't serve faithfully: + ``resid_mid`` for parallel-residual or norm-variant blocks, and ``attn_out``/ + ``mlp_out`` when their submodule isn't locatable (it warns when it gates one). + Head-split hooks (q/k/v/z, pattern), ``embed``, and ``ln_final`` are always + non-fireable — use ``boot_transformers()`` for those. + + For parity with ``boot_transformers`` the provider loads with the same dtype (fp32 by + default) and eager attention. Full-sequence logits ride on ``return_logits=True`` (the + default); pass ``return_logits=False`` to skip the (seq × d_vocab) payload for pure + activation capture (``run_with_cache`` keeps them since it returns logits). + """ + from inspect_ai.model import get_model + from transformers import AutoConfig, AutoTokenizer + + from . import ( # noqa: F401 — import registers @modelapi + transformers_provider as _provider, + ) + + hf_token = get_hf_token() + hf_config = AutoConfig.from_pretrained(model_name, token=hf_token) + architecture = hf_config.architectures[0] + # Default fp32 to match boot_transformers (which loads/casts fp32 regardless of the + # config's native dtype); an explicit dtype still wins. + resolved_dtype = dtype if dtype is not None else torch.float32 + + bridge_config = build_bridge_config_from_hf(hf_config, architecture, model_name, resolved_dtype) + adapter = ArchitectureAdapterFactory.select_architecture_adapter(bridge_config) + if tokenizer is None: + tokenizer = AutoTokenizer.from_pretrained(model_name, token=hf_token) + # Match boot_transformers' tokenizer setup so to_tokens(str) is token-identical. + tokenizer = setup_tokenizer( + tokenizer, default_padding_side=getattr(adapter.cfg, "default_padding_side", None) + ) + ( + adapter.cfg.tokenizer_prepends_bos, + adapter.cfg.tokenizer_appends_eos, + ) = detect_tokenizer_bos_eos(tokenizer) + + if provider == "tl_bridge": + # The provider's raw HF forward must match boot_transformers' load: same dtype, + # eager attention (TL forces eager — SDPA/flash diverge and accumulate with depth), + # and auth/remote-code so gated/custom models load at all. + inspect_kwargs["model_kwargs"] = _provider_model_kwargs( + dict(inspect_kwargs.get("model_kwargs", {})), adapter, resolved_dtype, hf_token + ) + + # memoize=False: inspect_ai caches get_model by name, which would (a) return a stale + # model ignoring a changed dtype/kwargs on re-boot and (b) keep weights resident past + # close(). Each boot must honor its own args and own its model's lifecycle. + model = get_model(f"{provider}/{model_name}", memoize=False, **inspect_kwargs) + # Both TL-bridge providers (HF + vLLM) restrict their profile to the boundaries the + # provider's structural self-check / overlay found this model can serve; warn only + # if it gated something. Third-party providers (e.g. vllm-lens) route via for_provider. + if provider in _TL_BRIDGE_PROVIDERS: + api = getattr(model, "api", None) + kinds = None + note = "" + # Default True for back-compat; vLLM provider sets False so RemoteBridge.forward + # rejects loss/both (otherwise loss over -inf earlier positions silently NaNs). + psl = True + if api is not None: + kinds = api.supported_kinds() if hasattr(api, "supported_kinds") else None + note = api.capability_note() if hasattr(api, "capability_note") else "" + psl = bool(getattr(api, "provides_sequence_logits", True)) + profile = profiles.TLBridgeProfile(supported_kinds=kinds, provides_sequence_logits=psl) + if note: + warnings.warn(note, UserWarning, stacklevel=2) + else: + profile = profiles.for_provider(provider) + driver = InspectDriver(model=model, adapter=adapter, tokenizer=tokenizer, profile=profile) + bridge = RemoteBridge(adapter=adapter, tokenizer=tokenizer, driver=driver) + _log_hook_summary(model_name, architecture, provider, driver) + return bridge + + +def _provider_model_kwargs( + model_kwargs: dict[str, Any], adapter: Any, dtype: torch.dtype, hf_token: Optional[str] +) -> dict[str, Any]: + """Load kwargs for the HF provider that mirror boot_transformers, so the provider's + raw forward matches the bridge. Caller-supplied keys win (setdefault).""" + model_kwargs.setdefault("torch_dtype", dtype) + # boot_transformers forces eager unless the adapter pins an implementation. + model_kwargs.setdefault( + "attn_implementation", getattr(adapter.cfg, "attn_implementation", None) or "eager" + ) + if hf_token: + model_kwargs.setdefault("token", hf_token) + return model_kwargs + + +def _log_hook_summary( + model_name: str, architecture: str, provider: str, driver: InspectDriver +) -> None: + log = logging.getLogger("transformer_lens.inspect") + fireable = sorted(driver.supported_hook_points) + log.info( + "Inspect source on %s (%s) via provider %r serves %d fireable hook(s).", + model_name, + architecture, + provider, + len(fireable), + ) + + +__all__ = ["boot_inspect"] diff --git a/transformer_lens/model_bridge/sources/inspect/transformers_provider.py b/transformer_lens/model_bridge/sources/inspect/transformers_provider.py new file mode 100644 index 0000000000..aa4faf356a --- /dev/null +++ b/transformer_lens/model_bridge/sources/inspect/transformers_provider.py @@ -0,0 +1,522 @@ +"""Our HF-transformers ``inspect_ai`` model provider, registered as ``tl_bridge``. + +The model-runner side of the Inspect driver (this file uses torch; the consuming +``InspectDriver`` does not). On ``generate`` it reads the request from +``config.extra_body["extra_args"]`` (token ids, which ``:`` boundaries +to capture, and intervention specs), runs an HF causal LM with forward hooks that +capture residual/attn/mlp boundaries and apply affine interventions, and returns a +``ModelOutput`` whose ``metadata`` carries the activations (encoded by ``wire``) +plus the full-sequence logits. +""" +from __future__ import annotations + +import contextvars +import uuid +from collections import defaultdict +from contextlib import contextmanager +from typing import Any, Iterator, Mapping + +import numpy as np +import torch +from inspect_ai.model import ( + ChatCompletionChoice, + ChatMessageAssistant, + GenerateConfig, + Logprobs, + ModelOutput, + ModelUsage, + StopReason, + modelapi, +) + +from . import wire +from ._provider_base import _InspectModelAPIBase, _parse_tool_calls, _require_served + +# NOT "transformer_lens" — inspect_ai ships a built-in provider by that name (the +# reverse direction: serving a HookedTransformer as an Inspect model for generation). +PROVIDER_NAME = "tl_bridge" + +# Per-call hook isolation. Capture/intervene hooks consult this contextvar and only fire +# for their own call's id — so concurrent inspect_eval samples (each running with their +# own contextvars copy via asyncio.to_thread) don't cross-pollute each other's activations. +_current_call_id: contextvars.ContextVar[str] = contextvars.ContextVar( + "tl_inspect_call_id", default="" +) + +# Decoder ModuleList by architecture family; each block's output is resid_post. +_LAYER_PATHS = ("model.layers", "transformer.h", "gpt_neox.layers", "model.decoder.layers") +# Attn/MLP submodule names within a block, by family. +_ATTN_ATTRS = ("self_attn", "attn", "attention") +_MLP_ATTRS = ("mlp", "feed_forward") + + +@modelapi(name=PROVIDER_NAME) +def transformer_lens_provider(): + """Lazy registration hook — returns the provider class on first use.""" + return TransformerLensTransformersModelAPI + + +class TransformerLensTransformersModelAPI(_InspectModelAPIBase): + """HF-backed provider: residual/attn/mlp capture + interventions + full logits. + + Inherits generate() dispatch, _messages_to_ids, _logprob_entry, and the per-turn + capture-config validation from :class:`_InspectModelAPIBase`; the HF-specific bits + here are the model load, the forward-hook capture machinery, and the structural probe. + """ + + # Real per-position logits via direct HF forward — loss/both via RemoteBridge work. + provides_sequence_logits = True + + def __init__( + self, + model_name: str, + base_url: str | None = None, + api_key: str | None = None, + config: GenerateConfig = GenerateConfig(), + **model_args: Any, + ) -> None: + super().__init__(model_name, base_url, api_key, [], config) + from transformers import AutoModelForCausalLM, AutoTokenizer + + self._device = model_args.pop("device", "cpu") + hf_kwargs = model_args.pop("model_kwargs", {}) + self._hf = ( + AutoModelForCausalLM.from_pretrained(model_name, **hf_kwargs).to(self._device).eval() + ) + self._tokenizer = AutoTokenizer.from_pretrained(model_name) + self._layers = _locate_layers(self._hf) + self._kinds, self._capability_note = _detect_capabilities(self._hf, self._layers) + # Per-turn capture during plain generation (agent rollouts): every _generate_eval + # stashes these hooks in ModelOutput.metadata. Gated by the structural self-check. + self._eval_capture = self._parse_eval_capture(model_args) + + def _generate_capture(self, input: Any, extra_args: Mapping[str, Any], config: GenerateConfig): + """TL-driven single forward: capture residual/attn/mlp boundaries + full logits.""" + input_ids = extra_args.get("input_ids") + if input_ids is None: + input_ids = self._messages_to_ids(input)[0].tolist() + # capture/interventions are keyed by ":" (hooks.wire_key). + capture_keys = list(extra_args.get("capture", [])) + for key in capture_keys: + _, _, kind = key.partition(":") + _require_served(kind, self._kinds, self._capability_note, f"capture {key!r}") + interventions: Mapping[str, Any] = extra_args.get("interventions", {}) + want_logits = bool(extra_args.get("return_logits", True)) + + capture, intervene = _plan(capture_keys, interventions) + raw: dict[tuple[int, str], np.ndarray] = {} + call_id = uuid.uuid4().hex + token = _current_call_id.set(call_id) + handles = self._install_hooks(capture, intervene, raw, call_id) + try: + with torch.no_grad(): + ids = torch.tensor([list(input_ids)], device=self._device) + logits = self._hf(ids).logits # (1, seq, vocab) + finally: + for handle in handles: + handle.remove() + _current_call_id.reset(token) + + captured = _assemble(raw, capture_keys) + metadata: dict[str, Any] = {"activations": wire.encode_activations(captured)} + if want_logits: + metadata["tl_logits"] = wire.encode_array(logits[0].float().cpu().numpy()) + + next_id = int(logits[0, -1].argmax()) + logprobs = ( + Logprobs(content=[self._logprob_entry(next_id, logits[0, -1], config.top_logprobs)]) + if config.logprobs + else None + ) + return ModelOutput( + model=self.model_name, + choices=[ + ChatCompletionChoice( + message=ChatMessageAssistant(content=str(self._tokenizer.decode([next_id]))), + stop_reason="stop", + logprobs=logprobs, + ) + ], + metadata=metadata, + ) + + def _generate_eval(self, input: Any, config: GenerateConfig, tools: Any): + """Plain Inspect generation: HF generate from the chat input (rendering ``tools`` + into the template), honoring max_tokens/sampling, with optional per-token logprobs, + token usage, parsed tool calls, and per-turn activation capture (agent rollouts).""" + ids = self._messages_to_ids(input, tools) + prompt_len = int(ids.shape[1]) + max_new = int(config.max_tokens) if config.max_tokens else 16 + temperature = config.temperature + do_sample = temperature is not None and temperature > 0 + gen: dict[str, Any] = { + "max_new_tokens": max_new, + "do_sample": do_sample, + "return_dict_in_generate": True, + "output_scores": True, + "pad_token_id": self._tokenizer.pad_token_id or self._tokenizer.eos_token_id, + } + if temperature is not None and temperature > 0: + gen["temperature"] = float(temperature) + if config.top_p is not None: + gen["top_p"] = float(config.top_p) + if config.top_k is not None: + gen["top_k"] = int(config.top_k) + + # Save BOTH CPU and CUDA RNG state — get_rng_state() is CPU-only, so seeding on a + # CUDA model would otherwise leak its CUDA seed past this generate. + rng_state = None + cuda_rng_state = None + on_cuda = "cuda" in str(self._device) + if do_sample and config.seed is not None: + rng_state = torch.get_rng_state() + if on_cuda: + cuda_rng_state = torch.cuda.get_rng_state_all() + torch.manual_seed(int(config.seed)) + # Install per-turn capture hooks AROUND generate (not pre-): first-write-wins lets + # the prompt forward populate them and decode forwards skip — no extra forward. + with self._eval_capture_scope() as metadata: + try: + with torch.no_grad(): + out = self._hf.generate(ids, **gen) + finally: + if rng_state is not None: + torch.set_rng_state(rng_state) + if cuda_rng_state is not None: + torch.cuda.set_rng_state_all(cuda_rng_state) + + new_ids = out.sequences[0, prompt_len:] + completion = str(self._tokenizer.decode(new_ids, skip_special_tokens=True)) + logprobs = None + if config.logprobs: + content = [ + self._logprob_entry(int(tok), step[0], config.top_logprobs) + for tok, step in zip(new_ids.tolist(), out.scores) + ] + logprobs = Logprobs(content=content) + n_new = int(new_ids.shape[0]) + tool_calls = _parse_tool_calls(completion) if len(tools) else None + eos = self._tokenizer.eos_token_id + stop_reason: StopReason + if tool_calls: + stop_reason = "tool_calls" + elif n_new and eos is not None and int(new_ids[-1]) == eos: + stop_reason = "stop" + else: + stop_reason = "max_tokens" + return ModelOutput( + model=self.model_name, + choices=[ + ChatCompletionChoice( + message=ChatMessageAssistant(content=completion, tool_calls=tool_calls), + stop_reason=stop_reason, + logprobs=logprobs, + ) + ], + usage=ModelUsage( + input_tokens=prompt_len, output_tokens=n_new, total_tokens=prompt_len + n_new + ), + metadata=metadata or None, + ) + + @contextmanager + def _eval_capture_scope(self) -> Iterator[dict[str, Any]]: + """Install per-turn ``capture=[...]`` hooks for the duration of a generate. First- + write-wins lets the prompt forward populate the raw dict (decode forwards find it + populated and skip), so this adds no extra forward. Yields a metadata dict (empty + when capture isn't configured) that the caller folds into ``ModelOutput.metadata``. + Contextvar-isolated so concurrent inspect_eval samples don't cross-pollute.""" + if not self._eval_capture: + yield {} + return + wire_keys = list(self._eval_capture) + capture, _ = _plan(wire_keys, {}) + raw: dict[tuple[int, str], np.ndarray] = {} + call_id = uuid.uuid4().hex + token = _current_call_id.set(call_id) + handles = self._install_hooks(capture, {}, raw, call_id) + metadata: dict[str, Any] = {} + try: + yield metadata + finally: + for handle in handles: + handle.remove() + _current_call_id.reset(token) + metadata["activations"] = wire.encode_activations(_assemble(raw, wire_keys)) + + def _install_hooks(self, capture, intervene, raw, call_id: str) -> list: + """Hook each block's pre/attn/mlp/post boundaries that need capture or intervention. + Hooks consult ``_current_call_id`` and only fire for ``call_id`` (concurrent calls).""" + handles = [] + for layer, block in enumerate(self._layers): + cap_kinds = capture.get(layer, set()) + iv_kinds = intervene.get(layer, {}) + if not cap_kinds and not iv_kinds: + continue + attn = _first_attr(block, _ATTN_ATTRS) + mlp = _first_attr(block, _MLP_ATTRS) + if "resid_pre" in cap_kinds or "resid_pre" in iv_kinds: + handles.append( + block.register_forward_pre_hook( + _pre_hook( + layer, "resid_pre" in cap_kinds, iv_kinds.get("resid_pre"), raw, call_id + ), + with_kwargs=True, + ) + ) + if attn is not None and ("attn_out" in cap_kinds or "attn_out" in iv_kinds): + handles.append( + attn.register_forward_hook( + _out_hook( + layer, + "attn_out", + "attn_out" in cap_kinds, + iv_kinds.get("attn_out"), + raw, + call_id, + ) + ) + ) + if mlp is not None and ("mlp_out" in cap_kinds or "mlp_out" in iv_kinds): + handles.append( + mlp.register_forward_hook( + _out_hook( + layer, + "mlp_out", + "mlp_out" in cap_kinds, + iv_kinds.get("mlp_out"), + raw, + call_id, + ) + ) + ) + if "resid_post" in cap_kinds or "resid_post" in iv_kinds: + handles.append( + block.register_forward_hook( + _out_hook( + layer, + "resid_post", + "resid_post" in cap_kinds, + iv_kinds.get("resid_post"), + raw, + call_id, + ) + ) + ) + return handles + + +def _plan(capture_keys, interventions): + """Resolve wire keys → per-layer kinds to capture (resid_mid needs pre+attn) and intervene.""" + capture: dict[int, set[str]] = defaultdict(set) + for key in capture_keys: + layer, _, kind = key.partition(":") + layer = int(layer) + if kind == "resid_mid": + capture[layer] |= {"resid_pre", "attn_out"} # derived = resid_pre + attn_out + else: + capture[layer].add(kind) + intervene: dict[int, dict[str, Any]] = defaultdict(dict) + for key, spec in interventions.items(): + layer, _, kind = key.partition(":") + intervene[int(layer)][kind] = spec + return capture, intervene + + +def _assemble(raw, capture_keys) -> dict[str, np.ndarray]: + """Build the emitted ``{wire_key: (seq, d)}`` map, deriving resid_mid as needed.""" + out: dict[str, np.ndarray] = {} + for key in capture_keys: + layer, _, kind = key.partition(":") + layer = int(layer) + if kind == "resid_mid": + pre, attn = raw.get((layer, "resid_pre")), raw.get((layer, "attn_out")) + if pre is not None and attn is not None: + out[key] = pre + attn + elif (layer, kind) in raw: + out[key] = raw[(layer, kind)] + return out + + +def _pre_hook(layer, want_capture, spec, raw, call_id): + # with_kwargs=True: hidden_states is args[0] for most decoders, but some pass it as + # the hidden_states kwarg — handle both so the right tensor is read/modified. + # First-write-wins on raw so install-around-generate captures the prompt forward + # (subsequent decode forwards find raw populated and skip — no extra prompt forward). + def hook(_module, args, kwargs): + if _current_call_id.get() != call_id: + return None # different concurrent call's hook + kw_key = None if args else "hidden_states" + hidden = args[0] if args else kwargs["hidden_states"] + if spec is not None: + hidden = _apply_affine(hidden, spec) + if want_capture and (layer, "resid_pre") not in raw: + raw[(layer, "resid_pre")] = hidden[0].detach().float().cpu().numpy() + if spec is None: + return None + if kw_key is None: + return (hidden, *args[1:]), kwargs + return args, {**kwargs, kw_key: hidden} + + return hook + + +def _out_hook(layer, kind, want_capture, spec, raw, call_id): + def hook(_module, _inputs, output): + if _current_call_id.get() != call_id: + return None # different concurrent call's hook + is_tuple = isinstance(output, tuple) + hidden = output[0] if is_tuple else output + if spec is not None: + hidden = _apply_affine(hidden, spec) + if want_capture and (layer, kind) not in raw: + raw[(layer, kind)] = hidden[0].detach().float().cpu().numpy() + if spec is None: + return None + return (hidden, *output[1:]) if is_tuple else hidden + + return hook + + +def _apply_affine(t: torch.Tensor, spec: Mapping[str, Any]) -> torch.Tensor: + """suppress→0, scale→·factor, add→+value, set→value (value scalar or width-shaped).""" + op = spec["op"] + if op == "suppress": + return torch.zeros_like(t) + if op == "scale": + return t * float(spec["factor"]) + value = torch.as_tensor(spec["value"], dtype=t.dtype, device=t.device) + if op == "add": + return t + value + return torch.zeros_like(t) + value # set + + +def _detect_capabilities(model: Any, layers: Any) -> tuple[frozenset, str]: + """Structural self-check: which boundary kinds this model can serve faithfully. + + resid_pre/resid_post are the block in/out (always); attn_out/mlp_out need their + submodules locatable; resid_mid is gated unless its derivation holds (see + :func:`_resid_mid_derivable`). Returns (kinds, note); note explains any gating, '' if none. + """ + block = layers[0] + attn = _first_attr(block, _ATTN_ATTRS) + mlp = _first_attr(block, _MLP_ATTRS) + kinds = {"resid_pre", "resid_post"} + gated = [] + if attn is not None: + kinds.add("attn_out") + else: + gated.append("attn_out (no attention submodule found)") + if mlp is not None: + kinds.add("mlp_out") + else: + gated.append("mlp_out (no MLP submodule found)") + if attn is not None and mlp is not None and _resid_mid_derivable(model, block, attn, mlp): + kinds.add("resid_mid") + else: + gated.append( + "resid_mid (resid_pre + attn_out doesn't hold — parallel or norm-variant block)" + ) + note = ( + "" + if not gated + else "InspectDriver: this architecture's block layout gates " + + ", ".join(gated) + + ". Remaining boundaries are served; use boot_transformers() for the gated ones." + ) + return frozenset(kinds), note + + +def _resid_mid_derivable(model: Any, block: Any, attn: Any, mlp: Any) -> bool: + """True iff ``resid_mid = resid_pre + attn_out`` holds, via two tiny probe forwards. + Requires both the linear identity ``resid_post = resid_pre + attn_out + mlp_out`` (broken + by post-norm/multiplier blocks — Gemma2/OLMo2/Granite) and attn feeding mlp (broken by + parallel blocks — GPTNeoX/GPT-J, where mlp reads resid_pre directly).""" + cap: dict[str, Any] = {} + + def grab(key: str): # type: ignore[no-untyped-def] + def hook(_m: Any, _i: Any, out: Any) -> None: + t = out[0] if isinstance(out, tuple) else out + cap[key] = t.detach().float() + + return hook + + def grab_in(key: str): # type: ignore[no-untyped-def] + def hook(_m: Any, args: Any, kwargs: Any) -> None: + t = args[0] if args else kwargs.get("hidden_states") + cap[key] = None if t is None else t.detach().float() + + return hook + + def perturb_attn(_m: Any, _i: Any, out: Any): # type: ignore[no-untyped-def] + is_tuple = isinstance(out, tuple) + h = out[0] if is_tuple else out + # Local generator: the probe must not reset the caller's global RNG. Non-uniform + # noise so layernorm's mean-subtraction can't cancel it (a constant would). + gen = torch.Generator(device=h.device).manual_seed(0) + h = h + torch.empty_like(h).normal_(generator=gen) + return (h, *out[1:]) if is_tuple else h + + ids = torch.tensor([[0, 1, 2]], device=next(model.parameters()).device) + try: + with torch.no_grad(): + handles = [ + block.register_forward_pre_hook(grab_in("resid_pre"), with_kwargs=True), + attn.register_forward_hook(grab("attn_out")), + mlp.register_forward_hook(grab("mlp_out")), + mlp.register_forward_pre_hook(grab_in("mlp_in"), with_kwargs=True), + block.register_forward_hook(grab("resid_post")), + ] + model(ids) + for h in handles: + h.remove() + mlp_in_clean = cap.pop("mlp_in", None) + handles = [ + mlp.register_forward_pre_hook(grab_in("mlp_in"), with_kwargs=True), + attn.register_forward_hook(perturb_attn), + ] + model(ids) + for h in handles: + h.remove() + mlp_in_perturbed = cap.get("mlp_in") + except Exception: + return False # can't probe (exotic signature) → conservatively gate resid_mid + + rp = cap.get("resid_pre") + ao = cap.get("attn_out") + mo = cap.get("mlp_out") + rpost = cap.get("resid_post") + if rp is None or ao is None or mo is None or rpost is None: + return False + if mlp_in_clean is None or mlp_in_perturbed is None: + return False + if not (rp.shape == ao.shape == mo.shape == rpost.shape == mlp_in_clean.shape): + return False + # (1) sub-block outputs add to the residual without intervening norm/scale. + identity = (rpost - rp - ao - mo).abs().max().item() + identity_ok = identity <= 1e-3 * (rpost.abs().max().item() + 1e-6) + # (2) perturbing attn moves mlp's input (sequential, not parallel). + causal_ok = (mlp_in_clean - mlp_in_perturbed).abs().max().item() > 1e-6 + return bool(identity_ok and causal_ok) + + +def _locate_layers(model: Any) -> Any: + for path in _LAYER_PATHS: + target: Any = model + for seg in path.split("."): + target = getattr(target, seg, None) + if target is None: + break + if target is not None: + return target + raise RuntimeError( + f"Could not locate decoder layers on {type(model).__name__}; tried {_LAYER_PATHS}." + ) + + +def _first_attr(obj: Any, names: tuple[str, ...]) -> Any: + for name in names: + found = getattr(obj, name, None) + if found is not None: + return found + return None diff --git a/transformer_lens/model_bridge/sources/inspect/vllm_provider.py b/transformer_lens/model_bridge/sources/inspect/vllm_provider.py new file mode 100644 index 0000000000..f04abb15a8 --- /dev/null +++ b/transformer_lens/model_bridge/sources/inspect/vllm_provider.py @@ -0,0 +1,439 @@ +"""vLLM-backed ``inspect_ai`` model provider, registered as ``tl_bridge_vllm``. + +A sibling to the HF-backed ``tl_bridge`` provider: instead of running an HF causal LM +locally, it generates via vLLM (PagedAttention + continuous batching) — so it scales to +parallel-sample evals and dataset-scale workloads where the HF provider serializes. + +Inherits ``generate()`` dispatch / message-rendering / per-turn-capture validation from +:class:`_InspectModelAPIBase`; this file owns the vLLM ``LLM`` construction (with the +plugin + worker_extension wiring needed for capture), eval-native generation via +``llm.generate(...)``, and the TL-driven capture path via ``collective_rpc`` to the +worker extension. The capture wire format matches the HF provider's, so the existing +``InspectDriver`` consumes it unchanged. + +vLLM is GPU-only and imported lazily inside ``__init__`` / ``_generate_*`` so this +module imports cleanly in environments without vLLM (matching the HF provider pattern). +""" +from __future__ import annotations + +import gc +import os +from typing import Any, Mapping + +import numpy as np +import torch +from inspect_ai.model import ( + ChatCompletionChoice, + ChatMessageAssistant, + GenerateConfig, + Logprob, + Logprobs, + ModelOutput, + ModelUsage, + StopReason, + TopLogprob, + modelapi, +) + +from . import hooks, wire +from ._provider_base import _InspectModelAPIBase, _parse_tool_calls, _require_served + +# Distinct from the HF ``tl_bridge`` provider and from inspect_ai's built-in ``vllm``. +PROVIDER_NAME = "tl_bridge_vllm" + +# Forced ``LLM(...)`` kwargs that the capture-hook design depends on (matches the same +# set in ``sources/vllm/source.py``). Multi-device / vLLM-owned tokenizer break the wire +# path; we own the tokenizer and the plugin assumes single-process workers. +_LOCKED_VLLM_KWARGS = { + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "skip_tokenizer_init": True, + "disable_log_stats": True, +} + +# Dotted path to the worker extension whose ``tl_*`` methods this provider drives via +# ``collective_rpc`` (capture reads, intervention specs, hook teardown). +_WORKER_EXTENSION_CLS = ( + "transformer_lens.model_bridge.sources.vllm.worker_extension.TLWorkerExtension" +) + + +def _kinds_from_specs(specs: dict[str, Any]) -> frozenset[str]: + """Boundary kinds (resid_post/attn_out/...) served by the overlay's ``capture_specs``. + + Non-block hooks (``embed.hook_out``, ``ln_final.hook_normalized``) don't resolve to + a kind and don't contribute — they're already in the InspectDriver's non-fireable set. + """ + kinds = set() + for name in specs: + resolved = hooks.resolve(name) + if resolved is not None: + kinds.add(resolved[1]) + return frozenset(kinds) + + +@modelapi(name=PROVIDER_NAME) +def transformer_lens_vllm_provider(): + """Lazy registration hook — returns the provider class on first use.""" + return TransformerLensVLLMModelAPI + + +class TransformerLensVLLMModelAPI(_InspectModelAPIBase): + """vLLM-backed Inspect provider. See module docstring for scope per increment.""" + + # vLLM's sampler bypasses lm_head; _synthesize_logits populates only the gen position, + # so earlier positions are -inf and loss would be NaN. RemoteBridge.forward must reject + # return_type ∈ {loss, both} — read by source.py → TLBridgeProfile → InspectDriver. + provides_sequence_logits = False + + def __init__( + self, + model_name: str, + base_url: str | None = None, + api_key: str | None = None, + config: GenerateConfig = GenerateConfig(), + **model_args: Any, + ) -> None: + super().__init__(model_name, base_url, api_key, [], config) + from transformers import AutoConfig, AutoTokenizer + from vllm import LLM + + from transformer_lens.utilities.hf_utils import get_hf_token + + from ..vllm import plugin + from ..vllm.internals import extract_hf_config + from ..vllm.overlays import get_overlay + + # Caller-overridable LLM kwargs go through ``vllm_kwargs``; the locked set above + # may not be overridden (multi-device / vLLM-owned tokenizer break our wire path). + vllm_kwargs = model_args.pop("vllm_kwargs", {}) + for key, locked in _LOCKED_VLLM_KWARGS.items(): + if key in vllm_kwargs and vllm_kwargs[key] != locked: + raise ValueError( + f"tl_bridge_vllm forces {key}={locked}; caller passed " + f"{key}={vllm_kwargs[key]}." + ) + gpu_memory_utilization = model_args.pop("gpu_memory_utilization", 0.5) + max_model_len = model_args.pop("max_model_len", None) + max_num_batched_tokens = int(model_args.pop("max_num_batched_tokens", 2048)) + dtype = model_args.pop("dtype", None) + + # vLLM is GPU-only in production; "device" stays caller-overridable mainly so + # mocked unit tests on CPU machines can build the prompt tensor without CUDA + # (the base's _messages_to_ids honors self._device; we drop to a list before + # handing the prompt to vLLM, so the device only governs the intermediate tensor). + self._device = model_args.pop("device", "cuda") + # skip_tokenizer_init=True ⇒ vLLM has no tokenizer; we own one for prompt rendering + # (via the base) and for decoding generated token ids back to strings. + hf_token = get_hf_token() + self._tokenizer = AutoTokenizer.from_pretrained(model_name, token=hf_token) + + # Pre-LLM: resolve architecture WITHOUT loading weights, then prime the plugin + # so its monkey-patched Worker.load_model installs capture hooks pre-compile. + hf_config_preview = AutoConfig.from_pretrained(model_name, token=hf_token) + architecture = hf_config_preview.architectures[0] + overlay = get_overlay(architecture) + resolved_dtype = dtype if dtype is not None else _dtype_from_hf_config(hf_config_preview) + capture_specs = overlay.capture_specs(hf_config_preview) + plugin.configure( + capture_specs=capture_specs, + max_num_batched_tokens=max_num_batched_tokens, + dtype=resolved_dtype, + enable_batching=False, # eager batched path is a later increment + ) + plugin.register() + # Single-process workers are required — otherwise the plugin's _config singleton + # isn't visible to worker subprocesses and hooks silently fail to install. + os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0" + + self._llm = LLM( + model=model_name, + gpu_memory_utilization=gpu_memory_utilization, + max_model_len=max_model_len, + max_num_batched_tokens=max_num_batched_tokens, + # Worker extension's tl_* methods are reachable via collective_rpc. + worker_extension_cls=_WORKER_EXTENSION_CLS, + # Full-vocab logprobs so _generate_capture can synthesize logits at the + # generated position (vLLM caps logprobs to this value; default 20 is too + # small for mech interp). + max_logprobs=int(hf_config_preview.vocab_size), + dtype=str(resolved_dtype).replace("torch.", "") if dtype is not None else "auto", + **_LOCKED_VLLM_KWARGS, + **vllm_kwargs, + ) + # Capture-path validity was enforced inside patched_load_model during LLM(...). + hf_config = extract_hf_config(self._llm) + # Don't leak our specs to a subsequent non-TL vllm.LLM(...) in the same process. + plugin._config.clear() + + # Capture-relevant constants used by _generate_capture. + self._d_vocab = int(hf_config.vocab_size) + self._max_logprobs = int(hf_config_preview.vocab_size) + self._max_num_batched_tokens = max_num_batched_tokens + + # Boundary kinds served by the vLLM overlay (decoder-only: resid_post / attn_out / + # mlp_out). vLLM's fused execution doesn't expose block input, so resid_pre and + # the derived resid_mid are gated — the InspectDriver consults this via the profile. + self._kinds = _kinds_from_specs(capture_specs) + self._capability_note = ( + "tl_bridge_vllm: vLLM's fused execution gates resid_pre (no block-input hook) " + "and the derived resid_mid. Use boot_inspect(provider='tl_bridge') for those." + ) + self._eval_capture = self._parse_eval_capture(model_args) + + def _generate_capture(self, input: Any, extra_args: Mapping[str, Any], config: GenerateConfig): + """TL-driven single-token capture: push interventions to the worker, run a + single-token generate (vLLM's prefill populates the capture buffers), read them + back via ``collective_rpc``, and return the wire-format ``metadata["activations"]`` + + synthesized ``tl_logits`` the InspectDriver expects.""" + from vllm import SamplingParams + from vllm.inputs import TokensPrompt + + input_ids = extra_args.get("input_ids") + if input_ids is None: + input_ids = self._messages_to_ids(input)[0].tolist() + n_tokens = len(input_ids) + if n_tokens > self._max_num_batched_tokens: + raise ValueError( + f"Prompt length {n_tokens} exceeds max_num_batched_tokens=" + f"{self._max_num_batched_tokens}; raise via the model_args kwarg " + "or shorten the prompt." + ) + + # Validate capture kinds against the structural self-check (same protection the + # HF provider gives — driver-path AND eval/extra_args entry points). + capture_keys = list(extra_args.get("capture", [])) + for key in capture_keys: + _, _, kind = key.partition(":") + _require_served(kind, self._kinds, self._capability_note, f"capture {key!r}") + + # Translate wire keys ↔ TL hook names. The worker extension is keyed by hook name + # (e.g. "blocks.0.hook_out"); the wire format uses ":". + name_by_wire = {wk: hooks.name_from_wire_key(wk) for wk in capture_keys} + if any(name is None for name in name_by_wire.values()): + unknown = sorted(wk for wk, name in name_by_wire.items() if name is None) + raise ValueError(f"unrecognised wire keys: {unknown}") + capture_names = list(name_by_wire.values()) + + interventions: Mapping[str, Any] = extra_args.get("interventions", {}) + intervention_specs: dict[str, Any] = {} + for wk, spec in interventions.items(): + name = hooks.name_from_wire_key(wk) + if name is None: + raise ValueError(f"intervention wire key {wk!r} is not a fireable hook.") + intervention_specs[name] = spec + + want_logits = bool(extra_args.get("return_logits", True)) + + # Push intervention state (possibly empty — also resets stale interventions from + # a prior call), open the per-hook capture gates (so the prefill below writes, + # and any later forward — should this driver be reused — would self-copy until + # the next explicit reset). Then run a single-token prefill; vLLM's prefill + # populates the capture buffers we registered via plugin.configure. + self._llm.collective_rpc("tl_set_interventions", args=(intervention_specs,)) + self._llm.collective_rpc("tl_reset_capture_flags") + outputs = self._llm.generate( + prompts=[TokensPrompt(prompt_token_ids=list(input_ids))], + sampling_params=SamplingParams( + max_tokens=1, + temperature=0.0, + logprobs=self._max_logprobs if want_logits else None, + ), + ) + # collective_rpc returns one result per worker; single-rank ⇒ [0]. + worker_captures = self._llm.collective_rpc( + "tl_read_captures", args=([n_tokens], capture_names) + )[0] + + # Convert TL-name-keyed (n_tokens, width) tensors → wire-key-keyed numpy arrays, + # then encode in the same envelope wire.decode_activations consumes on the driver. + captured_wire: dict[str, np.ndarray] = {} + for wk, name in name_by_wire.items(): + tensor = worker_captures.get(name) + if tensor is not None: + captured_wire[wk] = tensor.detach().float().cpu().numpy() + metadata: dict[str, Any] = {"activations": wire.encode_activations(captured_wire)} + + if want_logits: + logits = _synthesize_logits(outputs[0], n_tokens, self._d_vocab) + metadata["tl_logits"] = wire.encode_array(logits[0].cpu().numpy()) + + # The completion is the single generated token (matches the HF provider's shape). + next_id = int(outputs[0].outputs[0].token_ids[0]) + return ModelOutput( + model=self.model_name, + choices=[ + ChatCompletionChoice( + message=ChatMessageAssistant(content=str(self._tokenizer.decode([next_id]))), + stop_reason="stop", + ) + ], + metadata=metadata, + ) + + def _generate_eval(self, input: Any, config: GenerateConfig, tools: Any): + """vLLM generation: chat input → ``llm.generate`` → completion + Logprobs + usage. + If ``model_args['capture']`` was set, opens per-hook capture gates before the eval + generate so prefill captures the prompt activations and decode steps self-copy + (first-write-wins on the worker; no separate forward — see plugin._gated_capture).""" + from vllm import SamplingParams + from vllm.inputs import TokensPrompt + + # Worker-side intervention buffers are persistent: any prior capture-path call that + # pushed specs (e.g. bridge.forward(intervene=...)) would still be applied here + # without a reset. The HF provider installs hooks per-call so it's leak-immune. + self._llm.collective_rpc("tl_set_interventions", args=({},)) + ids = self._messages_to_ids(input, tools)[0].tolist() + prompt_len = len(ids) + if self._eval_capture: + if prompt_len > self._max_num_batched_tokens: + raise ValueError( + f"Prompt length {prompt_len} exceeds max_num_batched_tokens=" + f"{self._max_num_batched_tokens}; per-turn capture cannot snapshot it." + ) + self._llm.collective_rpc("tl_reset_capture_flags") + max_new = int(config.max_tokens) if config.max_tokens else 16 + temperature = float(config.temperature) if config.temperature is not None else 0.0 + + sp_kwargs: dict[str, Any] = {"max_tokens": max_new, "temperature": temperature} + if temperature > 0: + if config.top_p is not None: + sp_kwargs["top_p"] = float(config.top_p) + if config.top_k is not None: + sp_kwargs["top_k"] = int(config.top_k) + if config.seed is not None: + sp_kwargs["seed"] = int(config.seed) + if config.logprobs: + # vLLM returns this many top logprobs per generated token (incl. the chosen). + sp_kwargs["logprobs"] = int(config.top_logprobs) if config.top_logprobs else 1 + + outputs = self._llm.generate( + prompts=[TokensPrompt(prompt_token_ids=ids)], + sampling_params=SamplingParams(**sp_kwargs), + ) + request_output = outputs[0] + output = request_output.outputs[0] # one prompt, one sample + new_ids = list(output.token_ids) + n_new = len(new_ids) + completion = str(self._tokenizer.decode(new_ids, skip_special_tokens=True)) + + logprobs = None + if config.logprobs and output.logprobs: + logprobs = Logprobs( + content=[ + self._logprob_from_dict(int(tid), step, config.top_logprobs) + for tid, step in zip(new_ids, output.logprobs) + ] + ) + + tool_calls = _parse_tool_calls(completion) if len(tools) else None + finish = (output.finish_reason or "").lower() + stop_reason: StopReason + if tool_calls: + stop_reason = "tool_calls" + elif finish == "length": + stop_reason = "max_tokens" + elif finish == "stop": + stop_reason = "stop" + else: + stop_reason = "unknown" + + # Per-turn capture lands in metadata. First-write-wins gating made prefill the + # only forward that wrote to the capture buffer, so we read it now (decode steps + # left rows 1..prompt_len-1 untouched and row 0 self-copied). + eval_metadata: dict[str, Any] = {} + if self._eval_capture: + capture_names = list(self._eval_capture.values()) + worker_captures = self._llm.collective_rpc( + "tl_read_captures", args=([prompt_len], capture_names) + )[0] + captured_wire: dict[str, np.ndarray] = {} + for wk, name in self._eval_capture.items(): + tensor = worker_captures.get(name) + if tensor is not None: + captured_wire[wk] = tensor.detach().float().cpu().numpy() + eval_metadata = {"activations": wire.encode_activations(captured_wire)} + + return ModelOutput( + model=self.model_name, + choices=[ + ChatCompletionChoice( + message=ChatMessageAssistant(content=completion, tool_calls=tool_calls), + stop_reason=stop_reason, + logprobs=logprobs, + ) + ], + usage=ModelUsage( + input_tokens=prompt_len, output_tokens=n_new, total_tokens=prompt_len + n_new + ), + metadata=eval_metadata or None, + ) + + def _logprob_from_dict(self, token_id: int, step_logprobs: Any, top_n: Any) -> Logprob: + """vLLM per-step ``{token_id: Logprob(logprob, rank, decoded_token)}`` → + :class:`inspect_ai.model.Logprob` with optional top-k alternatives.""" + chosen = step_logprobs.get(token_id) + chosen_lp = float(chosen.logprob) if chosen is not None else float("-inf") + top: list[TopLogprob] = [] + if top_n: + ranked = sorted(step_logprobs.items(), key=lambda kv: -float(kv[1].logprob)) + for tid, lp in ranked[: int(top_n)]: + token = lp.decoded_token or self._tokenizer.decode([int(tid)]) + top.append(TopLogprob(token=str(token), logprob=float(lp.logprob), bytes=None)) + return Logprob( + token=str(self._tokenizer.decode([int(token_id)])), + logprob=chosen_lp, + bytes=None, + top_logprobs=top, + ) + + def close(self) -> None: + """Best-effort vLLM teardown — vLLM 0.20.2 has no ``LLM.shutdown()``, so weights + + KV cache stay resident until process exit unless we destroy the distributed env.""" + self._llm = None + try: + from vllm.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + ) + + destroy_model_parallel() + destroy_distributed_environment() + except Exception: + pass + gc.collect() + if torch.cuda.is_available(): + try: + torch.cuda.empty_cache() + except Exception: + pass + + +def _dtype_from_hf_config(hf_config: Any) -> torch.dtype: + """Best-effort dtype from an HF config — vLLM prefers fp16 on GPU.""" + raw = getattr(hf_config, "torch_dtype", None) + if isinstance(raw, torch.dtype): + return raw + if isinstance(raw, str): + return getattr(torch, raw, torch.float16) + return torch.float16 + + +def _synthesize_logits(request_output: Any, n_tokens: int, d_vocab: int) -> torch.Tensor: + """Build a ``(1, n_tokens, d_vocab)`` logits-like tensor from vLLM's sampler output — + log-probs (not raw logits), with earlier positions ``-inf`` (lm_head is bypassed so + only the generated position is populated). Matches the HF provider's ``tl_logits`` + shape so the InspectDriver consumes both the same way.""" + logits = torch.full((1, n_tokens, d_vocab), float("-inf"), dtype=torch.float32) + gen = request_output.outputs[0] if request_output.outputs else None + if gen is None: + return logits + if gen.logprobs: + for token_id, lp in gen.logprobs[0].items(): + logits[0, -1, int(token_id)] = float(lp.logprob) + elif gen.token_ids: + logits[0, -1, int(gen.token_ids[0])] = 0.0 + return logits + + +__all__ = ["TransformerLensVLLMModelAPI"] diff --git a/transformer_lens/model_bridge/sources/inspect/wire.py b/transformer_lens/model_bridge/sources/inspect/wire.py new file mode 100644 index 0000000000..bf527a5e34 --- /dev/null +++ b/transformer_lens/model_bridge/sources/inspect/wire.py @@ -0,0 +1,70 @@ +"""Serialization chokepoint for the Inspect activation wire format. + +Activations ride in ``ModelOutput.metadata["activations"]`` as a flat +``{":": {"data": , "dtype": str, "shape": [...]}}`` map (keys +are :func:`hooks.wire_key`). For vllm-lens interop, decode also understands its +*documented* nested ``{"residual_stream": {layer: ...}}`` shape (mapped to +``resid_post``) — unverified against a live vllm-lens provider. +Numpy-only (no torch) so both the torch-using provider and the torch-free driver +import it; the single place to patch on format drift. +""" +from __future__ import annotations + +import base64 +from typing import Any, Iterable, Mapping + +import numpy as np + +_RESIDUAL = "residual_stream" # vllm-lens's nested key (interop decode only) + + +def encode_array(arr: np.ndarray) -> dict[str, Any]: + """numpy array → ``{"data": b64, "dtype": str, "shape": [...]}``.""" + contiguous = np.ascontiguousarray(arr) + return { + "data": base64.b64encode(contiguous.tobytes()).decode("ascii"), + "dtype": str(contiguous.dtype), + "shape": list(contiguous.shape), + } + + +def decode_array(entry: Any) -> np.ndarray: + """Inverse of :func:`encode_array`; passes an already-decoded ndarray through.""" + if isinstance(entry, np.ndarray): + return entry + raw = base64.b64decode(entry["data"]) + # frombuffer is read-only; copy so the downstream torch tensor is writable. + return np.frombuffer(raw, dtype=np.dtype(entry["dtype"])).reshape(entry["shape"]).copy() + + +def encode_activations(captured: Mapping[str, np.ndarray]) -> dict[str, Any]: + """``{wire_key: array}`` → the ``metadata["activations"]`` payload.""" + return {key: encode_array(arr) for key, arr in captured.items()} + + +def decode_activations( + metadata: Mapping[str, Any] | None, wire_keys: Iterable[str] +) -> dict[str, np.ndarray]: + """Pull the requested ``:`` keys out of ``metadata["activations"]``, + falling back to the nested ``residual_stream`` for ``resid_post``. Missing keys are + skipped — the caller decides.""" + activations = (metadata or {}).get("activations") or {} + residual = activations.get(_RESIDUAL, {}) + out: dict[str, np.ndarray] = {} + for key in wire_keys: + if key in activations: + out[key] = decode_array(activations[key]) + continue + layer, _, kind = key.partition(":") + if kind == "resid_post": + entry = residual.get(layer, residual.get(_safe_int(layer))) + if entry is not None: + out[key] = decode_array(entry) + return out + + +def _safe_int(value: str) -> Any: + try: + return int(value) + except ValueError: + return value diff --git a/transformer_lens/model_bridge/sources/vllm/driver.py b/transformer_lens/model_bridge/sources/vllm/driver.py index cf231c3cc7..e78aa47ffa 100644 --- a/transformer_lens/model_bridge/sources/vllm/driver.py +++ b/transformer_lens/model_bridge/sources/vllm/driver.py @@ -95,6 +95,10 @@ def forward( # Push intervention state (possibly empty) before generate — this also # resets stale interventions from prior forwards. self._llm.collective_rpc("tl_set_interventions", args=(intervene_specs,)) + # Open per-hook capture gates; first-write-wins means a fresh prefill writes and + # subsequent forwards self-copy. Without this, repeated bridge.forward calls would + # see the gate closed from the prior call and read stale buffers. + self._llm.collective_rpc("tl_reset_capture_flags") # Full-vocab logprobs → position -1 of the synthesized logits (see _n_logprobs). outputs = self._llm.generate( prompts=[TokensPrompt(prompt_token_ids=ids_list)], diff --git a/transformer_lens/model_bridge/sources/vllm/plugin.py b/transformer_lens/model_bridge/sources/vllm/plugin.py index d2ee620b9b..6b54d46490 100644 --- a/transformer_lens/model_bridge/sources/vllm/plugin.py +++ b/transformer_lens/model_bridge/sources/vllm/plugin.py @@ -93,6 +93,11 @@ def patched_load_model(self): self._tl_buffers = {} self._tl_scale_buffers = {} self._tl_bias_buffers = {} + # Per-hook first-write-wins flag (compiled mode). 0 = open (next forward captures), + # 1 = closed (subsequent forwards self-copy and don't overwrite). Driver opens via + # tl_reset_capture_flags before any capture-needing call so a multi-token generate's + # prefill captures cleanly and decode steps don't overwrite row 0. + self._tl_capture_flags = {} self._tl_hook_handles = [] # Batched-mode per-(req_id, hook) accumulators + global spec dict. self._tl_accum = {} @@ -122,15 +127,20 @@ def patched_load_model(self): # to enable suppress/scale/add/set ops between forwards. scale_buf = torch.ones(width, device=device, dtype=dtype) bias_buf = torch.zeros(width, device=device, dtype=dtype) + # Default closed — opened explicitly by tl_reset_capture_flags for the + # next forward(s) that need to capture. + capture_flag = torch.ones(1, device=device, dtype=torch.int64) self._tl_buffers[canonical_name] = capture_buf self._tl_scale_buffers[canonical_name] = scale_buf self._tl_bias_buffers[canonical_name] = bias_buf + self._tl_capture_flags[canonical_name] = capture_flag handle = target.register_forward_hook( _make_capture_hook( capture_buf, scale_buf, bias_buf, self._tl_fire_counter, + capture_flag, materialize=materialize, ) ) @@ -145,10 +155,11 @@ def _make_capture_hook( scale_buf: torch.Tensor, bias_buf: torch.Tensor, fire_counter: torch.Tensor, + capture_flag: torch.Tensor, *, materialize: bool = False, ): - """GPU-only, dynamic-shape-safe affine + capture into pre-allocated buffers. + """GPU-only, dynamic-shape-safe affine + first-write-wins capture into pre-allocated buffers. When ``materialize=True`` (decoder layers), treat the module's output as vLLM's fused-residual ``(mlp_delta, residual)`` tuple: capture @@ -157,6 +168,12 @@ def _make_capture_hook( so the next layer's input_layernorm sees the same fused sum. Mutations propagate through both the capture and the downstream graph. + ``capture_flag`` (0 = open, 1 = closed) gates the buffer write — driver opens it + via ``tl_reset_capture_flags`` before each capture-needing forward, the hook closes + it on first fire, so a multi-token generate's prefill captures cleanly and decode + steps self-copy (no overwrite). Interventions still apply on every forward + regardless of the flag — the gate only affects the capture write. + ``fire_counter`` is incremented per call for the fire-once check. """ @@ -169,7 +186,7 @@ def hook(_module, _inputs, output): t = hidden + residual modified = t * scale_buf + bias_buf n = t.shape[0] - capture_buf.narrow(0, 0, n).copy_(modified) + _gated_capture(capture_buf, n, modified, capture_flag) # Reconstructs ``modified`` in the next layer's fused norm: exact at # identity, bounded fp16 error under intervention. return (modified - residual, residual) @@ -188,7 +205,7 @@ def hook(_module, _inputs, output): # narrow() keeps the dynamic shape; [:n] gets erased under fake-tensor # tracing and copy_ then sees the full buffer ("expand s72 -> max_n"). n = t.shape[0] - capture_buf.narrow(0, 0, n).copy_(modified) + _gated_capture(capture_buf, n, modified, capture_flag) # Gate on isinstance, not truthy tuple_tail — a 1-tuple has an empty tail. if isinstance(output, tuple): return (modified,) + tuple_tail @@ -197,6 +214,21 @@ def hook(_module, _inputs, output): return hook +def _gated_capture( + capture_buf: torch.Tensor, n: Any, modified: torch.Tensor, capture_flag: torch.Tensor +) -> None: + """First-write-wins via torch.where, compile-safe (no Python branching). + + When ``capture_flag == 0`` (open), writes ``modified`` to ``capture_buf[:n]``. + When ``capture_flag == 1`` (closed), self-copies ``capture_buf[:n]`` (no-op). + Always closes the flag — driver explicitly opens it before each capture forward. + """ + existing = capture_buf.narrow(0, 0, n) + to_write = torch.where(capture_flag.bool(), existing, modified) + capture_buf.narrow(0, 0, n).copy_(to_write) + capture_flag.fill_(1) + + def _make_batched_hook( worker: Any, canonical_name: str, diff --git a/transformer_lens/model_bridge/sources/vllm/worker_extension.py b/transformer_lens/model_bridge/sources/vllm/worker_extension.py index a462e826fd..5873be05c7 100644 --- a/transformer_lens/model_bridge/sources/vllm/worker_extension.py +++ b/transformer_lens/model_bridge/sources/vllm/worker_extension.py @@ -30,6 +30,8 @@ class TLWorkerExtension: _tl_buffers: Dict[str, torch.Tensor] _tl_scale_buffers: Dict[str, torch.Tensor] _tl_bias_buffers: Dict[str, torch.Tensor] + # Per-hook first-write-wins gates (compiled mode); see plugin._gated_capture. + _tl_capture_flags: Dict[str, torch.Tensor] _tl_fire_counter: torch.Tensor # Batched-mode state (eager). _tl_accum: Dict[tuple, List[torch.Tensor]] @@ -88,6 +90,17 @@ def tl_reset_counter(self) -> None: if counter is not None: counter.zero_() + def tl_reset_capture_flags(self) -> None: + """Open every per-hook capture gate so the next forward writes to the buffers. + + First-write-wins gating means decode-step forwards self-copy and never overwrite + prefill activations — the driver calls this once before any capture-needing + generate (single-forward or multi-token eval) and ``tl_read_captures`` afterward. + """ + flags: Dict[str, torch.Tensor] = getattr(self, "_tl_capture_flags", {}) + for flag in flags.values(): + flag.zero_() + def tl_read_counter(self) -> int: """Total hook fires since the last reset.""" counter = getattr(self, "_tl_fire_counter", None) @@ -132,6 +145,7 @@ def tl_remove_hooks(self) -> None: self._tl_buffers = {} self._tl_scale_buffers = {} self._tl_bias_buffers = {} + self._tl_capture_flags = {} self._tl_accum = {} self._tl_intervention_specs = {} diff --git a/uv.lock b/uv.lock index 69338a25aa..1e7b263f60 100644 --- a/uv.lock +++ b/uv.lock @@ -41,13 +41,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/1c/a17fb513aeb684fb83bef5f395910f53103ab30308bbdd77fd66d6698c46/accelerate-1.9.0-py3-none-any.whl", hash = "sha256:c24739a97ade1d54af4549a65f8b6b046adc87e2b3e4d6c66516e32c53d5a8f1", size = 367073, upload-time = "2025-07-16T16:24:52.957Z" }, ] +[[package]] +name = "agent-client-protocol" +version = "0.10.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/a0/3b96cd8374725c69bc3dae9fcc2082f3f6cafec1be35d24d7af0f8c3265f/agent_client_protocol-0.10.1.tar.gz", hash = "sha256:355c65ca19f0568344aafc2c1552b7066a8fc491df23ab28e7e253c6c9a85a25", size = 81924, upload-time = "2026-05-24T18:46:44.444Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/18/d8c7ff337cf621ea79a84006a7252ff057bfb5767549bb102cc6649f4ec2/agent_client_protocol-0.10.1-py3-none-any.whl", hash = "sha256:a03d3198f4d772f2e0ec012c00ac1cce131b4710220a3dc9fae3c991d047c750", size = 65401, upload-time = "2026-05-24T18:46:43.202Z" }, +] + +[[package]] +name = "aioboto3" +version = "15.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiobotocore", extra = ["boto3"] }, + { name = "aiofiles" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a2/01/92e9ab00f36e2899315f49eefcd5b4685fbb19016c7f19a9edf06da80bb0/aioboto3-15.5.0.tar.gz", hash = "sha256:ea8d8787d315594842fbfcf2c4dce3bac2ad61be275bc8584b2ce9a3402a6979", size = 255069, upload-time = "2025-10-30T13:37:16.122Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/3e/e8f5b665bca646d43b916763c901e00a07e40f7746c9128bdc912a089424/aioboto3-15.5.0-py3-none-any.whl", hash = "sha256:cc880c4d6a8481dd7e05da89f41c384dbd841454fc1998ae25ca9c39201437a6", size = 35913, upload-time = "2025-10-30T13:37:14.549Z" }, +] + +[[package]] +name = "aiobotocore" +version = "2.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "aioitertools" }, + { name = "botocore" }, + { name = "jmespath" }, + { name = "multidict" }, + { name = "python-dateutil" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/94/2e4ec48cf1abb89971cb2612d86f979a6240520f0a659b53a43116d344dc/aiobotocore-2.25.1.tar.gz", hash = "sha256:ea9be739bfd7ece8864f072ec99bb9ed5c7e78ebb2b0b15f29781fbe02daedbc", size = 120560, upload-time = "2025-10-28T22:33:21.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/2a/d275ec4ce5cd0096665043995a7d76f5d0524853c76a3d04656de49f8808/aiobotocore-2.25.1-py3-none-any.whl", hash = "sha256:eb6daebe3cbef5b39a0bb2a97cffbe9c7cb46b2fcc399ad141f369f3c2134b1f", size = 86039, upload-time = "2025-10-28T22:33:19.949Z" }, +] + +[package.optional-dependencies] +boto3 = [ + { name = "boto3" }, +] + [[package]] name = "aiofiles" -version = "22.1.0" +version = "25.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/86/26/6e5060a159a6131c430e8a01ec8327405a19a449a506224b394e36f2ebc9/aiofiles-22.1.0.tar.gz", hash = "sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6", size = 14669, upload-time = "2022-09-04T17:09:21.97Z" } +sdist = { url = "https://files.pythonhosted.org/packages/41/c3/534eac40372d8ee36ef40df62ec129bee4fdb5ad9706e58a29be53b2c970/aiofiles-25.1.0.tar.gz", hash = "sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2", size = 46354, upload-time = "2025-10-09T20:51:04.358Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/48/d5d1ab7cfe46e573c3694fa1365442a7d7cadc3abb03d8507e58a3755bb2/aiofiles-22.1.0-py3-none-any.whl", hash = "sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad", size = 14171, upload-time = "2022-09-04T17:09:19.625Z" }, + { url = "https://files.pythonhosted.org/packages/bc/8a/340a1555ae33d7354dbca4faa54948d76d89a27ceef032c8c3bc661d003e/aiofiles-25.1.0-py3-none-any.whl", hash = "sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695", size = 14668, upload-time = "2025-10-09T20:51:03.174Z" }, ] [[package]] @@ -145,6 +193,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/5f/8427618903343402fdafe2850738f735fd1d9409d2a8f9bcaae5e630d3ba/aiohttp-3.12.14-cp313-cp313-win_amd64.whl", hash = "sha256:3f8aad695e12edc9d571f878c62bedc91adf30c760c8632f09663e5f564f4baa", size = 448098, upload-time = "2025-07-10T13:04:53.999Z" }, ] +[[package]] +name = "aioitertools" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/3c/53c4a17a05fb9ea2313ee1777ff53f5e001aefd5cc85aa2f4c2d982e1e38/aioitertools-0.13.0.tar.gz", hash = "sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c", size = 19322, upload-time = "2025-11-06T22:17:07.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/a1/510b0a7fadc6f43a6ce50152e69dbd86415240835868bb0bd9b5b88b1e06/aioitertools-0.13.0-py3-none-any.whl", hash = "sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be", size = 24182, upload-time = "2025-11-06T22:17:06.502Z" }, +] + [[package]] name = "aiosignal" version = "1.4.0" @@ -158,18 +215,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] -[[package]] -name = "aiosqlite" -version = "0.21.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/13/7d/8bca2bf9a247c2c5dfeec1d7a5f40db6518f88d314b8bca9da29670d2671/aiosqlite-0.21.0.tar.gz", hash = "sha256:131bb8056daa3bc875608c631c678cda73922a2d4ba8aec373b19f18c17e7aa3", size = 13454, upload-time = "2025-02-03T07:30:16.235Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/10/6c25ed6de94c49f88a91fa5018cb4c0f3625f31d5be9f771ebe5cc7cd506/aiosqlite-0.21.0-py3-none-any.whl", hash = "sha256:2549cf4057f95f53dcba16f2b64e8e2791d7e1adedb13197dd8ed77bb226d7d0", size = 15792, upload-time = "2025-02-03T07:30:13.6Z" }, -] - [[package]] name = "alabaster" version = "0.7.16" @@ -285,6 +330,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl", hash = "sha256:15a3ebc0f43c2d0a50eeafea25e19046c68398e487b9f1f5b517f7c0f40f976a", size = 27047, upload-time = "2025-11-15T16:43:16.109Z" }, ] +[[package]] +name = "async-lru" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/1f/989ecfef8e64109a489fff357450cb73fa73a865a92bd8c272170a6922c2/async_lru-2.3.0.tar.gz", hash = "sha256:89bdb258a0140d7313cf8f4031d816a042202faa61d0ab310a0a538baa1c24b6", size = 16332, upload-time = "2026-03-19T01:04:32.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/e2/c2e3abf398f80732e58b03be77bde9022550d221dd8781bf586bd4d97cc1/async_lru-2.3.0-py3-none-any.whl", hash = "sha256:eea27b01841909316f2cc739807acea1c623df2be8c5cfad7583286397bb8315", size = 8403, upload-time = "2026-03-19T01:04:30.883Z" }, +] + [[package]] name = "async-timeout" version = "5.0.1" @@ -462,6 +519,34 @@ css = [ { name = "tinycss2" }, ] +[[package]] +name = "boto3" +version = "1.40.61" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/f9/6ef8feb52c3cce5ec3967a535a6114b57ac7949fd166b0f3090c2b06e4e5/boto3-1.40.61.tar.gz", hash = "sha256:d6c56277251adf6c2bdd25249feae625abe4966831676689ff23b4694dea5b12", size = 111535, upload-time = "2025-10-28T19:26:57.247Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/24/3bf865b07d15fea85b63504856e137029b6acbc73762496064219cdb265d/boto3-1.40.61-py3-none-any.whl", hash = "sha256:6b9c57b2a922b5d8c17766e29ed792586a818098efe84def27c8f582b33f898c", size = 139321, upload-time = "2025-10-28T19:26:55.007Z" }, +] + +[[package]] +name = "botocore" +version = "1.40.61" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/a3/81d3a47c2dbfd76f185d3b894f2ad01a75096c006a2dd91f237dca182188/botocore-1.40.61.tar.gz", hash = "sha256:a2487ad69b090f9cccd64cf07c7021cd80ee9c0655ad974f87045b02f3ef52cd", size = 14393956, upload-time = "2025-10-28T19:26:46.108Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/c5/f6ce561004db45f0b847c2cd9b19c67c6bf348a82018a48cb718be6b58b0/botocore-1.40.61-py3-none-any.whl", hash = "sha256:17ebae412692fd4824f99cde0f08d50126dc97954008e5ba2b522eb049238aa7", size = 14055973, upload-time = "2025-10-28T19:26:42.15Z" }, +] + [[package]] name = "brotli" version = "1.2.0" @@ -1145,6 +1230,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c9/7a/cef76fd8438a42f96db64ddaa85280485a9c395e7df3db8158cfec1eee34/dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7", size = 116252, upload-time = "2024-01-27T23:42:14.239Z" }, ] +[[package]] +name = "docstring-parser" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, +] + [[package]] name = "docutils" version = "0.20.1" @@ -1864,6 +1958,97 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, ] +[[package]] +name = "ijson" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/57/60d1a6a512f2f0508d0bc8b4f1cc5616fd3196619b66bd6a01f9155a1292/ijson-3.5.0.tar.gz", hash = "sha256:94688760720e3f5212731b3cb8d30267f9a045fb38fb3870254e7b9504246f31", size = 68658, upload-time = "2026-02-24T03:58:30.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/32/21c1b47a1afb7319944d0b9685c0997a9d574a77b030c82f6a1ac2cef4eb/ijson-3.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ea8dcac10d86adaeead454bc25c97b68d0bda573d5fd6f86f5e21cf8f7906f88", size = 88935, upload-time = "2026-02-24T03:56:40.591Z" }, + { url = "https://files.pythonhosted.org/packages/86/f7/6ac7ebbb3cd767c87cdcbb950a6754afd1c0977756347bfe03eb8e5b866d/ijson-3.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:92b0495bbb2150bbf14fc5d98fb6d76bcd1c526605a172709e602e6fedc96495", size = 60567, upload-time = "2026-02-24T03:56:41.919Z" }, + { url = "https://files.pythonhosted.org/packages/c4/98/1140de9ae872468a8bc2e87c171228e25e58b1eb696b7fb430f7590fea44/ijson-3.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7af0c4c8943be8b09a4e57bdc1da6001dae7b36526d4154fe5c8224738d0921f", size = 60620, upload-time = "2026-02-24T03:56:42.764Z" }, + { url = "https://files.pythonhosted.org/packages/60/e1/67dfe0774e4c7ca6ec8702e280e8764d356f3db54358999818cda6df7679/ijson-3.5.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:45887d5e84ff0d2b138c926cebd9071830733968afe8d9d12080b3c178c7f918", size = 126558, upload-time = "2026-02-24T03:56:43.922Z" }, + { url = "https://files.pythonhosted.org/packages/1f/ef/23d614fc773d428caeb6e197218b7e32adcc668ff5b98777039149571208/ijson-3.5.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9a70b575be8e57a28c80e90ed349ad3a851c3478524c70e36e07d6092ecd12c9", size = 133091, upload-time = "2026-02-24T03:56:45.291Z" }, + { url = "https://files.pythonhosted.org/packages/b8/80/99727603cd8a1d32edafa4392f4056b2420bf48c15afd34481c68a2d4435/ijson-3.5.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2adeecd45830bfd5580ca79a584154713aabef0b9607e16249133df5d2859813", size = 130249, upload-time = "2026-02-24T03:56:46.333Z" }, + { url = "https://files.pythonhosted.org/packages/0b/94/3a3d623ca80768e834be8a834ef05960e3b9e79af1a911704ff10c9e8792/ijson-3.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d873e72889e7fc5962ab58909f1adff338d7c2f49e450e5b5fe844eff8155a14", size = 133501, upload-time = "2026-02-24T03:56:47.54Z" }, + { url = "https://files.pythonhosted.org/packages/cf/f6/df2c14ad340834eccee379046f155e4b66a16ddafd445429dee7b3323614/ijson-3.5.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9a88c559456a79708592234d697645d92b599718f4cbbeaa6515f83ac63ca0ae", size = 128438, upload-time = "2026-02-24T03:56:48.455Z" }, + { url = "https://files.pythonhosted.org/packages/0c/7e/9ff5b8b5fee113f5607bc4149b707382a898eeb545153189b075e5ec8d59/ijson-3.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cf83f58ad50dc0d39a2105cb26d4f359b38f42cef68b913170d4d47d97d97ba5", size = 131116, upload-time = "2026-02-24T03:56:49.737Z" }, + { url = "https://files.pythonhosted.org/packages/64/20/954ce0d440d7cf72a3d8361b14406f9cdbf624b1625c10f8488857c769d6/ijson-3.5.0-cp310-cp310-win32.whl", hash = "sha256:aec4580a7712a19b1f95cd41bed260fc6a31266d37ef941827772a4c199e8143", size = 52724, upload-time = "2026-02-24T03:56:50.932Z" }, + { url = "https://files.pythonhosted.org/packages/24/33/ece87d60502c6115642cbabeb8c122fa982212b392bc4f4ff5aab8e02dac/ijson-3.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:9a9c4c70501e23e8eb1675330686d1598eebfa14b6f0dbc8f00c2e081cc628fa", size = 55125, upload-time = "2026-02-24T03:56:51.942Z" }, + { url = "https://files.pythonhosted.org/packages/65/da/644343198abca5e0f6e2486063f8d8f3c443ca0ef5e5c890e51ef6032e33/ijson-3.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5616311404b858d32740b7ad8b9a799c62165f5ecb85d0a8ed16c21665a90533", size = 88964, upload-time = "2026-02-24T03:56:53.099Z" }, + { url = "https://files.pythonhosted.org/packages/5b/63/8621190aa2baf96156dfd4c632b6aa9f1464411e50b98750c09acc0505ea/ijson-3.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e9733f94029dd41702d573ef64752e2556e72aea14623d6dbb7a44ca1ccf30fd", size = 60582, upload-time = "2026-02-24T03:56:54.261Z" }, + { url = "https://files.pythonhosted.org/packages/20/31/6a3f041fdd17dacff33b7d7d3ba3df6dca48740108340c6042f974b2ad20/ijson-3.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db8398c6721b98412a4f618da8022550c8b9c5d9214040646071b5deb4d4a393", size = 60632, upload-time = "2026-02-24T03:56:55.159Z" }, + { url = "https://files.pythonhosted.org/packages/e4/68/474541998abbdecfd46a744536878335de89aceb9f085bff1aaf35575ceb/ijson-3.5.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c061314845c08163b1784b6076ea5f075372461a32e6916f4e5f211fd4130b64", size = 131988, upload-time = "2026-02-24T03:56:56.35Z" }, + { url = "https://files.pythonhosted.org/packages/cd/32/e05ff8b72a44fe9d192f41c5dcbc35cfa87efc280cdbfe539ffaf4a7535e/ijson-3.5.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1111a1c5ac79119c5d6e836f900c1a53844b50a18af38311baa6bb61e2645aca", size = 138669, upload-time = "2026-02-24T03:56:57.555Z" }, + { url = "https://files.pythonhosted.org/packages/49/b5/955a83b031102c7a602e2c06d03aff0a0e584212f09edb94ccc754d203ac/ijson-3.5.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e74aff8c681c24002b61b1822f9511d4c384f324f7dbc08c78538e01fdc9fcb", size = 135093, upload-time = "2026-02-24T03:56:59.267Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f2/30250cfcb4d2766669b31f6732689aab2bb91de426a15a3ebe482df7ee48/ijson-3.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:739a7229b1b0cc5f7e2785a6e7a5fc915e850d3fed9588d0e89a09f88a417253", size = 138715, upload-time = "2026-02-24T03:57:00.491Z" }, + { url = "https://files.pythonhosted.org/packages/a2/05/785a145d7e75e04e04480d59b6323cd4b1d9013a6cd8643fa635fbc93490/ijson-3.5.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ef88712160360cab3ca6471a4e5418243f8b267cf1fe1620879d1b5558babc71", size = 133194, upload-time = "2026-02-24T03:57:01.759Z" }, + { url = "https://files.pythonhosted.org/packages/14/eb/80d6f8a748dead4034cea0939494a67d10ccf88d6413bf6e860393139676/ijson-3.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6ca0d1b6b5f8166a6248f4309497585fb8553b04bc8179a0260fad636cfdb798", size = 135588, upload-time = "2026-02-24T03:57:03.131Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a8/bbc21f9400ebdbca48fab272593e0d1f875691be1e927d264d90d48b8c47/ijson-3.5.0-cp311-cp311-win32.whl", hash = "sha256:966039cf9047c7967febf7b9a52ec6f38f5464a4c7fbb5565e0224b7376fefff", size = 52721, upload-time = "2026-02-24T03:57:04.365Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2e/4e8c0208b8f920ee80c88c956f93e78318f2cfb646455353b182738b490c/ijson-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:6bad6a1634cb7c9f3f4c7e52325283b35b565f5b6cc27d42660c6912ce883422", size = 55121, upload-time = "2026-02-24T03:57:05.498Z" }, + { url = "https://files.pythonhosted.org/packages/aa/17/9c63c7688025f3a8c47ea717b8306649c8c7244e49e20a2be4e3515dc75c/ijson-3.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1ebefbe149a6106cc848a3eaf536af51a9b5ccc9082de801389f152dba6ab755", size = 88536, upload-time = "2026-02-24T03:57:06.809Z" }, + { url = "https://files.pythonhosted.org/packages/6f/dd/e15c2400244c117b06585452ebc63ae254f5a6964f712306afd1422daae0/ijson-3.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:19e30d9f00f82e64de689c0b8651b9cfed879c184b139d7e1ea5030cec401c21", size = 60499, upload-time = "2026-02-24T03:57:09.155Z" }, + { url = "https://files.pythonhosted.org/packages/77/a9/bf4fe3538a0c965f16b406f180a06105b875da83f0743e36246be64ef550/ijson-3.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a04a33ee78a6f27b9b8528c1ca3c207b1df3b8b867a4cf2fcc4109986f35c227", size = 60330, upload-time = "2026-02-24T03:57:10.574Z" }, + { url = "https://files.pythonhosted.org/packages/31/76/6f91bdb019dd978fce1bc5ea1cd620cfc096d258126c91db2c03a20a7f34/ijson-3.5.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7d48dc2984af02eb3c56edfb3f13b3f62f2f3e4fe36f058c8cfc75d93adf4fed", size = 138977, upload-time = "2026-02-24T03:57:11.932Z" }, + { url = "https://files.pythonhosted.org/packages/11/be/bbc983059e48a54b0121ee60042979faed7674490bbe7b2c41560db3f436/ijson-3.5.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1e73a44844d9adbca9cf2c4132cd875933e83f3d4b23881fcaf82be83644c7d", size = 149785, upload-time = "2026-02-24T03:57:13.255Z" }, + { url = "https://files.pythonhosted.org/packages/6d/81/2fee58f9024a3449aee83edfa7167fb5ccd7e1af2557300e28531bb68e16/ijson-3.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7389a56b8562a19948bdf1d7bae3a2edc8c7f86fb59834dcb1c4c722818e645a", size = 149729, upload-time = "2026-02-24T03:57:14.191Z" }, + { url = "https://files.pythonhosted.org/packages/c7/56/f1706761fcc096c9d414b3dcd000b1e6e5c24364c21cfba429837f98ee8d/ijson-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3176f23f8ebec83f374ed0c3b4e5a0c4db7ede54c005864efebbed46da123608", size = 150697, upload-time = "2026-02-24T03:57:15.855Z" }, + { url = "https://files.pythonhosted.org/packages/d9/6e/ee0d9c875a0193b632b3e9ccd1b22a50685fb510256ad57ba483b6529f77/ijson-3.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6babd88e508630c6ef86c9bebaaf13bb2fb8ec1d8f8868773a03c20253f599bc", size = 142873, upload-time = "2026-02-24T03:57:16.831Z" }, + { url = "https://files.pythonhosted.org/packages/d2/bf/f9d4399d0e6e3fd615035290a71e97c843f17f329b43638c0a01cf112d73/ijson-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dc1b3836b174b6db2fa8319f1926fb5445abd195dc963368092103f8579cb8ed", size = 151583, upload-time = "2026-02-24T03:57:17.757Z" }, + { url = "https://files.pythonhosted.org/packages/b2/71/a7254a065933c0e2ffd3586f46187d84830d3d7b6f41cfa5901820a4f87d/ijson-3.5.0-cp312-cp312-win32.whl", hash = "sha256:6673de9395fb9893c1c79a43becd8c8fbee0a250be6ea324bfd1487bb5e9ee4c", size = 53079, upload-time = "2026-02-24T03:57:18.703Z" }, + { url = "https://files.pythonhosted.org/packages/8f/7b/2edca79b359fc9f95d774616867a03ecccdf333797baf5b3eea79733918c/ijson-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:f4f7fabd653459dcb004175235f310435959b1bb5dfa8878578391c6cc9ad944", size = 55500, upload-time = "2026-02-24T03:57:20.428Z" }, + { url = "https://files.pythonhosted.org/packages/a2/71/d67e764a712c3590627480643a3b51efcc3afa4ef3cb54ee4c989073c97e/ijson-3.5.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e9cedc10e40dd6023c351ed8bfc7dcfce58204f15c321c3c1546b9c7b12562a4", size = 88544, upload-time = "2026-02-24T03:57:21.293Z" }, + { url = "https://files.pythonhosted.org/packages/1a/39/f1c299371686153fa3cf5c0736b96247a87a1bee1b7145e6d21f359c505a/ijson-3.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3647649f782ee06c97490b43680371186651f3f69bebe64c6083ee7615d185e5", size = 60495, upload-time = "2026-02-24T03:57:22.501Z" }, + { url = "https://files.pythonhosted.org/packages/16/94/b1438e204d75e01541bebe3e668fe3e68612d210e9931ae1611062dd0a56/ijson-3.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:90e74be1dce05fce73451c62d1118671f78f47c9f6be3991c82b91063bf01fc9", size = 60325, upload-time = "2026-02-24T03:57:23.332Z" }, + { url = "https://files.pythonhosted.org/packages/30/e2/4aa9c116fa86cc8b0f574f3c3a47409edc1cd4face05d0e589a5a176b05d/ijson-3.5.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:78e9ad73e7be2dd80627504bd5cbf512348c55ce2c06e362ed7683b5220e8568", size = 138774, upload-time = "2026-02-24T03:57:24.683Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d2/738b88752a70c3be1505faa4dcd7110668c2712e582a6a36488ed1e295d4/ijson-3.5.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9577449313cc94be89a4fe4b3e716c65f09cc19636d5a6b2861c4e80dddebd58", size = 149820, upload-time = "2026-02-24T03:57:26.062Z" }, + { url = "https://files.pythonhosted.org/packages/ed/df/0b3ab9f393ca8f72ea03bc896ba9fdc987e90ae08cdb51c32a4ee0c14d5e/ijson-3.5.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e4c1178fb50aff5f5701a30a5152ead82a14e189ce0f6102fa1b5f10b2f54ff", size = 149747, upload-time = "2026-02-24T03:57:27.308Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a3/b0037119f75131b78cb00acc2657b1a9d0435475f1f2c5f8f5a170b66b9c/ijson-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0eb402ab026ffb37a918d75af2b7260fe6cfbce13232cc83728a714dd30bd81d", size = 151027, upload-time = "2026-02-24T03:57:28.522Z" }, + { url = "https://files.pythonhosted.org/packages/22/a0/cb344de1862bf09d8f769c9d25c944078c87dd59a1b496feec5ad96309a4/ijson-3.5.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:5b08ee08355f9f729612a8eb9bf69cc14f9310c3b2a487c6f1c3c65d85216ec4", size = 142996, upload-time = "2026-02-24T03:57:29.774Z" }, + { url = "https://files.pythonhosted.org/packages/ca/32/a8ffd67182e02ea61f70f62daf43ded4fa8a830a2520a851d2782460aba8/ijson-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bda62b6d48442903e7bf56152108afb7f0f1293c2b9bef2f2c369defea76ab18", size = 152068, upload-time = "2026-02-24T03:57:30.969Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d1/3578df8e75d446aab0ae92e27f641341f586b85e1988536adebc65300cb4/ijson-3.5.0-cp313-cp313-win32.whl", hash = "sha256:8d073d9b13574cfa11083cc7267c238b7a6ed563c2661e79192da4a25f09c82c", size = 53065, upload-time = "2026-02-24T03:57:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/fb/a2/f7cdaf5896710da3e69e982e44f015a83d168aa0f3a89b6f074b5426779d/ijson-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:2419f9e32e0968a876b04d8f26aeac042abd16f582810b576936bbc4c6015069", size = 55499, upload-time = "2026-02-24T03:57:32.773Z" }, + { url = "https://files.pythonhosted.org/packages/42/65/13e2492d17e19a2084523e18716dc2809159f2287fd2700c735f311e76c4/ijson-3.5.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:4d4b0cd676b8c842f7648c1a783448fac5cd3b98289abd83711b3e275e143524", size = 93019, upload-time = "2026-02-24T03:57:33.976Z" }, + { url = "https://files.pythonhosted.org/packages/33/92/483fc97ece0c3f1cecabf48f6a7a36e89d19369eec462faaeaa34c788992/ijson-3.5.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:252dec3680a48bb82d475e36b4ae1b3a9d7eb690b951bb98a76c5fe519e30188", size = 62714, upload-time = "2026-02-24T03:57:34.819Z" }, + { url = "https://files.pythonhosted.org/packages/4b/88/793fe020a0fe9d9eed4c285cf4a5cfdb0a935708b3bde0d72f35c794b513/ijson-3.5.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:aa1b5dca97d323931fde2501172337384c958914d81a9dac7f00f0d4bfc76bc7", size = 62460, upload-time = "2026-02-24T03:57:35.874Z" }, + { url = "https://files.pythonhosted.org/packages/51/69/f1a2690aa8d4df1f4e262b385e65a933ffdc250b091531bac9a449c19e16/ijson-3.5.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7a5ec7fd86d606094bba6f6f8f87494897102fa4584ef653f3005c51a784c320", size = 199273, upload-time = "2026-02-24T03:57:37.07Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a2/f1346d5299e79b988ab472dc773d5381ec2d57c23cb2f1af3ede4a810e62/ijson-3.5.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:009f41443e1521847701c6d87fa3923c0b1961be3c7e7de90947c8cb92ea7c44", size = 216884, upload-time = "2026-02-24T03:57:38.346Z" }, + { url = "https://files.pythonhosted.org/packages/28/3c/8b637e869be87799e6c2c3c275a30a546f086b1aed77e2b7f11512168c5a/ijson-3.5.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e4c3651d1f9fe2839a93fdf8fd1d5ca3a54975349894249f3b1b572bcc4bd577", size = 207306, upload-time = "2026-02-24T03:57:39.718Z" }, + { url = "https://files.pythonhosted.org/packages/7f/7c/18b1c1df6951ca056782d7580ec40cea4ff9a27a0947d92640d1cc8c4ae3/ijson-3.5.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:945b7abcfcfeae2cde17d8d900870f03536494245dda7ad4f8d056faa303256c", size = 211364, upload-time = "2026-02-24T03:57:40.953Z" }, + { url = "https://files.pythonhosted.org/packages/f3/55/e795812e82851574a9dba8a53fde045378f531ef14110c6fb55dbd23b443/ijson-3.5.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:0574b0a841ff97495c13e9d7260fbf3d85358b061f540c52a123db9dbbaa2ed6", size = 200608, upload-time = "2026-02-24T03:57:42.272Z" }, + { url = "https://files.pythonhosted.org/packages/5c/cd/013c85b4749b57a4cb4c2670014d1b32b8db4ab1a7be92ea7aeb5d7fe7b5/ijson-3.5.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f969ffb2b89c5cdf686652d7fb66252bc72126fa54d416317411497276056a18", size = 205127, upload-time = "2026-02-24T03:57:43.286Z" }, + { url = "https://files.pythonhosted.org/packages/0e/7c/faf643733e3ab677f180018f6a855c4ef70b7c46540987424c563c959e42/ijson-3.5.0-cp313-cp313t-win32.whl", hash = "sha256:59d3f9f46deed1332ad669518b8099920512a78bda64c1f021fcd2aff2b36693", size = 55282, upload-time = "2026-02-24T03:57:44.353Z" }, + { url = "https://files.pythonhosted.org/packages/69/22/94ddb47c24b491377aca06cd8fc9202cad6ab50619842457d2beefde21ea/ijson-3.5.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5c2839fa233746d8aad3b8cd2354e441613f5df66d721d59da4a09394bd1db2b", size = 58016, upload-time = "2026-02-24T03:57:45.237Z" }, + { url = "https://files.pythonhosted.org/packages/7a/93/0868efe753dc1df80cc405cf0c1f2527a6991643607c741bff8dcb899b3b/ijson-3.5.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:25a5a6b2045c90bb83061df27cfa43572afa43ba9408611d7bfe237c20a731a9", size = 89094, upload-time = "2026-02-24T03:57:46.115Z" }, + { url = "https://files.pythonhosted.org/packages/24/94/fd5a832a0df52ef5e4e740f14ac8640725d61034a1b0c561e8b5fb424706/ijson-3.5.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:8976c54c0b864bc82b951bae06567566ac77ef63b90a773a69cd73aab47f4f4f", size = 60715, upload-time = "2026-02-24T03:57:47.552Z" }, + { url = "https://files.pythonhosted.org/packages/70/79/1b9a90af5732491f9eec751ee211b86b11011e1158c555c06576d52c3919/ijson-3.5.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:859eb2038f7f1b0664df4241957694cc35e6295992d71c98659b22c69b3cbc10", size = 60638, upload-time = "2026-02-24T03:57:48.428Z" }, + { url = "https://files.pythonhosted.org/packages/23/6f/2c551ea980fe56f68710a8d5389cfbd015fc45aaafd17c3c52c346db6aa1/ijson-3.5.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c911aa02991c7c0d3639b6619b93a93210ff1e7f58bf7225d613abea10adc78e", size = 140667, upload-time = "2026-02-24T03:57:49.314Z" }, + { url = "https://files.pythonhosted.org/packages/25/0e/27b887879ba6a5bc29766e3c5af4942638c952220fd63e1e442674f7883a/ijson-3.5.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:903cbdc350173605220edc19796fbea9b2203c8b3951fb7335abfa8ed37afda8", size = 149850, upload-time = "2026-02-24T03:57:50.329Z" }, + { url = "https://files.pythonhosted.org/packages/da/1e/23e10e1bc04bf31193b21e2960dce14b17dbd5d0c62204e8401c59d62c08/ijson-3.5.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4549d96ded5b8efa71639b2160235415f6bdb8c83367615e2dbabcb72755c33", size = 149206, upload-time = "2026-02-24T03:57:51.261Z" }, + { url = "https://files.pythonhosted.org/packages/8e/90/e552f6495063b235cf7fa2c592f6597c057077195e517b842a0374fd470c/ijson-3.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6b2dcf6349e6042d83f3f8c39ce84823cf7577eba25bac5aae5e39bbbbbe9c1c", size = 150438, upload-time = "2026-02-24T03:57:52.198Z" }, + { url = "https://files.pythonhosted.org/packages/5c/18/45bf8f297c41b42a1c231d261141097babd953d2c28a07be57ae4c3a1a02/ijson-3.5.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:e44af39e6f8a17e5627dcd89715d8279bf3474153ff99aae031a936e5c5572e5", size = 144369, upload-time = "2026-02-24T03:57:53.22Z" }, + { url = "https://files.pythonhosted.org/packages/9b/3a/deb9772bb2c0cead7ad64f00c3598eec9072bdf511818e70e2c512eeabbe/ijson-3.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9260332304b7e7828db56d43f08fc970a3ab741bf84ff10189361ea1b60c395b", size = 151352, upload-time = "2026-02-24T03:57:54.375Z" }, + { url = "https://files.pythonhosted.org/packages/e4/51/67f4d80cd58ad7eab0cd1af5fe28b961886338956b2f88c0979e21914346/ijson-3.5.0-cp314-cp314-win32.whl", hash = "sha256:63bc8121bb422f6969ced270173a3fa692c29d4ae30c860a2309941abd81012a", size = 53610, upload-time = "2026-02-24T03:57:55.655Z" }, + { url = "https://files.pythonhosted.org/packages/70/d3/263672ea22983ba3940f1534316dbc9200952c1c2a2332d7a664e4eaa7ae/ijson-3.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:01b6dad72b7b7df225ef970d334556dfad46c696a2c6767fb5d9ed8889728bca", size = 56301, upload-time = "2026-02-24T03:57:56.584Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d9/86f7fac35e0835faa188085ae0579e813493d5261ce056484015ad533445/ijson-3.5.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:2ea4b676ec98e374c1df400a47929859e4fa1239274339024df4716e802aa7e4", size = 93069, upload-time = "2026-02-24T03:57:57.849Z" }, + { url = "https://files.pythonhosted.org/packages/33/d2/e7366ed9c6e60228d35baf4404bac01a126e7775ea8ce57f560125ed190a/ijson-3.5.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:014586eec043e23c80be9a923c56c3a0920a0f1f7d17478ce7bc20ba443968ef", size = 62767, upload-time = "2026-02-24T03:57:58.758Z" }, + { url = "https://files.pythonhosted.org/packages/35/8b/3e703e8cc4b3ada79f13b28070b51d9550c578f76d1968657905857b2ddd/ijson-3.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5b8b886b0248652d437f66e7c5ac318bbdcb2c7137a7e5327a68ca00b286f5f", size = 62467, upload-time = "2026-02-24T03:58:00.261Z" }, + { url = "https://files.pythonhosted.org/packages/21/42/0c91af32c1ee8a957fdac2e051b5780756d05fd34e4b60d94a08d51bac1d/ijson-3.5.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:498fd46ae2349297e43acf97cdc421e711dbd7198418677259393d2acdc62d78", size = 200447, upload-time = "2026-02-24T03:58:01.591Z" }, + { url = "https://files.pythonhosted.org/packages/f9/80/796ea0e391b7e2d45c5b1b451734bba03f81c2984cf955ea5eaa6c4920ad/ijson-3.5.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22a51b4f9b81f12793731cf226266d1de2112c3c04ba4a04117ad4e466897e05", size = 217820, upload-time = "2026-02-24T03:58:02.598Z" }, + { url = "https://files.pythonhosted.org/packages/38/14/52b6613fdda4078c62eb5b4fe3efc724ddc55a4ad524c93de51830107aa3/ijson-3.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9636c710dc4ac4a281baa266a64f323b4cc165cec26836af702c44328b59a515", size = 208310, upload-time = "2026-02-24T03:58:04.759Z" }, + { url = "https://files.pythonhosted.org/packages/6a/ad/8b3105a78774fd4a65e534a21d975ef3a77e189489fe3029ebcaeba5e243/ijson-3.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f7168a39e8211107666d71b25693fd1b2bac0b33735ef744114c403c6cac21e1", size = 211843, upload-time = "2026-02-24T03:58:05.836Z" }, + { url = "https://files.pythonhosted.org/packages/36/ab/a2739f6072d6e1160581bc3ed32da614c8cced023dcd519d9c5fa66e0425/ijson-3.5.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8696454245415bc617ab03b0dc3ae4c86987df5dc6a90bad378fe72c5409d89e", size = 200906, upload-time = "2026-02-24T03:58:07.788Z" }, + { url = "https://files.pythonhosted.org/packages/6d/5e/e06c2de3c3d4a9cfb655c1ad08a68fb72838d271072cdd3196576ac4431a/ijson-3.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c21bfb61f71f191565885bf1bc29e0a186292d866b4880637b833848360bdc1b", size = 205495, upload-time = "2026-02-24T03:58:09.163Z" }, + { url = "https://files.pythonhosted.org/packages/7c/11/778201eb2e202ddd76b36b0fb29bf3d8e3c167389d8aa883c62524e49f47/ijson-3.5.0-cp314-cp314t-win32.whl", hash = "sha256:a2619460d6795b70d0155e5bf016200ac8a63ab5397aa33588bb02b6c21759e6", size = 56280, upload-time = "2026-02-24T03:58:10.116Z" }, + { url = "https://files.pythonhosted.org/packages/23/28/96711503245339084c8086b892c47415895eba49782d6cc52d9f4ee50301/ijson-3.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:4f24b78d4ef028d17eb57ad1b16c0aed4a17bdd9badbf232dc5d9305b7e13854", size = 58965, upload-time = "2026-02-24T03:58:11.278Z" }, + { url = "https://files.pythonhosted.org/packages/d9/3b/d31ecfa63a218978617446159f3d77aab2417a5bd2885c425b176353ff78/ijson-3.5.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d64c624da0e9d692d6eb0ff63a79656b59d76bf80773a17c5b0f835e4e8ef627", size = 57715, upload-time = "2026-02-24T03:58:24.545Z" }, + { url = "https://files.pythonhosted.org/packages/30/51/b170e646d378e8cccf9637c05edb5419b00c2c4df64b0258c3af5355608e/ijson-3.5.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:876f7df73b7e0d6474f9caa729b9cdbfc8e76de9075a4887dfd689e29e85c4ca", size = 57205, upload-time = "2026-02-24T03:58:25.681Z" }, + { url = "https://files.pythonhosted.org/packages/ef/83/44dbd0231b0a8c6c14d27473d10c4e27dfbce7d5d9a833c79e3e6c33eb40/ijson-3.5.0-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e7dbff2c8d9027809b0cde663df44f3210da10ea377121d42896fb6ee405dd31", size = 71229, upload-time = "2026-02-24T03:58:27.103Z" }, + { url = "https://files.pythonhosted.org/packages/c8/98/cf84048b7c6cec888826e696a31f45bee7ebcac15e532b6be1fc4c2c9608/ijson-3.5.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4217a1edc278660679e1197c83a1a2a2d367792bfbb2a3279577f4b59b93730d", size = 71217, upload-time = "2026-02-24T03:58:28.021Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0a/e34c729a87ff67dc6540f6bcc896626158e691d433ab57db0086d73decd2/ijson-3.5.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04f0fc740311388ee745ba55a12292b722d6f52000b11acbb913982ba5fbdf87", size = 68618, upload-time = "2026-02-24T03:58:28.918Z" }, + { url = "https://files.pythonhosted.org/packages/c1/0f/e849d072f2e0afe49627de3995fc9dae54b4c804c70c0840f928d95c10e1/ijson-3.5.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:fdeee6957f92e0c114f65c55cf8fe7eabb80cfacab64eea6864060913173f66d", size = 55369, upload-time = "2026-02-24T03:58:29.839Z" }, +] + [[package]] name = "imageio" version = "2.37.3" @@ -1916,9 +2101,60 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, ] +[[package]] +name = "inspect-ai" +version = "0.3.235" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "agent-client-protocol" }, + { name = "aioboto3" }, + { name = "anyio" }, + { name = "beautifulsoup4" }, + { name = "boto3" }, + { name = "click" }, + { name = "debugpy" }, + { name = "docstring-parser" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "fastapi" }, + { name = "fsspec" }, + { name = "httpx" }, + { name = "ijson" }, + { name = "jsonlines" }, + { name = "jsonpatch" }, + { name = "jsonpath-ng" }, + { name = "jsonref" }, + { name = "jsonschema" }, + { name = "mmh3" }, + { name = "nest-asyncio2" }, + { name = "numpy" }, + { name = "platformdirs" }, + { name = "psutil" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "rich" }, + { name = "s3fs" }, + { name = "semver" }, + { name = "shortuuid" }, + { name = "sniffio" }, + { name = "tenacity" }, + { name = "textual" }, + { name = "tiktoken" }, + { name = "typing-extensions" }, + { name = "universal-pathlib" }, + { name = "uvicorn" }, + { name = "zipfile-zstd", marker = "python_full_version < '3.14'" }, + { name = "zipp" }, + { name = "zstandard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d3/9f/9b3f384517723880a84c6a37a7d881541a91dc284403f403e81d1504256b/inspect_ai-0.3.235.tar.gz", hash = "sha256:14b5fd878fe4a14109d2e4a46548c6af5fbdc06e3425bb86f13dc154e342d9cb", size = 46965608, upload-time = "2026-06-03T22:55:19.49Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/cc/93f35bc14c6036f774dab0d6c379cc3a72efd9153e19ac7c6b9f88f5cf99/inspect_ai-0.3.235-py3-none-any.whl", hash = "sha256:3690a20dc4d7fe0f52b395d0a4d2286749195a27f03350319a66d6116ececbc4", size = 36437159, upload-time = "2026-06-03T22:55:08.961Z" }, +] + [[package]] name = "ipykernel" -version = "6.30.0" +version = "7.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "appnope", marker = "sys_platform == 'darwin'" }, @@ -1936,9 +2172,9 @@ dependencies = [ { name = "tornado" }, { name = "traitlets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/38/27/9e6e30ed92f2ac53d29f70b09da8b2dc456e256148e289678fa0e825f46a/ipykernel-6.30.0.tar.gz", hash = "sha256:b7b808ddb2d261aae2df3a26ff3ff810046e6de3dfbc6f7de8c98ea0a6cb632c", size = 165125, upload-time = "2025-07-21T10:36:09.259Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/8d/b68b728e2d06b9e0051019640a40a9eb7a88fcd82c2e1b5ce70bef5ff044/ipykernel-7.2.0.tar.gz", hash = "sha256:18ed160b6dee2cbb16e5f3575858bc19d8f1fe6046a9a680c708494ce31d909e", size = 176046, upload-time = "2026-02-06T16:43:27.403Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/3d/00813c3d9b46e3dcd88bd4530e0a3c63c0509e5d8c9eff34723ea243ab04/ipykernel-6.30.0-py3-none-any.whl", hash = "sha256:fd2936e55c4a1c2ee8b1e5fa6a372b8eecc0ab1338750dee76f48fa5cca1301e", size = 117264, upload-time = "2025-07-21T10:36:06.854Z" }, + { url = "https://files.pythonhosted.org/packages/82/b9/e73d5d9f405cba7706c539aa8b311b49d4c2f3d698d9c12f815231169c71/ipykernel-7.2.0-py3-none-any.whl", hash = "sha256:3bbd4420d2b3cc105cbdf3756bfc04500b1e52f090a90716851f3916c62e1661", size = 118788, upload-time = "2026-02-06T16:43:25.149Z" }, ] [[package]] @@ -2096,6 +2332,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "joblib" version = "1.5.3" @@ -2126,6 +2371,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/62/d9ba6323b9202dd2fe166beab8a86d29465c41a0288cbe229fac60c1ab8d/jsonlines-4.0.0-py3-none-any.whl", hash = "sha256:185b334ff2ca5a91362993f42e83588a360cf95ce4b71a73548502bda52a7c55", size = 8701, upload-time = "2023-09-01T12:34:42.563Z" }, ] +[[package]] +name = "jsonpatch" +version = "1.33" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonpointer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/78/18813351fe5d63acad16aec57f94ec2b70a09e53ca98145589e185423873/jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c", size = 21699, upload-time = "2023-06-26T12:07:29.144Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/07/02e16ed01e04a374e644b575638ec7987ae846d25ad97bcc9945a3ee4b0e/jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade", size = 12898, upload-time = "2023-06-16T21:01:28.466Z" }, +] + +[[package]] +name = "jsonpath-ng" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/32/58/250751940d75c8019659e15482d548a4aa3b6ce122c515102a4bfdac50e3/jsonpath_ng-1.8.0.tar.gz", hash = "sha256:54252968134b5e549ea5b872f1df1168bd7defe1a52fed5a358c194e1943ddc3", size = 74513, upload-time = "2026-02-24T14:42:06.182Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/99/33c7d78a3fb70d545fd5411ac67a651c81602cc09c9cf0df383733f068c5/jsonpath_ng-1.8.0-py3-none-any.whl", hash = "sha256:b8dde192f8af58d646fc031fac9c99fe4d00326afc4148f1f043c601a8cfe138", size = 67844, upload-time = "2026-02-28T00:53:19.637Z" }, +] + [[package]] name = "jsonpointer" version = "3.0.0" @@ -2135,6 +2401,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942", size = 7595, upload-time = "2024-06-10T19:24:40.698Z" }, ] +[[package]] +name = "jsonref" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/0d/c1f3277e90ccdb50d33ed5ba1ec5b3f0a242ed8c1b1a85d3afeb68464dca/jsonref-1.1.0.tar.gz", hash = "sha256:32fe8e1d85af0fdefbebce950af85590b22b60f9e95443176adbde4e1ecea552", size = 8814, upload-time = "2023-01-16T16:10:04.455Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/ec/e1db9922bceb168197a558a2b8c03a7963f1afe93517ddd3cf99f202f996/jsonref-1.1.0-py3-none-any.whl", hash = "sha256:590dc7773df6c21cbf948b5dac07a72a251db28b0238ceecce0a2abfa8ec30a9", size = 9425, upload-time = "2023-01-16T16:10:02.255Z" }, +] + [[package]] name = "jsonschema" version = "4.25.0" @@ -2194,7 +2469,7 @@ wheels = [ [[package]] name = "jupyter-client" -version = "8.6.3" +version = "8.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jupyter-core" }, @@ -2202,10 +2477,11 @@ dependencies = [ { name = "pyzmq" }, { name = "tornado" }, { name = "traitlets" }, + { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/71/22/bf9f12fdaeae18019a468b68952a60fe6dbab5d67cd2a103cac7659b41ca/jupyter_client-8.6.3.tar.gz", hash = "sha256:35b3a0947c4a6e9d589eb97d7d4cd5e90f910ee73101611f01283732bd6d9419", size = 342019, upload-time = "2024-09-17T10:44:17.613Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/a2/9ef7832e6c7d619a35bf6732d199a850ef3b6db4af1cd783a71f81eaeab0/jupyter_client-8.9.0.tar.gz", hash = "sha256:23c0c182e1901ffdab96b5a02cb7bc6f0b04524fd7fc43688a14c4ff2308fb77", size = 358714, upload-time = "2026-06-05T12:17:44.014Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/11/85/b0394e0b6fcccd2c1eeefc230978a6f8cb0c5df1e4cd3e7625735a0d7d1e/jupyter_client-8.6.3-py3-none-any.whl", hash = "sha256:e8a19cc986cc45905ac3362915f410f3af85424b4c0905e94fa5f2cb08e8f23f", size = 106105, upload-time = "2024-09-17T10:44:15.218Z" }, + { url = "https://files.pythonhosted.org/packages/ef/ce/93ccaca54d41327491b1f6d7341d0eef49f71e8929f875b53c45446335ca/jupyter_client-8.9.0-py3-none-any.whl", hash = "sha256:a0efc16adcec2bb6669d2cf91e3ba5337b338bd1ecd0d9c70940752fcb1144b2", size = 109723, upload-time = "2026-06-05T12:17:42.135Z" }, ] [[package]] @@ -2261,6 +2537,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e2/48/577993f1f99c552f18a0428731a755e06171f9902fa118c379eb7c04ea22/jupyter_events-0.12.0-py3-none-any.whl", hash = "sha256:6464b2fa5ad10451c3d35fabc75eab39556ae1e2853ad0c0cc31b656731a97fb", size = 19430, upload-time = "2025-02-03T17:23:38.643Z" }, ] +[[package]] +name = "jupyter-lsp" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-server" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/ff/1e4a61f5170a9a1d978f3ac3872449de6c01fc71eaf89657824c878b1549/jupyter_lsp-2.3.1.tar.gz", hash = "sha256:fdf8a4aa7d85813976d6e29e95e6a2c8f752701f926f2715305249a3829805a6", size = 55677, upload-time = "2026-04-02T08:10:06.749Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/e8/9d61dcbd1dce8ef418f06befd4ac084b4720429c26b0b1222bc218685eff/jupyter_lsp-2.3.1-py3-none-any.whl", hash = "sha256:71b954d834e85ff3096400554f2eefaf7fe37053036f9a782b0f7c5e42dadb81", size = 77513, upload-time = "2026-04-02T08:10:01.753Z" }, +] + [[package]] name = "jupyter-server" version = "2.16.0" @@ -2291,19 +2579,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/1f/5ebbced977171d09a7b0c08a285ff9a20aafb9c51bde07e52349ff1ddd71/jupyter_server-2.16.0-py3-none-any.whl", hash = "sha256:3d8db5be3bc64403b1c65b400a1d7f4647a5ce743f3b20dbdefe8ddb7b55af9e", size = 386904, upload-time = "2025-05-12T16:44:43.335Z" }, ] -[[package]] -name = "jupyter-server-fileid" -version = "0.9.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jupyter-events" }, - { name = "jupyter-server" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0d/eb/7c2c09454bbf66b3727ba8c431d16159d642c0eb1aa179412a4f7af721cf/jupyter_server_fileid-0.9.3.tar.gz", hash = "sha256:521608bb87f606a8637fcbdce2f3d24a8b3cc89d2eef61751cb40e468d4e54be", size = 54959, upload-time = "2024-09-06T07:18:40.412Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/d6/5e5bca083664b1dd368e261107cbe2d350e3bdc62bdba8720fdbb9b9db39/jupyter_server_fileid-0.9.3-py3-none-any.whl", hash = "sha256:f73c01c19f90005d3fff93607b91b4955ba4e1dccdde9bfe8026646f94053791", size = 16922, upload-time = "2024-09-06T07:18:38.445Z" }, -] - [[package]] name = "jupyter-server-terminals" version = "0.5.3" @@ -2317,54 +2592,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/2d/2b32cdbe8d2a602f697a649798554e4f072115438e92249624e532e8aca6/jupyter_server_terminals-0.5.3-py3-none-any.whl", hash = "sha256:41ee0d7dc0ebf2809c668e0fc726dfaf258fcd3e769568996ca731b6194ae9aa", size = 13656, upload-time = "2024-03-12T14:37:00.708Z" }, ] -[[package]] -name = "jupyter-server-ydoc" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jupyter-server-fileid" }, - { name = "jupyter-ydoc" }, - { name = "ypy-websocket" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2b/a5/a2f366d772d7da8bc36f67eadd08707610512685e266305f5e59fe317c26/jupyter_server_ydoc-0.8.0.tar.gz", hash = "sha256:a6fe125091792d16c962cc3720c950c2b87fcc8c3ecf0c54c84e9a20b814526c", size = 25769, upload-time = "2023-03-05T11:22:23.828Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/e4/b9e9b31e5b0d91b4ef749195a19a02f565f6edfb3a845d8dd457031578e3/jupyter_server_ydoc-0.8.0-py3-none-any.whl", hash = "sha256:969a3a1a77ed4e99487d60a74048dc9fa7d3b0dcd32e60885d835bbf7ba7be11", size = 11515, upload-time = "2023-03-05T11:22:21.199Z" }, -] - -[[package]] -name = "jupyter-ydoc" -version = "0.2.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "y-py" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/01/11/4bc1c63aad398095171848d66940f7cf057cc7c1d04ecbeb4119a4a2b22d/jupyter_ydoc-0.2.5.tar.gz", hash = "sha256:5a02ca7449f0d875f73e8cb8efdf695dddef15a8e71378b1f4eda6b7c90f5382", size = 64912, upload-time = "2023-07-18T10:25:51.044Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/36/66e6cb851a43c95f00f47b36d2cb3e17d37f862449dc8258b2c04f02544b/jupyter_ydoc-0.2.5-py3-none-any.whl", hash = "sha256:5759170f112c70320a84217dd98d287699076ae65a7f88d458d57940a9f2b882", size = 6196, upload-time = "2023-07-18T10:25:49.011Z" }, -] - [[package]] name = "jupyterlab" -version = "3.6.8" +version = "4.5.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "ipython", version = "9.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "async-lru" }, + { name = "httpx" }, + { name = "ipykernel" }, { name = "jinja2" }, { name = "jupyter-core" }, + { name = "jupyter-lsp" }, { name = "jupyter-server" }, - { name = "jupyter-server-ydoc" }, - { name = "jupyter-ydoc" }, { name = "jupyterlab-server" }, - { name = "nbclassic" }, - { name = "notebook" }, + { name = "notebook-shim" }, { name = "packaging" }, + { name = "setuptools" }, { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "tornado" }, + { name = "traitlets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e2/b8/b4aeb5f2a6a6e1c05867b16ac436945e711bde15083cc2c8cb968b9e0c2b/jupyterlab-3.6.8.tar.gz", hash = "sha256:a2477383e23f20009188bd9dac7e6e38dbc54307bc36d716bea6ced450647c97", size = 16854256, upload-time = "2024-08-26T20:20:40.593Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0e/74/089613e6099e851a6130816f2df592c839d8565f8746a701edada05a33e4/jupyterlab-4.5.8.tar.gz", hash = "sha256:af54d7242cc689a1e6c3ad213cc9b6d9781787d9ec67c52ec9a8f4707088cadd", size = 23994076, upload-time = "2026-06-04T12:32:12.906Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/09/34b872cee73d02a960523bb8edccdcc5b76d4fc75ffa3f0f04f701cd5137/jupyterlab-3.6.8-py3-none-any.whl", hash = "sha256:891284e75158998e23eb7a23ecc4caaf27b365e41adca374109b1305b9f769db", size = 8864730, upload-time = "2024-08-26T20:20:35.645Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d1/56a400100559cbf154a23cd29989261941ae5c9f743898fc10e8a5508b7c/jupyterlab-4.5.8-py3-none-any.whl", hash = "sha256:7d514c856d0d607601ec7692374da4f26e2aaf3b6e7cd363136b422a50588d6c", size = 12449443, upload-time = "2026-06-04T12:32:08.442Z" }, ] [[package]] @@ -2378,7 +2628,7 @@ wheels = [ [[package]] name = "jupyterlab-server" -version = "2.27.3" +version = "2.28.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "babel" }, @@ -2389,9 +2639,9 @@ dependencies = [ { name = "packaging" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0a/c9/a883ce65eb27905ce77ace410d83587c82ea64dc85a48d1f7ed52bcfa68d/jupyterlab_server-2.27.3.tar.gz", hash = "sha256:eb36caca59e74471988f0ae25c77945610b887f777255aa21f8065def9e51ed4", size = 76173, upload-time = "2024-07-16T17:02:04.149Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/2c/90153f189e421e93c4bb4f9e3f59802a1f01abd2ac5cf40b152d7f735232/jupyterlab_server-2.28.0.tar.gz", hash = "sha256:35baa81898b15f93573e2deca50d11ac0ae407ebb688299d3a5213265033712c", size = 76996, upload-time = "2025-10-22T13:59:18.37Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/09/2032e7d15c544a0e3cd831c51d77a8ca57f7555b2e1b2922142eddb02a84/jupyterlab_server-2.27.3-py3-none-any.whl", hash = "sha256:e697488f66c3db49df675158a77b3b017520d772c6e1548c7d9bcc5df7944ee4", size = 59700, upload-time = "2024-07-16T17:02:01.115Z" }, + { url = "https://files.pythonhosted.org/packages/e0/07/a000fe835f76b7e1143242ab1122e6362ef1c03f23f83a045c38859c2ae0/jupyterlab_server-2.28.0-py3-none-any.whl", hash = "sha256:e4355b148fdcf34d312bbbc80f22467d6d20460e8b8736bf235577dd18506968", size = 59830, upload-time = "2025-10-22T13:59:16.767Z" }, ] [[package]] @@ -2703,6 +2953,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d3/8f/da755d6d517eb8ec9664afae967b00a9b8dd567bbbb350e261359c1b47fc/libcst-1.8.2-cp313-cp313t-win_arm64.whl", hash = "sha256:4f14f5045766646ed9e8826b959c6d07194788babed1e0ba08c94ea4f39517e3", size = 1974355, upload-time = "2025-06-13T20:56:18.064Z" }, ] +[[package]] +name = "linkify-it-py" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "uc-micro-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/c9/06ea13676ef354f0af6169587ae292d3e2406e212876a413bf9eece4eb23/linkify_it_py-2.1.0.tar.gz", hash = "sha256:43360231720999c10e9328dc3691160e27a718e280673d444c38d7d3aaa3b98b", size = 29158, upload-time = "2026-03-01T07:48:47.683Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/de/88b3be5c31b22333b3ca2f6ff1de4e863d8fe45aaea7485f591970ec1d3e/linkify_it_py-2.1.0-py3-none-any.whl", hash = "sha256:0d252c1594ecba2ecedc444053db5d3a9b7ec1b0dd929c8f1d74dce89f86c05e", size = 19878, upload-time = "2026-03-01T07:48:46.098Z" }, +] + [[package]] name = "lit-nlp" version = "1.3.1" @@ -2944,6 +3206,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload-time = "2023-06-03T06:41:11.019Z" }, ] +[package.optional-dependencies] +linkify = [ + { name = "linkify-it-py" }, +] +plugins = [ + { name = "mdit-py-plugins" }, +] + [[package]] name = "markupsafe" version = "3.0.2" @@ -3146,6 +3416,120 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/8a/18d4ff2c7bd83f30d6924bd4ad97abf418488c3f908dea228d6f0961ad68/ml_collections-1.1.0-py3-none-any.whl", hash = "sha256:23b6fa4772aac1ae745a96044b925a5746145a70734f087eaca6626e92c05cbc", size = 76707, upload-time = "2025-04-17T08:24:59.038Z" }, ] +[[package]] +name = "mmh3" +version = "5.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/1a/edb23803a168f070ded7a3014c6d706f63b90c84ccc024f89d794a3b7a6d/mmh3-5.2.1.tar.gz", hash = "sha256:bbea5b775f0ac84945191fb83f845a6fd9a21a03ea7f2e187defac7e401616ad", size = 33775, upload-time = "2026-03-05T15:55:57.716Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/bb/88ee54afa5644b0f35ab5b435f208394feb963e5bb47c4e404deb625ffa4/mmh3-5.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5d87a3584093e1a89987e3d36d82c98d9621b2cb944e22a420aa1401e096758f", size = 56080, upload-time = "2026-03-05T15:53:40.452Z" }, + { url = "https://files.pythonhosted.org/packages/cc/bf/5404c2fd6ac84819e8ff1b7e34437b37cf55a2b11318894909e7bb88de3f/mmh3-5.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:30e4d2084df019880d55f6f7bea35328d9b464ebee090baa372c096dc77556fb", size = 40462, upload-time = "2026-03-05T15:53:41.751Z" }, + { url = "https://files.pythonhosted.org/packages/de/0b/52bffad0b52ae4ea53e222b594bd38c08ecac1fc410323220a7202e43da5/mmh3-5.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0bbc17250b10d3466875a40a52520a6bac3c02334ca709207648abd3c223ed5c", size = 40077, upload-time = "2026-03-05T15:53:42.753Z" }, + { url = "https://files.pythonhosted.org/packages/a0/9e/326c93d425b9fa4cbcdc71bc32aaba520db37577d632a24d25d927594eca/mmh3-5.2.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:76219cd1eefb9bf4af7856e3ae563d15158efa145c0aab01e9933051a1954045", size = 95302, upload-time = "2026-03-05T15:53:43.867Z" }, + { url = "https://files.pythonhosted.org/packages/c6/b1/e20d5f0d19c4c0f3df213fa7dcfa0942c4fb127d38e11f398ae8ddf6cccc/mmh3-5.2.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb9d44c25244e11c8be3f12c938ca8ba8404620ef8092245d2093c6ab3df260f", size = 101174, upload-time = "2026-03-05T15:53:45.194Z" }, + { url = "https://files.pythonhosted.org/packages/7f/4a/1a9bb3e33c18b1e1cee2c249a3053c4d4d9c93ecb30738f39a62249a7e86/mmh3-5.2.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d5d542bf2abd0fd0361e8017d03f7cb5786214ceb4a40eef1539d6585d93386", size = 103979, upload-time = "2026-03-05T15:53:46.334Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8d/dab9ee7545429e7acdd38d23d0104471d31de09a0c695f1b751e0ff34532/mmh3-5.2.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:08043f7cb1fb9467c3fbbbaea7896986e7fbc81f4d3fd9289a73d9110ab6207a", size = 110898, upload-time = "2026-03-05T15:53:47.443Z" }, + { url = "https://files.pythonhosted.org/packages/72/08/408f11af7fe9e76b883142bb06536007cc7f237be2a5e9ad4e837716e627/mmh3-5.2.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:add7ac388d1e0bf57259afbcf9ed05621a3bf11ce5ee337e7536f1e1aaf056b0", size = 118308, upload-time = "2026-03-05T15:53:49.1Z" }, + { url = "https://files.pythonhosted.org/packages/86/2d/0551be7fe0000736d9ad12ffa1f130d7a0c17b49193d6dc41c82bd9404c6/mmh3-5.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:41105377f6282e8297f182e393a79cfffd521dde37ace52b106373bdcd9ca5cb", size = 101671, upload-time = "2026-03-05T15:53:50.317Z" }, + { url = "https://files.pythonhosted.org/packages/44/17/6e4f80c4e6ad590139fa2017c3aeca54e7cc9ef68e08aa142a0c90f40a97/mmh3-5.2.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3cb61db880ec11e984348227b333259994c2c85caa775eb7875decb3768db890", size = 96682, upload-time = "2026-03-05T15:53:51.48Z" }, + { url = "https://files.pythonhosted.org/packages/ad/a7/b82fccd38c1fa815de72e94ebe9874562964a10e21e6c1bc3b01d3f15a0e/mmh3-5.2.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b5378de2b139c3a830f0209c1e91f7705919a4b3e563a10955104f5097a70a", size = 110287, upload-time = "2026-03-05T15:53:52.68Z" }, + { url = "https://files.pythonhosted.org/packages/a8/a1/2644069031c8cec0be46f0346f568a53f42fddd843f03cc890306699c1e2/mmh3-5.2.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e904f2417f0d6f6d514f3f8b836416c360f306ddaee1f84de8eef1e722d212e5", size = 111899, upload-time = "2026-03-05T15:53:53.791Z" }, + { url = "https://files.pythonhosted.org/packages/51/7b/6614f3eb8fb33f931fa7616c6d477247e48ec6c5082b02eeeee998cffa94/mmh3-5.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f1fbb0a99125b1287c6d9747f937dc66621426836d1a2d50d05aecfc81911b57", size = 100078, upload-time = "2026-03-05T15:53:55.234Z" }, + { url = "https://files.pythonhosted.org/packages/27/9a/dd4d5a5fb893e64f71b42b69ecae97dd78db35075412488b24036bc5599c/mmh3-5.2.1-cp310-cp310-win32.whl", hash = "sha256:b4cce60d0223074803c9dbe0721ad3fa51dafe7d462fee4b656a1aa01ee07518", size = 40756, upload-time = "2026-03-05T15:53:56.319Z" }, + { url = "https://files.pythonhosted.org/packages/c9/34/0b25889450f8aeffcec840aa73251e853f059c1b72ed1d1c027b956f95f5/mmh3-5.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:6f01f044112d43a20be2f13a11683666d87151542ad627fe41a18b9791d2802f", size = 41519, upload-time = "2026-03-05T15:53:57.41Z" }, + { url = "https://files.pythonhosted.org/packages/fd/31/8fd42e3c526d0bcb1db7f569c0de6729e180860a0495e387a53af33c2043/mmh3-5.2.1-cp310-cp310-win_arm64.whl", hash = "sha256:7501e9be34cb21e72fcfe672aafd0eee65c16ba2afa9dcb5500a587d3a0580f0", size = 39285, upload-time = "2026-03-05T15:53:58.697Z" }, + { url = "https://files.pythonhosted.org/packages/65/d7/3312a59df3c1cdd783f4cf0c4ee8e9decff9c5466937182e4cc7dbbfe6c5/mmh3-5.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:dae0f0bd7d30c0ad61b9a504e8e272cb8391eed3f1587edf933f4f6b33437450", size = 56082, upload-time = "2026-03-05T15:53:59.702Z" }, + { url = "https://files.pythonhosted.org/packages/61/96/6f617baa098ca0d2989bfec6d28b5719532cd8d8848782662f5b755f657f/mmh3-5.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9aeaf53eaa075dd63e81512522fd180097312fb2c9f476333309184285c49ce0", size = 40458, upload-time = "2026-03-05T15:54:01.548Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b4/9cd284bd6062d711e13d26c04d4778ab3f690c1c38a4563e3c767ec8802e/mmh3-5.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0634581290e6714c068f4aa24020acf7880927d1f0084fa753d9799ae9610082", size = 40079, upload-time = "2026-03-05T15:54:02.743Z" }, + { url = "https://files.pythonhosted.org/packages/f6/09/a806334ce1d3d50bf782b95fcee8b3648e1e170327d4bb7b4bad2ad7d956/mmh3-5.2.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e080c0637aea036f35507e803a4778f119a9b436617694ae1c5c366805f1e997", size = 97242, upload-time = "2026-03-05T15:54:04.536Z" }, + { url = "https://files.pythonhosted.org/packages/ee/93/723e317dd9e041c4dc4566a2eb53b01ad94de31750e0b834f1643905e97c/mmh3-5.2.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:db0562c5f71d18596dcd45e854cf2eeba27d7543e1a3acdafb7eef728f7fe85d", size = 103082, upload-time = "2026-03-05T15:54:06.387Z" }, + { url = "https://files.pythonhosted.org/packages/61/b5/f96121e69cc48696075071531cf574f112e1ffd08059f4bffb41210e6fc5/mmh3-5.2.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d9f9a3ce559a5267014b04b82956993270f63ec91765e13e9fd73daf2d2738e", size = 106054, upload-time = "2026-03-05T15:54:07.506Z" }, + { url = "https://files.pythonhosted.org/packages/82/49/192b987ec48d0b2aecf8ac285a9b11fbc00030f6b9c694664ae923458dde/mmh3-5.2.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:960b1b3efa39872ac8b6cc3a556edd6fb90ed74f08c9c45e028f1005b26aa55d", size = 112910, upload-time = "2026-03-05T15:54:09.403Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a1/03e91fd334ed0144b83343a76eb11f17434cd08f746401488cfeafb2d241/mmh3-5.2.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d30b650595fdbe32366b94cb14f30bb2b625e512bd4e1df00611f99dc5c27fd4", size = 120551, upload-time = "2026-03-05T15:54:10.587Z" }, + { url = "https://files.pythonhosted.org/packages/93/b9/b89a71d2ff35c3a764d1c066c7313fc62c7cc48fa48a4b3b0304a4a0146f/mmh3-5.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:82f3802bfc4751f420d591c5c864de538b71cea117fce67e4595c2afede08a15", size = 99096, upload-time = "2026-03-05T15:54:11.76Z" }, + { url = "https://files.pythonhosted.org/packages/36/b5/613772c1c6ed5f7b63df55eb131e887cc43720fec392777b95a79d34e640/mmh3-5.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:915e7a2418f10bd1151b1953df06d896db9783c9cfdb9a8ee1f9b3a4331ab503", size = 98524, upload-time = "2026-03-05T15:54:13.122Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0e/1524566fe8eaf871e4f7bc44095929fcd2620488f402822d848df19d679c/mmh3-5.2.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:fc78739b5ec6e4fb02301984a3d442a91406e7700efbe305071e7fd1c78278f2", size = 106239, upload-time = "2026-03-05T15:54:14.601Z" }, + { url = "https://files.pythonhosted.org/packages/04/94/21adfa7d90a7a697137ad6de33eeff6445420ca55e433a5d4919c79bc3b5/mmh3-5.2.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:41aac7002a749f08727cb91babff1daf8deac317c0b1f317adc69be0e6c375d1", size = 109797, upload-time = "2026-03-05T15:54:15.819Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e6/1aacc3a219e1aa62fa65669995d4a3562b35be5200ec03680c7e4bec9676/mmh3-5.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9d8089d853c7963a8ce87fff93e2a67075c0bc08684a08ea6ad13577c38ffc38", size = 97228, upload-time = "2026-03-05T15:54:16.992Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b9/5e4cca8dcccf298add0a27f3c357bc8cf8baf821d35cdc6165e4bd5a48b0/mmh3-5.2.1-cp311-cp311-win32.whl", hash = "sha256:baeb47635cb33375dee4924cd93d7f5dcaa786c740b08423b0209b824a1ee728", size = 40751, upload-time = "2026-03-05T15:54:18.714Z" }, + { url = "https://files.pythonhosted.org/packages/72/fc/5b11d49247f499bcda591171e9cf3b6ee422b19e70aa2cef2e0ae65ca3b9/mmh3-5.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:1e4ecee40ba19e6975e1120829796770325841c2f153c0e9aecca927194c6a2a", size = 41517, upload-time = "2026-03-05T15:54:19.764Z" }, + { url = "https://files.pythonhosted.org/packages/8a/5f/2a511ee8a1c2a527c77726d5231685b72312c5a1a1b7639ad66a9652aa84/mmh3-5.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:c302245fd6c33d96bd169c7ccf2513c20f4c1e417c07ce9dce107c8bc3f8411f", size = 39287, upload-time = "2026-03-05T15:54:20.904Z" }, + { url = "https://files.pythonhosted.org/packages/92/94/bc5c3b573b40a328c4d141c20e399039ada95e5e2a661df3425c5165fd84/mmh3-5.2.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0cc21533878e5586b80d74c281d7f8da7932bc8ace50b8d5f6dbf7e3935f63f1", size = 56087, upload-time = "2026-03-05T15:54:21.92Z" }, + { url = "https://files.pythonhosted.org/packages/f6/80/64a02cc3e95c3af0aaa2590849d9ed24a9f14bb93537addde688e039b7c3/mmh3-5.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4eda76074cfca2787c8cf1bec603eaebdddd8b061ad5502f85cddae998d54f00", size = 40500, upload-time = "2026-03-05T15:54:22.953Z" }, + { url = "https://files.pythonhosted.org/packages/8b/72/e6d6602ce18adf4ddcd0e48f2e13590cc92a536199e52109f46f259d3c46/mmh3-5.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:eee884572b06bbe8a2b54f424dbd996139442cf83c76478e1ec162512e0dd2c7", size = 40034, upload-time = "2026-03-05T15:54:23.943Z" }, + { url = "https://files.pythonhosted.org/packages/59/c2/bf4537a8e58e21886ef16477041238cab5095c836496e19fafc34b7445d2/mmh3-5.2.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0d0b7e803191db5f714d264044e06189c8ccd3219e936cc184f07106bd17fd7b", size = 97292, upload-time = "2026-03-05T15:54:25.335Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e2/51ed62063b44d10b06d975ac87af287729eeb5e3ed9772f7584a17983e90/mmh3-5.2.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8e6c219e375f6341d0959af814296372d265a8ca1af63825f65e2e87c618f006", size = 103274, upload-time = "2026-03-05T15:54:26.44Z" }, + { url = "https://files.pythonhosted.org/packages/75/ce/12a7524dca59eec92e5b31fdb13ede1e98eda277cf2b786cf73bfbc24e81/mmh3-5.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26fb5b9c3946bf7f1daed7b37e0c03898a6f062149127570f8ede346390a0825", size = 106158, upload-time = "2026-03-05T15:54:28.578Z" }, + { url = "https://files.pythonhosted.org/packages/86/1f/d3ba6dd322d01ab5d44c46c8f0c38ab6bbbf9b5e20e666dfc05bf4a23604/mmh3-5.2.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3c38d142c706201db5b2345166eeef1e7740e3e2422b470b8ba5c8727a9b4c7a", size = 113005, upload-time = "2026-03-05T15:54:29.767Z" }, + { url = "https://files.pythonhosted.org/packages/b6/a9/15d6b6f913294ea41b44d901741298e3718e1cb89ee626b3694625826a43/mmh3-5.2.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50885073e2909251d4718634a191c49ae5f527e5e1736d738e365c3e8be8f22b", size = 120744, upload-time = "2026-03-05T15:54:30.931Z" }, + { url = "https://files.pythonhosted.org/packages/76/b3/70b73923fd0284c439860ff5c871b20210dfdbe9a6b9dd0ee6496d77f174/mmh3-5.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b3f99e1756fc48ad507b95e5d86f2fb21b3d495012ff13e6592ebac14033f166", size = 99111, upload-time = "2026-03-05T15:54:32.353Z" }, + { url = "https://files.pythonhosted.org/packages/dd/38/99f7f75cd27d10d8b899a1caafb9d531f3903e4d54d572220e3d8ac35e89/mmh3-5.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62815d2c67f2dd1be76a253d88af4e1da19aeaa1820146dec52cf8bee2958b16", size = 98623, upload-time = "2026-03-05T15:54:33.801Z" }, + { url = "https://files.pythonhosted.org/packages/fd/68/6e292c0853e204c44d2f03ea5f090be3317a0e2d9417ecb62c9eb27687df/mmh3-5.2.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8f767ba0911602ddef289404e33835a61168314ebd3c729833db2ed685824211", size = 106437, upload-time = "2026-03-05T15:54:35.177Z" }, + { url = "https://files.pythonhosted.org/packages/dd/c6/fedd7284c459cfb58721d461fcf5607a4c1f5d9ab195d113d51d10164d16/mmh3-5.2.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:67e41a497bac88cc1de96eeba56eeb933c39d54bc227352f8455aa87c4ca4000", size = 110002, upload-time = "2026-03-05T15:54:36.673Z" }, + { url = "https://files.pythonhosted.org/packages/3b/ac/ca8e0c19a34f5b71390171d2ff0b9f7f187550d66801a731bb68925126a4/mmh3-5.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3d74a03fb57757ece25aa4b3c1c60157a1cece37a020542785f942e2f827eed5", size = 97507, upload-time = "2026-03-05T15:54:37.804Z" }, + { url = "https://files.pythonhosted.org/packages/df/94/6ebb9094cfc7ac5e7950776b9d13a66bb4a34f83814f32ba2abc9494fc68/mmh3-5.2.1-cp312-cp312-win32.whl", hash = "sha256:7374d6e3ef72afe49697ecd683f3da12f4fc06af2d75433d0580c6746d2fa025", size = 40773, upload-time = "2026-03-05T15:54:40.077Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/cd3527198cf159495966551c84a5f36805a10ac17b294f41f67b83f6a4d6/mmh3-5.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:3a9fed49c6ce4ed7e73f13182760c65c816da006debe67f37635580dfb0fae00", size = 41560, upload-time = "2026-03-05T15:54:41.148Z" }, + { url = "https://files.pythonhosted.org/packages/15/96/6fe5ebd0f970a076e3ed5512871ce7569447b962e96c125528a2f9724470/mmh3-5.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:bbfcb95d9a744e6e2827dfc66ad10e1020e0cac255eb7f85652832d5a264c2fc", size = 39313, upload-time = "2026-03-05T15:54:42.171Z" }, + { url = "https://files.pythonhosted.org/packages/25/a5/9daa0508a1569a54130f6198d5462a92deda870043624aa3ea72721aa765/mmh3-5.2.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:723b2681ed4cc07d3401bbea9c201ad4f2a4ca6ba8cddaff6789f715dd2b391e", size = 40832, upload-time = "2026-03-05T15:54:43.212Z" }, + { url = "https://files.pythonhosted.org/packages/0a/6b/3230c6d80c1f4b766dedf280a92c2241e99f87c1504ff74205ec8cebe451/mmh3-5.2.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:3619473a0e0d329fd4aec8075628f8f616be2da41605300696206d6f36920c3d", size = 41964, upload-time = "2026-03-05T15:54:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/62/fb/648bfddb74a872004b6ee751551bfdda783fe6d70d2e9723bad84dbe5311/mmh3-5.2.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e48d4dbe0f88e53081da605ae68644e5182752803bbc2beb228cca7f1c4454d6", size = 39114, upload-time = "2026-03-05T15:54:45.205Z" }, + { url = "https://files.pythonhosted.org/packages/95/c2/ab7901f87af438468b496728d11264cb397b3574d41506e71b92128e0373/mmh3-5.2.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a482ac121de6973897c92c2f31defc6bafb11c83825109275cffce54bb64933f", size = 39819, upload-time = "2026-03-05T15:54:46.509Z" }, + { url = "https://files.pythonhosted.org/packages/2f/ed/6f88dda0df67de1612f2e130ffea34cf84aaee5bff5b0aff4dbff2babe34/mmh3-5.2.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:17fbb47f0885ace8327ce1235d0416dc86a211dcd8cc1e703f41523be32cfec8", size = 40330, upload-time = "2026-03-05T15:54:47.864Z" }, + { url = "https://files.pythonhosted.org/packages/3d/66/7516d23f53cdf90f43fce24ab80c28f45e6851d78b46bef8c02084edf583/mmh3-5.2.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d51fde50a77f81330523562e3c2734ffdca9c4c9e9d355478117905e1cfe16c6", size = 56078, upload-time = "2026-03-05T15:54:48.9Z" }, + { url = "https://files.pythonhosted.org/packages/bc/34/4d152fdf4a91a132cb226b671f11c6b796eada9ab78080fb5ce1e95adaab/mmh3-5.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:19bbd3b841174ae6ed588536ab5e1b1fe83d046e668602c20266547298d939a9", size = 40498, upload-time = "2026-03-05T15:54:49.942Z" }, + { url = "https://files.pythonhosted.org/packages/d4/4c/8e3af1b6d85a299767ec97bd923f12b06267089c1472c27c1696870d1175/mmh3-5.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:be77c402d5e882b6fbacfd90823f13da8e0a69658405a39a569c6b58fdb17b03", size = 40033, upload-time = "2026-03-05T15:54:50.994Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f2/966ea560e32578d453c9e9db53d602cbb1d0da27317e232afa7c38ceba11/mmh3-5.2.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fd96476f04db5ceba1cfa0f21228f67c1f7402296f0e73fee3513aa680ad237b", size = 97320, upload-time = "2026-03-05T15:54:52.072Z" }, + { url = "https://files.pythonhosted.org/packages/bb/0d/2c5f9893b38aeb6b034d1a44ecd55a010148054f6a516abe53b5e4057297/mmh3-5.2.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:707151644085dd0f20fe4f4b573d28e5130c4aaa5f587e95b60989c5926653b5", size = 103299, upload-time = "2026-03-05T15:54:53.569Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fc/2ebaef4a4d4376f89761274dc274035ffd96006ab496b4ee5af9b08f21a9/mmh3-5.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3737303ca9ea0f7cb83028781148fcda4f1dac7821db0c47672971dabcf63593", size = 106222, upload-time = "2026-03-05T15:54:55.092Z" }, + { url = "https://files.pythonhosted.org/packages/57/09/ea7ffe126d0ba0406622602a2d05e1e1a6841cc92fc322eb576c95b27fad/mmh3-5.2.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2778fed822d7db23ac5008b181441af0c869455b2e7d001f4019636ac31b6fe4", size = 113048, upload-time = "2026-03-05T15:54:56.305Z" }, + { url = "https://files.pythonhosted.org/packages/85/57/9447032edf93a64aa9bef4d9aa596400b1756f40411890f77a284f6293ca/mmh3-5.2.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d57dea657357230cc780e13920d7fa7db059d58fe721c80020f94476da4ca0a1", size = 120742, upload-time = "2026-03-05T15:54:57.453Z" }, + { url = "https://files.pythonhosted.org/packages/53/82/a86cc87cc88c92e9e1a598fee509f0409435b57879a6129bf3b3e40513c7/mmh3-5.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:169e0d178cb59314456ab30772429a802b25d13227088085b0d49b9fe1533104", size = 99132, upload-time = "2026-03-05T15:54:58.583Z" }, + { url = "https://files.pythonhosted.org/packages/54/f7/6b16eb1b40ee89bb740698735574536bc20d6cdafc65ae702ea235578e05/mmh3-5.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7e4e1f580033335c6f76d1e0d6b56baf009d1a64d6a4816347e4271ba951f46d", size = 98686, upload-time = "2026-03-05T15:55:00.078Z" }, + { url = "https://files.pythonhosted.org/packages/e8/88/a601e9f32ad1410f438a6d0544298ea621f989bd34a0731a7190f7dec799/mmh3-5.2.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2bd9f19f7f1fcebd74e830f4af0f28adad4975d40d80620be19ffb2b2af56c9f", size = 106479, upload-time = "2026-03-05T15:55:01.532Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5c/ce29ae3dfc4feec4007a437a1b7435fb9507532a25147602cd5b52be86db/mmh3-5.2.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:c88653877aeb514c089d1b3d473451677b8b9a6d1497dbddf1ae7934518b06d2", size = 110030, upload-time = "2026-03-05T15:55:02.934Z" }, + { url = "https://files.pythonhosted.org/packages/13/30/ae444ef2ff87c805d525da4fa63d27cda4fe8a48e77003a036b8461cfd5c/mmh3-5.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fceef7fe67c81e1585198215e42ad3fdba3a25644beda8fbdaf85f4d7b93175a", size = 97536, upload-time = "2026-03-05T15:55:04.135Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f9/dc3787ee5c813cc27fe79f45ad4500d9b5437f23a7402435cc34e07c7718/mmh3-5.2.1-cp313-cp313-win32.whl", hash = "sha256:54b64fb2433bc71488e7a449603bf8bd31fbcf9cb56fbe1eb6d459e90b86c37b", size = 40769, upload-time = "2026-03-05T15:55:05.277Z" }, + { url = "https://files.pythonhosted.org/packages/43/67/850e0b5a1e97799822ebfc4ca0e8c6ece3ed8baf7dcdf64de817dfdda2ca/mmh3-5.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:cae6383181f1e345317742d2ddd88f9e7d2682fa4c9432e3a74e47d92dce0229", size = 41563, upload-time = "2026-03-05T15:55:06.283Z" }, + { url = "https://files.pythonhosted.org/packages/c0/cc/98c90b28e1da5458e19fbfaf4adb5289208d3bfccd45dd14eab216a2f0bb/mmh3-5.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:022aa1a528604e6c83d0a7705fdef0b5355d897a9e0fa3a8d26709ceaa06965d", size = 39310, upload-time = "2026-03-05T15:55:07.323Z" }, + { url = "https://files.pythonhosted.org/packages/63/b4/65bc1fb2bb7f83e91c30865023b1847cf89a5f237165575e8c83aa536584/mmh3-5.2.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:d771f085fcdf4035786adfb1d8db026df1eb4b41dac1c3d070d1e49512843227", size = 40794, upload-time = "2026-03-05T15:55:09.773Z" }, + { url = "https://files.pythonhosted.org/packages/c4/86/7168b3d83be8eb553897b1fac9da8bbb06568e5cfe555ffc329ebb46f59d/mmh3-5.2.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:7f196cd7910d71e9d9860da0ff7a77f64d22c1ad931f1dd18559a06e03109fc0", size = 41923, upload-time = "2026-03-05T15:55:10.924Z" }, + { url = "https://files.pythonhosted.org/packages/bf/9b/b653ab611c9060ce8ff0ba25c0226757755725e789292f3ca138a58082cd/mmh3-5.2.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b1f12bd684887a0a5d55e6363ca87056f361e45451105012d329b86ec19dbe0b", size = 39131, upload-time = "2026-03-05T15:55:11.961Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b4/5a2e0d34ab4d33543f01121e832395ea510132ea8e52cdf63926d9d81754/mmh3-5.2.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d106493a60dcb4aef35a0fac85105e150a11cf8bc2b0d388f5a33272d756c966", size = 39825, upload-time = "2026-03-05T15:55:13.013Z" }, + { url = "https://files.pythonhosted.org/packages/bd/69/81699a8f39a3f8d368bec6443435c0c392df0d200ad915bf0d222b588e03/mmh3-5.2.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:44983e45310ee5b9f73397350251cdf6e63a466406a105f1d16cb5baa659270b", size = 40344, upload-time = "2026-03-05T15:55:14.026Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b3/71c8c775807606e8fd8acc5c69016e1caf3200d50b50b6dd4b40ce10b76c/mmh3-5.2.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:368625fb01666655985391dbad3860dc0ba7c0d6b9125819f3121ee7292b4ac8", size = 56291, upload-time = "2026-03-05T15:55:15.137Z" }, + { url = "https://files.pythonhosted.org/packages/6f/75/2c24517d4b2ce9e4917362d24f274d3d541346af764430249ddcc4cb3a08/mmh3-5.2.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:72d1cc63bcc91e14933f77d51b3df899d6a07d184ec515ea7f56bff659e124d7", size = 40575, upload-time = "2026-03-05T15:55:16.518Z" }, + { url = "https://files.pythonhosted.org/packages/bf/b9/e4a360164365ac9f07a25f0f7928e3a66eb9ecc989384060747aa170e6aa/mmh3-5.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e8b4b5580280b9265af3e0409974fb79c64cf7523632d03fbf11df18f8b0181e", size = 40052, upload-time = "2026-03-05T15:55:17.735Z" }, + { url = "https://files.pythonhosted.org/packages/97/ca/120d92223a7546131bbbc31c9174168ee7a73b1366f5463ffe69d9e691fe/mmh3-5.2.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4cbbde66f1183db040daede83dd86c06d663c5bb2af6de1142b7c8c37923dd74", size = 97311, upload-time = "2026-03-05T15:55:18.959Z" }, + { url = "https://files.pythonhosted.org/packages/b6/71/c1a60c1652b8813ef9de6d289784847355417ee0f2980bca002fe87f4ae5/mmh3-5.2.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8ff038d52ef6aa0f309feeba00c5095c9118d0abf787e8e8454d6048db2037fc", size = 103279, upload-time = "2026-03-05T15:55:20.448Z" }, + { url = "https://files.pythonhosted.org/packages/48/29/ad97f4be1509cdcb28ae32c15593ce7c415db47ace37f8fad35b493faa9a/mmh3-5.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4130d0b9ce5fad6af07421b1aecc7e079519f70d6c05729ab871794eded8617", size = 106290, upload-time = "2026-03-05T15:55:21.6Z" }, + { url = "https://files.pythonhosted.org/packages/77/29/1f86d22e281bd8827ba373600a4a8b0c0eae5ca6aa55b9a8c26d2a34decc/mmh3-5.2.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6e0bfe77d238308839699944164b96a2eeccaf55f2af400f54dc20669d8d5f2", size = 113116, upload-time = "2026-03-05T15:55:22.826Z" }, + { url = "https://files.pythonhosted.org/packages/a7/7c/339971ea7ed4c12d98f421f13db3ea576a9114082ccb59d2d1a0f00ccac1/mmh3-5.2.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f963eafc0a77a6c0562397da004f5876a9bcf7265a7bcc3205e29636bc4a1312", size = 120740, upload-time = "2026-03-05T15:55:24.3Z" }, + { url = "https://files.pythonhosted.org/packages/e4/92/3c7c4bdb8e926bb3c972d1e2907d77960c1c4b250b41e8366cf20c6e4373/mmh3-5.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:92883836caf50d5255be03d988d75bc93e3f86ba247b7ca137347c323f731deb", size = 99143, upload-time = "2026-03-05T15:55:25.456Z" }, + { url = "https://files.pythonhosted.org/packages/df/0a/33dd8706e732458c8375eae63c981292de07a406bad4ec03e5269654aa2c/mmh3-5.2.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:57b52603e89355ff318025dd55158f6e71396c0f1f609d548e9ea9c94cc6ce0a", size = 98703, upload-time = "2026-03-05T15:55:26.723Z" }, + { url = "https://files.pythonhosted.org/packages/51/04/76bbce05df76cbc3d396f13b2ea5b1578ef02b6a5187e132c6c33f99d596/mmh3-5.2.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f40a95186a72fa0b67d15fef0f157bfcda00b4f59c8a07cbe5530d41ac35d105", size = 106484, upload-time = "2026-03-05T15:55:28.214Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8f/c6e204a2c70b719c1f62ffd9da27aef2dddcba875ea9c31ca0e87b975a46/mmh3-5.2.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:58370d05d033ee97224c81263af123dea3d931025030fd34b61227a768a8858a", size = 110012, upload-time = "2026-03-05T15:55:29.532Z" }, + { url = "https://files.pythonhosted.org/packages/e3/37/7181efd8e39db386c1ebc3e6b7d1f702a09d7c1197a6f2742ed6b5c16597/mmh3-5.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7be6dfb49e48fd0a7d91ff758a2b51336f1cd21f9d44b20f6801f072bd080cdd", size = 97508, upload-time = "2026-03-05T15:55:31.01Z" }, + { url = "https://files.pythonhosted.org/packages/42/0f/afa7ca2615fd85e1469474bb860e381443d0b868c083b62b41cb1d7ca32f/mmh3-5.2.1-cp314-cp314-win32.whl", hash = "sha256:54fe8518abe06a4c3852754bfd498b30cc58e667f376c513eac89a244ce781a4", size = 41387, upload-time = "2026-03-05T15:55:32.403Z" }, + { url = "https://files.pythonhosted.org/packages/71/0d/46d42a260ee1357db3d486e6c7a692e303c017968e14865e00efa10d09fc/mmh3-5.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:3f796b535008708846044c43302719c6956f39ca2d93f2edda5319e79a29efbb", size = 42101, upload-time = "2026-03-05T15:55:33.646Z" }, + { url = "https://files.pythonhosted.org/packages/a4/7b/848a8378059d96501a41159fca90d6a99e89736b0afbe8e8edffeac8c74b/mmh3-5.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:cd471ede0d802dd936b6fab28188302b2d497f68436025857ca72cd3810423fe", size = 39836, upload-time = "2026-03-05T15:55:35.026Z" }, + { url = "https://files.pythonhosted.org/packages/27/61/1dabea76c011ba8547c25d30c91c0ec22544487a8750997a27a0c9e1180b/mmh3-5.2.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:5174a697ce042fa77c407e05efe41e03aa56dae9ec67388055820fb48cf4c3ba", size = 57727, upload-time = "2026-03-05T15:55:36.162Z" }, + { url = "https://files.pythonhosted.org/packages/b7/32/731185950d1cf2d5e28979cc8593016ba1619a295faba10dda664a4931b5/mmh3-5.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0a3984146e414684a6be2862d84fcb1035f4984851cb81b26d933bab6119bf00", size = 41308, upload-time = "2026-03-05T15:55:37.254Z" }, + { url = "https://files.pythonhosted.org/packages/76/aa/66c76801c24b8c9418b4edde9b5e57c75e72c94e29c48f707e3962534f18/mmh3-5.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bd6e7d363aa93bd3421b30b6af97064daf47bc96005bddba67c5ffbc6df426b8", size = 40758, upload-time = "2026-03-05T15:55:38.61Z" }, + { url = "https://files.pythonhosted.org/packages/9e/bb/79a1f638a02f0ae389f706d13891e2fbf7d8c0a22ecde67ba828951bb60a/mmh3-5.2.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:113f78e7463a36dbbcea05bfe688efd7fa759d0f0c56e73c974d60dcfec3dfcc", size = 109670, upload-time = "2026-03-05T15:55:40.13Z" }, + { url = "https://files.pythonhosted.org/packages/26/94/8cd0e187a288985bcfc79bf5144d1d712df9dee74365f59d26e3a1865be6/mmh3-5.2.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7e8ec5f606e0809426d2440e0683509fb605a8820a21ebd120dcdba61b74ef7f", size = 117399, upload-time = "2026-03-05T15:55:42.076Z" }, + { url = "https://files.pythonhosted.org/packages/42/94/dfea6059bd5c5beda565f58a4096e43f4858fb6d2862806b8bbd12cbb284/mmh3-5.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22b0f9971ec4e07e8223f2beebe96a6cfc779d940b6f27d26604040dd74d3a44", size = 120386, upload-time = "2026-03-05T15:55:43.481Z" }, + { url = "https://files.pythonhosted.org/packages/47/cb/f9c45e62aaa67220179f487772461d891bb582bb2f9783c944832c60efd9/mmh3-5.2.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85ffc9920ffc39c5eee1e3ac9100c913a0973996fbad5111f939bbda49204bb7", size = 125924, upload-time = "2026-03-05T15:55:44.638Z" }, + { url = "https://files.pythonhosted.org/packages/a5/83/fe54a4a7c11bc9f623dfc1707decd034245602b076dfc1dcc771a4163170/mmh3-5.2.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7aec798c2b01aaa65a55f1124f3405804184373abb318a3091325aece235f67c", size = 135280, upload-time = "2026-03-05T15:55:45.866Z" }, + { url = "https://files.pythonhosted.org/packages/97/67/fe7e9e9c143daddd210cd22aef89cbc425d58ecf238d2b7d9eb0da974105/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:55dbbd8ffbc40d1697d5e2d0375b08599dae8746b0b08dea05eee4ce81648fac", size = 110050, upload-time = "2026-03-05T15:55:47.074Z" }, + { url = "https://files.pythonhosted.org/packages/43/c4/6d4b09fcbef80794de447c9378e39eefc047156b290fa3dd2d5257ca8227/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:6c85c38a279ca9295a69b9b088a2e48aa49737bb1b34e6a9dc6297c110e8d912", size = 111158, upload-time = "2026-03-05T15:55:48.239Z" }, + { url = "https://files.pythonhosted.org/packages/81/a6/ca51c864bdb30524beb055a6d8826db3906af0834ec8c41d097a6e8573d5/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:6290289fa5fb4c70fd7f72016e03633d60388185483ff3b162912c81205ae2cf", size = 116890, upload-time = "2026-03-05T15:55:49.405Z" }, + { url = "https://files.pythonhosted.org/packages/cc/04/5a1fe2e2ad843d03e89af25238cbc4f6840a8bb6c4329a98ab694c71deda/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4fc6cd65dc4d2fdb2625e288939a3566e36127a84811a4913f02f3d5931da52d", size = 123121, upload-time = "2026-03-05T15:55:50.61Z" }, + { url = "https://files.pythonhosted.org/packages/af/4d/3c820c6f4897afd25905270a9f2330a23f77a207ea7356f7aadace7273c0/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:623f938f6a039536cc02b7582a07a080f13fdfd48f87e63201d92d7e34d09a18", size = 110187, upload-time = "2026-03-05T15:55:52.143Z" }, + { url = "https://files.pythonhosted.org/packages/21/54/1d71cd143752361c0aebef16ad3f55926a6faf7b112d355745c1f8a25f7f/mmh3-5.2.1-cp314-cp314t-win32.whl", hash = "sha256:29bc3973676ae334412efdd367fcd11d036b7be3efc1ce2407ef8676dabfeb82", size = 41934, upload-time = "2026-03-05T15:55:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e4/63a2a88f31d93dea03947cccc2a076946857e799ea4f7acdecbf43b324aa/mmh3-5.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:28cfab66577000b9505a0d068c731aee7ca85cd26d4d63881fab17857e0fe1fb", size = 43036, upload-time = "2026-03-05T15:55:55.252Z" }, + { url = "https://files.pythonhosted.org/packages/a0/0f/59204bf136d1201f8d7884cfbaf7498c5b4674e87a4c693f9bde63741ce1/mmh3-5.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:dfd51b4c56b673dfbc43d7d27ef857dd91124801e2806c69bb45585ce0fa019b", size = 40391, upload-time = "2026-03-05T15:55:56.697Z" }, +] + [[package]] name = "more-itertools" version = "11.0.2" @@ -3479,6 +3863,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, ] +[[package]] +name = "nest-asyncio2" +version = "1.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/73/731debf26e27e0a0323d7bda270dc2f634b398e38f040a09da1f4351d0aa/nest_asyncio2-1.7.2.tar.gz", hash = "sha256:1921d70b92cc4612c374928d081552efb59b83d91b2b789d935c665fa01729a8", size = 14743, upload-time = "2026-02-13T00:34:04.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/3c/3179b85b0e1c3659f0369940200cd6d0fa900e6cefcc7ea0bc6dd0e29ffb/nest_asyncio2-1.7.2-py3-none-any.whl", hash = "sha256:f5dfa702f3f81f6a03857e9a19e2ba578c0946a4ad417b4c50a24d7ba641fe01", size = 7843, upload-time = "2026-02-13T00:34:02.691Z" }, +] + [[package]] name = "networkx" version = "3.4.2" @@ -3989,6 +4382,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/ac/dac4a63f978e4dcb3c6d3a78c4d8e0192a113d288502a1216950c41b1027/parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18", size = 103650, upload-time = "2024-04-05T09:43:53.299Z" }, ] +[[package]] +name = "pathlib-abc" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/cb/448649d7f25d228bf0be3a04590ab7afa77f15e056f8fa976ed05ec9a78f/pathlib_abc-0.5.2.tar.gz", hash = "sha256:fcd56f147234645e2c59c7ae22808b34c364bb231f685ddd9f96885aed78a94c", size = 33342, upload-time = "2025-10-10T18:37:20.524Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/29/c028a0731e202035f0e2e0bfbf1a3e46ad6c628cbb17f6f1cc9eea5d9ff1/pathlib_abc-0.5.2-py3-none-any.whl", hash = "sha256:4c9d94cf1b23af417ce7c0417b43333b06a106c01000b286c99de230d95eefbb", size = 19070, upload-time = "2025-10-10T18:37:19.437Z" }, +] + [[package]] name = "pathspec" version = "0.12.1" @@ -4465,7 +4867,7 @@ wheels = [ [[package]] name = "pydantic" -version = "2.11.7" +version = "2.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, @@ -4473,96 +4875,125 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/00/dd/4325abf92c39ba8623b5af936ddb36ffcfe0beae70405d456ab1fb2f5b8c/pydantic-2.11.7.tar.gz", hash = "sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db", size = 788350, upload-time = "2025-06-14T08:33:17.137Z" } +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/c0/ec2b1c8712ca690e5d61979dee872603e92b8a32f94cc1b72d53beab008a/pydantic-2.11.7-py3-none-any.whl", hash = "sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b", size = 444782, upload-time = "2025-06-14T08:33:14.905Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, ] [[package]] name = "pydantic-core" -version = "2.33.2" +version = "2.46.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ad/88/5f2260bdfae97aabf98f1778d43f69574390ad787afb646292a638c923d4/pydantic_core-2.33.2.tar.gz", hash = "sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc", size = 435195, upload-time = "2025-04-23T18:33:52.104Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/92/b31726561b5dae176c2d2c2dc43a9c5bfba5d32f96f8b4c0a600dd492447/pydantic_core-2.33.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8", size = 2028817, upload-time = "2025-04-23T18:30:43.919Z" }, - { url = "https://files.pythonhosted.org/packages/a3/44/3f0b95fafdaca04a483c4e685fe437c6891001bf3ce8b2fded82b9ea3aa1/pydantic_core-2.33.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d", size = 1861357, upload-time = "2025-04-23T18:30:46.372Z" }, - { url = "https://files.pythonhosted.org/packages/30/97/e8f13b55766234caae05372826e8e4b3b96e7b248be3157f53237682e43c/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d", size = 1898011, upload-time = "2025-04-23T18:30:47.591Z" }, - { url = "https://files.pythonhosted.org/packages/9b/a3/99c48cf7bafc991cc3ee66fd544c0aae8dc907b752f1dad2d79b1b5a471f/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572", size = 1982730, upload-time = "2025-04-23T18:30:49.328Z" }, - { url = "https://files.pythonhosted.org/packages/de/8e/a5b882ec4307010a840fb8b58bd9bf65d1840c92eae7534c7441709bf54b/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02", size = 2136178, upload-time = "2025-04-23T18:30:50.907Z" }, - { url = "https://files.pythonhosted.org/packages/e4/bb/71e35fc3ed05af6834e890edb75968e2802fe98778971ab5cba20a162315/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b", size = 2736462, upload-time = "2025-04-23T18:30:52.083Z" }, - { url = "https://files.pythonhosted.org/packages/31/0d/c8f7593e6bc7066289bbc366f2235701dcbebcd1ff0ef8e64f6f239fb47d/pydantic_core-2.33.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2", size = 2005652, upload-time = "2025-04-23T18:30:53.389Z" }, - { url = "https://files.pythonhosted.org/packages/d2/7a/996d8bd75f3eda405e3dd219ff5ff0a283cd8e34add39d8ef9157e722867/pydantic_core-2.33.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a", size = 2113306, upload-time = "2025-04-23T18:30:54.661Z" }, - { url = "https://files.pythonhosted.org/packages/ff/84/daf2a6fb2db40ffda6578a7e8c5a6e9c8affb251a05c233ae37098118788/pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac", size = 2073720, upload-time = "2025-04-23T18:30:56.11Z" }, - { url = "https://files.pythonhosted.org/packages/77/fb/2258da019f4825128445ae79456a5499c032b55849dbd5bed78c95ccf163/pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a", size = 2244915, upload-time = "2025-04-23T18:30:57.501Z" }, - { url = "https://files.pythonhosted.org/packages/d8/7a/925ff73756031289468326e355b6fa8316960d0d65f8b5d6b3a3e7866de7/pydantic_core-2.33.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b", size = 2241884, upload-time = "2025-04-23T18:30:58.867Z" }, - { url = "https://files.pythonhosted.org/packages/0b/b0/249ee6d2646f1cdadcb813805fe76265745c4010cf20a8eba7b0e639d9b2/pydantic_core-2.33.2-cp310-cp310-win32.whl", hash = "sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22", size = 1910496, upload-time = "2025-04-23T18:31:00.078Z" }, - { url = "https://files.pythonhosted.org/packages/66/ff/172ba8f12a42d4b552917aa65d1f2328990d3ccfc01d5b7c943ec084299f/pydantic_core-2.33.2-cp310-cp310-win_amd64.whl", hash = "sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640", size = 1955019, upload-time = "2025-04-23T18:31:01.335Z" }, - { url = "https://files.pythonhosted.org/packages/3f/8d/71db63483d518cbbf290261a1fc2839d17ff89fce7089e08cad07ccfce67/pydantic_core-2.33.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7", size = 2028584, upload-time = "2025-04-23T18:31:03.106Z" }, - { url = "https://files.pythonhosted.org/packages/24/2f/3cfa7244ae292dd850989f328722d2aef313f74ffc471184dc509e1e4e5a/pydantic_core-2.33.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246", size = 1855071, upload-time = "2025-04-23T18:31:04.621Z" }, - { url = "https://files.pythonhosted.org/packages/b3/d3/4ae42d33f5e3f50dd467761304be2fa0a9417fbf09735bc2cce003480f2a/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f", size = 1897823, upload-time = "2025-04-23T18:31:06.377Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f3/aa5976e8352b7695ff808599794b1fba2a9ae2ee954a3426855935799488/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc", size = 1983792, upload-time = "2025-04-23T18:31:07.93Z" }, - { url = "https://files.pythonhosted.org/packages/d5/7a/cda9b5a23c552037717f2b2a5257e9b2bfe45e687386df9591eff7b46d28/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de", size = 2136338, upload-time = "2025-04-23T18:31:09.283Z" }, - { url = "https://files.pythonhosted.org/packages/2b/9f/b8f9ec8dd1417eb9da784e91e1667d58a2a4a7b7b34cf4af765ef663a7e5/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a", size = 2730998, upload-time = "2025-04-23T18:31:11.7Z" }, - { url = "https://files.pythonhosted.org/packages/47/bc/cd720e078576bdb8255d5032c5d63ee5c0bf4b7173dd955185a1d658c456/pydantic_core-2.33.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef", size = 2003200, upload-time = "2025-04-23T18:31:13.536Z" }, - { url = "https://files.pythonhosted.org/packages/ca/22/3602b895ee2cd29d11a2b349372446ae9727c32e78a94b3d588a40fdf187/pydantic_core-2.33.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e", size = 2113890, upload-time = "2025-04-23T18:31:15.011Z" }, - { url = "https://files.pythonhosted.org/packages/ff/e6/e3c5908c03cf00d629eb38393a98fccc38ee0ce8ecce32f69fc7d7b558a7/pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d", size = 2073359, upload-time = "2025-04-23T18:31:16.393Z" }, - { url = "https://files.pythonhosted.org/packages/12/e7/6a36a07c59ebefc8777d1ffdaf5ae71b06b21952582e4b07eba88a421c79/pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30", size = 2245883, upload-time = "2025-04-23T18:31:17.892Z" }, - { url = "https://files.pythonhosted.org/packages/16/3f/59b3187aaa6cc0c1e6616e8045b284de2b6a87b027cce2ffcea073adf1d2/pydantic_core-2.33.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf", size = 2241074, upload-time = "2025-04-23T18:31:19.205Z" }, - { url = "https://files.pythonhosted.org/packages/e0/ed/55532bb88f674d5d8f67ab121a2a13c385df382de2a1677f30ad385f7438/pydantic_core-2.33.2-cp311-cp311-win32.whl", hash = "sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51", size = 1910538, upload-time = "2025-04-23T18:31:20.541Z" }, - { url = "https://files.pythonhosted.org/packages/fe/1b/25b7cccd4519c0b23c2dd636ad39d381abf113085ce4f7bec2b0dc755eb1/pydantic_core-2.33.2-cp311-cp311-win_amd64.whl", hash = "sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab", size = 1952909, upload-time = "2025-04-23T18:31:22.371Z" }, - { url = "https://files.pythonhosted.org/packages/49/a9/d809358e49126438055884c4366a1f6227f0f84f635a9014e2deb9b9de54/pydantic_core-2.33.2-cp311-cp311-win_arm64.whl", hash = "sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65", size = 1897786, upload-time = "2025-04-23T18:31:24.161Z" }, - { url = "https://files.pythonhosted.org/packages/18/8a/2b41c97f554ec8c71f2a8a5f85cb56a8b0956addfe8b0efb5b3d77e8bdc3/pydantic_core-2.33.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc", size = 2009000, upload-time = "2025-04-23T18:31:25.863Z" }, - { url = "https://files.pythonhosted.org/packages/a1/02/6224312aacb3c8ecbaa959897af57181fb6cf3a3d7917fd44d0f2917e6f2/pydantic_core-2.33.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7", size = 1847996, upload-time = "2025-04-23T18:31:27.341Z" }, - { url = "https://files.pythonhosted.org/packages/d6/46/6dcdf084a523dbe0a0be59d054734b86a981726f221f4562aed313dbcb49/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025", size = 1880957, upload-time = "2025-04-23T18:31:28.956Z" }, - { url = "https://files.pythonhosted.org/packages/ec/6b/1ec2c03837ac00886ba8160ce041ce4e325b41d06a034adbef11339ae422/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011", size = 1964199, upload-time = "2025-04-23T18:31:31.025Z" }, - { url = "https://files.pythonhosted.org/packages/2d/1d/6bf34d6adb9debd9136bd197ca72642203ce9aaaa85cfcbfcf20f9696e83/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f", size = 2120296, upload-time = "2025-04-23T18:31:32.514Z" }, - { url = "https://files.pythonhosted.org/packages/e0/94/2bd0aaf5a591e974b32a9f7123f16637776c304471a0ab33cf263cf5591a/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88", size = 2676109, upload-time = "2025-04-23T18:31:33.958Z" }, - { url = "https://files.pythonhosted.org/packages/f9/41/4b043778cf9c4285d59742281a769eac371b9e47e35f98ad321349cc5d61/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1", size = 2002028, upload-time = "2025-04-23T18:31:39.095Z" }, - { url = "https://files.pythonhosted.org/packages/cb/d5/7bb781bf2748ce3d03af04d5c969fa1308880e1dca35a9bd94e1a96a922e/pydantic_core-2.33.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b", size = 2100044, upload-time = "2025-04-23T18:31:41.034Z" }, - { url = "https://files.pythonhosted.org/packages/fe/36/def5e53e1eb0ad896785702a5bbfd25eed546cdcf4087ad285021a90ed53/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1", size = 2058881, upload-time = "2025-04-23T18:31:42.757Z" }, - { url = "https://files.pythonhosted.org/packages/01/6c/57f8d70b2ee57fc3dc8b9610315949837fa8c11d86927b9bb044f8705419/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6", size = 2227034, upload-time = "2025-04-23T18:31:44.304Z" }, - { url = "https://files.pythonhosted.org/packages/27/b9/9c17f0396a82b3d5cbea4c24d742083422639e7bb1d5bf600e12cb176a13/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea", size = 2234187, upload-time = "2025-04-23T18:31:45.891Z" }, - { url = "https://files.pythonhosted.org/packages/b0/6a/adf5734ffd52bf86d865093ad70b2ce543415e0e356f6cacabbc0d9ad910/pydantic_core-2.33.2-cp312-cp312-win32.whl", hash = "sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290", size = 1892628, upload-time = "2025-04-23T18:31:47.819Z" }, - { url = "https://files.pythonhosted.org/packages/43/e4/5479fecb3606c1368d496a825d8411e126133c41224c1e7238be58b87d7e/pydantic_core-2.33.2-cp312-cp312-win_amd64.whl", hash = "sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2", size = 1955866, upload-time = "2025-04-23T18:31:49.635Z" }, - { url = "https://files.pythonhosted.org/packages/0d/24/8b11e8b3e2be9dd82df4b11408a67c61bb4dc4f8e11b5b0fc888b38118b5/pydantic_core-2.33.2-cp312-cp312-win_arm64.whl", hash = "sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab", size = 1888894, upload-time = "2025-04-23T18:31:51.609Z" }, - { url = "https://files.pythonhosted.org/packages/46/8c/99040727b41f56616573a28771b1bfa08a3d3fe74d3d513f01251f79f172/pydantic_core-2.33.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f", size = 2015688, upload-time = "2025-04-23T18:31:53.175Z" }, - { url = "https://files.pythonhosted.org/packages/3a/cc/5999d1eb705a6cefc31f0b4a90e9f7fc400539b1a1030529700cc1b51838/pydantic_core-2.33.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6", size = 1844808, upload-time = "2025-04-23T18:31:54.79Z" }, - { url = "https://files.pythonhosted.org/packages/6f/5e/a0a7b8885c98889a18b6e376f344da1ef323d270b44edf8174d6bce4d622/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef", size = 1885580, upload-time = "2025-04-23T18:31:57.393Z" }, - { url = "https://files.pythonhosted.org/packages/3b/2a/953581f343c7d11a304581156618c3f592435523dd9d79865903272c256a/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a", size = 1973859, upload-time = "2025-04-23T18:31:59.065Z" }, - { url = "https://files.pythonhosted.org/packages/e6/55/f1a813904771c03a3f97f676c62cca0c0a4138654107c1b61f19c644868b/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916", size = 2120810, upload-time = "2025-04-23T18:32:00.78Z" }, - { url = "https://files.pythonhosted.org/packages/aa/c3/053389835a996e18853ba107a63caae0b9deb4a276c6b472931ea9ae6e48/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a", size = 2676498, upload-time = "2025-04-23T18:32:02.418Z" }, - { url = "https://files.pythonhosted.org/packages/eb/3c/f4abd740877a35abade05e437245b192f9d0ffb48bbbbd708df33d3cda37/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d", size = 2000611, upload-time = "2025-04-23T18:32:04.152Z" }, - { url = "https://files.pythonhosted.org/packages/59/a7/63ef2fed1837d1121a894d0ce88439fe3e3b3e48c7543b2a4479eb99c2bd/pydantic_core-2.33.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56", size = 2107924, upload-time = "2025-04-23T18:32:06.129Z" }, - { url = "https://files.pythonhosted.org/packages/04/8f/2551964ef045669801675f1cfc3b0d74147f4901c3ffa42be2ddb1f0efc4/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5", size = 2063196, upload-time = "2025-04-23T18:32:08.178Z" }, - { url = "https://files.pythonhosted.org/packages/26/bd/d9602777e77fc6dbb0c7db9ad356e9a985825547dce5ad1d30ee04903918/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e", size = 2236389, upload-time = "2025-04-23T18:32:10.242Z" }, - { url = "https://files.pythonhosted.org/packages/42/db/0e950daa7e2230423ab342ae918a794964b053bec24ba8af013fc7c94846/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162", size = 2239223, upload-time = "2025-04-23T18:32:12.382Z" }, - { url = "https://files.pythonhosted.org/packages/58/4d/4f937099c545a8a17eb52cb67fe0447fd9a373b348ccfa9a87f141eeb00f/pydantic_core-2.33.2-cp313-cp313-win32.whl", hash = "sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849", size = 1900473, upload-time = "2025-04-23T18:32:14.034Z" }, - { url = "https://files.pythonhosted.org/packages/a0/75/4a0a9bac998d78d889def5e4ef2b065acba8cae8c93696906c3a91f310ca/pydantic_core-2.33.2-cp313-cp313-win_amd64.whl", hash = "sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9", size = 1955269, upload-time = "2025-04-23T18:32:15.783Z" }, - { url = "https://files.pythonhosted.org/packages/f9/86/1beda0576969592f1497b4ce8e7bc8cbdf614c352426271b1b10d5f0aa64/pydantic_core-2.33.2-cp313-cp313-win_arm64.whl", hash = "sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9", size = 1893921, upload-time = "2025-04-23T18:32:18.473Z" }, - { url = "https://files.pythonhosted.org/packages/a4/7d/e09391c2eebeab681df2b74bfe6c43422fffede8dc74187b2b0bf6fd7571/pydantic_core-2.33.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac", size = 1806162, upload-time = "2025-04-23T18:32:20.188Z" }, - { url = "https://files.pythonhosted.org/packages/f1/3d/847b6b1fed9f8ed3bb95a9ad04fbd0b212e832d4f0f50ff4d9ee5a9f15cf/pydantic_core-2.33.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5", size = 1981560, upload-time = "2025-04-23T18:32:22.354Z" }, - { url = "https://files.pythonhosted.org/packages/6f/9a/e73262f6c6656262b5fdd723ad90f518f579b7bc8622e43a942eec53c938/pydantic_core-2.33.2-cp313-cp313t-win_amd64.whl", hash = "sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9", size = 1935777, upload-time = "2025-04-23T18:32:25.088Z" }, - { url = "https://files.pythonhosted.org/packages/30/68/373d55e58b7e83ce371691f6eaa7175e3a24b956c44628eb25d7da007917/pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa", size = 2023982, upload-time = "2025-04-23T18:32:53.14Z" }, - { url = "https://files.pythonhosted.org/packages/a4/16/145f54ac08c96a63d8ed6442f9dec17b2773d19920b627b18d4f10a061ea/pydantic_core-2.33.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29", size = 1858412, upload-time = "2025-04-23T18:32:55.52Z" }, - { url = "https://files.pythonhosted.org/packages/41/b1/c6dc6c3e2de4516c0bb2c46f6a373b91b5660312342a0cf5826e38ad82fa/pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d", size = 1892749, upload-time = "2025-04-23T18:32:57.546Z" }, - { url = "https://files.pythonhosted.org/packages/12/73/8cd57e20afba760b21b742106f9dbdfa6697f1570b189c7457a1af4cd8a0/pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e", size = 2067527, upload-time = "2025-04-23T18:32:59.771Z" }, - { url = "https://files.pythonhosted.org/packages/e3/d5/0bb5d988cc019b3cba4a78f2d4b3854427fc47ee8ec8e9eaabf787da239c/pydantic_core-2.33.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c", size = 2108225, upload-time = "2025-04-23T18:33:04.51Z" }, - { url = "https://files.pythonhosted.org/packages/f1/c5/00c02d1571913d496aabf146106ad8239dc132485ee22efe08085084ff7c/pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec", size = 2069490, upload-time = "2025-04-23T18:33:06.391Z" }, - { url = "https://files.pythonhosted.org/packages/22/a8/dccc38768274d3ed3a59b5d06f59ccb845778687652daa71df0cab4040d7/pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052", size = 2237525, upload-time = "2025-04-23T18:33:08.44Z" }, - { url = "https://files.pythonhosted.org/packages/d4/e7/4f98c0b125dda7cf7ccd14ba936218397b44f50a56dd8c16a3091df116c3/pydantic_core-2.33.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c", size = 2238446, upload-time = "2025-04-23T18:33:10.313Z" }, - { url = "https://files.pythonhosted.org/packages/ce/91/2ec36480fdb0b783cd9ef6795753c1dea13882f2e68e73bce76ae8c21e6a/pydantic_core-2.33.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808", size = 2066678, upload-time = "2025-04-23T18:33:12.224Z" }, - { url = "https://files.pythonhosted.org/packages/7b/27/d4ae6487d73948d6f20dddcd94be4ea43e74349b56eba82e9bdee2d7494c/pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8", size = 2025200, upload-time = "2025-04-23T18:33:14.199Z" }, - { url = "https://files.pythonhosted.org/packages/f1/b8/b3cb95375f05d33801024079b9392a5ab45267a63400bf1866e7ce0f0de4/pydantic_core-2.33.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593", size = 1859123, upload-time = "2025-04-23T18:33:16.555Z" }, - { url = "https://files.pythonhosted.org/packages/05/bc/0d0b5adeda59a261cd30a1235a445bf55c7e46ae44aea28f7bd6ed46e091/pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612", size = 1892852, upload-time = "2025-04-23T18:33:18.513Z" }, - { url = "https://files.pythonhosted.org/packages/3e/11/d37bdebbda2e449cb3f519f6ce950927b56d62f0b84fd9cb9e372a26a3d5/pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7", size = 2067484, upload-time = "2025-04-23T18:33:20.475Z" }, - { url = "https://files.pythonhosted.org/packages/8c/55/1f95f0a05ce72ecb02a8a8a1c3be0579bbc29b1d5ab68f1378b7bebc5057/pydantic_core-2.33.2-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e", size = 2108896, upload-time = "2025-04-23T18:33:22.501Z" }, - { url = "https://files.pythonhosted.org/packages/53/89/2b2de6c81fa131f423246a9109d7b2a375e83968ad0800d6e57d0574629b/pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8", size = 2069475, upload-time = "2025-04-23T18:33:24.528Z" }, - { url = "https://files.pythonhosted.org/packages/b8/e9/1f7efbe20d0b2b10f6718944b5d8ece9152390904f29a78e68d4e7961159/pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf", size = 2239013, upload-time = "2025-04-23T18:33:26.621Z" }, - { url = "https://files.pythonhosted.org/packages/3c/b2/5309c905a93811524a49b4e031e9851a6b00ff0fb668794472ea7746b448/pydantic_core-2.33.2-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb", size = 2238715, upload-time = "2025-04-23T18:33:28.656Z" }, - { url = "https://files.pythonhosted.org/packages/32/56/8a7ca5d2cd2cda1d245d34b1c9a942920a718082ae8e54e5f3e5a58b7add/pydantic_core-2.33.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1", size = 2066757, upload-time = "2025-04-23T18:33:30.645Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/08/f1ba952f1c8ae5581c70fa9c6da89f247b83e3dd8c09c035d5d7931fc23d/pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4", size = 2113146, upload-time = "2026-05-06T13:37:36.537Z" }, + { url = "https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5", size = 1949769, upload-time = "2026-05-06T13:37:46.365Z" }, + { url = "https://files.pythonhosted.org/packages/64/ba/bfb1d928fd5b49e1258935ff104ae356e9fd89384a55bf9f847e9193ad40/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba", size = 1974958, upload-time = "2026-05-06T13:37:28.611Z" }, + { url = "https://files.pythonhosted.org/packages/4e/74/76223bfb117b64af743c9b6670d1364516f5c0604f96b48f3272f6af6cc6/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b", size = 2042118, upload-time = "2026-05-06T13:36:55.216Z" }, + { url = "https://files.pythonhosted.org/packages/cb/7b/848732968bc8f48f3187542f08358b9d842db564147b256669426ebb1652/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c", size = 2222876, upload-time = "2026-05-06T13:38:25.455Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2f/e90b63ee2e14bd8d3db8f705a6d75d64e6ee1b7c2c8833747ce706e1e0ce/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50", size = 2286703, upload-time = "2026-05-06T13:37:53.304Z" }, + { url = "https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd", size = 2092042, upload-time = "2026-05-06T13:38:46.981Z" }, + { url = "https://files.pythonhosted.org/packages/a9/da/0a422b57bf8504102bf3c4ccea9c41bab5a5cee6a54650acf8faf67f5a24/pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01", size = 2117231, upload-time = "2026-05-06T13:39:23.146Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2a/2ac13c3af305843e23c5078c53d135656b3f05a2fd78cb7bbbb12e97b473/pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d", size = 2168388, upload-time = "2026-05-06T13:40:08.06Z" }, + { url = "https://files.pythonhosted.org/packages/72/04/2beacf7e1607e93eefe4aed1b4709f079b905fb77530179d4f7c71745f22/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4", size = 2184769, upload-time = "2026-05-06T13:38:13.901Z" }, + { url = "https://files.pythonhosted.org/packages/9e/29/d2b9fd9f539133548eaf622c06a4ce176cb46ac59f32d0359c4abc0de047/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f", size = 2319312, upload-time = "2026-05-06T13:39:08.24Z" }, + { url = "https://files.pythonhosted.org/packages/7c/af/0f7a5b85fec6075bea96e3ef9187de38fccced0de92c1e7feda8d5cc7bb9/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39", size = 2361817, upload-time = "2026-05-06T13:38:43.2Z" }, + { url = "https://files.pythonhosted.org/packages/25/a4/73363fec545fd3ec025490bdda2743c56d0dd5b6266b1a53bbe9e4265375/pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d", size = 1987085, upload-time = "2026-05-06T13:39:25.497Z" }, + { url = "https://files.pythonhosted.org/packages/01/aa/62f082da2c91fac1c234bc9ee0066257ce83f0604abd72e4c9d5991f2d84/pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf", size = 2074311, upload-time = "2026-05-06T13:39:59.922Z" }, + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, ] [[package]] @@ -4693,6 +5124,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + [[package]] name = "python-json-logger" version = "3.3.0" @@ -5307,6 +5747,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/ed/9de62c2150ca8e2e5858acf3f4f4d0d180a38feef9fdab4078bea63d8dba/rpds_py-0.26.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:e99685fc95d386da368013e7fb4269dd39c30d99f812a8372d62f244f662709c", size = 555334, upload-time = "2025-07-01T15:56:51.703Z" }, ] +[[package]] +name = "s3fs" +version = "2025.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiobotocore" }, + { name = "aiohttp" }, + { name = "fsspec" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/cd/5dde2fed1699ff48120336249d9857a574e39feb8afaff694568ab1499b3/s3fs-2025.3.0.tar.gz", hash = "sha256:446dd539eb0d0678209723cb7ad1bedbb172185b0d34675b09be1ad81843a644", size = 77153, upload-time = "2025-03-07T21:58:32.114Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/3f/35f4041a82a68df89fe4af97c8bb44aa492dad924799cbb02078e9e303e6/s3fs-2025.3.0-py3-none-any.whl", hash = "sha256:88d803615baa04945156ca0e1498009b7acd3132c07198bd81b3e874846e0aa2", size = 30454, upload-time = "2025-03-07T21:58:30.998Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/74/8d69dcb7a9efe8baa2046891735e5dfe433ad558ae23d9e3c14c633d1d58/s3transfer-0.14.0.tar.gz", hash = "sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125", size = 151547, upload-time = "2025-09-09T19:23:31.089Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" }, +] + [[package]] name = "sacrebleu" version = "2.6.0" @@ -5749,6 +6215,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177", size = 15552, upload-time = "2022-05-26T13:35:21.206Z" }, ] +[[package]] +name = "semver" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/d1/d3159231aec234a59dd7d601e9dd9fe96f3afff15efd33c1070019b26132/semver-3.0.4.tar.gz", hash = "sha256:afc7d8c584a5ed0a11033af086e8af226a9c0b206f313e0301f8dd7b6b589602", size = 269730, upload-time = "2025-01-24T13:19:27.617Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl", hash = "sha256:9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746", size = 17912, upload-time = "2025-01-24T13:19:24.949Z" }, +] + [[package]] name = "send2trash" version = "1.8.3" @@ -5892,6 +6367,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "shortuuid" +version = "1.0.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/e2/bcf761f3bff95856203f9559baf3741c416071dd200c0fc19fad7f078f86/shortuuid-1.0.13.tar.gz", hash = "sha256:3bb9cf07f606260584b1df46399c0b87dd84773e7b25912b7e391e30797c5e72", size = 9662, upload-time = "2024-03-11T20:11:06.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/44/21d6bf170bf40b41396480d8d49ad640bca3f2b02139cd52aa1e272830a5/shortuuid-1.0.13-py3-none-any.whl", hash = "sha256:a482a497300b49b4953e15108a7913244e1bb0d41f9d332f5e9925dba33a3c5a", size = 10529, upload-time = "2024-03-11T20:11:04.807Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -6143,6 +6627,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/05/a2/ed023f2edd1e011b4d99b6727bce8253842d66c3fbf9ed0a26fc09a92571/tcolorpy-0.1.7-py3-none-any.whl", hash = "sha256:26a59d52027e175a37e0aba72efc99dda43f074db71f55b316d3de37d3251378", size = 8096, upload-time = "2024-12-29T15:24:21.33Z" }, ] +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + [[package]] name = "termcolor" version = "3.3.0" @@ -6166,6 +6659,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl", hash = "sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0", size = 14154, upload-time = "2024-03-12T14:34:36.569Z" }, ] +[[package]] +name = "textual" +version = "6.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py", extra = ["linkify", "plugins"] }, + { name = "platformdirs" }, + { name = "pygments" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a2/30/38b615f7d4b16f6fdd73e4dcd8913e2d880bbb655e68a076e3d91181a7ee/textual-6.2.1.tar.gz", hash = "sha256:4699d8dfae43503b9c417bd2a6fb0da1c89e323fe91c4baa012f9298acaa83e1", size = 1570645, upload-time = "2025-10-01T16:11:24.467Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/93/02c7adec57a594af28388d85da9972703a4af94ae1399542555cd9581952/textual-6.2.1-py3-none-any.whl", hash = "sha256:3c7190633cd4d8bfe6049ae66808b98da91ded2edb85cef54e82bf77b03d2a54", size = 710702, upload-time = "2025-10-01T16:11:22.161Z" }, +] + [[package]] name = "threadpoolctl" version = "3.6.0" @@ -6213,6 +6722,67 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/e4/e804505f87627cd8cdae9c010c47c4485fd8c1ce31a7dd0ab7fcc4707377/tifffile-2026.3.3-py3-none-any.whl", hash = "sha256:e8be15c94273113d31ecb7aa3a39822189dd11c4967e3cc88c178f1ad2fd1170", size = 243960, upload-time = "2026-03-03T19:14:35.808Z" }, ] +[[package]] +name = "tiktoken" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/e5/5f3cb2159769d0f4324c0e9e87f9de3c4b1cd45848a96b2eb3566ad5ca77/tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1", size = 38986, upload-time = "2026-05-15T04:51:27.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/e3/03c90dadcf5b3f82b83cee9adee60ef666b329c654f58c066af44eae0287/tiktoken-0.13.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:47b1df8d73390a24f94980c75158cdd5c56d256f16d55f30cb49c230caba9ba4", size = 1036627, upload-time = "2026-05-15T04:50:11.229Z" }, + { url = "https://files.pythonhosted.org/packages/5e/30/760463e5b2e8ad2bc229ae0a17ecb06727b6cbc094f08d8f65844315632e/tiktoken-0.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7d40c6c5aab171dcd6eb8455bc567bde404bb9def60cdb8c1299cc782b242bb9", size = 984699, upload-time = "2026-05-15T04:50:12.874Z" }, + { url = "https://files.pythonhosted.org/packages/de/8a/8895f342a6b6aabd1a358e672f6f077b3ae51d0c63ca605d142db3bcd8ab/tiktoken-0.13.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:9b842981fa91accdffd48ff6408a977b7a91c3fbda55d353c3c68114d5c9d69e", size = 1118690, upload-time = "2026-05-15T04:50:14.234Z" }, + { url = "https://files.pythonhosted.org/packages/51/e0/92557768fb0801f0d9dd9243cb9b6d342900b05e4b1006d4771f49ce233e/tiktoken-0.13.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:ed5a30027cb4d8c7ca8b273d4766f3db3cf58fad9e9f3b1a68a351ffb54873d5", size = 1138423, upload-time = "2026-05-15T04:50:15.668Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b9/a3d99feeedb032ffd09cd6652077f86bdee9a70dd0b990b2b272b445d4c3/tiktoken-0.13.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7ab10f4a21c2999846940113f6dbd72e0fa06a24119feddd74cc47e85818e06d", size = 1185077, upload-time = "2026-05-15T04:50:17.19Z" }, + { url = "https://files.pythonhosted.org/packages/cc/93/bab868277d475dc6d2aaacd34cdd239c282f4908dcc8702e0a3311a8e032/tiktoken-0.13.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a2937ad042d49d50eac6e1ba07c5661d4bd3942a5b1e0c0d08475c4df83676e1", size = 1241702, upload-time = "2026-05-15T04:50:18.772Z" }, + { url = "https://files.pythonhosted.org/packages/c3/16/27e9f7e0ed76e501cfefc9fb2112df4c7bf70ca96945b15ecb7615aac860/tiktoken-0.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:44733b99bfd72b590cd0936b1c01b3b4dd73122db2d544bc1ceeb18a7678c910", size = 876565, upload-time = "2026-05-15T04:50:20.268Z" }, + { url = "https://files.pythonhosted.org/packages/1a/4c/1bc81f4cd53e827c4ee67ca951b5935724716049452d8dfa09b8b82372bb/tiktoken-0.13.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7bfe1849caa65d1e1d9871817170ec497bbb7984e182012e1bdce72f66608cdb", size = 1036353, upload-time = "2026-05-15T04:50:21.757Z" }, + { url = "https://files.pythonhosted.org/packages/75/91/10b9c7076bc02c246c853201fdbbe300a4b8c5ed7b84c25f7403f4e32655/tiktoken-0.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:91c180fe255bd5a86d8316210d2833a1d4d33d026cd86a67812f4773743c8d26", size = 984644, upload-time = "2026-05-15T04:50:23.256Z" }, + { url = "https://files.pythonhosted.org/packages/4e/e4/fceae98015fab47fcd49b8bd7f46145bcd187a47e0add1e5378ed67ef980/tiktoken-0.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:059c8ecf554eb5b41e6e054ba467b871b03277d267dee7244380aca4359747d4", size = 1119261, upload-time = "2026-05-15T04:50:24.348Z" }, + { url = "https://files.pythonhosted.org/packages/f9/39/fe42ad00de01a8c4a49ad8649a2c8a316835a9cad5961b11d21eac0020a5/tiktoken-0.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:36217497eaffc158607a3b26f065300db2aefd43b115263f3b9688ce38146173", size = 1138253, upload-time = "2026-05-15T04:50:25.505Z" }, + { url = "https://files.pythonhosted.org/packages/03/c4/ccee1ecccca107e9a16efcecdeeb964c325305038554d466ece65b42338f/tiktoken-0.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:303f7d91b4fce3baddbcde05c139091d4caa5026ac7214c1dc7ff7a71ee429ff", size = 1185747, upload-time = "2026-05-15T04:50:27.02Z" }, + { url = "https://files.pythonhosted.org/packages/9d/03/cd0cba295522b91eb55c6b2704f1df895f8226cfe60ab10d4d51d0cc9e69/tiktoken-0.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5d48843bee149630eb735a99e1f4a85b47308d21868ea63163f6e87768d3cfed", size = 1241265, upload-time = "2026-05-15T04:50:28.815Z" }, + { url = "https://files.pythonhosted.org/packages/7e/25/a10efd564402d82c2ff50d12057353ace447aa8007deceaa48641f63d35c/tiktoken-0.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:fc1c44cd37b43fc46bae593129164f4f281e82ea116b57a85aa81bda57eafc94", size = 876509, upload-time = "2026-05-15T04:50:30.026Z" }, + { url = "https://files.pythonhosted.org/packages/85/8e/144bde4e01df66b34bb865557c7cd754ed08b036217ebd79c9db5e9048a9/tiktoken-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32ac870a806cfb260a02d0cb70426aef02e038297f8ad50df5040bb5af360791", size = 1034888, upload-time = "2026-05-15T04:50:31.579Z" }, + { url = "https://files.pythonhosted.org/packages/36/18/d4ac9d20956cdebca04841316660ed584c2fecdc2b81722a28bc7ad3b1e4/tiktoken-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d9980f11429ed2d737c463bb1fb78cf330caa026adf002f714aced7849a687b", size = 982970, upload-time = "2026-05-15T04:50:32.961Z" }, + { url = "https://files.pythonhosted.org/packages/74/ed/6bb8d05b9f731f749fee5c6f5ca63e981143c826a5985877330507bd13b7/tiktoken-0.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3f277ebea5edd7b8bf03c6f9431e1d67d517530115572b2dc1d465326e8f88c7", size = 1115741, upload-time = "2026-05-15T04:50:34.475Z" }, + { url = "https://files.pythonhosted.org/packages/34/de/2ca96b07a82d972b74fe4b46de055b79c904e45c7eab699354a0bfa697dc/tiktoken-0.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a116178fa7e1b4065bff05214360373a65cac22f965be7b3f73d00a0dbfe7649", size = 1136523, upload-time = "2026-05-15T04:50:35.782Z" }, + { url = "https://files.pythonhosted.org/packages/ee/dc/9dafec002c2d4424378563cf4cf5c7fb93631d2a55013c8b87554ee4012c/tiktoken-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c397ddda233208345b01bd30f2fca79ff730e55731d0108a603f9bc57f6af3b", size = 1181954, upload-time = "2026-05-15T04:50:36.99Z" }, + { url = "https://files.pythonhosted.org/packages/a1/d0/1f8578c45b2f24759b46f0b50d31878c63c73e6bf0f2227e10ec5c5408dc/tiktoken-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:95097e4f89b06403976e498abf61a0ee73a7497e73fb599cb211d8197a054d91", size = 1240069, upload-time = "2026-05-15T04:50:38.221Z" }, + { url = "https://files.pythonhosted.org/packages/aa/90/28d7f154888610aa9237e541986beb62b479df29d193a5a0617dbb1514d0/tiktoken-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f2d16e7a7c783ad81f36e457d046d1f1c8af70b22aec8a13238efe531977c41", size = 874748, upload-time = "2026-05-15T04:50:39.587Z" }, + { url = "https://files.pythonhosted.org/packages/9c/83/b096c859c2a47c11731bf2f5885f4028b809dfe2396582883eed9cae372f/tiktoken-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5df5d1507bd245f1ccad4a074698240021239e455eb0bb4ced4e3d7181872154", size = 1034228, upload-time = "2026-05-15T04:50:40.988Z" }, + { url = "https://files.pythonhosted.org/packages/53/61/c68e123b6d753e3fc2751e9b18e732c9d8bf1e1926762e736eee935d931c/tiktoken-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fe806a50664e83a6ffd56cbd1e4f5dcc6cd32a3e7538f70dc38b1a271384545", size = 982978, upload-time = "2026-05-15T04:50:42.195Z" }, + { url = "https://files.pythonhosted.org/packages/ef/8b/96cc178cc584e65d363134500f297790b06cd48cdeb1e8fcf7bbe60f4715/tiktoken-0.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:125bc05005e747f993a83dc67934249932d6e4209854452cd4c0b1d53fba3ba2", size = 1116355, upload-time = "2026-05-15T04:50:43.564Z" }, + { url = "https://files.pythonhosted.org/packages/86/f5/bab735d2c72ea55404b295d02d092644eb5f7cc6205e34d35eb9abfb9ab2/tiktoken-0.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5e6358911cab4adee6712da27d65573496a4f68cf8a2b5fca6a4ad10fc5748cf", size = 1135772, upload-time = "2026-05-15T04:50:44.782Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b9/6de04ebdf904edfaad87788011b3735087a0c9ea671b9027e1e4e965e8c8/tiktoken-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:975cbd78d085d75d26b59660e262736dcaed1e35f8f142cd6291025c01d25486", size = 1182415, upload-time = "2026-05-15T04:50:46.422Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9c/470a05f3b1caf038f44880e334d47ab674e0c80d514c66b375d14d5afa10/tiktoken-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ab9bc99fa020a4c283424590ecd7f3afd70c1c281cb3fa3192a6c3af9f9615", size = 1239879, upload-time = "2026-05-15T04:50:48.052Z" }, + { url = "https://files.pythonhosted.org/packages/42/a6/c1936d16055436cb32e6c6128d68629622e00f4768562f55653752d34768/tiktoken-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:6b1615f0ff71953d19729ceb18865429c185b0a23c5353f1bbca34a394bf60f7", size = 874829, upload-time = "2026-05-15T04:50:49.202Z" }, + { url = "https://files.pythonhosted.org/packages/d6/07/acb5992c3772b5a36284f742cfb7a5895aa4471d1848ac31464ad50d7fdf/tiktoken-0.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6eb4a5bfbc6426938026b1a334e898ac53541360d62d8c689870160cc80abd67", size = 1033600, upload-time = "2026-05-15T04:50:50.4Z" }, + { url = "https://files.pythonhosted.org/packages/14/e9/742e9aec30f59b9f161f7ff7cd072e02ea836c9e1c0854a8076dfcd40d5c/tiktoken-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:43cee3e5400573b2046fbf092cc7a5bc30164f9e4c95ce20714da929df48737a", size = 982516, upload-time = "2026-05-15T04:50:52.03Z" }, + { url = "https://files.pythonhosted.org/packages/72/74/ca1541b053e7648254d2e4b42a253e1bb4359f2c91a0a8d49228c794e1a0/tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7de52e3f566d19b3b11bd37eea552c6c305ad74081f736882bd44d148ed4c48d", size = 1115518, upload-time = "2026-05-15T04:50:53.543Z" }, + { url = "https://files.pythonhosted.org/packages/46/e3/93825eaf5a4a504795b787e5d5dea07fbeb3dabf97aa7b450be8bde59c89/tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:51384448aa508e4df84c0f7c1dc3211c7f7b8096325660ee5fc82f3e11b381ce", size = 1136867, upload-time = "2026-05-15T04:50:55.191Z" }, + { url = "https://files.pythonhosted.org/packages/8c/46/002b68de6827091d5ae90b048f326e8aad8d953520950e5ce1508879414f/tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e28157350f7ebf35008dd8e9e0fdb621f976e4230c881099c85e8cf07eaa50e2", size = 1181826, upload-time = "2026-05-15T04:50:56.296Z" }, + { url = "https://files.pythonhosted.org/packages/db/c6/d393e3185a276505182f7abd93fe714f3c444a2be9180798fa052347504e/tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:165cf1820ea4a354985c2490a5205d4cc74661c934aca79dd0368232fff94e0f", size = 1239489, upload-time = "2026-05-15T04:50:57.918Z" }, + { url = "https://files.pythonhosted.org/packages/b7/4d/bc07d1f1635d4897a202acc0ae11c2886eaa7325c359ba4741b47bf8e225/tiktoken-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6c43a675ca14f6f2749ba7f12075d37456015a24b859f2517b9beb4ef30807ec", size = 873820, upload-time = "2026-05-15T04:50:59.528Z" }, + { url = "https://files.pythonhosted.org/packages/8c/93/0dd6adca026a616c3a92974566b43381eea4b475ce1f36c062b8271a9ac5/tiktoken-0.13.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaaaef47c2406277181d2086484c317bf7fc433e2d5d03ff94f56b0dcec87471", size = 1034977, upload-time = "2026-05-15T04:51:00.957Z" }, + { url = "https://files.pythonhosted.org/packages/d9/77/5ec6e6bc5b30bed6d93f7f2162d8f6b32437b3ba27cb527cfe004f6109c9/tiktoken-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ca8b310bd93b3772cb1b7922d915446864860f562bdfe4825c63a0aed3fb28cd", size = 983635, upload-time = "2026-05-15T04:51:02.629Z" }, + { url = "https://files.pythonhosted.org/packages/94/b0/c8ae9aff00d625c50659b4513e707a0462c4bf5d4d6cc1b802103225c02e/tiktoken-0.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32e0c12305105002c047b3bb1070b0dd9a73b0cb3b2856a8972b810e7a4f5881", size = 1116036, upload-time = "2026-05-15T04:51:04.082Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ac/6a5dddd1d0a6018ecb389bd0353e6b4a515eb4d2286611bd0ace1937b9e1/tiktoken-0.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:5ba5fd62507a932d1241346179e3b39bc7bf7408f03c272652d93b3bedf5db24", size = 1135544, upload-time = "2026-05-15T04:51:05.229Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b8/585032b4384b2f7dcdaddcb52865c83a701a420d09e3c2b4a2be1c450c57/tiktoken-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d108bc2d470fc53c8ecd24f2c0fd2b5f98c33e87cdb6aa2e9b8c5dced703d273", size = 1182217, upload-time = "2026-05-15T04:51:06.517Z" }, + { url = "https://files.pythonhosted.org/packages/cd/b6/993ff1ded3958215fd341a847b8e5ffeb5de473f435296870d314fc91ac4/tiktoken-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cb99cb5127449f58d0a2d5f5ccfb390d8dbdfd919c221246caaee29d8725ed51", size = 1239404, upload-time = "2026-05-15T04:51:07.843Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3d/fef7e06e3b33e7538db0ced734cf9fe23b6832d2ac4990c119c377aec55e/tiktoken-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:115c4f26ffa11caac8b54eea35c2ad38c612c20a48d35dd15d70a02ac6f51f58", size = 918686, upload-time = "2026-05-15T04:51:08.925Z" }, + { url = "https://files.pythonhosted.org/packages/c1/82/a7fc44582bc32ab00de988a2299bf77c077f59068b233109e34b7d6ca7e6/tiktoken-0.13.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:472527e9132952f2fbf77cd290658bacf003d4d5a3fabc18e5fbd407cbae4d9b", size = 1034454, upload-time = "2026-05-15T04:51:10.035Z" }, + { url = "https://files.pythonhosted.org/packages/37/d0/24d8a890c14f432a05cea669c17bebeaa99f96a7c79523b590f564246411/tiktoken-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e2f67d27c9626cdd25fe33d9313c5cdb3d8d82da646b68d6eb8e7e9c20e6448", size = 982976, upload-time = "2026-05-15T04:51:11.23Z" }, + { url = "https://files.pythonhosted.org/packages/49/b7/2ab43f62788a9266187a9bfc1d3af99ad83e5eaa25fbef168a69cd5ad14f/tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2b920b35805cd64585a37c3dc7ce65fba4d2d36016be01e1d7942482ca29093a", size = 1115526, upload-time = "2026-05-15T04:51:12.608Z" }, + { url = "https://files.pythonhosted.org/packages/64/39/1494321ed323ce7a14d88e3cd6cb9058625977df1c6961ddc492bd10a9f3/tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:493af3aa28a4aaf2e3d2600a2ee717252c9bf5ab38fff94eb5a02db5ab77e5ad", size = 1136466, upload-time = "2026-05-15T04:51:13.926Z" }, + { url = "https://files.pythonhosted.org/packages/96/d9/dfd086aa2d918c563a140720e0ce296cada1634efd2783d5cf51e05f984e/tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6644c9c2b5cf3916f5a3641d7d12fdb3f006a7b3d9ff6acdaec44e29ab1ff91e", size = 1181863, upload-time = "2026-05-15T04:51:15.025Z" }, + { url = "https://files.pythonhosted.org/packages/2f/68/a18b4f307086954fdae32714cb4f85562e34f9d34ab206e61f1816aa6018/tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cb65b60b9408563676d874a3a4ee573370066f0dc4e29d84e82e989c6517424", size = 1239218, upload-time = "2026-05-15T04:51:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/16/5b/f2aa703a4fc5d2dff73460a7d46cc2f3f44aa0f3dd8eeb20d2a0ecf68862/tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07", size = 918110, upload-time = "2026-05-15T04:51:17.237Z" }, +] + [[package]] name = "timm" version = "1.0.27" @@ -6472,6 +7042,9 @@ evals = [ { name = "chardet" }, { name = "lm-eval" }, ] +inspect = [ + { name = "inspect-ai" }, +] lit = [ { name = "lit-nlp" }, ] @@ -6534,6 +7107,7 @@ requires-dist = [ { name = "einops", specifier = ">=0.6.0" }, { name = "fancy-einsum", specifier = ">=0.0.3" }, { name = "huggingface-hub", specifier = ">=0.23.2" }, + { name = "inspect-ai", marker = "extra == 'inspect'", specifier = ">=0.3" }, { name = "jaxtyping", specifier = ">=0.2.11" }, { name = "lit-nlp", marker = "extra == 'lit'", specifier = ">=1.3" }, { name = "lm-eval", marker = "extra == 'evals'", specifier = ">=0.4" }, @@ -6553,7 +7127,7 @@ requires-dist = [ { name = "typing-extensions" }, { name = "wandb", specifier = ">=0.13.5" }, ] -provides-extras = ["evals", "lit"] +provides-extras = ["evals", "inspect", "lit"] [package.metadata.requires-dev] demo = [ @@ -6731,6 +7305,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, ] +[[package]] +name = "uc-micro-py" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/78/67/9a363818028526e2d4579334460df777115bdec1bb77c08f9db88f6389f2/uc_micro_py-2.0.0.tar.gz", hash = "sha256:c53691e495c8db60e16ffc4861a35469b0ba0821fe409a8a7a0a71864d33a811", size = 6611, upload-time = "2026-03-01T06:31:27.526Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/73/d21edf5b204d1467e06500080a50f79d49ef2b997c79123a536d4a17d97c/uc_micro_py-2.0.0-py3-none-any.whl", hash = "sha256:3603a3859af53e5a39bc7677713c78ea6589ff188d70f4fee165db88e22b242c", size = 6383, upload-time = "2026-03-01T06:31:26.257Z" }, +] + +[[package]] +name = "universal-pathlib" +version = "0.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fsspec" }, + { name = "pathlib-abc" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/6e/d997a70ee8f4c61f9a7e2f4f8af721cf072a3326848fc881b05187e52558/universal_pathlib-0.3.10.tar.gz", hash = "sha256:4487cbc90730a48cfb64f811d99e14b6faed6d738420cd5f93f59f48e6930bfb", size = 261110, upload-time = "2026-02-22T14:40:58.87Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/1a/5d9a402b39ec892d856bbdd9db502ff73ce28cdf4aff72eb1ce1d6843506/universal_pathlib-0.3.10-py3-none-any.whl", hash = "sha256:dfaf2fb35683d2eb1287a3ed7b215e4d6016aa6eaf339c607023d22f90821c66", size = 83528, upload-time = "2026-02-22T14:40:57.316Z" }, +] + [[package]] name = "uri-template" version = "1.3.0" @@ -6856,6 +7452,75 @@ version = "1.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/4a/29/a31940c848521f0725f0df6b25dca8917f13a2025b0e8fcbe5d0457e45e6/word2number-1.1.zip", hash = "sha256:70e27a5d387f67b04c71fbb7621c05930b19bfd26efd6851e6e0f9969dcde7d0", size = 9723, upload-time = "2017-06-02T15:45:14.488Z" } +[[package]] +name = "wrapt" +version = "1.17.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547, upload-time = "2025-08-12T05:53:21.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/23/bb82321b86411eb51e5a5db3fb8f8032fd30bd7c2d74bfe936136b2fa1d6/wrapt-1.17.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88bbae4d40d5a46142e70d58bf664a89b6b4befaea7b2ecc14e03cedb8e06c04", size = 53482, upload-time = "2025-08-12T05:51:44.467Z" }, + { url = "https://files.pythonhosted.org/packages/45/69/f3c47642b79485a30a59c63f6d739ed779fb4cc8323205d047d741d55220/wrapt-1.17.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6b13af258d6a9ad602d57d889f83b9d5543acd471eee12eb51f5b01f8eb1bc2", size = 38676, upload-time = "2025-08-12T05:51:32.636Z" }, + { url = "https://files.pythonhosted.org/packages/d1/71/e7e7f5670c1eafd9e990438e69d8fb46fa91a50785332e06b560c869454f/wrapt-1.17.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd341868a4b6714a5962c1af0bd44f7c404ef78720c7de4892901e540417111c", size = 38957, upload-time = "2025-08-12T05:51:54.655Z" }, + { url = "https://files.pythonhosted.org/packages/de/17/9f8f86755c191d6779d7ddead1a53c7a8aa18bccb7cea8e7e72dfa6a8a09/wrapt-1.17.3-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f9b2601381be482f70e5d1051a5965c25fb3625455a2bf520b5a077b22afb775", size = 81975, upload-time = "2025-08-12T05:52:30.109Z" }, + { url = "https://files.pythonhosted.org/packages/f2/15/dd576273491f9f43dd09fce517f6c2ce6eb4fe21681726068db0d0467096/wrapt-1.17.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:343e44b2a8e60e06a7e0d29c1671a0d9951f59174f3709962b5143f60a2a98bd", size = 83149, upload-time = "2025-08-12T05:52:09.316Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c4/5eb4ce0d4814521fee7aa806264bf7a114e748ad05110441cd5b8a5c744b/wrapt-1.17.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:33486899acd2d7d3066156b03465b949da3fd41a5da6e394ec49d271baefcf05", size = 82209, upload-time = "2025-08-12T05:52:10.331Z" }, + { url = "https://files.pythonhosted.org/packages/31/4b/819e9e0eb5c8dc86f60dfc42aa4e2c0d6c3db8732bce93cc752e604bb5f5/wrapt-1.17.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e6f40a8aa5a92f150bdb3e1c44b7e98fb7113955b2e5394122fa5532fec4b418", size = 81551, upload-time = "2025-08-12T05:52:31.137Z" }, + { url = "https://files.pythonhosted.org/packages/f8/83/ed6baf89ba3a56694700139698cf703aac9f0f9eb03dab92f57551bd5385/wrapt-1.17.3-cp310-cp310-win32.whl", hash = "sha256:a36692b8491d30a8c75f1dfee65bef119d6f39ea84ee04d9f9311f83c5ad9390", size = 36464, upload-time = "2025-08-12T05:53:01.204Z" }, + { url = "https://files.pythonhosted.org/packages/2f/90/ee61d36862340ad7e9d15a02529df6b948676b9a5829fd5e16640156627d/wrapt-1.17.3-cp310-cp310-win_amd64.whl", hash = "sha256:afd964fd43b10c12213574db492cb8f73b2f0826c8df07a68288f8f19af2ebe6", size = 38748, upload-time = "2025-08-12T05:53:00.209Z" }, + { url = "https://files.pythonhosted.org/packages/bd/c3/cefe0bd330d389c9983ced15d326f45373f4073c9f4a8c2f99b50bfea329/wrapt-1.17.3-cp310-cp310-win_arm64.whl", hash = "sha256:af338aa93554be859173c39c85243970dc6a289fa907402289eeae7543e1ae18", size = 36810, upload-time = "2025-08-12T05:52:51.906Z" }, + { url = "https://files.pythonhosted.org/packages/52/db/00e2a219213856074a213503fdac0511203dceefff26e1daa15250cc01a0/wrapt-1.17.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:273a736c4645e63ac582c60a56b0acb529ef07f78e08dc6bfadf6a46b19c0da7", size = 53482, upload-time = "2025-08-12T05:51:45.79Z" }, + { url = "https://files.pythonhosted.org/packages/5e/30/ca3c4a5eba478408572096fe9ce36e6e915994dd26a4e9e98b4f729c06d9/wrapt-1.17.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5531d911795e3f935a9c23eb1c8c03c211661a5060aab167065896bbf62a5f85", size = 38674, upload-time = "2025-08-12T05:51:34.629Z" }, + { url = "https://files.pythonhosted.org/packages/31/25/3e8cc2c46b5329c5957cec959cb76a10718e1a513309c31399a4dad07eb3/wrapt-1.17.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0610b46293c59a3adbae3dee552b648b984176f8562ee0dba099a56cfbe4df1f", size = 38959, upload-time = "2025-08-12T05:51:56.074Z" }, + { url = "https://files.pythonhosted.org/packages/5d/8f/a32a99fc03e4b37e31b57cb9cefc65050ea08147a8ce12f288616b05ef54/wrapt-1.17.3-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b32888aad8b6e68f83a8fdccbf3165f5469702a7544472bdf41f582970ed3311", size = 82376, upload-time = "2025-08-12T05:52:32.134Z" }, + { url = "https://files.pythonhosted.org/packages/31/57/4930cb8d9d70d59c27ee1332a318c20291749b4fba31f113c2f8ac49a72e/wrapt-1.17.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cccf4f81371f257440c88faed6b74f1053eef90807b77e31ca057b2db74edb1", size = 83604, upload-time = "2025-08-12T05:52:11.663Z" }, + { url = "https://files.pythonhosted.org/packages/a8/f3/1afd48de81d63dd66e01b263a6fbb86e1b5053b419b9b33d13e1f6d0f7d0/wrapt-1.17.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8a210b158a34164de8bb68b0e7780041a903d7b00c87e906fb69928bf7890d5", size = 82782, upload-time = "2025-08-12T05:52:12.626Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d7/4ad5327612173b144998232f98a85bb24b60c352afb73bc48e3e0d2bdc4e/wrapt-1.17.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:79573c24a46ce11aab457b472efd8d125e5a51da2d1d24387666cd85f54c05b2", size = 82076, upload-time = "2025-08-12T05:52:33.168Z" }, + { url = "https://files.pythonhosted.org/packages/bb/59/e0adfc831674a65694f18ea6dc821f9fcb9ec82c2ce7e3d73a88ba2e8718/wrapt-1.17.3-cp311-cp311-win32.whl", hash = "sha256:c31eebe420a9a5d2887b13000b043ff6ca27c452a9a22fa71f35f118e8d4bf89", size = 36457, upload-time = "2025-08-12T05:53:03.936Z" }, + { url = "https://files.pythonhosted.org/packages/83/88/16b7231ba49861b6f75fc309b11012ede4d6b0a9c90969d9e0db8d991aeb/wrapt-1.17.3-cp311-cp311-win_amd64.whl", hash = "sha256:0b1831115c97f0663cb77aa27d381237e73ad4f721391a9bfb2fe8bc25fa6e77", size = 38745, upload-time = "2025-08-12T05:53:02.885Z" }, + { url = "https://files.pythonhosted.org/packages/9a/1e/c4d4f3398ec073012c51d1c8d87f715f56765444e1a4b11e5180577b7e6e/wrapt-1.17.3-cp311-cp311-win_arm64.whl", hash = "sha256:5a7b3c1ee8265eb4c8f1b7d29943f195c00673f5ab60c192eba2d4a7eae5f46a", size = 36806, upload-time = "2025-08-12T05:52:53.368Z" }, + { url = "https://files.pythonhosted.org/packages/9f/41/cad1aba93e752f1f9268c77270da3c469883d56e2798e7df6240dcb2287b/wrapt-1.17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0", size = 53998, upload-time = "2025-08-12T05:51:47.138Z" }, + { url = "https://files.pythonhosted.org/packages/60/f8/096a7cc13097a1869fe44efe68dace40d2a16ecb853141394047f0780b96/wrapt-1.17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba", size = 39020, upload-time = "2025-08-12T05:51:35.906Z" }, + { url = "https://files.pythonhosted.org/packages/33/df/bdf864b8997aab4febb96a9ae5c124f700a5abd9b5e13d2a3214ec4be705/wrapt-1.17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd", size = 39098, upload-time = "2025-08-12T05:51:57.474Z" }, + { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036, upload-time = "2025-08-12T05:52:34.784Z" }, + { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156, upload-time = "2025-08-12T05:52:13.599Z" }, + { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102, upload-time = "2025-08-12T05:52:14.56Z" }, + { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732, upload-time = "2025-08-12T05:52:36.165Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b1/43b286ca1392a006d5336412d41663eeef1ad57485f3e52c767376ba7e5a/wrapt-1.17.3-cp312-cp312-win32.whl", hash = "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe", size = 36705, upload-time = "2025-08-12T05:53:07.123Z" }, + { url = "https://files.pythonhosted.org/packages/28/de/49493f962bd3c586ab4b88066e967aa2e0703d6ef2c43aa28cb83bf7b507/wrapt-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c", size = 38877, upload-time = "2025-08-12T05:53:05.436Z" }, + { url = "https://files.pythonhosted.org/packages/f1/48/0f7102fe9cb1e8a5a77f80d4f0956d62d97034bbe88d33e94699f99d181d/wrapt-1.17.3-cp312-cp312-win_arm64.whl", hash = "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6", size = 36885, upload-time = "2025-08-12T05:52:54.367Z" }, + { url = "https://files.pythonhosted.org/packages/fc/f6/759ece88472157acb55fc195e5b116e06730f1b651b5b314c66291729193/wrapt-1.17.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a47681378a0439215912ef542c45a783484d4dd82bac412b71e59cf9c0e1cea0", size = 54003, upload-time = "2025-08-12T05:51:48.627Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a9/49940b9dc6d47027dc850c116d79b4155f15c08547d04db0f07121499347/wrapt-1.17.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:54a30837587c6ee3cd1a4d1c2ec5d24e77984d44e2f34547e2323ddb4e22eb77", size = 39025, upload-time = "2025-08-12T05:51:37.156Z" }, + { url = "https://files.pythonhosted.org/packages/45/35/6a08de0f2c96dcdd7fe464d7420ddb9a7655a6561150e5fc4da9356aeaab/wrapt-1.17.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:16ecf15d6af39246fe33e507105d67e4b81d8f8d2c6598ff7e3ca1b8a37213f7", size = 39108, upload-time = "2025-08-12T05:51:58.425Z" }, + { url = "https://files.pythonhosted.org/packages/0c/37/6faf15cfa41bf1f3dba80cd3f5ccc6622dfccb660ab26ed79f0178c7497f/wrapt-1.17.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6fd1ad24dc235e4ab88cda009e19bf347aabb975e44fd5c2fb22a3f6e4141277", size = 88072, upload-time = "2025-08-12T05:52:37.53Z" }, + { url = "https://files.pythonhosted.org/packages/78/f2/efe19ada4a38e4e15b6dff39c3e3f3f73f5decf901f66e6f72fe79623a06/wrapt-1.17.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ed61b7c2d49cee3c027372df5809a59d60cf1b6c2f81ee980a091f3afed6a2d", size = 88214, upload-time = "2025-08-12T05:52:15.886Z" }, + { url = "https://files.pythonhosted.org/packages/40/90/ca86701e9de1622b16e09689fc24b76f69b06bb0150990f6f4e8b0eeb576/wrapt-1.17.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:423ed5420ad5f5529db9ce89eac09c8a2f97da18eb1c870237e84c5a5c2d60aa", size = 87105, upload-time = "2025-08-12T05:52:17.914Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e0/d10bd257c9a3e15cbf5523025252cc14d77468e8ed644aafb2d6f54cb95d/wrapt-1.17.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e01375f275f010fcbf7f643b4279896d04e571889b8a5b3f848423d91bf07050", size = 87766, upload-time = "2025-08-12T05:52:39.243Z" }, + { url = "https://files.pythonhosted.org/packages/e8/cf/7d848740203c7b4b27eb55dbfede11aca974a51c3d894f6cc4b865f42f58/wrapt-1.17.3-cp313-cp313-win32.whl", hash = "sha256:53e5e39ff71b3fc484df8a522c933ea2b7cdd0d5d15ae82e5b23fde87d44cbd8", size = 36711, upload-time = "2025-08-12T05:53:10.074Z" }, + { url = "https://files.pythonhosted.org/packages/57/54/35a84d0a4d23ea675994104e667ceff49227ce473ba6a59ba2c84f250b74/wrapt-1.17.3-cp313-cp313-win_amd64.whl", hash = "sha256:1f0b2f40cf341ee8cc1a97d51ff50dddb9fcc73241b9143ec74b30fc4f44f6cb", size = 38885, upload-time = "2025-08-12T05:53:08.695Z" }, + { url = "https://files.pythonhosted.org/packages/01/77/66e54407c59d7b02a3c4e0af3783168fff8e5d61def52cda8728439d86bc/wrapt-1.17.3-cp313-cp313-win_arm64.whl", hash = "sha256:7425ac3c54430f5fc5e7b6f41d41e704db073309acfc09305816bc6a0b26bb16", size = 36896, upload-time = "2025-08-12T05:52:55.34Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/cd864b2a14f20d14f4c496fab97802001560f9f41554eef6df201cd7f76c/wrapt-1.17.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cf30f6e3c077c8e6a9a7809c94551203c8843e74ba0c960f4a98cd80d4665d39", size = 54132, upload-time = "2025-08-12T05:51:49.864Z" }, + { url = "https://files.pythonhosted.org/packages/d5/46/d011725b0c89e853dc44cceb738a307cde5d240d023d6d40a82d1b4e1182/wrapt-1.17.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e228514a06843cae89621384cfe3a80418f3c04aadf8a3b14e46a7be704e4235", size = 39091, upload-time = "2025-08-12T05:51:38.935Z" }, + { url = "https://files.pythonhosted.org/packages/2e/9e/3ad852d77c35aae7ddebdbc3b6d35ec8013af7d7dddad0ad911f3d891dae/wrapt-1.17.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5ea5eb3c0c071862997d6f3e02af1d055f381b1d25b286b9d6644b79db77657c", size = 39172, upload-time = "2025-08-12T05:51:59.365Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f7/c983d2762bcce2326c317c26a6a1e7016f7eb039c27cdf5c4e30f4160f31/wrapt-1.17.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:281262213373b6d5e4bb4353bc36d1ba4084e6d6b5d242863721ef2bf2c2930b", size = 87163, upload-time = "2025-08-12T05:52:40.965Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0f/f673f75d489c7f22d17fe0193e84b41540d962f75fce579cf6873167c29b/wrapt-1.17.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4a8d2b25efb6681ecacad42fca8859f88092d8732b170de6a5dddd80a1c8fa", size = 87963, upload-time = "2025-08-12T05:52:20.326Z" }, + { url = "https://files.pythonhosted.org/packages/df/61/515ad6caca68995da2fac7a6af97faab8f78ebe3bf4f761e1b77efbc47b5/wrapt-1.17.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:373342dd05b1d07d752cecbec0c41817231f29f3a89aa8b8843f7b95992ed0c7", size = 86945, upload-time = "2025-08-12T05:52:21.581Z" }, + { url = "https://files.pythonhosted.org/packages/d3/bd/4e70162ce398462a467bc09e768bee112f1412e563620adc353de9055d33/wrapt-1.17.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d40770d7c0fd5cbed9d84b2c3f2e156431a12c9a37dc6284060fb4bec0b7ffd4", size = 86857, upload-time = "2025-08-12T05:52:43.043Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b8/da8560695e9284810b8d3df8a19396a6e40e7518059584a1a394a2b35e0a/wrapt-1.17.3-cp314-cp314-win32.whl", hash = "sha256:fbd3c8319de8e1dc79d346929cd71d523622da527cca14e0c1d257e31c2b8b10", size = 37178, upload-time = "2025-08-12T05:53:12.605Z" }, + { url = "https://files.pythonhosted.org/packages/db/c8/b71eeb192c440d67a5a0449aaee2310a1a1e8eca41676046f99ed2487e9f/wrapt-1.17.3-cp314-cp314-win_amd64.whl", hash = "sha256:e1a4120ae5705f673727d3253de3ed0e016f7cd78dc463db1b31e2463e1f3cf6", size = 39310, upload-time = "2025-08-12T05:53:11.106Z" }, + { url = "https://files.pythonhosted.org/packages/45/20/2cda20fd4865fa40f86f6c46ed37a2a8356a7a2fde0773269311f2af56c7/wrapt-1.17.3-cp314-cp314-win_arm64.whl", hash = "sha256:507553480670cab08a800b9463bdb881b2edeed77dc677b0a5915e6106e91a58", size = 37266, upload-time = "2025-08-12T05:52:56.531Z" }, + { url = "https://files.pythonhosted.org/packages/77/ed/dd5cf21aec36c80443c6f900449260b80e2a65cf963668eaef3b9accce36/wrapt-1.17.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:ed7c635ae45cfbc1a7371f708727bf74690daedc49b4dba310590ca0bd28aa8a", size = 56544, upload-time = "2025-08-12T05:51:51.109Z" }, + { url = "https://files.pythonhosted.org/packages/8d/96/450c651cc753877ad100c7949ab4d2e2ecc4d97157e00fa8f45df682456a/wrapt-1.17.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:249f88ed15503f6492a71f01442abddd73856a0032ae860de6d75ca62eed8067", size = 40283, upload-time = "2025-08-12T05:51:39.912Z" }, + { url = "https://files.pythonhosted.org/packages/d1/86/2fcad95994d9b572db57632acb6f900695a648c3e063f2cd344b3f5c5a37/wrapt-1.17.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a03a38adec8066d5a37bea22f2ba6bbf39fcdefbe2d91419ab864c3fb515454", size = 40366, upload-time = "2025-08-12T05:52:00.693Z" }, + { url = "https://files.pythonhosted.org/packages/64/0e/f4472f2fdde2d4617975144311f8800ef73677a159be7fe61fa50997d6c0/wrapt-1.17.3-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d4478d72eb61c36e5b446e375bbc49ed002430d17cdec3cecb36993398e1a9e", size = 108571, upload-time = "2025-08-12T05:52:44.521Z" }, + { url = "https://files.pythonhosted.org/packages/cc/01/9b85a99996b0a97c8a17484684f206cbb6ba73c1ce6890ac668bcf3838fb/wrapt-1.17.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223db574bb38637e8230eb14b185565023ab624474df94d2af18f1cdb625216f", size = 113094, upload-time = "2025-08-12T05:52:22.618Z" }, + { url = "https://files.pythonhosted.org/packages/25/02/78926c1efddcc7b3aa0bc3d6b33a822f7d898059f7cd9ace8c8318e559ef/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e405adefb53a435f01efa7ccdec012c016b5a1d3f35459990afc39b6be4d5056", size = 110659, upload-time = "2025-08-12T05:52:24.057Z" }, + { url = "https://files.pythonhosted.org/packages/dc/ee/c414501ad518ac3e6fe184753632fe5e5ecacdcf0effc23f31c1e4f7bfcf/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:88547535b787a6c9ce4086917b6e1d291aa8ed914fdd3a838b3539dc95c12804", size = 106946, upload-time = "2025-08-12T05:52:45.976Z" }, + { url = "https://files.pythonhosted.org/packages/be/44/a1bd64b723d13bb151d6cc91b986146a1952385e0392a78567e12149c7b4/wrapt-1.17.3-cp314-cp314t-win32.whl", hash = "sha256:41b1d2bc74c2cac6f9074df52b2efbef2b30bdfe5f40cb78f8ca22963bc62977", size = 38717, upload-time = "2025-08-12T05:53:15.214Z" }, + { url = "https://files.pythonhosted.org/packages/79/d9/7cfd5a312760ac4dd8bf0184a6ee9e43c33e47f3dadc303032ce012b8fa3/wrapt-1.17.3-cp314-cp314t-win_amd64.whl", hash = "sha256:73d496de46cd2cdbdbcce4ae4bcdb4afb6a11234a1df9c085249d55166b95116", size = 41334, upload-time = "2025-08-12T05:53:14.178Z" }, + { url = "https://files.pythonhosted.org/packages/46/78/10ad9781128ed2f99dbc474f43283b13fea8ba58723e98844367531c18e9/wrapt-1.17.3-cp314-cp314t-win_arm64.whl", hash = "sha256:f38e60678850c42461d4202739f9bf1e3a737c7ad283638251e79cc49effb6b6", size = 38471, upload-time = "2025-08-12T05:52:57.784Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, +] + [[package]] name = "xxhash" version = "3.5.0" @@ -6929,42 +7594,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/e3/bef7b82c1997579c94de9ac5ea7626d01ae5858aa22bf4fcb38bf220cb3e/xxhash-3.5.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5a74f23335b9689b66eb6dbe2a931a88fcd7a4c2cc4b1cb0edba8ce381c7a1da", size = 30064, upload-time = "2024-08-17T09:20:15.925Z" }, ] -[[package]] -name = "y-py" -version = "0.6.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7a/4f/af6e0c02d6876fc466f0ae74ac01693f00d822a93830a9c3e84d17b03f8d/y_py-0.6.2.tar.gz", hash = "sha256:4757a82a50406a0b3a333aa0122019a331bd6f16e49fed67dca423f928b3fd4d", size = 53013, upload-time = "2023-10-05T06:00:28.253Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/fb/cc2f4d626f4b2daf7adb5c118f7c384da37e5fe27a7e79a5754ba687f25d/y_py-0.6.2-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:c26bada6cd109095139237a46f50fc4308f861f0d304bc9e70acbc6c4503d158", size = 757240, upload-time = "2023-10-05T05:57:47.817Z" }, - { url = "https://files.pythonhosted.org/packages/07/00/2e3f449ac8eb9d0630fc83b2b45ec15b0454f6077b32c9b19e092c7b4ca1/y_py-0.6.2-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:bae1b1ad8d2b8cf938a60313f8f7461de609621c5dcae491b6e54975f76f83c5", size = 1468898, upload-time = "2023-10-05T05:57:50.555Z" }, - { url = "https://files.pythonhosted.org/packages/32/09/917e0c31bf3a714621a5f4c4a95ef674a64857a0410738a006f1fdd8413d/y_py-0.6.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e794e44fa260300b8850246c6371d94014753c73528f97f6ccb42f5e7ce698ae", size = 1689362, upload-time = "2023-10-05T05:57:53.216Z" }, - { url = "https://files.pythonhosted.org/packages/dc/f9/5038e74a773c2ee89ed5f93474a5baef8275ac3d37a2dde6fd5b5a9f5849/y_py-0.6.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b2686d7d8ca31531458a48e08b0344a8eec6c402405446ce7d838e2a7e43355a", size = 1693938, upload-time = "2023-10-05T05:57:55.37Z" }, - { url = "https://files.pythonhosted.org/packages/ef/42/b429fae61604c7b051719c4ebfe4114fd63aa53ea6f44d3e61c61cbe394a/y_py-0.6.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d917f5bc27b85611ceee4eb85f0e4088b0a03b4eed22c472409933a94ee953cf", size = 1850649, upload-time = "2023-10-05T05:57:58.207Z" }, - { url = "https://files.pythonhosted.org/packages/9e/71/d1eaa4a8b459f7e85f834b2c9e378f3438248b38c591483b84e3e920b6f6/y_py-0.6.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8f6071328aad06fdcc0a4acc2dc4839396d645f5916de07584af807eb7c08407", size = 2053480, upload-time = "2023-10-05T05:58:00.838Z" }, - { url = "https://files.pythonhosted.org/packages/81/1b/06a65269836058d91603034b85ec7c7dbf710f790a8b3a136e371690a653/y_py-0.6.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:266ec46ab9f9cb40fbb5e649f55c329fc4620fa0b1a8117bdeefe91595e182dc", size = 1704103, upload-time = "2023-10-05T05:58:03.995Z" }, - { url = "https://files.pythonhosted.org/packages/21/9d/041e36707f5ef7643e50ca48ad4dcc02e0b9f68228e71affc064b4918e46/y_py-0.6.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ce15a842c2a0bf46180ae136743b561fa276300dd7fa61fe76daf00ec7dc0c2d", size = 1759469, upload-time = "2023-10-05T05:58:06.619Z" }, - { url = "https://files.pythonhosted.org/packages/cc/be/49423b6ac8869d52a8e455173532d0167150b43fd6a6bc041a2c82df2eef/y_py-0.6.2-cp310-none-win32.whl", hash = "sha256:1d5b544e79ace93fdbd0b36ed329c86e346898153ac7ba2ec62bc9b4c6b745c9", size = 521410, upload-time = "2023-10-05T05:58:08.354Z" }, - { url = "https://files.pythonhosted.org/packages/9a/19/3fb6e0e998aa610b9a4da4649569cfea1353986c58abe161e25d7cddda9e/y_py-0.6.2-cp310-none-win_amd64.whl", hash = "sha256:80a827e173372682959a57e6b8cc4f6468b1a4495b4bc7a775ef6ca05ae3e8e8", size = 550365, upload-time = "2023-10-05T05:58:10.842Z" }, - { url = "https://files.pythonhosted.org/packages/a0/82/8884c8184f563c3bd78dcd010bfd88d9e9add84ba16f4a4c2fd7c8d5e27c/y_py-0.6.2-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:a21148b8ea09a631b752d975f9410ee2a31c0e16796fdc113422a6d244be10e5", size = 757236, upload-time = "2023-10-05T05:58:12.819Z" }, - { url = "https://files.pythonhosted.org/packages/53/07/ca7950843650b38a22cf65fdf0a50271771624aa5ff1daf6b6ea5c17d40f/y_py-0.6.2-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:898fede446ca1926b8406bdd711617c2aebba8227ee8ec1f0c2f8568047116f7", size = 1468889, upload-time = "2023-10-05T05:58:15.389Z" }, - { url = "https://files.pythonhosted.org/packages/02/44/2cf2ca14c93a0fc2793bc3c010e26284ced6705327367665e476297f9a2d/y_py-0.6.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce7c20b9395696d3b5425dccf2706d374e61ccf8f3656bff9423093a6df488f5", size = 1689431, upload-time = "2023-10-05T05:58:17.191Z" }, - { url = "https://files.pythonhosted.org/packages/c1/c1/0eec08de1104f21dc5a04e3179452d11cae46c18857c7ca556f70fbe6278/y_py-0.6.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a3932f53418b408fa03bd002e6dc573a74075c2c092926dde80657c39aa2e054", size = 1693932, upload-time = "2023-10-05T05:58:19.385Z" }, - { url = "https://files.pythonhosted.org/packages/67/a7/756acaa1fbc81f855a02df50f7a71d52ccf266bc9cc67f877ac9333f84ec/y_py-0.6.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:df35ea436592eb7e30e59c5403ec08ec3a5e7759e270cf226df73c47b3e739f5", size = 1850768, upload-time = "2023-10-05T05:58:21.846Z" }, - { url = "https://files.pythonhosted.org/packages/b1/dc/e82efbd5ae0e4be22c3f2234f79d0151ce8c1b5f9de6c96a717977871e9f/y_py-0.6.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26cb1307c3ca9e21a3e307ab2c2099677e071ae9c26ec10ddffb3faceddd76b3", size = 2053681, upload-time = "2023-10-05T05:58:23.589Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d1/6431bcbf33ba8d3055812e3dc1ccc6bf21ad5491ae7113a78b43383ad0be/y_py-0.6.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:863e175ce5585f9ff3eba2aa16626928387e2a576157f02c8eb247a218ecdeae", size = 1704093, upload-time = "2023-10-05T05:58:25.749Z" }, - { url = "https://files.pythonhosted.org/packages/9f/fc/79e1bc21400d111cce64417c0db1b1e3844484086dcdf9054448c3c68421/y_py-0.6.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:35fcb9def6ce137540fdc0e91b08729677548b9c393c0151a6359fd199da3bd7", size = 1759685, upload-time = "2023-10-05T05:58:27.937Z" }, - { url = "https://files.pythonhosted.org/packages/66/7f/235d923bcf3fca6ddffe5ffa69d593be4c0b7b258f69a9e3f5919669f64b/y_py-0.6.2-cp311-none-win32.whl", hash = "sha256:86422c6090f34906c062fd3e4fdfdccf3934f2922021e979573ae315050b4288", size = 521409, upload-time = "2023-10-05T05:58:30.448Z" }, - { url = "https://files.pythonhosted.org/packages/a2/87/a81ae35a5d8ee94866b99211cd163dc09318f37bab97f1b05e2619f56a66/y_py-0.6.2-cp311-none-win_amd64.whl", hash = "sha256:6c2f2831c5733b404d2f2da4bfd02bb4612ae18d0822e14ae79b0b92436b816d", size = 550361, upload-time = "2023-10-05T05:58:32.271Z" }, - { url = "https://files.pythonhosted.org/packages/19/e0/7baf1d43bf922e91e54e9fa04858619d1957ec63292191e524456d146648/y_py-0.6.2-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:7cbefd4f1060f05768227ddf83be126397b1d430b026c64e0eb25d3cf50c5734", size = 752650, upload-time = "2023-10-05T05:58:34.025Z" }, - { url = "https://files.pythonhosted.org/packages/05/41/b242e358c530d88053172617489a702cabaa00a4ea6462e02857f7ee173a/y_py-0.6.2-cp312-cp312-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:032365dfe932bfab8e80937ad6093b4c22e67d63ad880096b5fa8768f8d829ba", size = 1461497, upload-time = "2023-10-05T05:58:35.904Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ff/4cbe7961c0ddb421cee48a2dc754a43dde05f31ee15d76e089eeb4f541b8/y_py-0.6.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a70aee572da3994238c974694767365f237fc5949a550bee78a650fe16f83184", size = 1687776, upload-time = "2023-10-05T05:58:37.766Z" }, - { url = "https://files.pythonhosted.org/packages/ff/ff/7190c93d6f0f63d40514a10116512b788b2363a3f8a960851f435ccbab3f/y_py-0.6.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ae80d505aee7b3172cdcc2620ca6e2f85586337371138bb2b71aa377d2c31e9a", size = 1693306, upload-time = "2023-10-05T05:58:40.354Z" }, - { url = "https://files.pythonhosted.org/packages/4e/29/df7d9b506deff4158b80433c19294889951afe0cef911ab99dbbcf8704d5/y_py-0.6.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a497ebe617bec6a420fc47378856caae40ab0652e756f3ed40c5f1fe2a12220", size = 1844205, upload-time = "2023-10-05T05:58:42.409Z" }, - { url = "https://files.pythonhosted.org/packages/1d/a1/be12ce76ae783838c4df08f37dc0b65e85fe762f49c8510e2ca388d4eb68/y_py-0.6.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e8638355ae2f996356f7f281e03a3e3ce31f1259510f9d551465356532e0302c", size = 2029472, upload-time = "2023-10-05T05:58:44.186Z" }, - { url = "https://files.pythonhosted.org/packages/cc/60/cbf95e42656fd84af4bb341f253add1030184b39f86962434920e121a2e4/y_py-0.6.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8448da4092265142662bbd3fc46cb8b0796b1e259189c020bc8f738899abd0b5", size = 1701647, upload-time = "2023-10-05T05:58:46.146Z" }, - { url = "https://files.pythonhosted.org/packages/c9/84/31998386aa81982fac3097816e45db238c8259480a228b497c3b3dc5d57a/y_py-0.6.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:69cfbcbe0a05f43e780e6a198080ba28034bf2bb4804d7d28f71a0379bfd1b19", size = 1758480, upload-time = "2023-10-05T05:58:48.771Z" }, -] - [[package]] name = "yarl" version = "1.20.1" @@ -7065,17 +7694,15 @@ wheels = [ ] [[package]] -name = "ypy-websocket" -version = "0.8.4" +name = "zipfile-zstd" +version = "0.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiofiles" }, - { name = "aiosqlite" }, - { name = "y-py" }, + { name = "zstandard", marker = "python_full_version < '3.14'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b1/0b/8e936a41e15b8a8a034c9708062ee82f18f4ca6a969c443d3ea10a54f1ea/ypy_websocket-0.8.4.tar.gz", hash = "sha256:43a001473f5c8abcf182f603049cf305cbc855ad8deaa9dfa0f3b5a7cea9d0ff", size = 11142, upload-time = "2023-02-21T16:33:52.059Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f7/2a/2e0941bc0058d10ab37d8c578b94a19f611f6ae54f124140f2fb451f0932/zipfile-zstd-0.0.4.tar.gz", hash = "sha256:c1498e15b7922a3d1af0ea55df8b11b2af4e8f7e0e80e414e25d66899f7def89", size = 4603, upload-time = "2021-12-08T07:38:16.245Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bd/f3/f6a8dfcae1716d260e6cf3ecea864a6abfddadd6a059bed80bd5618b67c1/ypy_websocket-0.8.4-py3-none-any.whl", hash = "sha256:b1ba0dfcc9762f0ca168d2378062d3ca1299d39076b0f145d961359121042be5", size = 10465, upload-time = "2023-02-21T16:33:49.668Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3a/bc3011d26bbb490741f58c28a2df559445c59e8524cbbb71ecf33db23bb7/zipfile_zstd-0.0.4-py3-none-any.whl", hash = "sha256:c8e07be35765c072eb7b1be715c89ecb248a1127b014e12a9b8ac7db2600c166", size = 4058, upload-time = "2021-12-08T07:38:14.715Z" }, ] [[package]] From 3e8da1eb08d43bc977043e1c68f2a55c95a2c06f Mon Sep 17 00:00:00 2001 From: jlarson4 Date: Mon, 6 Jul 2026 13:39:17 -0500 Subject: [PATCH 06/87] Setup a parity script --- scripts/vllm_parity_report.py | 222 ++++++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 scripts/vllm_parity_report.py diff --git a/scripts/vllm_parity_report.py b/scripts/vllm_parity_report.py new file mode 100644 index 0000000000..bad526806d --- /dev/null +++ b/scripts/vllm_parity_report.py @@ -0,0 +1,222 @@ +"""Empirical parity report: boot_vllm vs boot_transformers per architecture. + +Closes the vLLM capture trust gap: every vLLM test is otherwise mocked, so captured +activations are numerically unverified. For each model this boots both backends in +fp32 (isolating the boundary mapping from dtype/kernel-precision differences) and diffs +the fireable capture points the vLLM driver OFFERS (``driver.supported_hook_points``): + +- ``embed.hook_out`` and per-layer ``blocks.{i}.hook_out`` / ``attn.hook_out`` / + ``mlp.hook_out`` are semantically identical to boot_transformers and compared directly. +- ``ln_final.hook_normalized`` carries vLLM's POST-weight RMSNorm value under a hook + whose HT convention is PRE-weight; it is un-folded (÷ weight, or ÷ (1+weight) for + Gemma) before comparison. The raw (un-un-folded) diff is also reported so a wrong + un-fold direction is visible rather than silent. + +Non-fireable points (fused attention pattern/scores/rope, unembed) are withheld and +reported, not compared. A final-position argmax agreement check sanity-checks logits. + +GPU-ONLY: requires a CUDA device and a working ``vllm`` install (pinned build). This is +NOT a CPU/per-PR CI job — it SKIPs cleanly when vLLM or a GPU is unavailable. + +Run: uv run python scripts/vllm_parity_report.py +Env: TL_PARITY_MODELS="id1,id2,..." overrides the model list. + TL_VLLM_ATOL / TL_VLLM_RTOL override tolerance (defaults 2e-2). +""" +from __future__ import annotations + +import gc +import os +import sys +import warnings + +warnings.filterwarnings("ignore") + +import torch + +# vLLM-supported decoder-only checkpoints small enough to hold model + capture buffers + +# KV cache on one GPU. gemma-2-2b is included specifically to exercise the (1 + weight) +# ln_final un-fold. Gated ids need HF_TOKEN (sourced from .env); bad/gated ids -> SKIP. +DEFAULT_MODELS = [ + "HuggingFaceTB/SmolLM2-135M", # Llama (tiny, ungated — the smoke test) + "Qwen/Qwen2.5-0.5B", # Qwen2 + "Qwen/Qwen3-0.6B", # Qwen3 + "meta-llama/Llama-3.2-1B", # Llama (gated) + "google/gemma-2-2b", # Gemma2 (gated) — exercises the (1 + weight) ln_final un-fold +] + +PROMPT = "The quick brown fox" +# vLLM's fused kernels (PagedAttention, fused RMSNorm/RoPE) differ numerically from HF's +# eager path even in fp32, so the mapping-correct band is looser than the HF-vs-HF 1e-3. +# A wrong hook mapping (e.g. un-un-folded ln_final) diverges by O(1), well outside this. +ATOL = float(os.environ.get("TL_VLLM_ATOL", "2e-2")) +RTOL = float(os.environ.get("TL_VLLM_RTOL", "2e-2")) + +# vLLM capture kind -> TransformerBridge-native hook name (per-layer uses {i}). +DIRECT_KINDS = { + "resid_post": "blocks.{i}.hook_out", + "attn_out": "blocks.{i}.attn.hook_out", + "mlp_out": "blocks.{i}.mlp.hook_out", +} +EMBED_HOOK = "embed.hook_out" +LNF_HOOK = "ln_final.hook_normalized" + + +def _to2d(t: torch.Tensor) -> torch.Tensor: + """Collapse [batch, seq, d] or [seq, d] to a [tokens, d] float CPU tensor for diffing.""" + t = t.detach().to("cpu", torch.float32) + return t.reshape(-1, t.shape[-1]) + + +def _diff(a: torch.Tensor, b: torch.Tensor) -> tuple[float, bool]: + a2, b2 = _to2d(a), _to2d(b) + if a2.shape != b2.shape: + return float("inf"), False + d = (a2 - b2).abs().max().item() + return d, torch.allclose(a2, b2, atol=ATOL, rtol=RTOL) + + +def _unfold_lnf(vllm_val: torch.Tensor, weight: torch.Tensor, is_gemma: bool) -> torch.Tensor: + """vLLM ln_final is post-weight (x·rsqrt(var+eps)·w); recover the pre-weight value HT + exposes by dividing out the norm weight (Gemma folds 1 + weight).""" + w = weight.detach().to(vllm_val.device, torch.float32) + denom = (1.0 + w) if is_gemma else w + # Guard against near-zero weight entries producing spurious blow-ups. + denom = torch.where(denom.abs() < 1e-6, torch.ones_like(denom), denom) + return vllm_val.to(torch.float32) / denom + + +def verify(model_id: str) -> dict: + from transformer_lens.model_bridge.sources.vllm.source import boot_vllm + from transformer_lens.model_bridge.transformer_bridge import TransformerBridge + + result: dict = {"model": model_id, "arch": "?", "status": "", "detail": ""} + hf = vllm = None + try: + # Matched fp32 both sides so the comparison isolates the boundary mapping, not + # dtype. HF ref on the same CUDA device as vLLM; tensors are moved to CPU to diff. + hf = TransformerBridge.boot_transformers(model_id, device="cuda", dtype=torch.float32) + arch = getattr(hf.cfg, "architecture", "?") + result["arch"] = arch + is_gemma = "gemma" in arch.lower() or "gemma" in model_id.lower() + n_layers = int(hf.cfg.n_layers) + toks = hf.to_tokens(PROMPT) + + vllm = boot_vllm(model_id, dtype=torch.float32, max_model_len=2048) + offered = vllm._driver.supported_hook_points + + hf_logits, hf_cache = hf.run_with_cache(toks) + v_logits, v_cache = vllm.run_with_cache(toks) + + worst = 0.0 + mism: list[str] = [] + notes: list[str] = [] + + # embed + the three per-layer direct boundaries (first & last layer). + checks = [(EMBED_HOOK, EMBED_HOOK)] + for i in sorted({0, n_layers - 1}): + for name in DIRECT_KINDS.values(): + hk = name.format(i=i) + checks.append((hk, hk)) + for hk, _ in checks: + if hk not in offered: + continue # not a fireable point for this overlay + if hk not in v_cache or hk not in hf_cache: + mism.append(f"{hk} missing") + continue + d, ok = _diff(hf_cache[hk], v_cache[hk]) + worst = max(worst, d if d != float("inf") else worst) + if not ok: + mism.append(f"{hk} maxdiff={d:.2e}") + + # ln_final: compare un-folded; also report the raw diff so a wrong un-fold shows. + if LNF_HOOK in offered and LNF_HOOK in v_cache and LNF_HOOK in hf_cache: + raw_d, raw_ok = _diff(hf_cache[LNF_HOOK], v_cache[LNF_HOOK]) + weight = vllm._driver.get_param("model.norm.weight") + if weight is None: + notes.append(f"ln_final raw={raw_d:.2e} (no norm weight to un-fold)") + if not raw_ok: + mism.append(f"{LNF_HOOK} maxdiff={raw_d:.2e} (un-fold unavailable)") + else: + unfolded = _unfold_lnf(v_cache[LNF_HOOK], weight, is_gemma) + uf_d, uf_ok = _diff(hf_cache[LNF_HOOK], unfolded) + worst = max(worst, uf_d if uf_d != float("inf") else worst) + notes.append(f"ln_final unfold={uf_d:.2e} raw={raw_d:.2e}") + if not uf_ok: + mism.append(f"{LNF_HOOK} unfold maxdiff={uf_d:.2e} (raw={raw_d:.2e})") + + # Logit sanity: vLLM synthesizes final-position log-probs only; check top-1 agrees + # (reusing the run_with_cache logits above — no second forward). + try: + hf_top = int(torch.as_tensor(hf_logits)[0, -1].argmax()) + v_top = int(torch.as_tensor(v_logits)[0, -1].argmax()) + notes.append(f"argmax {'==' if hf_top == v_top else '!='}({hf_top},{v_top})") + if hf_top != v_top: + mism.append(f"final-token argmax {hf_top}!={v_top}") + except Exception as e: # logit synth is best-effort; don't fail parity on it + notes.append(f"logit-check err: {type(e).__name__}") + + note_str = "; ".join(notes) + if mism: + result["status"] = "FAIL" + result["detail"] = "; ".join(mism[:3]) + (f" | {note_str}" if note_str else "") + else: + result["status"] = "PASS" + result["detail"] = f"maxdiff={worst:.2e} (L={n_layers}); {note_str}" + except Exception as e: # download/gating/OOM/vLLM-unavailable are not parity failures + result["status"] = "SKIP" + result["detail"] = f"{type(e).__name__}: {str(e).splitlines()[0][:140]}" + finally: + for bridge in (vllm, hf): + try: + if bridge is not None: + bridge.close() + except Exception: + pass + del hf, vllm + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return result + + +def _preflight() -> str | None: + """Return a human reason to abort (no GPU / no vllm), or None if runnable.""" + if not torch.cuda.is_available(): + return "no CUDA device — vLLM capture only materializes in a real GPU forward" + try: + import vllm # noqa: F401 + except Exception as e: + return f"vllm not importable ({type(e).__name__}: {e})" + return None + + +def main() -> None: + reason = _preflight() + if reason is not None: + print(f"SKIP ALL: {reason}", flush=True) + print("This harness is GPU-only and requires a working vllm install.", flush=True) + sys.exit(0) + + ids = os.environ.get("TL_PARITY_MODELS") + models = [m.strip() for m in ids.split(",")] if ids else DEFAULT_MODELS + rows = [] + for m in models: + r = verify(m) + rows.append(r) + print(f"[{r['status']:4}] {r['arch']:28} {r['model']:40} {r['detail']}", flush=True) + + print("\n================ vLLM PARITY REPORT CARD ================") + for status in ("PASS", "FAIL", "SKIP"): + sel = [r for r in rows if r["status"] == status] + print(f"\n{status} ({len(sel)}):") + for r in sel: + print(f" {r['arch']:28} {r['model']:40} {r['detail']}") + passed = sorted({r["arch"] for r in rows if r["status"] == "PASS"}) + print("\nvLLM-CAPTURE-VERIFIED ARCHITECTURES (measured PASS):") + print(" " + (", ".join(passed) if passed else "(none)")) + # Non-zero exit if anything actually ran and failed — lets a manual GPU run gate. + sys.exit(1 if any(r["status"] == "FAIL" for r in rows) else 0) + + +if __name__ == "__main__": + main() From bb5be715eadc5a70733a73cdaa776a607e2659fe Mon Sep 17 00:00:00 2001 From: jlarson4 Date: Mon, 6 Jul 2026 14:15:01 -0500 Subject: [PATCH 07/87] Close gap that prevent falcon attention in inspect --- .../model_bridge/sources/inspect/transformers_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformer_lens/model_bridge/sources/inspect/transformers_provider.py b/transformer_lens/model_bridge/sources/inspect/transformers_provider.py index aa4faf356a..be8a26cf89 100644 --- a/transformer_lens/model_bridge/sources/inspect/transformers_provider.py +++ b/transformer_lens/model_bridge/sources/inspect/transformers_provider.py @@ -46,7 +46,7 @@ # Decoder ModuleList by architecture family; each block's output is resid_post. _LAYER_PATHS = ("model.layers", "transformer.h", "gpt_neox.layers", "model.decoder.layers") # Attn/MLP submodule names within a block, by family. -_ATTN_ATTRS = ("self_attn", "attn", "attention") +_ATTN_ATTRS = ("self_attn", "attn", "attention", "self_attention") # self_attention: Falcon _MLP_ATTRS = ("mlp", "feed_forward") From 0b20b6aec0ebc0e995d7cfa2f7b9755f9a8c0ecc Mon Sep 17 00:00:00 2001 From: jlarson4 Date: Mon, 6 Jul 2026 14:38:54 -0500 Subject: [PATCH 08/87] fixed some logit resolution issues,add logit reconstruction demo for collab --- demos/vLLM_Bridge_Integration_Test.ipynb | 14 ++++++++++++++ .../model_bridge/sources/inspect/driver.py | 16 +++++++++++++++- .../model_bridge/sources/inspect/profiles.py | 10 ++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/demos/vLLM_Bridge_Integration_Test.ipynb b/demos/vLLM_Bridge_Integration_Test.ipynb index 106ec40986..031b5c4942 100644 --- a/demos/vLLM_Bridge_Integration_Test.ipynb +++ b/demos/vLLM_Bridge_Integration_Test.ipynb @@ -194,6 +194,20 @@ ], "id": "a59824b0e3be" }, + { + "cell_type": "markdown", + "id": "f9e987ba", + "source": "## Step 4½ — Logit-reconstruction experiment (help finalize `_reconstruct_logits`)\n\nvLLM's sampler bypasses `lm_head`, so the bridge only returns **final-position log-probs** (`provides_sequence_logits=False`) — no full-sequence logits, no loss, no logit-lens magnitude. Those are recoverable host-side as `ln_final @ lm_head.weightᵀ`, since the captured `ln_final` plus `lm_head.weight` (via `get_param`) are already available.\n\nThe open question this cell answers: **which transform of the captured `ln_final` reproduces HF's logits?** HF's `hook_normalized` is the *pre*-weight RMSNorm value; vLLM's capture is documented *post*-weight and Step 5 flags it reading **~2× HF's** (unresolved). `lm_head` consumes the *post*-weight value, so this searches the candidates (`raw`, `÷ norm_w`, `÷ 2`, …) for the one whose `@ W_U` matches HF's next-token argmax across **all** positions.\n\nThat result nails down two things at once: the driver-side reconstruction formula, **and** the Step-5 `ln_final` ~2× divergence. **Run the next cell and paste its full output back.**\n", + "metadata": {} + }, + { + "cell_type": "code", + "id": "3774dd88", + "source": "# ==========================================================================\n# EXPERIMENT — reconstruct full-sequence logits from the captured ln_final\n# ==========================================================================\n# vLLM bypasses lm_head, so the bridge returns final-position log-probs only\n# (provides_sequence_logits=False). Real logits = @ lm_head.weight.T\n# — but WHICH ln_final? HF's hook_normalized is PRE-weight; vLLM's capture is\n# POST-weight and Step 5 notes it reads ~2x HF's. lm_head consumes the POST-weight\n# value. This cell finds the transform of the captured ln_final whose @ W_U\n# reproduces HF's argmax across ALL positions: that formula is what a driver-side\n# _reconstruct_logits should use, and it also pins down the Step-5 ln_final ~2x.\nimport torch\n\nLNF = \"ln_final.hook_normalized\"\nassert LNF in cache_vllm and LNF in cache_hf_cpu, \"need ln_final captured on both sides\"\nv_lnf = cache_vllm[LNF][0].float().cpu() # (seq, d_model) vLLM capture\nhf_lnf = cache_hf_cpu[LNF][0].float() # (seq, d_model) HF capture (pre-weight)\n\nW = bridge._driver.get_param(\"lm_head.weight\")\ntied = W is None\nif tied:\n W = bridge._driver.get_param(\"model.embed_tokens.weight\") # tie fallback\nW = W.float().cpu() # (vocab, d_model)\n_nw = bridge._driver.get_param(\"model.norm.weight\")\nnorm_w = _nw.float().cpu() if _nw is not None else torch.ones(W.shape[1])\n\nprint(f\"ln_final: vLLM {tuple(v_lnf.shape)} | HF {tuple(hf_lnf.shape)}\")\nprint(f\"lm_head.weight: {'TIED->embed_tokens' if tied else 'present'} {tuple(W.shape)}\"\n f\" | norm_w {tuple(norm_w.shape)} | d_vocab(bridge)={bridge.cfg.d_vocab}\")\n\n# (1) Diagnose vLLM-vs-HF ln_final (Step 5's open ~2x question).\nm = hf_lnf.abs() > 1e-2\nratio = v_lnf[m] / hf_lnf[m]\nprint(f\"\\nln_final ratio vLLM/HF (|hf|>1e-2): median={ratio.median():.3f} mean={ratio.mean():.3f}\")\nprint(f\"norm_w: median={norm_w.median():.3f} (a pure post/pre-weight gap would be ~norm_w)\")\n\n# (2) Which reconstruction reproduces HF's next-token argmax?\ndef recon(lnf, label):\n lg = lnf @ W.T # (seq, vocab)\n top = int(lg[-1].argmax())\n print(f\" {label:<32} final argmax={top}{' <== ==HF' if top == argmax_hf else ''}\")\n return lg\n\nprint(f\"\\nTargets: HF argmax={argmax_hf} vLLM argmax={argmax_vllm}\")\nprint(\"Candidates (lnf @ lm_head.weight.T):\")\ncands = {\n \"HF ln_final (pre-weight)\": hf_lnf,\n \"HF ln_final * norm_w\": hf_lnf * norm_w,\n \"vLLM ln_final (raw)\": v_lnf,\n \"vLLM ln_final / norm_w\": v_lnf / norm_w,\n \"vLLM ln_final / 2\": v_lnf / 2.0,\n}\nlg = {k: recon(v, k) for k, v in cands.items()}\n\n# (3) Full-sequence argmax agreement between the HF-truth recon and each vLLM recon.\ntruth = lg[\"HF ln_final * norm_w\"]\nif int(truth[-1].argmax()) != argmax_hf:\n truth = lg[\"HF ln_final (pre-weight)\"]\ndef agree(a):\n return float((truth.argmax(-1) == a.argmax(-1)).float().mean()) * 100\nprint(\"\\nFull-sequence argmax agreement vs HF-truth recon (all positions):\")\nfor k in (\"vLLM ln_final (raw)\", \"vLLM ln_final / norm_w\", \"vLLM ln_final / 2\"):\n print(f\" {k:<32} {agree(lg[k]):.1f}%\")\n\nprint(\"\\n>>> Paste this whole output back. I need: which candidate is '==HF', the\")\nprint(\">>> ln_final ratio vs norm_w, and the full-sequence agreement %s — that fixes\")\nprint(\">>> both the driver reconstruction formula AND the Step-5 ln_final divergence.\")\n", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/transformer_lens/model_bridge/sources/inspect/driver.py b/transformer_lens/model_bridge/sources/inspect/driver.py index f30240cf96..b7dc954ed9 100644 --- a/transformer_lens/model_bridge/sources/inspect/driver.py +++ b/transformer_lens/model_bridge/sources/inspect/driver.py @@ -10,8 +10,10 @@ import asyncio import logging +import os import threading import warnings +from concurrent.futures import TimeoutError as FutureTimeout from typing import Any, Mapping import numpy as np @@ -26,6 +28,10 @@ from . import hooks, wire from .profiles import TLBridgeProfile +# Cap a single provider call so a hung remote/provider forward unblocks the sync caller +# instead of stalling it forever. Generous by default; override for slow remote backends. +_PROVIDER_TIMEOUT_S = float(os.environ.get("TL_INSPECT_TIMEOUT_S", "300")) + class InspectDriver(DriverBase): """Driver wrapping an ``inspect_ai`` model; capture + interventions via a Profile.""" @@ -160,7 +166,15 @@ def _ensure_loop(self) -> asyncio.AbstractEventLoop: def _run_coro(self, coro: Any) -> Any: future = asyncio.run_coroutine_threadsafe(coro, self._ensure_loop()) - return future.result() # blocks the sync caller; re-raises provider errors + try: + return future.result(timeout=_PROVIDER_TIMEOUT_S) # re-raises provider errors + except FutureTimeout: + future.cancel() + raise TimeoutError( + f"Inspect provider call exceeded {_PROVIDER_TIMEOUT_S:.0f}s " + "(set TL_INSPECT_TIMEOUT_S to change) — the remote/provider forward looks " + "hung; unblocking the caller rather than stalling indefinitely." + ) from None def _wire_keys(self, names: list[str]) -> list[str]: """Unique ``:`` keys for the requested hook names (aliases collapse).""" diff --git a/transformer_lens/model_bridge/sources/inspect/profiles.py b/transformer_lens/model_bridge/sources/inspect/profiles.py index 20222a529c..225317bb7a 100644 --- a/transformer_lens/model_bridge/sources/inspect/profiles.py +++ b/transformer_lens/model_bridge/sources/inspect/profiles.py @@ -13,6 +13,7 @@ """ from __future__ import annotations +import warnings from typing import Any, Mapping import numpy as np @@ -55,6 +56,15 @@ def decode_logits(self, output, n_tokens, d_vocab, tokenizer): entry = (getattr(output, "metadata", None) or {}).get("tl_logits") if entry is not None: return wire.decode_array(entry)[np.newaxis, ...] # full (1, seq, d_vocab) + # Absent tl_logits despite a logits request means the forward produced none — warn + # rather than silently hand back an all -inf tensor (whose argmax is a bogus token 0). + warnings.warn( + "tl_bridge provider returned no 'tl_logits' in output metadata; emitting an " + "all -inf placeholder (argmax would be token 0). Check the provider/wire path " + "rather than trusting these logits.", + RuntimeWarning, + stacklevel=2, + ) return np.full((1, n_tokens, d_vocab), -np.inf, dtype=np.float32) From 2bb3c02dd3d498e082b3ab8e606c641d2347a7d5 Mon Sep 17 00:00:00 2001 From: Jonah Larson Date: Tue, 7 Jul 2026 13:25:10 -0500 Subject: [PATCH 09/87] Inspect & vLLM driver Features & Bugs (#1492) * notebook fix * Rebuilding real logits * improve buffers * Fixed softcap bug * per-position interventino * Inspect HF provider servers head-split attention hooks * Resolving final gaps in interp skills --- demos/vLLM_Bridge_Integration_Test.ipynb | 56 ++- scripts/inspect_parity_report.py | 15 +- .../model_bridge/test_inspect_provider.py | 154 +++++++- .../test_symbolic_bridge_hooks.py | 56 +++ .../unit/model_bridge/test_inspect_driver.py | 118 +++++++ tests/unit/model_bridge/test_vllm_boot.py | 14 + tests/unit/model_bridge/test_vllm_driver.py | 145 +++++++- .../test_vllm_worker_extension.py | 65 ++++ .../model_bridge/component_setup.py | 23 ++ .../model_bridge/sources/inspect/driver.py | 14 +- .../model_bridge/sources/inspect/eval.py | 21 +- .../model_bridge/sources/inspect/hooks.py | 68 +++- .../sources/inspect/intervention.py | 11 +- .../model_bridge/sources/inspect/source.py | 16 +- .../sources/inspect/transformers_provider.py | 332 ++++++++++++++++-- .../model_bridge/sources/vllm/driver.py | 183 ++++++++-- .../model_bridge/sources/vllm/plugin.py | 38 +- .../model_bridge/sources/vllm/source.py | 16 + .../sources/vllm/worker_extension.py | 61 +++- 19 files changed, 1250 insertions(+), 156 deletions(-) create mode 100644 tests/acceptance/model_bridge/test_symbolic_bridge_hooks.py diff --git a/demos/vLLM_Bridge_Integration_Test.ipynb b/demos/vLLM_Bridge_Integration_Test.ipynb index 031b5c4942..ade7aa1e2c 100644 --- a/demos/vLLM_Bridge_Integration_Test.ipynb +++ b/demos/vLLM_Bridge_Integration_Test.ipynb @@ -42,7 +42,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Install vllm and TransformerLens @ feature/vllm-batched. ~3-5 minutes.\n# Branch must match this notebook: feature/vllm-batched has the hook-fire counter\n# (Step 7) and the batched capture surface (Step 10). feature/driver-system lacks\n# both, so collective_rpc raises NotImplementedError on tl_reset_counter / the\n# batched RPCs.\n# vllm pinned to 0.20.2 — the version the internal-API walks in\n# transformer_lens/model_bridge/sources/vllm/{plugin,internals}.py were validated against.\n# vLLM rearranges its internal class paths every 4-6 weeks; re-validate before bumping.\n%pip install -q \"vllm==0.20.2\"\n%pip install -q git+https://github.com/TransformerLensOrg/TransformerLens.git@feature/vllm-batched", + "source": "# Install vllm and TransformerLens @ feature/vllm-batched. ~3-5 minutes.\n# Branch must match this notebook: feature/vllm-batched has the hook-fire counter\n# (Step 7) and the batched capture surface (Step 10). feature/driver-system lacks\n# both, so collective_rpc raises NotImplementedError on tl_reset_counter / the\n# batched RPCs.\n#\n# vLLM pinned to 0.20.2 — the version the internal-API walks in\n# transformer_lens/model_bridge/sources/vllm/{plugin,internals}.py were validated\n# against. Installed from the CUDA-12.9 *release wheel*, NOT PyPI's default\n# `vllm==0.20.2`: the PyPI wheel is a CUDA-13 build that fails on Colab's CUDA-12.8\n# image with `libcudart.so.13: cannot open shared object file` and drags in\n# cuda-toolkit 13, which conflicts with Colab's cu12 RAPIDS stack. The cu129 wheel\n# uses libcudart.so.12 (present) and stays in the cu12 family. vLLM has no cu128\n# wheel for 0.20.2; cu129 is CUDA-12.x-compatible with the cu128 runtime.\n# Re-validate the internals walks before bumping vLLM (it moves them every 4-6 weeks).\n%pip install -q \"https://github.com/vllm-project/vllm/releases/download/v0.20.2/vllm-0.20.2%2Bcu129-cp38-abi3-manylinux_2_31_x86_64.whl\"\n%pip install -q git+https://github.com/TransformerLensOrg/TransformerLens.git@feature/vllm-batched", "id": "4ab1eb60e6b1" }, { @@ -50,7 +50,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "import gc\nimport os\nimport sys\n\nimport torch\n\n# HF_TOKEN comes from Colab secrets. Falls back to env var for non-Colab runs.\ntry:\n from google.colab import userdata\n os.environ.setdefault(\"HF_TOKEN\", userdata.get(\"HF_TOKEN\"))\nexcept (ImportError, Exception):\n pass\nassert os.environ.get(\"HF_TOKEN\"), \"Set HF_TOKEN in Colab Secrets (gear icon, left sidebar).\"\n\n# Colab/Jupyter compatibility: ipykernel's stdout doesn't expose a fileno();\n# vLLM's worker init calls sys.stdout.fileno() during parallel-state setup\n# and crashes with UnsupportedOperation: fileno. Patch fileno to return the\n# underlying process FDs (1, 2) — Colab writes back to those anyway.\nif \"ipykernel\" in sys.modules:\n sys.stdout.fileno = lambda: 1 # type: ignore[method-assign]\n sys.stderr.fileno = lambda: 2 # type: ignore[method-assign]\n\n# Read the installed vllm version from package metadata, NOT `import vllm` —\n# importing vllm loads its CUDA C extension (vllm._C), which is exactly what\n# fails with `libcudart.so.NN not found` when a newer wheel built for a CUDA\n# version Colab doesn't ship gets installed. Metadata read works regardless.\nfrom importlib.metadata import PackageNotFoundError\nfrom importlib.metadata import version as _pkg_version\n\n_PINNED_VLLM = \"0.20.2\"\ntry:\n _vllm_ver = _pkg_version(\"vllm\")\n print(f\"vllm version: {_vllm_ver}\")\n if _vllm_ver != _PINNED_VLLM:\n print(\n f\"⚠ expected vllm=={_PINNED_VLLM} (the version the capture plugin is validated \"\n f\"against); got {_vllm_ver}. Newer wheels target a CUDA the Colab image may not \"\n \"ship (→ libcudart.so error at boot) and may move vLLM internals the plugin walks. \"\n f\"Re-pin with %pip install -q 'vllm=={_PINNED_VLLM}' and restart the runtime.\"\n )\nexcept PackageNotFoundError:\n print(\"⚠ vllm is not installed — run the install cell above.\")\n\nMODEL = \"meta-llama/Llama-3.2-1B\"\nPROMPT = \"The quick brown fox jumps over the\"\nDTYPE = torch.float16\ntorch.manual_seed(0)\n\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nprint(f\"Device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU only — abort'}\")\nassert torch.cuda.is_available(), \"GPU runtime required.\"", + "source": "import gc\nimport os\nimport sys\n\nimport torch\n\n# HF_TOKEN comes from Colab secrets. Falls back to env var for non-Colab runs.\ntry:\n from google.colab import userdata\n os.environ.setdefault(\"HF_TOKEN\", userdata.get(\"HF_TOKEN\"))\nexcept (ImportError, Exception):\n pass\nassert os.environ.get(\"HF_TOKEN\"), \"Set HF_TOKEN in Colab Secrets (gear icon, left sidebar).\"\n\n# Colab/Jupyter compatibility: ipykernel's stdout doesn't expose a fileno();\n# vLLM's worker init calls sys.stdout.fileno() during parallel-state setup\n# and crashes with UnsupportedOperation: fileno. Patch fileno to return the\n# underlying process FDs (1, 2) — Colab writes back to those anyway.\nif \"ipykernel\" in sys.modules:\n sys.stdout.fileno = lambda: 1 # type: ignore[method-assign]\n sys.stderr.fileno = lambda: 2 # type: ignore[method-assign]\n\n# Read the installed vllm version from package metadata, NOT `import vllm` —\n# importing vllm loads its CUDA C extension (vllm._C), which is exactly what\n# fails with `libcudart.so.NN not found` when a wheel built for a CUDA version\n# Colab doesn't ship gets installed. Metadata read works regardless.\nfrom importlib.metadata import PackageNotFoundError\nfrom importlib.metadata import version as _pkg_version\n\n_PINNED_VLLM = \"0.20.2\"\ntry:\n _vllm_ver = _pkg_version(\"vllm\")\n print(f\"vllm version: {_vllm_ver}\")\n # The install cell pins the cu129 release wheel, so the local tag is\n # \"0.20.2+cu129\" — compare the base version only.\n if _vllm_ver.split(\"+\")[0] != _PINNED_VLLM:\n print(\n f\"⚠ expected vllm base {_PINNED_VLLM} (the version the capture plugin is validated \"\n f\"against); got {_vllm_ver}. A mismatched wheel may target a CUDA the Colab image \"\n \"doesn't ship (→ libcudart.so error at boot) and may move vLLM internals the plugin \"\n \"walks. Re-run the install cell above (it fetches the cu129 release wheel) and \"\n \"restart the runtime. Do NOT `pip install vllm==0.20.2` — PyPI's default is the \"\n \"CUDA-13 build that fails on Colab's CUDA-12.8 image.\"\n )\n elif \"+cu\" in _vllm_ver and \"cu129\" not in _vllm_ver:\n print(\n f\"⚠ got {_vllm_ver}: not the cu129 wheel. PyPI's default vllm==0.20.2 is a CUDA-13 \"\n \"build (libcudart.so.13 missing on Colab's CUDA-12.8). Re-run the install cell above.\"\n )\nexcept PackageNotFoundError:\n print(\"⚠ vllm is not installed — run the install cell above.\")\n\nMODEL = \"meta-llama/Llama-3.2-1B\"\nPROMPT = \"The quick brown fox jumps over the\"\nDTYPE = torch.float16\ntorch.manual_seed(0)\n\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nprint(f\"Device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU only — abort'}\")\nassert torch.cuda.is_available(), \"GPU runtime required.\"", "id": "1748b73ca8b0" }, { @@ -121,23 +121,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "from transformer_lens.model_bridge.remote_bridge import RemoteBridge\n", - "\n", - "# max_model_len=2048 caps the KV cache reservation. Llama-3.2-1B's native\n", - "# context is 131072 (128k) — the default reservation is ~4 GiB and overshoots\n", - "# the free T4 budget. The test prompt is ~10 tokens, so 2048 is plenty.\n", - "bridge = RemoteBridge.boot_vllm(\n", - " MODEL,\n", - " dtype=DTYPE,\n", - " gpu_memory_utilization=0.5,\n", - " max_model_len=2048,\n", - ")\n", - "assert isinstance(bridge, RemoteBridge), f\"Expected RemoteBridge, got {type(bridge).__name__}\"\n", - "print(f\"Architecture: {bridge.cfg.architecture}\")\n", - "print(f\"Fireable hooks: {len(bridge._driver.supported_hook_points)}\")\n", - "print(f\"Non-fireable hooks (fused kernels): {len(bridge._driver.non_fireable_hook_points)}\")" - ], + "source": "from transformer_lens.model_bridge.remote_bridge import RemoteBridge\n\n# max_model_len=2048 caps the KV cache reservation. Llama-3.2-1B's native\n# context is 131072 (128k) — the default reservation is ~4 GiB and overshoots\n# the free T4 budget. The test prompt is ~10 tokens, so 2048 is plenty.\n#\n# enable_position_interventions=True widens each hook's affine scale/bias buffers\n# from (width,) to (max_num_batched_tokens, width) so Step 6½ can patch specific\n# sequence positions via a spec 'pos' field. Costs ~2× the affine buffers' resident\n# memory (~0.8 GB here); the buffers stay identity until an intervention writes them,\n# so every other step is unchanged. Drop the flag (and skip Step 6½) to run leaner.\nbridge = RemoteBridge.boot_vllm(\n MODEL,\n dtype=DTYPE,\n gpu_memory_utilization=0.5,\n max_model_len=2048,\n enable_position_interventions=True,\n)\nassert isinstance(bridge, RemoteBridge), f\"Expected RemoteBridge, got {type(bridge).__name__}\"\nprint(f\"Architecture: {bridge.cfg.architecture}\")\nprint(f\"Fireable hooks: {len(bridge._driver.supported_hook_points)}\")\nprint(f\"Non-fireable hooks (fused kernels): {len(bridge._driver.non_fireable_hook_points)}\")", "id": "49d50957a4c7" }, { @@ -208,18 +192,18 @@ "execution_count": null, "outputs": [] }, + { + "cell_type": "code", + "id": "85900fee", + "source": "# ============================================================================\n# VERIFY — the driver returns real full-sequence logits (branch: review/dev-4.x)\n# ============================================================================\n# With VLLMDriver._reconstruct_logits + provides_sequence_logits=True, run_with_cache's\n# logits should be (1, seq, d_vocab) real logits at EVERY position (old: final-only\n# log-probs, -inf elsewhere). Reuses v_lnf / hf_lnf / W / norm_w from the Step 4½ cell.\nlg = logits_vllm.float()\nseq = tokens.shape[1]\nprint(f\"driver logits shape: {tuple(lg.shape)} (expect (1, {seq}, {bridge.cfg.d_vocab}))\")\nfinite_pos = torch.isfinite(lg[0]).all(-1)\nprint(f\"all-finite positions: {int(finite_pos.sum())}/{lg.shape[1]} (old behavior: 1, final only)\")\nassert lg.shape == (1, seq, bridge.cfg.d_vocab), \"driver logits are not full-sequence\"\nassert finite_pos.all(), \"some positions non-finite — reconstruction didn't fill them\"\n\ndrv = lg[0, :, : W.shape[0]] # trim any vocab padding to compare against W's real vocab\n\n# (a) driver output == the inline formula (Step 4½) → the driver wired the reconstruction right.\nd_inline = (drv - (v_lnf @ W.T)).abs().max().item()\nprint(f\"driver vs inline (ln_final @ W_U): max|Δ|={d_inline:.2e} (~0 => correct formula in driver)\")\n\n# (b) driver logits vs HF's real logits (HF post-weight = HF_ln_final * norm_w) @ W_U, all positions.\nhf_real = (hf_lnf * norm_w) @ W.T\nper_pos = (drv - hf_real).abs().amax(-1)\nagree = (drv.argmax(-1) == hf_real.argmax(-1)).float().mean().item() * 100\nprint(f\"driver vs HF real logits: max|Δ|={per_pos.max():.2e} (fp16) argmax agreement={agree:.1f}%\")\nprint(\"\\n>>> full-sequence + inline~0 + argmax 100% => VLLMDriver._reconstruct_logits is correct end-to-end.\")\n", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, { "cell_type": "markdown", "metadata": {}, - "source": [ - "## Step 5 — Per-hook L2 (acceptance gate)\n", - "\n", - "Target is relative L2 < 5e-3 in fp16 for every fireable hook. One hook remains exempted from the strict gate:\n", - "\n", - "- **`ln_final.hook_normalized`** — vLLM's `model.norm` is invoked as part of the fused-residual norm kernel and the captured value scales ~2× HF's. Open investigation (likely a residual-fusion semantic discrepancy at the model boundary rather than a real divergence; argmax + downstream parity work correctly).\n", - "\n", - "All other fireable hooks (including `blocks.{i}.hook_out`, which is now materialized from vLLM's `(mlp_delta, residual)` tuple) must be within target." - ], + "source": "## Step 5 — Per-hook L2 (acceptance gate)\n\nTarget is relative L2 < 5e-3 in fp16 for every fireable hook. One hook remains exempted from the strict gate:\n\n- **`ln_final.hook_normalized`** — vLLM exposes the **post-weight** RMSNorm value (`normed × norm_weight`), while HF/HookedTransformer exposes the **pre-weight** value, so vLLM's capture reads larger by exactly the final-norm weight. **Resolved** by Step 4½: the vLLM/HF ratio equals `norm_w` (median ≈ 2.45 for Llama-3.2-1B — the earlier \"~2×\" was just this weight's magnitude, not a fixed factor). This is the documented post/pre-weight convention gap (see `sources.vllm.overlays.decoder_only`), **not** a divergence — and it's exactly why the Step 4½ logit reconstruction (`ln_final @ lm_head.weightᵀ`) works directly off the raw capture. Still exempted from the strict L2 gate below because the *values* legitimately differ from HF's pre-weight hook.\n\nAll other fireable hooks (including `blocks.{i}.hook_out`, which is now materialized from vLLM's `(mlp_delta, residual)` tuple) must be within target.", "id": "3e66a40c8c0a" }, { @@ -321,6 +305,20 @@ ], "id": "8c94e6f538f4" }, + { + "cell_type": "markdown", + "id": "57e058a4", + "source": "## Step 6½ — Per-position interventions (`pos`)\n\nWhole-sequence interventions (Step 6) edit every position. With the driver booted\n`enable_position_interventions=True`, a spec can carry a **`pos`** field (int or\n`list[int]`) that scopes the affine to specific sequence rows — position-scoped\nactivation patching / tensor injection, the same surface the Inspect/HF backend\nalready offers. This exercises the `(max_num_batched_tokens, width)` affine buffers\nand the per-row `narrow()` path in the compiled hook.\n\nThe test edits `embed.hook_out` at chosen positions and verifies, from the captured\n(post-affine) cache, that **only** those rows changed and every other row stayed\nidentical to the clean run.", + "metadata": {} + }, + { + "cell_type": "code", + "id": "91576f5a", + "source": "# Per-position interventions: a spec's 'pos' (int or list[int]) scopes the affine to\n# specific sequence rows. Verified against the captured (post-affine) embed.hook_out:\n# only the targeted rows change; every other row matches the clean baseline (fp16).\nseq = tokens.shape[1]\nd_model = bridge.cfg.d_model\nHOOK = \"embed.hook_out\"\n\n_, cache_base = bridge.run_with_cache(tokens)\nbase = cache_base[HOOK][0].float().clone()\n\n# (a) add a constant vector at ONE position; all other rows must be untouched.\nPOS = 2\n_, cache_add = bridge.run_with_cache(\n tokens, intervene={HOOK: {\"op\": \"add\", \"value\": [3.0] * d_model, \"pos\": POS}}\n)\nadd = cache_add[HOOK][0].float()\noff = [p for p in range(seq) if p != POS]\nprint(f\"add@{POS}: Δ@POS mean={(add[POS] - base[POS]).mean():.3f} (→3.0) \"\n f\"max|Δ| off-POS={(add[off] - base[off]).abs().max():.2e} (→0)\")\nassert torch.allclose(add[POS], base[POS] + 3.0, atol=1e-2), \"add@pos wrong at the target row\"\nassert torch.allclose(add[off], base[off], atol=1e-2), \"add@pos leaked to other rows\"\n\n# (b) suppress a LIST of positions; only those rows go to zero, the rest pass through.\nPOSL = [0, seq - 1]\n_, cache_sup = bridge.run_with_cache(tokens, intervene={HOOK: {\"op\": \"suppress\", \"pos\": POSL}})\nsup = cache_sup[HOOK][0].float()\nkeep = [p for p in range(seq) if p not in POSL]\nprint(f\"suppress@{POSL}: |max|@POS={sup[POSL].abs().max():.2e} (→0) \"\n f\"max|Δ| kept-rows={(sup[keep] - base[keep]).abs().max():.2e} (→0)\")\nassert sup[POSL].abs().max() < 1e-4, \"suppress@pos did not zero the target rows\"\nassert torch.allclose(sup[keep], base[keep], atol=1e-2), \"suppress@pos leaked to kept rows\"\n\n# (c) reset: a clean forward reverts — pos interventions are per-forward, not sticky.\n_, cache_reset = bridge.run_with_cache(tokens)\nassert torch.allclose(cache_reset[HOOK][0].float(), base, atol=1e-2), \"pos intervention persisted\"\nprint(\"✅ per-position add / suppress / reset all correct\")", + "metadata": {}, + "execution_count": null, + "outputs": [] + }, { "cell_type": "markdown", "id": "a21f3155a525", @@ -555,7 +553,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": "## Summary\n\nIf all asserts above passed, the v4 Driver-protocol vLLM backend is sound on this architecture:\n\n- `boot_vllm` returns `RemoteBridge` end-to-end.\n- `collective_rpc → tl_read_captures` populates the cache.\n- Greedy argmax matches HF (greedy parity).\n- Per-hook L2 < 5e-3 vs HF across every fireable hook (hook-fidelity gate; `ln_final.hook_normalized` exempted — see overlay docstring).\n- Affine interventions (suppress / scale / add / set) apply per-forward and reset cleanly.\n- Each hook fires exactly once per forward under compile (no double-fire / silent overwrite).\n- Back-to-back captures are deterministic (stream-safe).\n- GPU lifetime behaves as expected for vLLM 0.20.x (full release needs a runtime restart).\n- **Batched mode** (`enable_batching=True`, Step 10): `batch_size > 1` parity vs single-prompt runs, chunked-prefill accumulation reconstructs full sequences, and global interventions apply + reset across the batch.\n\nKnown divergences (documented, not bugs): `ln_final.hook_normalized` post-weight vs pre-weight, Gemma `embed.hook_out` scaling. Both reconciled by conversions noted in `sources.vllm.overlays.decoder_only`.\n\nOut of scope: multi-token generation (`max_new_tokens > 1`), per-request (non-global) batched interventions, tensor/pipeline parallelism.", + "source": "## Summary\n\nIf all asserts above passed, the v4 Driver-protocol vLLM backend is sound on this architecture:\n\n- `boot_vllm` returns `RemoteBridge` end-to-end.\n- `collective_rpc → tl_read_captures` populates the cache.\n- Greedy argmax matches HF (greedy parity).\n- Per-hook L2 < 5e-3 vs HF across every fireable hook (hook-fidelity gate; `ln_final.hook_normalized` exempted — see overlay docstring).\n- Affine interventions (suppress / scale / add / set) apply per-forward and reset cleanly — both whole-sequence (Step 6) and scoped to specific positions via a spec `pos` field (Step 6½, `enable_position_interventions=True`).\n- Each hook fires exactly once per forward under compile (no double-fire / silent overwrite).\n- Back-to-back captures are deterministic (stream-safe).\n- GPU lifetime behaves as expected for vLLM 0.20.x (full release needs a runtime restart).\n- **Batched mode** (`enable_batching=True`, Step 10): `batch_size > 1` parity vs single-prompt runs, chunked-prefill accumulation reconstructs full sequences, and global interventions apply + reset across the batch.\n\nKnown divergences (documented, not bugs): `ln_final.hook_normalized` post-weight vs pre-weight, Gemma `embed.hook_out` scaling. Both reconciled by conversions noted in `sources.vllm.overlays.decoder_only`.\n\nOut of scope: multi-token generation (`max_new_tokens > 1`), per-request (non-global) batched interventions, tensor/pipeline parallelism.", "id": "fedb17c9a539" } ], diff --git a/scripts/inspect_parity_report.py b/scripts/inspect_parity_report.py index 4d643f0c47..9a6083ca8f 100644 --- a/scripts/inspect_parity_report.py +++ b/scripts/inspect_parity_report.py @@ -60,13 +60,21 @@ ATOL, RTOL = 1e-3, 1e-3 -# Boundary kind -> TransformerBridge-native hook suffix. +# Kind -> TransformerBridge-native hook suffix: the five d_model boundaries plus the +# head-split kinds (served where the structural probe finds the projections — q/k/v are +# gated on fused-qkv archs, pattern under non-eager attention; gated kinds show up in +# the report's `gated=` field, not as failures). KIND_SUFFIX = { "resid_pre": "hook_in", "resid_mid": "ln2.hook_in", "resid_post": "hook_out", "attn_out": "attn.hook_out", "mlp_out": "mlp.hook_out", + "q": "attn.hook_q", + "k": "attn.hook_k", + "v": "attn.hook_v", + "z": "attn.hook_z", + "pattern": "attn.hook_pattern", } @@ -103,6 +111,11 @@ def verify(model_id: str) -> dict: mism.append(f"{hk} missing") continue a, b = hf_cache[hk].float(), i_cache[hk].float() + if a.shape != b.shape and a.numel() == b.numel(): + # OPT-style blocks flatten the FFN to (batch·seq, d) and the bridge + # caches that raw 2-D layout; the driver emits the conventional + # (1, seq, d). Same values, pure reshape — normalize to compare. + a = a.reshape(b.shape) if a.shape != b.shape: mism.append(f"{hk} shape {tuple(a.shape)}!={tuple(b.shape)}") continue diff --git a/tests/acceptance/model_bridge/test_inspect_provider.py b/tests/acceptance/model_bridge/test_inspect_provider.py index 6bc51e7259..46bbac2a74 100644 --- a/tests/acceptance/model_bridge/test_inspect_provider.py +++ b/tests/acceptance/model_bridge/test_inspect_provider.py @@ -35,6 +35,10 @@ "blocks.0.hook_out", # resid_post "blocks.6.attn.hook_out", "blocks.11.hook_out", + # Head-split kinds gpt2 serves despite its fused c_attn (q/k/v are gated): z reads + # the c_proj (Conv1D) input, pattern rides output_attentions under eager. + "blocks.0.attn.hook_z", + "blocks.0.attn.hook_pattern", ] # Boundary kind -> TransformerBridge-native hook suffix. @@ -48,12 +52,15 @@ # Token-free tiny-random checkpoints, one per structural-check code path: standard # sequential (nothing gated), parallel-residual (resid_mid gated by the causal probe), -# post-norm (resid_mid gated by the linear-identity probe). Pins cross-family behavior -# so a detector/load-path regression on non-gpt2 archs fails CI instead of shipping silently. +# post-norm (resid_mid gated by the linear-identity probe), and fc-split FFN (OPT/XGLM: +# no mlp container — fc1/fc2 on the block; mlp_out must still be located and match). Pins +# cross-family behavior so a detector/load-path regression on non-gpt2 archs fails CI. STRUCTURAL_FAMILIES = [ ("hf-internal-testing/tiny-random-LlamaForCausalLM", frozenset()), ("hf-internal-testing/tiny-random-GPTJForCausalLM", frozenset({"resid_mid"})), ("hf-internal-testing/tiny-random-Gemma2ForCausalLM", frozenset({"resid_mid"})), + ("hf-internal-testing/tiny-random-OPTForCausalLM", frozenset()), + ("hf-internal-testing/tiny-random-XGLMForCausalLM", frozenset()), ] @@ -245,9 +252,11 @@ def test_sequential_offers_resid_mid(self): ) m = self._toy_model() # standard sequential, linear residual - kinds, note = _detect_capabilities(m, m.layers) + kinds, note = _detect_capabilities(m, m.layers, (None, None, None)) assert "resid_mid" in kinds - assert note == "" # nothing gated + # No boundary kind gated. (Head kinds ARE gated — the toy has no head geometry — + # so the note mentions only those.) + assert "resid_mid" not in note and "attn_out" not in note and "mlp_out" not in note def test_parallel_gates_resid_mid(self): from transformer_lens.model_bridge.sources.inspect.transformers_provider import ( @@ -255,7 +264,7 @@ def test_parallel_gates_resid_mid(self): ) m = self._toy_model(parallel=True) # mlp reads block input, not attn output - kinds, note = _detect_capabilities(m, m.layers) + kinds, note = _detect_capabilities(m, m.layers, (None, None, None)) assert "resid_mid" not in kinds assert {"resid_pre", "resid_post", "attn_out", "mlp_out"} <= kinds # rest still served assert "resid_mid" in note # note explains the gate @@ -268,7 +277,7 @@ def test_nonlinear_residual_gates_resid_mid(self): # Sequential (attn feeds mlp) but outputs are scaled before the residual add, so # resid_post != resid_pre + attn_out + mlp_out — resid_mid must still be gated. m = self._toy_model(resid_scale=2.0) - kinds, note = _detect_capabilities(m, m.layers) + kinds, note = _detect_capabilities(m, m.layers, (None, None, None)) assert "resid_mid" not in kinds assert {"resid_pre", "resid_post", "attn_out", "mlp_out"} <= kinds assert "resid_mid" in note @@ -282,7 +291,7 @@ def test_probe_leaves_global_rng_untouched(self): m = self._toy_model() before = torch.get_rng_state() - _detect_capabilities(m, m.layers) + _detect_capabilities(m, m.layers, (None, None, None)) assert torch.equal(torch.get_rng_state(), before) @@ -327,6 +336,137 @@ def test_gating_and_parity(self, model_id, expected_gated): inspect.close() +HEAD_MODEL = "hf-internal-testing/tiny-random-LlamaForCausalLM" +HEAD_HOOKS = [ + "blocks.0.attn.hook_q", + "blocks.0.attn.hook_k", + "blocks.0.attn.hook_v", + "blocks.0.attn.hook_z", + "blocks.0.attn.hook_pattern", + "blocks.1.attn.hook_z", +] + + +class TestHeadSplitHooks: + """Head-split q/k/v/z/pattern on a separate-projection arch (tiny-random Llama): + capture parity vs boot_transformers, interventions, and the fused-qkv gating path + (via the module-scoped gpt2 fixtures).""" + + @pytest.fixture(scope="class") + def head_pair(self): + from transformer_lens.model_bridge.remote_bridge import RemoteBridge + from transformer_lens.model_bridge.transformer_bridge import TransformerBridge + + hf = TransformerBridge.boot_transformers(HEAD_MODEL, device="cpu", dtype=torch.float32) + inspect = RemoteBridge.boot_inspect(HEAD_MODEL, dtype=torch.float32) + yield hf, inspect + inspect.close() + + def test_all_head_hooks_served(self, head_pair): + _, inspect = head_pair + for hook in HEAD_HOOKS: + assert hook in inspect._driver.supported_hook_points, f"{hook} not served" + + def test_head_capture_parity(self, head_pair): + hf, inspect = head_pair + toks = hf.to_tokens(PROMPT) + _, hf_cache = hf.run_with_cache(toks) + _, i_cache = inspect.run_with_cache(toks) + for hook in HEAD_HOOKS: + a, b = hf_cache[hook].float(), i_cache[hook].float() + assert a.shape == b.shape, f"{hook}: {tuple(b.shape)} vs bridge {tuple(a.shape)}" + assert torch.allclose( + a, b, atol=1e-4, rtol=1e-4 + ), f"{hook} diverges: max {(a - b).abs().max().item():.2e}" + + def test_suppress_v_zeroes_capture_and_moves_logits(self, head_pair): + _, inspect = head_pair + toks = inspect.to_tokens(PROMPT) + base = inspect.forward(toks) + logits, cache = inspect.run_with_cache( + toks, intervene={"blocks.0.attn.hook_v": {"op": "suppress"}} + ) + assert cache["blocks.0.attn.hook_v"].abs().max().item() == 0.0 + assert not torch.allclose(base, logits) + + def test_per_position_q_patch_is_position_scoped(self, head_pair): + _, inspect = head_pair + toks = inspect.to_tokens(PROMPT) + _, base_cache = inspect.run_with_cache(toks) + _, cache = inspect.run_with_cache( + toks, intervene={"blocks.0.attn.hook_q": {"op": "add", "value": 5.0, "pos": 1}} + ) + q_base, q_new = base_cache["blocks.0.attn.hook_q"][0], cache["blocks.0.attn.hook_q"][0] + others = [p for p in range(q_base.shape[0]) if p != 1] + assert torch.allclose(q_new[1], q_base[1] + 5.0, atol=1e-5) + assert torch.equal(q_new[others], q_base[others]) + + def test_pattern_intervention_rejected(self, head_pair): + _, inspect = head_pair + with pytest.raises(ValueError, match="capture-only"): + inspect.forward( + inspect.to_tokens(PROMPT), + intervene={"blocks.0.attn.hook_pattern": {"op": "suppress"}}, + ) + + def test_gpt2_fused_qkv_gated_but_z_pattern_served(self, inspect_bridge): + supported = inspect_bridge._driver.supported_hook_points + nonfireable = inspect_bridge._driver.non_fireable_hook_points + for hook in ("blocks.0.attn.hook_q", "blocks.0.attn.hook_k", "blocks.0.attn.hook_v"): + assert hook not in supported and hook in nonfireable + assert "blocks.0.attn.hook_z" in supported + assert "blocks.0.attn.hook_pattern" in supported + + def test_gptneo_wrapper_attention_serves_head_hooks(self): + """GPT-Neo wraps the real attention (standard q_proj/out_proj) at attn.attention — + the projection-host descent must find it, and captures must match the bridge.""" + from transformer_lens.model_bridge.remote_bridge import RemoteBridge + from transformer_lens.model_bridge.transformer_bridge import TransformerBridge + + mid = "hf-internal-testing/tiny-random-GPTNeoForCausalLM" + hf = TransformerBridge.boot_transformers(mid, device="cpu", dtype=torch.float32) + inspect = RemoteBridge.boot_inspect(mid, dtype=torch.float32) + try: + for kind in ("q", "k", "v", "z", "pattern"): + assert f"blocks.0.attn.hook_{kind}" in inspect._driver.supported_hook_points + toks = hf.to_tokens(PROMPT) + _, hf_cache = hf.run_with_cache(toks) + _, i_cache = inspect.run_with_cache(toks) + for hook in ("blocks.0.attn.hook_q", "blocks.0.attn.hook_z"): + a, b = hf_cache[hook].float(), i_cache[hook].float() + assert a.shape == b.shape + assert torch.allclose(a, b, atol=1e-4, rtol=1e-4) + finally: + inspect.close() + + def test_provider_direct_interventions_validated(self, inspect_bridge, tokens): + """The provider's documented extra_args interface must reject gated and + capture-only intervention kinds instead of silently no-op'ing (the driver path + already rejects; this covers callers that speak to the provider directly).""" + import asyncio + + from inspect_ai.model import GenerateConfig + + api = inspect_bridge._driver._model.api + + def call(interventions): + cfg = GenerateConfig( + extra_body={ + "extra_args": { + "input_ids": tokens[0].tolist(), + "capture": [], + "interventions": interventions, + } + } + ) + return asyncio.run(api.generate("", [], None, cfg)) + + with pytest.raises(ValueError, match="gated"): # q gated on gpt2's fused c_attn + call({"0:q": {"op": "suppress"}}) + with pytest.raises(ValueError, match="capture-only"): + call({"0:resid_mid": {"op": "suppress"}}) + + class TestRebootSemantics: """inspect_ai memoizes get_model by name; boot_inspect passes memoize=False so each boot honors its own args and independently owns (and frees) its model.""" diff --git a/tests/acceptance/model_bridge/test_symbolic_bridge_hooks.py b/tests/acceptance/model_bridge/test_symbolic_bridge_hooks.py new file mode 100644 index 0000000000..8a28edf74e --- /dev/null +++ b/tests/acceptance/model_bridge/test_symbolic_bridge_hooks.py @@ -0,0 +1,56 @@ +"""SymbolicBridge hook mirroring: fc-split archs (OPT/XGLM) must fire the placeholder's +own ``mlp.hook_in``/``mlp.hook_out`` — wired at setup from the ``in``/``out`` subcomponents +(see ``component_setup._wire_symbolic_hooks``). Without the mirror those HookPoints exist +in the registry but never fire, so caching misses them and interventions silently no-op. + +Runs on CPU with token-free tiny-random checkpoints. +""" +from __future__ import annotations + +import pytest +import torch + +OPT = "hf-internal-testing/tiny-random-OPTForCausalLM" +XGLM = "hf-internal-testing/tiny-random-XGLMForCausalLM" + + +@pytest.fixture(scope="module") +def opt_bridge(): + from transformer_lens.model_bridge.transformer_bridge import TransformerBridge + + return TransformerBridge.boot_transformers(OPT, device="cpu", dtype=torch.float32) + + +class TestSymbolicMirror: + def test_mlp_hooks_fire_and_match_subcomponents(self, opt_bridge): + tokens = opt_bridge.to_tokens("Hello world") + _, cache = opt_bridge.run_with_cache(tokens) + assert "blocks.0.mlp.hook_out" in cache.cache_dict + assert "blocks.0.mlp.hook_in" in cache.cache_dict + # The mirror re-fires the subcomponent tensors: out = fc2's output, in = fc1's input. + assert torch.equal(cache["blocks.0.mlp.hook_out"], cache["blocks.0.mlp.out.hook_out"]) + assert torch.equal(cache["blocks.0.mlp.hook_in"], cache["blocks.0.mlp.in.hook_in"]) + + def test_intervention_via_symbolic_hook_propagates(self, opt_bridge): + tokens = opt_bridge.to_tokens("Hello world") + base = opt_bridge.forward(tokens) + patched = opt_bridge.run_with_hooks( + tokens, fwd_hooks=[("blocks.0.mlp.hook_out", lambda t, hook: torch.zeros_like(t))] + ) + assert not torch.allclose(base, patched), "symbolic-hook edit did not reach the logits" + + def test_mirror_survives_reset_hooks(self, opt_bridge): + """The wiring is permanent — a reset_hooks (any run_with_* teardown) must not + sever it, else the second run_with_cache silently loses the mlp entries.""" + tokens = opt_bridge.to_tokens("Hello world") + opt_bridge.reset_hooks() + _, cache = opt_bridge.run_with_cache(tokens) + assert "blocks.0.mlp.hook_out" in cache.cache_dict + + def test_xglm_mirror_fires(self): + from transformer_lens.model_bridge.transformer_bridge import TransformerBridge + + bridge = TransformerBridge.boot_transformers(XGLM, device="cpu", dtype=torch.float32) + _, cache = bridge.run_with_cache(bridge.to_tokens("Hi")) + assert "blocks.0.mlp.hook_out" in cache.cache_dict + assert torch.equal(cache["blocks.0.mlp.hook_out"], cache["blocks.0.mlp.out.hook_out"]) diff --git a/tests/unit/model_bridge/test_inspect_driver.py b/tests/unit/model_bridge/test_inspect_driver.py index b4b7894459..1ee943f37a 100644 --- a/tests/unit/model_bridge/test_inspect_driver.py +++ b/tests/unit/model_bridge/test_inspect_driver.py @@ -423,6 +423,124 @@ def test_context_manager_closes(self): assert driver._model is None +class TestHeadSplitKinds: + """Head-split q/k/v/z/pattern: registry entries, driver assembly, intervention gating. + + The provider serves these only when its structural probe finds the projections; here + a profile with an explicit kind set stands in for that detection. + """ + + def _head_profile(self): + return profiles.TLBridgeProfile(supported_kinds=hooks.ALL_KINDS | hooks.HEAD_KINDS) + + def test_resolve_head_names(self): + assert hooks.resolve("blocks.3.attn.hook_q") == (3, "q") + assert hooks.resolve("blocks.0.attn.hook_z") == (0, "z") + assert hooks.resolve("blocks.1.attn.hook_pattern") == (1, "pattern") + assert hooks.resolve("blocks.0.attn.hook_attn_scores") is None # never served + + def test_default_supported_excludes_head_kinds(self): + """kinds=None stays boundary-only — head kinds are opt-in via detection, so + profiles that pass None (vllm-lens, defaults) don't silently claim them.""" + default = hooks.supported_hook_points(N_LAYERS) + assert "blocks.0.attn.hook_q" not in default + assert "blocks.0.attn.hook_pattern" not in default + + def test_all_hook_points_is_boundary_plus_head(self): + universe = hooks.all_hook_points(N_LAYERS) + assert hooks.supported_hook_points(N_LAYERS) < universe + assert "blocks.0.attn.hook_q" in universe + assert len(universe) == (len(hooks.ALL_KINDS) + len(hooks.HEAD_KINDS)) * N_LAYERS + + def test_interveneable_includes_qkvz_not_pattern(self): + assert {"q", "k", "v", "z"} <= hooks.INTERVENEABLE_KINDS + assert "pattern" not in hooks.INTERVENEABLE_KINDS + + def test_attn_scores_always_nonfireable(self): + driver = InspectDriver( + _fake_model(), _adapter(), tokenizer=None, profile=self._head_profile() + ) + assert "blocks.0.attn.hook_attn_scores" in driver.non_fireable_hook_points + + def test_driver_serves_head_kinds_via_profile(self): + driver = InspectDriver( + _fake_model(), _adapter(), tokenizer=None, profile=self._head_profile() + ) + for name in ("blocks.0.attn.hook_q", "blocks.1.attn.hook_z", "blocks.0.attn.hook_pattern"): + assert name in driver.supported_hook_points + assert driver.supported_hook_points.isdisjoint(driver.non_fireable_hook_points) + + def test_default_driver_moves_head_kinds_to_nonfireable(self): + driver = _driver() # default profile: boundary kinds only + assert "blocks.0.attn.hook_q" in driver.non_fireable_hook_points + assert "blocks.0.attn.hook_pattern" in driver.non_fireable_hook_points + + def test_head_captures_get_batch_dim(self): + """3-D wire arrays (seq,heads,d_head) / (heads,q,k) unsqueeze to exactly one batch dim.""" + heads, d_head, seq = 2, 2, 3 + caps = { + "0:q": np.ones((seq, heads, d_head), np.float32), + "0:pattern": np.ones((heads, seq, seq), np.float32), + "0:resid_post": np.ones((seq, D_MODEL), np.float32), + } + driver = InspectDriver( + _fake_model(captures=caps, logits=np.zeros((seq, D_VOCAB), np.float32)), + _adapter(), + tokenizer=None, + profile=self._head_profile(), + ) + result = driver.forward( + torch.tensor([[1, 2, 3]]), + capture=("blocks.0.attn.hook_q", "blocks.0.attn.hook_pattern", "blocks.0.hook_out"), + ) + assert tuple(result.captured["blocks.0.attn.hook_q"].shape) == (1, seq, heads, d_head) + assert tuple(result.captured["blocks.0.attn.hook_pattern"].shape) == (1, heads, seq, seq) + assert tuple(result.captured["blocks.0.hook_out"].shape) == (1, seq, D_MODEL) + + def test_qkvz_intervention_translates_to_wire_key(self): + supported = hooks.supported_hook_points(N_LAYERS, hooks.ALL_KINDS | hooks.HEAD_KINDS) + out = intervention.build_interventions( + {"blocks.0.attn.hook_v": {"op": "suppress"}}, supported + ) + assert out == {"0:v": {"op": "suppress"}} + + def test_pattern_intervention_rejected(self): + supported = hooks.supported_hook_points(N_LAYERS, hooks.ALL_KINDS | hooks.HEAD_KINDS) + with pytest.raises(ValueError, match="capture-only"): + intervention.build_interventions( + {"blocks.0.attn.hook_pattern": {"op": "suppress"}}, supported + ) + + def test_head_hook_rejected_when_gated(self): + """A gated head hook (default boundary-only profile) can't be intervened on.""" + supported = hooks.supported_hook_points(N_LAYERS) # no head kinds + with pytest.raises(ValueError, match="not in supported_hook_points"): + intervention.build_interventions( + {"blocks.0.attn.hook_q": {"op": "suppress"}}, supported + ) + + def test_turn_activations_batch_dim_is_rank_aware(self): + """Mixed 2-D boundary and 3-D head-split arrays in one turn all get exactly one + batch dim — a rank-2-only rule would leave head-split arrays batchless, and a + batchless (seq, heads, d_head) is shape-indistinguishable from a batched 2-D.""" + from transformer_lens.model_bridge.sources.inspect.eval import turn_activations + + payload = wire.encode_activations( + { + "0:resid_post": np.ones((3, D_MODEL), np.float32), + "0:q": np.ones((3, 2, 2), np.float32), + "0:pattern": np.ones((2, 3, 3), np.float32), + } + ) + sample = SimpleNamespace( + events=[SimpleNamespace(output=SimpleNamespace(metadata={"activations": payload}))] + ) + (turn,) = turn_activations(sample) + assert turn["blocks.0.hook_out"].shape == (1, 3, D_MODEL) + assert turn["blocks.0.attn.hook_q"].shape == (1, 3, 2, 2) + assert turn["blocks.0.attn.hook_pattern"].shape == (1, 2, 3, 3) + + def test_driver_imports_no_torch(): """The acceptance gate: the driver file must not import torch (data-only boundary).""" import transformer_lens.model_bridge.sources.inspect.driver as drv diff --git a/tests/unit/model_bridge/test_vllm_boot.py b/tests/unit/model_bridge/test_vllm_boot.py index cd0edaff03..7d79bbc639 100644 --- a/tests/unit/model_bridge/test_vllm_boot.py +++ b/tests/unit/model_bridge/test_vllm_boot.py @@ -107,6 +107,20 @@ def test_rejects_locked_kwarg_override(): boot_vllm("any-model", tensor_parallel_size=2) +def test_rejects_position_interventions_with_batching(): + """Position interventions need the compiled path's affine buffers — fails fast.""" + with pytest.raises(ValueError, match="incompatible with enable_batching"): + boot_vllm("any-model", enable_position_interventions=True, enable_batching=True) + + +def test_position_interventions_flag_reaches_driver(mocked_boot): + """boot_vllm threads enable_position_interventions through to the driver.""" + bridge = boot_vllm("any-model", enable_position_interventions=True) + assert bridge._driver._enable_position_interventions is True + # Default stays off. + assert boot_vllm("any-model")._driver._enable_position_interventions is False + + @pytest.mark.parametrize( "raw, expected", [ diff --git a/tests/unit/model_bridge/test_vllm_driver.py b/tests/unit/model_bridge/test_vllm_driver.py index f676a005d7..80140367c4 100644 --- a/tests/unit/model_bridge/test_vllm_driver.py +++ b/tests/unit/model_bridge/test_vllm_driver.py @@ -89,6 +89,7 @@ def _driver( max_num_batched_tokens=2048, generated_token=None, top_logprobs=None, + enable_position_interventions=False, ) -> VLLMDriver: """Build a VLLMDriver. ``captures`` populates llm.collective_rpc; ``generated_token`` and ``top_logprobs`` populate llm.generate's RequestOutput so _synthesize_logits @@ -104,6 +105,7 @@ def _driver( overlay=_overlay(), hf_config=hf_config or _hf_config(), max_num_batched_tokens=max_num_batched_tokens, + enable_position_interventions=enable_position_interventions, ) @@ -115,11 +117,19 @@ def test_passes_validate_driver(self): assert isinstance(driver, Driver) validate_driver(driver) - def test_no_torch_capability_flags(self): - """vLLM owns the model in a worker — no torch-specific capability surface.""" + def test_capability_flags(self): + """vLLM owns the model in a worker — no torch module surface (parameters/ + state_dict/grads), but named-weight reads ARE served via the get_param RPC.""" driver = _driver() - for feature in ("parameters", "state_dict", "gradients", "weight_access"): + for feature in ("parameters", "state_dict", "gradients"): assert driver.supports(feature) is False, f"vLLM shouldn't support {feature!r}" + assert driver.supports("weight_access") is True + + def test_zero_layers_config_fails_loud(self): + """A missing/zero num_hidden_layers must raise at boot, not leave a raw '{i}' + template in non_fireable_hook_points.""" + with pytest.raises(ValueError, match="num_hidden_layers"): + _driver(hf_config=_hf_config(num_hidden_layers=0)) def test_non_fireable_expanded_per_layer(self): """``blocks.{i}.attn.hook_pattern`` template expands to one entry per layer.""" @@ -168,7 +178,7 @@ def test_forward_logits_from_sampler_logprobs(self): result = _driver( captures={"embed.hook_out": torch.randn(3, 4)}, top_logprobs={7: 2.5, 3: 1.0, 11: -0.5}, - ).forward(torch.tensor([[1, 2, 3]])) + ).forward(torch.tensor([[1, 2, 3]]), capture=("embed.hook_out",)) assert isinstance(result, ForwardResult) assert tuple(result.captured["embed.hook_out"].shape) == (1, 3, 4) assert result.logits is not None and tuple(result.logits.shape) == (1, 3, 16) @@ -234,6 +244,67 @@ def test_forward_rejects_set_missing_value(self): torch.tensor([[1]]), intervene={"embed.hook_out": {"op": "set"}} ) + +class TestVLLMDriverPositionInterventions: + """`pos`-scoped interventions require the opt-in (max_n, width) affine buffers.""" + + def test_pos_rejected_without_flag(self): + with pytest.raises(NotImplementedError, match="enable_position_interventions=True"): + _driver()._validate_interventions({"embed.hook_out": {"op": "suppress", "pos": 0}}) + + def test_pos_accepted_and_preserved_with_flag(self): + driver = _driver(enable_position_interventions=True) + out = driver._validate_interventions( + {"embed.hook_out": {"op": "add", "value": 1.0, "pos": [0, 2]}} + ) + assert out["embed.hook_out"]["pos"] == [0, 2] + + def test_pos_int_accepted_with_flag(self): + driver = _driver(enable_position_interventions=True) + out = driver._validate_interventions({"embed.hook_out": {"op": "suppress", "pos": 3}}) + assert out["embed.hook_out"]["pos"] == 3 + + def test_pos_wrong_type_raises(self): + driver = _driver(enable_position_interventions=True) + with pytest.raises(ValueError, match="must be an int or list of ints"): + driver._validate_interventions({"embed.hook_out": {"op": "suppress", "pos": "last"}}) + + def test_pos_negative_raises(self): + driver = _driver(enable_position_interventions=True) + with pytest.raises(ValueError, match="must be non-negative"): + driver._validate_interventions({"embed.hook_out": {"op": "suppress", "pos": [-1]}}) + + def test_pos_rejected_on_batched_path(self): + """The batched/eager path has no affine buffers, so 'pos' is unsupported there.""" + driver = _driver() + driver._enable_batching = True + with pytest.raises(NotImplementedError, match="batched/eager path"): + driver._validate_interventions({"embed.hook_out": {"op": "suppress", "pos": 0}}) + + def test_no_pos_still_works_with_flag(self): + """A whole-sequence spec (no pos) is unaffected by the flag.""" + driver = _driver(enable_position_interventions=True) + out = driver._validate_interventions({"embed.hook_out": {"op": "suppress"}}) + assert "pos" not in out["embed.hook_out"] + + def test_pos_beyond_prompt_length_fails_loud(self): + """pos within buffer capacity but past the prompt length must raise, not silently no-op. + + The rejection fires before driver.forward reaches the vllm import, so no install needed. + """ + driver = _driver(captures={}, enable_position_interventions=True) + with pytest.raises(ValueError, match="beyond the prompt length"): + driver.forward( # 3-token prompt, pos=50 is unreadable by the hook + torch.tensor([[1, 2, 3]]), + intervene={"embed.hook_out": {"op": "suppress", "pos": 50}}, + ) + + def test_reject_pos_beyond_seq_bounds_against_actual_length(self): + driver = _driver(enable_position_interventions=True) + driver._reject_pos_beyond_seq({"embed.hook_out": {"op": "suppress", "pos": [0, 2]}}, 3) + with pytest.raises(ValueError, match="beyond the prompt length"): + driver._reject_pos_beyond_seq({"embed.hook_out": {"op": "suppress", "pos": 3}}, 3) + def test_forward_pushes_interventions_before_generate(self): """The driver pushes spec dicts via collective_rpc('tl_set_interventions', ...).""" pytest.importorskip("vllm") @@ -267,6 +338,29 @@ def test_forward_rejects_prompt_exceeding_buffer(self): with pytest.raises(ValueError, match="exceeds max_num_batched_tokens"): _driver(captures={}, max_num_batched_tokens=4).forward(torch.tensor([[1, 2, 3, 4, 5]])) + def test_zero_capture_skips_worker_read(self): + """capture=() must not trigger a GPU→CPU copy: with logits off there is no + tl_read_captures RPC at all (an empty tuple used to collapse to None = read + every buffer); with logits on, only the forced ln_final is read.""" + pytest.importorskip("vllm") + driver = _driver(captures={"embed.hook_out": torch.zeros(3, 4)}) + driver.forward(torch.tensor([[1, 2, 3]]), return_logits=False) + reads = [ + c.args + for c in driver._llm.collective_rpc.call_args_list + if c.args[0] == "tl_read_captures" + ] + assert reads == [], f"hookless forward still read the buffers: {reads}" + + driver2 = _driver(captures={"ln_final.hook_normalized": torch.zeros(3, 4)}) + driver2.forward(torch.tensor([[1, 2, 3]])) # return_logits=True default + reads2 = [ + c.args + for c in driver2._llm.collective_rpc.call_args_list + if c.args[0] == "tl_read_captures" + ] + assert len(reads2) == 1 and reads2[0][1][1] == ["ln_final.hook_normalized"] + def _batched_request_output(request_id, generated_token=None, top_logprobs=None): """RequestOutput-shaped mock carrying a request_id for the accumulator join.""" @@ -338,7 +432,7 @@ def test_req_id_join_not_positional(self): "req-A": {"embed.hook_out": torch.full((3, 4), 1.0)}, } result = _batched_driver(outputs=outputs, captures_by_req=captures_by_req).forward( - [[1, 2, 3], [4, 5]] + [[1, 2, 3], [4, 5]], capture=("embed.hook_out",) ) emb = result.captured["embed.hook_out"] assert tuple(emb.shape) == (2, 3, 4) # (batch, max_seq, width) @@ -360,7 +454,7 @@ def test_join_strips_internal_req_id_suffix(self): "11-a1b2c3d4": {"embed.hook_out": torch.full((2, 4), 2.0)}, } result = _batched_driver(outputs=outputs, captures_by_req=captures_by_req).forward( - [[1, 2, 3], [4, 5]] + [[1, 2, 3], [4, 5]], capture=("embed.hook_out",) ) emb = result.captured["embed.hook_out"] assert tuple(emb.shape) == (2, 3, 4) @@ -379,7 +473,7 @@ def test_join_prefix_does_not_collide_on_numeric_ids(self): "10-bbbbbbbb": {"embed.hook_out": torch.full((3, 4), 9.0)}, } result = _batched_driver(outputs=outputs, captures_by_req=captures_by_req).forward( - [[1, 2], [1, 2, 3]] + [[1, 2], [1, 2, 3]], capture=("embed.hook_out",) ) emb = result.captured["embed.hook_out"] assert torch.equal(emb[0, :2], torch.full((2, 4), 1.0)) # req "1" @@ -396,12 +490,31 @@ def test_shorter_rows_right_padded_with_zeros(self): "r1": {"embed.hook_out": torch.ones(1, 4)}, } result = _batched_driver(outputs=outputs, captures_by_req=captures_by_req).forward( - [[1, 2, 3], [9]] + [[1, 2, 3], [9]], capture=("embed.hook_out",) ) emb = result.captured["embed.hook_out"] # Row 1 has 1 real token; positions 1,2 are zero pad. assert torch.equal(emb[1, 1:], torch.zeros(2, 4)) + def test_zero_capture_skips_batched_join(self): + """capture=() with logits off must return empty captures, not crash: _assemble_padded + needs one worker key per request, so it can't run on the empty read — mirror the + single path. (Regression: the batched join was called unconditionally.)""" + pytest.importorskip("vllm") + outputs = [ + _batched_request_output("r0", top_logprobs={1: 1.0}), + _batched_request_output("r1", top_logprobs={1: 1.0}), + ] + driver = _batched_driver(outputs=outputs, captures_by_req={}) + result = driver.forward([[1, 2, 3], [9]], return_logits=False) # capture=() default + assert result.captured == {} + reads = [ + c.args + for c in driver._llm.collective_rpc.call_args_list + if c.args[0] == "tl_read_batched_captures" + ] + assert reads == [], f"batched hookless forward still read captures: {reads}" + def test_logits_at_per_row_last_token(self): """Next-token logits land at prompt_len-1 per row, never -1 (a pad row).""" pytest.importorskip("vllm") @@ -414,7 +527,7 @@ def test_logits_at_per_row_last_token(self): "r1": {"embed.hook_out": torch.ones(1, 4)}, } result = _batched_driver(outputs=outputs, captures_by_req=captures_by_req).forward( - [[1, 2, 3], [9]] + [[1, 2, 3], [9]], capture=("embed.hook_out",) ) logits = result.logits assert logits is not None and tuple(logits.shape) == (2, 3, 16) @@ -481,12 +594,18 @@ def test_bridge_replays_vllm_captures(self): assert len(fired) == 1 assert tuple(fired[0].shape) == (1, 3, 4) - def test_forward_rejects_loss_return_types(self): - """Synthesized logits are last-position-only; loss/both would be nan, so refuse. + def test_loss_return_type_gated_on_sequence_logits(self): + """The bridge's loss/both guard fires only when the driver lacks full-sequence + logits. The vLLM driver now reconstructs them (provides_sequence_logits=True), + so loss/both are permitted; a driver without that capability is still refused. - The guard is the first thing forward() does, so this needs no vllm install. + The rejection path fires before driver.forward, so it needs no vllm install. """ - bridge = RemoteBridge(adapter=_adapter(), tokenizer=None, driver=_driver(captures={})) + driver = _driver(captures={}) + assert driver.provides_sequence_logits is True # reconstruction path is live + bridge = RemoteBridge(adapter=_adapter(), tokenizer=None, driver=driver) + # Drop the capability → the guard must refuse loss/both (nan over -inf positions). + driver.provides_sequence_logits = False for rt in ("loss", "both"): with pytest.raises(NotImplementedError, match="return_type"): bridge.forward(torch.tensor([[1, 2, 3]]), return_type=rt) diff --git a/tests/unit/model_bridge/test_vllm_worker_extension.py b/tests/unit/model_bridge/test_vllm_worker_extension.py index 5d06ca40a6..3d254347c2 100644 --- a/tests/unit/model_bridge/test_vllm_worker_extension.py +++ b/tests/unit/model_bridge/test_vllm_worker_extension.py @@ -135,6 +135,71 @@ def test_value_shape_mismatch_raises(self): _apply_intervention(scale, bias, {"op": "add", "value": [1.0, 2.0]}) +class TestApplyInterventionPerPosition: + """2-D (max_n, width) affine buffers let a spec's 'pos' scope the edit to rows. + + Buffers start at identity (scale=1, bias=0), mirroring tl_set_interventions' + reset before each apply, so a pos-scoped edit must leave the other rows untouched. + """ + + def _buffers(self, max_n=5, width=4): + return torch.ones(max_n, width), torch.zeros(max_n, width) + + def test_set_at_single_pos_leaves_others_identity(self): + scale, bias = self._buffers() + _apply_intervention(scale, bias, {"op": "set", "value": 2.0, "pos": 2}) + assert torch.equal(scale[2], torch.zeros(4)) + assert torch.equal(bias[2], torch.full((4,), 2.0)) + for r in (0, 1, 3, 4): + assert torch.equal(scale[r], torch.ones(4)) + assert torch.equal(bias[r], torch.zeros(4)) + + def test_add_vector_at_pos_list(self): + scale, bias = self._buffers() + vec = [1.0, 2.0, 3.0, 4.0] + _apply_intervention(scale, bias, {"op": "add", "value": vec, "pos": [0, 3]}) + for r in (0, 3): + assert torch.equal(scale[r], torch.ones(4)) + assert torch.equal(bias[r], torch.tensor(vec)) + for r in (1, 2, 4): + assert torch.equal(bias[r], torch.zeros(4)) + + def test_suppress_at_pos(self): + scale, bias = self._buffers() + _apply_intervention(scale, bias, {"op": "suppress", "pos": 1}) + assert torch.equal(scale[1], torch.zeros(4)) + assert torch.equal(bias[1], torch.zeros(4)) + assert torch.equal(scale[0], torch.ones(4)) # untouched + + def test_scale_at_pos(self): + scale, bias = self._buffers() + _apply_intervention(scale, bias, {"op": "scale", "factor": 0.5, "pos": 4}) + assert torch.equal(scale[4], torch.full((4,), 0.5)) + assert torch.equal(scale[0], torch.ones(4)) + + def test_no_pos_writes_all_rows(self): + """A whole-sequence spec on 2-D buffers broadcasts across every row.""" + scale, bias = self._buffers() + _apply_intervention(scale, bias, {"op": "add", "value": [1.0, 2.0, 3.0, 4.0]}) + for r in range(5): + assert torch.equal(bias[r], torch.tensor([1.0, 2.0, 3.0, 4.0])) + + def test_pos_out_of_range_raises(self): + import pytest + + scale, bias = self._buffers(max_n=3) + with pytest.raises(ValueError, match="out of range"): + _apply_intervention(scale, bias, {"op": "suppress", "pos": 5}) + + def test_pos_on_1d_buffer_raises(self): + """A 'pos' spec against 1-D buffers (flag off) is a misconfiguration.""" + import pytest + + scale, bias = torch.ones(4), torch.zeros(4) + with pytest.raises(ValueError, match="requires 2-D affine buffers"): + _apply_intervention(scale, bias, {"op": "suppress", "pos": 0}) + + class TestApplyOp: """_apply_op is the eager batched path's tensor-level intervention.""" diff --git a/transformer_lens/model_bridge/component_setup.py b/transformer_lens/model_bridge/component_setup.py index 7821d03542..cb7b026a5c 100644 --- a/transformer_lens/model_bridge/component_setup.py +++ b/transformer_lens/model_bridge/component_setup.py @@ -58,6 +58,22 @@ def set_original_components( setup_components(component_mapping, bridge_module, architecture_adapter, original_model) +def _wire_symbolic_hooks(symbolic: SymbolicBridge) -> None: + """Fire a SymbolicBridge's own hook_in/hook_out from its ``in``/``out`` subcomponents. + + Permanent hooks (survive reset_hooks) re-fire the subcomponent activation through the + placeholder's HookPoint, so ``blocks.{i}.mlp.hook_in/hook_out`` (and their compat + aliases) behave like every non-symbolic arch's. A hook that returns a modified tensor + on the symbolic point propagates: the mirror returns it into the subcomponent's chain. + """ + sub_in = symbolic.submodules.get("in") + sub_out = symbolic.submodules.get("out") + if sub_in is not None: + sub_in.hook_in.add_hook(lambda tensor, hook: symbolic.hook_in(tensor), is_permanent=True) + if sub_out is not None: + sub_out.hook_out.add_hook(lambda tensor, hook: symbolic.hook_out(tensor), is_permanent=True) + + def setup_submodules( component: GeneralizedComponent, architecture_adapter: ArchitectureAdapter, @@ -92,6 +108,13 @@ def setup_submodules( for sub_name, (sub_path, sub_comp) in submodule.real_components.items(): prefixed_key = f"{module_name}.{sub_name}" component.real_components[prefixed_key] = (sub_path, sub_comp) + + # The placeholder has no forward, so its own hook_in/hook_out would never + # fire — mirror them from the designated subcomponents (fc-split archs: + # mlp.hook_in = fc1's input, mlp.hook_out = fc2's output). Firing through + # the parent HookPoint keeps caching AND interventions working: the return + # value feeds back into the subcomponent's hook chain. + _wire_symbolic_hooks(submodule) else: # Set up original_component if not already set if submodule.original_component is None: diff --git a/transformer_lens/model_bridge/sources/inspect/driver.py b/transformer_lens/model_bridge/sources/inspect/driver.py index b7dc954ed9..518036b276 100644 --- a/transformer_lens/model_bridge/sources/inspect/driver.py +++ b/transformer_lens/model_bridge/sources/inspect/driver.py @@ -48,9 +48,11 @@ def __init__(self, model: Any, adapter: Any, tokenizer: Any, profile: Any = None # Provider-specific: loss/both allowed only if the provider returns full logits. self.provides_sequence_logits = self._profile.provides_sequence_logits self.supported_hook_points = self._profile.supported_hooks(self._n_layers) - full = hooks.supported_hook_points(self._n_layers) + # Everything the registry could serve (boundaries + head-split) that this + # provider/model doesn't, plus the never-fireable set (embed, ln_final, scores). + universe = hooks.all_hook_points(self._n_layers) self.non_fireable_hook_points = hooks.nonfireable_hook_points(self._n_layers) | ( - full - self.supported_hook_points + universe - self.supported_hook_points ) # Background event loop, created lazily on first forward. self._loop: asyncio.AbstractEventLoop | None = None @@ -106,8 +108,9 @@ async def _generate(self, prompt: Any, extra_args: dict[str, Any]) -> Any: return await self._model.generate(prompt, config=config) def _assemble_captures(self, output: Any, names: list[str]) -> dict[str, np.ndarray]: - """Decode the requested boundaries → ``{hook_name: (1, seq, d_model)}``; names the - provider didn't return are skipped (and warned once).""" + """Decode the requested hooks → ``{hook_name: (1, ...)}`` (batch dim added onto the + provider's batchless array — rank 2 for boundaries, 3 for head-split/pattern); + names the provider didn't return are skipped (and warned once).""" metadata = getattr(output, "metadata", None) or {} decoded = wire.decode_activations(metadata, self._wire_keys(names)) captured: dict[str, np.ndarray] = {} @@ -120,7 +123,8 @@ def _assemble_captures(self, output: Any, names: list[str]) -> dict[str, np.ndar if arr is None: missing.append(name) continue - captured[name] = arr[np.newaxis, ...] if arr.ndim == 2 else arr + batchless = hooks.WIRE_BATCHLESS_NDIM.get(resolved[1], 2) + captured[name] = arr[np.newaxis, ...] if arr.ndim == batchless else arr self._warn_missing(missing) return captured diff --git a/transformer_lens/model_bridge/sources/inspect/eval.py b/transformer_lens/model_bridge/sources/inspect/eval.py index eebad35931..176d6036a4 100644 --- a/transformer_lens/model_bridge/sources/inspect/eval.py +++ b/transformer_lens/model_bridge/sources/inspect/eval.py @@ -84,9 +84,10 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: def turn_activations(sample: Any) -> list[dict[str, np.ndarray]]: """Per-turn activations from an eval sample's model events, for a provider booted with - ``capture=[...]`` (e.g. ``model_args={"capture": [...]}``). Returns one - ``{hook_name: (1, seq, d)}`` dict per model generation, in turn order — the activations - of an agentic/multi-turn rollout. + ``capture=[...]`` (e.g. ``model_args={"capture": [...]}``). Returns one dict per model + generation, in turn order — the activations of an agentic/multi-turn rollout. Every + array gets a leading batch dim: boundaries are ``(1, seq, d_model)``, head-split + q/k/v/z ``(1, seq, heads, d_head)``. """ turns = [] for event in getattr(sample, "events", []) or []: @@ -94,11 +95,15 @@ def turn_activations(sample: Any) -> list[dict[str, np.ndarray]]: if not metadata or "activations" not in metadata: continue decoded = wire.decode_activations(metadata, list(metadata["activations"])) - named = { - name: arr[np.newaxis, ...] if arr.ndim == 2 else arr - for wk, arr in decoded.items() - if (name := hooks.name_from_wire_key(wk)) is not None - } + named = {} + for wk, arr in decoded.items(): + name = hooks.name_from_wire_key(wk) + if name is None: + continue + # Rank-aware batch dim (mirrors driver._assemble_captures): boundary kinds + # arrive rank-2, head-split kinds rank-3 — unsqueeze exactly once either way. + batchless = hooks.WIRE_BATCHLESS_NDIM.get(wk.partition(":")[2], 2) + named[name] = arr[np.newaxis, ...] if arr.ndim == batchless else arr if named: turns.append(named) return turns diff --git a/transformer_lens/model_bridge/sources/inspect/hooks.py b/transformer_lens/model_bridge/sources/inspect/hooks.py index ed287bca0a..2282472216 100644 --- a/transformer_lens/model_bridge/sources/inspect/hooks.py +++ b/transformer_lens/model_bridge/sources/inspect/hooks.py @@ -1,8 +1,10 @@ """Canonical hook names ↔ (layer, kind) for the Inspect HF provider. Torch-free; shared by the provider (capture/intervene) and the driver (supported set, -decode). Covers the ``d_model``-shaped decoder-layer boundaries; head-split hooks -(q/k/v/z, pattern), ``embed``, and ``ln_final`` (fold-LN convention) are non-fireable. +decode). Covers the ``d_model``-shaped decoder-layer boundaries plus the head-split +attention hooks (q/k/v/z, pattern) where the provider's structural probe finds the +projections; ``attn_scores``, ``embed``, and ``ln_final`` (fold-LN convention) stay +non-fireable. Names are TransformerBridge-native (``blocks.{i}.hook_out``, ``.attn.hook_out``, ...), not the HookedTransformer aliases. A bridge cache carries both with identical values, @@ -10,8 +12,9 @@ Which boundaries are actually fireable is decided per-model by the provider's structural self-check, not a hand-kept architecture list: it locates attn/mlp and probes whether the -``resid_pre + attn_out`` derivation holds, gating ``resid_mid`` otherwise. -``supported_hook_points(n_layers, kinds=...)`` filters to that detected set. +``resid_pre + attn_out`` derivation holds, gating ``resid_mid`` otherwise; head-split +kinds need separate q/k/v projections (q/k/v), a locatable out-projection (z), or eager +attention (pattern). ``supported_hook_points(n_layers, kinds=...)`` filters to that set. """ from __future__ import annotations @@ -19,6 +22,10 @@ from typing import Iterable, Optional ALL_KINDS = frozenset({"resid_pre", "resid_mid", "resid_post", "attn_out", "mlp_out"}) +# Head-split attention kinds, served only when structurally detected (never by default): +# q/k/v are the pre-RoPE projection outputs, z is the out-projection input (all +# ``(seq, heads, d_head)``); pattern is post-softmax attention ``(heads, q_pos, k_pos)``. +HEAD_KINDS = frozenset({"q", "k", "v", "z", "pattern"}) # One canonical TransformerBridge name per boundary (no aliases — avoids duplicate # HookPoints/cache entries). resid_mid (ln2.hook_in) is derived (resid_pre + @@ -29,8 +36,27 @@ "resid_post": "blocks.{i}.hook_out", "attn_out": "blocks.{i}.attn.hook_out", "mlp_out": "blocks.{i}.mlp.hook_out", + "q": "blocks.{i}.attn.hook_q", + "k": "blocks.{i}.attn.hook_k", + "v": "blocks.{i}.attn.hook_v", + "z": "blocks.{i}.attn.hook_z", + "pattern": "blocks.{i}.attn.hook_pattern", +} +# pattern is read from the forward's output_attentions — nothing to write back, so it's +# capture-only (like the derived resid_mid). +INTERVENEABLE_KINDS = frozenset( + {"resid_pre", "attn_out", "mlp_out", "resid_post", "q", "k", "v", "z"} +) + +# Rank of each kind's batchless wire array; the driver unsqueezes exactly one batch dim. +WIRE_BATCHLESS_NDIM = { + **{kind: 2 for kind in ALL_KINDS}, # (seq, d_model) + "q": 3, + "k": 3, + "v": 3, + "z": 3, # (seq, heads, d_head) + "pattern": 3, # (heads, q_pos, k_pos) } -INTERVENEABLE_KINDS = frozenset({"resid_pre", "attn_out", "mlp_out", "resid_post"}) _SUFFIX_TO_KIND = { "hook_in": "resid_pre", @@ -38,29 +64,35 @@ "hook_out": "resid_post", "attn.hook_out": "attn_out", "mlp.hook_out": "mlp_out", + "attn.hook_q": "q", + "attn.hook_k": "k", + "attn.hook_v": "v", + "attn.hook_z": "z", + "attn.hook_pattern": "pattern", } _BLOCK = re.compile(r"^blocks\.(\d+)\.(.+)$") def supported_hook_points(n_layers: int, kinds: Optional[Iterable[str]] = None) -> frozenset[str]: - """Fireable hook names across all layers. ``kinds=None`` means all boundaries; + """Fireable hook names across all layers. ``kinds=None`` means all *boundary* kinds + (head-split kinds are opt-in — a provider must detect and list them explicitly); pass the provider's detected kinds to gate (e.g. drop ``resid_mid`` for parallel).""" - selected = _KIND_NAMES if kinds is None else {k: _KIND_NAMES[k] for k in kinds} - return frozenset(name.format(i=i) for i in range(n_layers) for name in selected.values()) + selected = ALL_KINDS if kinds is None else kinds + return frozenset(_KIND_NAMES[k].format(i=i) for i in range(n_layers) for k in selected) + + +def all_hook_points(n_layers: int) -> frozenset[str]: + """Every hook name the registry can serve (boundaries + head-split) — the universe a + driver subtracts its supported set from to build ``non_fireable_hook_points``.""" + return supported_hook_points(n_layers, ALL_KINDS | HEAD_KINDS) def nonfireable_hook_points(n_layers: int) -> frozenset[str]: - """Hooks the residual/attn/mlp provider can't fire (head-split, embed, ln_final).""" + """Hooks no provider configuration can fire (embed, ln_final, pre-softmax scores). + Head-split q/k/v/z/pattern are *conditionally* fireable and belong here only when a + model's structural probe gates them — the driver handles that subtraction.""" names = ["embed.hook_out", "ln_final.hook_normalized", "unembed.hook_out"] - for i in range(n_layers): - names += [ - f"blocks.{i}.attn.hook_pattern", - f"blocks.{i}.attn.hook_attn_scores", - f"blocks.{i}.attn.hook_q", - f"blocks.{i}.attn.hook_k", - f"blocks.{i}.attn.hook_v", - f"blocks.{i}.attn.hook_z", - ] + names += [f"blocks.{i}.attn.hook_attn_scores" for i in range(n_layers)] return frozenset(names) diff --git a/transformer_lens/model_bridge/sources/inspect/intervention.py b/transformer_lens/model_bridge/sources/inspect/intervention.py index fcda9bc802..0cd1ee9dc8 100644 --- a/transformer_lens/model_bridge/sources/inspect/intervention.py +++ b/transformer_lens/model_bridge/sources/inspect/intervention.py @@ -47,7 +47,7 @@ def build_interventions( if resolved is None or resolved[1] not in hooks.INTERVENEABLE_KINDS: raise ValueError( f"Cannot intervene on {hook_name!r}: capture-only " - f"(intervene on resid_pre/attn_out/mlp_out/resid_post instead)." + f"(intervene on resid_pre/attn_out/mlp_out/resid_post or attn q/k/v/z instead)." ) if op == "scale" and "factor" not in spec: raise ValueError(f"Intervention {hook_name!r}: op='scale' requires 'factor' (float).") @@ -55,6 +55,15 @@ def build_interventions( raise ValueError( f"Intervention {hook_name!r}: op={op!r} requires 'value' (scalar or width-shaped)." ) + pos = spec.get("pos") + if pos is not None and not ( + isinstance(pos, int) + or (isinstance(pos, (list, tuple)) and all(isinstance(p, int) for p in pos)) + ): + raise ValueError( + f"Intervention {hook_name!r}: 'pos' must be an int or list of ints " + f"(sequence positions to patch); got {pos!r}." + ) layer, kind = resolved out[hooks.wire_key(layer, kind)] = dict(spec) return out diff --git a/transformer_lens/model_bridge/sources/inspect/source.py b/transformer_lens/model_bridge/sources/inspect/source.py index b23f565c83..656d31558b 100644 --- a/transformer_lens/model_bridge/sources/inspect/source.py +++ b/transformer_lens/model_bridge/sources/inspect/source.py @@ -43,12 +43,16 @@ def boot_inspect( Fireable hooks (``tl_bridge``, TransformerBridge-native names): ``blocks.{i}.hook_in`` (resid_pre) / ``ln2.hook_in`` (resid_mid) / - ``hook_out`` (resid_post) / ``attn.hook_out`` / ``mlp.hook_out``. The provider runs - a structural self-check per model and gates any boundary it can't serve faithfully: - ``resid_mid`` for parallel-residual or norm-variant blocks, and ``attn_out``/ - ``mlp_out`` when their submodule isn't locatable (it warns when it gates one). - Head-split hooks (q/k/v/z, pattern), ``embed``, and ``ln_final`` are always - non-fireable — use ``boot_transformers()`` for those. + ``hook_out`` (resid_post) / ``attn.hook_out`` / ``mlp.hook_out``, plus the head-split + attention hooks where the structural probe finds them: ``attn.hook_q/k/v`` (pre-RoPE + projection outputs; separate-projection archs only — fused qkv gates them), + ``attn.hook_z`` (out-projection input), and ``attn.hook_pattern`` (post-softmax, + capture-only, eager attention required). The provider runs a structural self-check per + model and gates any boundary it can't serve faithfully: ``resid_mid`` for + parallel-residual or norm-variant blocks, ``attn_out``/``mlp_out`` when their submodule + isn't locatable (it warns when it gates one). ``embed``, ``ln_final``, and + ``attn.hook_attn_scores`` are always non-fireable — use ``boot_transformers()`` for + those. For parity with ``boot_transformers`` the provider loads with the same dtype (fp32 by default) and eager attention. Full-sequence logits ride on ``return_logits=True`` (the diff --git a/transformer_lens/model_bridge/sources/inspect/transformers_provider.py b/transformer_lens/model_bridge/sources/inspect/transformers_provider.py index be8a26cf89..74ca10b513 100644 --- a/transformer_lens/model_bridge/sources/inspect/transformers_provider.py +++ b/transformer_lens/model_bridge/sources/inspect/transformers_provider.py @@ -29,7 +29,7 @@ modelapi, ) -from . import wire +from . import hooks, wire from ._provider_base import _InspectModelAPIBase, _parse_tool_calls, _require_served # NOT "transformer_lens" — inspect_ai ships a built-in provider by that name (the @@ -48,6 +48,17 @@ # Attn/MLP submodule names within a block, by family. _ATTN_ATTRS = ("self_attn", "attn", "attention", "self_attention") # self_attention: Falcon _MLP_ATTRS = ("mlp", "feed_forward") +# fc-split blocks (OPT/XGLM) have no mlp container — fc1/fc2 sit on the block directly: +# mlp_in boundary = fc1's input, mlp_out boundary = fc2's output. +_MLP_SPLIT_ATTRS = ("fc1", "fc2") +# Separate q/k/v projections (Llama/Mistral/Qwen/OPT-family). Fused-qkv archs +# (GPT-2 c_attn, Falcon/GPTNeoX query_key_value) gate q/k/v — their packed layouts +# vary per family, so slicing them is per-arch work we don't hand-maintain here. +_Q_PROJ_ATTRS = ("q_proj", "query") +_K_PROJ_ATTRS = ("k_proj", "key") +_V_PROJ_ATTRS = ("v_proj", "value") +# Attention out-projection; its input is z (works for fused-qkv archs too). +_O_PROJ_ATTRS = ("o_proj", "out_proj", "dense", "c_proj") @modelapi(name=PROVIDER_NAME) @@ -85,10 +96,24 @@ def __init__( ) self._tokenizer = AutoTokenizer.from_pretrained(model_name) self._layers = _locate_layers(self._hf) - self._kinds, self._capability_note = _detect_capabilities(self._hf, self._layers) + # Head-split reshape geometry; None per-field when the config lacks it (head + # kinds are then gated by _detect_capabilities' width checks). + self._geometry = _attn_geometry(self._hf.config) + self._kinds, self._capability_note = _detect_capabilities( + self._hf, self._layers, self._geometry + ) # Per-turn capture during plain generation (agent rollouts): every _generate_eval # stashes these hooks in ModelOutput.metadata. Gated by the structural self-check. self._eval_capture = self._parse_eval_capture(model_args) + for key in self._eval_capture: + if key.endswith(":pattern"): + # pattern rides the forward's output_attentions, which the eval path's + # hf.generate doesn't thread — reject rather than silently omit. + raise ValueError( + "Per-turn eval capture of attn.hook_pattern is not supported (it needs " + "output_attentions on the forward). Use the driver path " + "(bridge.run_with_cache) for pattern capture." + ) def _generate_capture(self, input: Any, extra_args: Mapping[str, Any], config: GenerateConfig): """TL-driven single forward: capture residual/attn/mlp boundaries + full logits.""" @@ -101,9 +126,22 @@ def _generate_capture(self, input: Any, extra_args: Mapping[str, Any], config: G _, _, kind = key.partition(":") _require_served(kind, self._kinds, self._capability_note, f"capture {key!r}") interventions: Mapping[str, Any] = extra_args.get("interventions", {}) + # The driver validates before sending, but this direct interface (extra_args) is + # documented — validate here too so a gated/capture-only kind fails loud instead + # of silently no-op'ing when no hook installs for it. + for key in interventions: + _, _, kind = key.partition(":") + _require_served(kind, self._kinds, self._capability_note, f"intervention {key!r}") + if kind not in hooks.INTERVENEABLE_KINDS: + raise ValueError( + f"intervention {key!r}: kind {kind!r} is capture-only " + f"(interveneable: {sorted(hooks.INTERVENEABLE_KINDS)})." + ) want_logits = bool(extra_args.get("return_logits", True)) capture, intervene = _plan(capture_keys, interventions) + # pattern comes from the forward's output_attentions, not a module hook. + pattern_layers = [layer for layer, kinds in capture.items() if "pattern" in kinds] raw: dict[tuple[int, str], np.ndarray] = {} call_id = uuid.uuid4().hex token = _current_call_id.set(call_id) @@ -111,12 +149,19 @@ def _generate_capture(self, input: Any, extra_args: Mapping[str, Any], config: G try: with torch.no_grad(): ids = torch.tensor([list(input_ids)], device=self._device) - logits = self._hf(ids).logits # (1, seq, vocab) + outputs = self._hf(ids, output_attentions=bool(pattern_layers)) + logits = outputs.logits # (1, seq, vocab) finally: for handle in handles: handle.remove() _current_call_id.reset(token) + attentions = getattr(outputs, "attentions", None) + for layer in pattern_layers: + attn_l = attentions[layer] if attentions is not None else None + if attn_l is not None: # None → driver's missing-hook warning handles it + raw[(layer, "pattern")] = attn_l[0].detach().float().cpu().numpy() + captured = _assemble(raw, capture_keys) metadata: dict[str, Any] = {"activations": wire.encode_activations(captured)} if want_logits: @@ -254,7 +299,7 @@ def _install_hooks(self, capture, intervene, raw, call_id: str) -> list: if not cap_kinds and not iv_kinds: continue attn = _first_attr(block, _ATTN_ATTRS) - mlp = _first_attr(block, _MLP_ATTRS) + _, mlp_out_mod = _locate_mlp(block) if "resid_pre" in cap_kinds or "resid_pre" in iv_kinds: handles.append( block.register_forward_pre_hook( @@ -277,9 +322,13 @@ def _install_hooks(self, capture, intervene, raw, call_id: str) -> list: ) ) ) - if mlp is not None and ("mlp_out" in cap_kinds or "mlp_out" in iv_kinds): + if attn is not None: + handles.extend( + self._install_head_hooks(layer, attn, cap_kinds, iv_kinds, raw, call_id) + ) + if mlp_out_mod is not None and ("mlp_out" in cap_kinds or "mlp_out" in iv_kinds): handles.append( - mlp.register_forward_hook( + mlp_out_mod.register_forward_hook( _out_hook( layer, "mlp_out", @@ -305,6 +354,58 @@ def _install_hooks(self, capture, intervene, raw, call_id: str) -> list: ) return handles + def _install_head_hooks(self, layer, attn, cap_kinds, iv_kinds, raw, call_id: str) -> list: + """Hooks for the head-split kinds: q/k/v on their projection outputs, z on the + out-projection's input. pattern isn't hooked (it rides output_attentions). + + Interventions apply to the module's natural *flat* tensor ``(..., seq, + heads·d_head)`` — a spec ``value`` is scalar, ``(heads·d_head,)``, or per-position + ``(len(pos), heads·d_head)`` (flatten a captured head-split tensor to build one). + Captures are emitted head-split ``(seq, heads, d_head)`` to match the bridge's + ``hook_q/k/v/z``. + """ + handles: list = [] + d_head = self._geometry[2] + if d_head is None: # geometry undetectable → head kinds were gated at detection + return handles + host = _projection_host(attn) + for kind, attrs in (("q", _Q_PROJ_ATTRS), ("k", _K_PROJ_ATTRS), ("v", _V_PROJ_ATTRS)): + if kind not in cap_kinds and kind not in iv_kinds: + continue + proj = _first_attr(host, attrs) + if proj is None: + # Detection ran on layers[0]; a later layer missing the projection would + # silently skip a validated intervention — fail loud (capture-only misses + # surface through the driver's missing-hook warning instead). + if kind in iv_kinds: + raise RuntimeError( + f"Intervention on blocks.{layer}.attn.hook_{kind} cannot be applied: " + f"layer {layer} has no {kind} projection (heterogeneous layers)." + ) + continue + handles.append( + proj.register_forward_hook( + _proj_hook( + layer, kind, d_head, kind in cap_kinds, iv_kinds.get(kind), raw, call_id + ) + ) + ) + if "z" in cap_kinds or "z" in iv_kinds: + o_proj = _first_attr(host, _O_PROJ_ATTRS) + if o_proj is None and "z" in iv_kinds: + raise RuntimeError( + f"Intervention on blocks.{layer}.attn.hook_z cannot be applied: layer " + f"{layer} has no out-projection (heterogeneous layers)." + ) + if o_proj is not None: + handles.append( + o_proj.register_forward_pre_hook( + _zin_hook(layer, d_head, "z" in cap_kinds, iv_kinds.get("z"), raw, call_id), + with_kwargs=True, + ) + ) + return handles + def _plan(capture_keys, interventions): """Resolve wire keys → per-layer kinds to capture (resid_mid needs pre+attn) and intervene.""" @@ -370,7 +471,10 @@ def hook(_module, _inputs, output): if spec is not None: hidden = _apply_affine(hidden, spec) if want_capture and (layer, kind) not in raw: - raw[(layer, kind)] = hidden[0].detach().float().cpu().numpy() + # OPT-style blocks flatten the FFN to (batch·seq, d) — with batch_size=1 + # that IS (seq, d) already; only 3-D (batch, seq, d) needs the batch strip. + flat = hidden if hidden.ndim == 2 else hidden[0] + raw[(layer, kind)] = flat.detach().float().cpu().numpy() if spec is None: return None return (hidden, *output[1:]) if is_tuple else hidden @@ -378,45 +482,114 @@ def hook(_module, _inputs, output): return hook -def _apply_affine(t: torch.Tensor, spec: Mapping[str, Any]) -> torch.Tensor: - """suppress→0, scale→·factor, add→+value, set→value (value scalar or width-shaped).""" +def _proj_hook(layer, kind, d_head, want_capture, spec, raw, call_id): + """q/k/v projection output: affine on the flat ``(..., seq, heads·d_head)`` tensor, + captured head-split ``(seq, heads, d_head)`` (pre-RoPE — matches the bridge's + ``hook_q``/``hook_k``/``hook_v``). Mutations feed the downstream attention math.""" + + def hook(_module, _inputs, output): + if _current_call_id.get() != call_id: + return None # different concurrent call's hook + hidden = output + if spec is not None: + hidden = _apply_affine(hidden, spec) + if want_capture and (layer, kind) not in raw: + flat = hidden[0].detach().float().cpu() + raw[(layer, kind)] = flat.reshape(flat.shape[0], -1, d_head).numpy() + return hidden if spec is not None else None + + return hook + + +def _zin_hook(layer, d_head, want_capture, spec, raw, call_id): + """z — the out-projection's *input* (attention-weighted values, all heads): pre-hook, + affine on the flat tensor, captured head-split to match the bridge's ``hook_z``.""" + + def hook(_module, args, kwargs): + if _current_call_id.get() != call_id: + return None # different concurrent call's hook + z = args[0] + if spec is not None: + z = _apply_affine(z, spec) + if want_capture and (layer, "z") not in raw: + flat = z[0].detach().float().cpu() + raw[(layer, "z")] = flat.reshape(flat.shape[0], -1, d_head).numpy() + if spec is None: + return None + return (z, *args[1:]), kwargs + + return hook + + +def _affine_op(sub: torch.Tensor, spec: Mapping[str, Any]) -> torch.Tensor: + """One affine op: suppress→0, scale→·factor, add→+value, set→value. ``value`` broadcasts + (scalar, width-shaped, or per-position ``(n_pos, width)``).""" op = spec["op"] if op == "suppress": - return torch.zeros_like(t) + return torch.zeros_like(sub) if op == "scale": - return t * float(spec["factor"]) - value = torch.as_tensor(spec["value"], dtype=t.dtype, device=t.device) + return sub * float(spec["factor"]) + value = torch.as_tensor(spec["value"], dtype=sub.dtype, device=sub.device) if op == "add": - return t + value - return torch.zeros_like(t) + value # set + return sub + value + return torch.zeros_like(sub) + value # set + + +def _apply_affine(t: torch.Tensor, spec: Mapping[str, Any]) -> torch.Tensor: + """Affine intervention on a captured tensor ``(..., seq, width)``. + Without ``pos`` the op spans every position (the original width-broadcast form). With + ``pos`` (an int or list of sequence indices) it touches only those positions — the + activation-patching primitive — and ``value`` may be per-position ``(len(pos), width)`` + to transplant a captured activation (path/causal tracing) rather than a single vector. + """ + pos = spec.get("pos") + if pos is None: + return _affine_op(t, spec) + idx = [pos] if isinstance(pos, int) else list(pos) + out = t.clone() + out[..., idx, :] = _affine_op(t[..., idx, :], spec) + return out -def _detect_capabilities(model: Any, layers: Any) -> tuple[frozenset, str]: - """Structural self-check: which boundary kinds this model can serve faithfully. + +def _detect_capabilities( + model: Any, layers: Any, geometry: tuple[Any, Any, Any] +) -> tuple[frozenset, str]: + """Structural self-check: which kinds this model can serve faithfully. resid_pre/resid_post are the block in/out (always); attn_out/mlp_out need their submodules locatable; resid_mid is gated unless its derivation holds (see - :func:`_resid_mid_derivable`). Returns (kinds, note); note explains any gating, '' if none. + :func:`_resid_mid_derivable`). Head-split kinds: q/k/v need separate projections whose + widths match ``heads·d_head``; z needs an out-projection of in-width ``n_heads·d_head``; + pattern needs eager attention (output_attentions is a no-op under sdpa/flash). + Returns (kinds, note); note explains any gating, '' if none. """ block = layers[0] attn = _first_attr(block, _ATTN_ATTRS) - mlp = _first_attr(block, _MLP_ATTRS) + mlp_in_mod, mlp_out_mod = _locate_mlp(block) kinds = {"resid_pre", "resid_post"} gated = [] if attn is not None: kinds.add("attn_out") else: gated.append("attn_out (no attention submodule found)") - if mlp is not None: + if mlp_out_mod is not None: kinds.add("mlp_out") else: gated.append("mlp_out (no MLP submodule found)") - if attn is not None and mlp is not None and _resid_mid_derivable(model, block, attn, mlp): + if ( + attn is not None + and mlp_out_mod is not None + and _resid_mid_derivable(model, block, attn, mlp_in_mod, mlp_out_mod) + ): kinds.add("resid_mid") else: gated.append( "resid_mid (resid_pre + attn_out doesn't hold — parallel or norm-variant block)" ) + head_kinds, head_gated = _detect_head_capabilities(model, attn, geometry) + kinds |= head_kinds + gated += head_gated note = ( "" if not gated @@ -427,11 +600,85 @@ def _detect_capabilities(model: Any, layers: Any) -> tuple[frozenset, str]: return frozenset(kinds), note -def _resid_mid_derivable(model: Any, block: Any, attn: Any, mlp: Any) -> bool: +def _detect_head_capabilities( + model: Any, attn: Any, geometry: tuple[Any, Any, Any] +) -> tuple[set, list]: + """Head-split kinds this model serves: q/k/v iff separate projections with the + expected widths, z iff the out-projection's in-width is ``n_heads·d_head``, pattern + iff attention runs eager (otherwise ``output_attentions`` returns None/garbage).""" + n_heads, n_kv_heads, d_head = geometry + kinds: set = set() + gated: list = [] + if attn is None or d_head is None: + gated.append("q/k/v/z/pattern (no attention submodule or head geometry in config)") + return kinds, gated + + host = _projection_host(attn) + q = _first_attr(host, _Q_PROJ_ATTRS) + k = _first_attr(host, _K_PROJ_ATTRS) + v = _first_attr(host, _V_PROJ_ATTRS) + expected = {"q": n_heads * d_head, "k": n_kv_heads * d_head, "v": n_kv_heads * d_head} + if all( + proj is not None and _out_width(proj) == expected[kind] + for kind, proj in (("q", q), ("k", k), ("v", v)) + ): + kinds |= {"q", "k", "v"} + else: + gated.append("q/k/v (fused or nonstandard qkv projections)") + + o_proj = _first_attr(host, _O_PROJ_ATTRS) + if o_proj is not None and _in_width(o_proj) == n_heads * d_head: + kinds.add("z") + else: + gated.append("z (out-projection missing or nonstandard width)") + + if getattr(model.config, "_attn_implementation", "eager") == "eager": + kinds.add("pattern") + else: + gated.append("pattern (attention implementation is not eager)") + return kinds, gated + + +def _out_width(module: Any) -> Any: + """Output width of a projection: ``nn.Linear.out_features`` or GPT-2 ``Conv1D.nf``.""" + out = getattr(module, "out_features", None) + if out is not None: + return int(out) + nf = getattr(module, "nf", None) # transformers Conv1D + return int(nf) if nf is not None else None + + +def _in_width(module: Any) -> Any: + """Input width of a projection: ``nn.Linear.in_features`` or Conv1D ``weight.shape[0]``.""" + in_f = getattr(module, "in_features", None) + if in_f is not None: + return int(in_f) + if getattr(module, "nf", None) is not None and hasattr(module, "weight"): + return int(module.weight.shape[0]) # Conv1D stores weight (in, out) + return None + + +def _attn_geometry(config: Any) -> tuple[Any, Any, Any]: + """(n_heads, n_kv_heads, d_head) from an HF config; (None, None, None) if underivable.""" + n_heads = getattr(config, "num_attention_heads", None) or getattr(config, "n_head", None) + hidden = getattr(config, "hidden_size", None) or getattr(config, "n_embd", None) + if n_heads is None: + return None, None, None + n_kv = getattr(config, "num_key_value_heads", None) or n_heads + d_head = getattr(config, "head_dim", None) + if d_head is None and hidden is not None: + d_head = hidden // n_heads + return (int(n_heads), int(n_kv), int(d_head) if d_head is not None else None) + + +def _resid_mid_derivable( + model: Any, block: Any, attn: Any, mlp_in_mod: Any, mlp_out_mod: Any +) -> bool: """True iff ``resid_mid = resid_pre + attn_out`` holds, via two tiny probe forwards. Requires both the linear identity ``resid_post = resid_pre + attn_out + mlp_out`` (broken by post-norm/multiplier blocks — Gemma2/OLMo2/Granite) and attn feeding mlp (broken by - parallel blocks — GPTNeoX/GPT-J, where mlp reads resid_pre directly).""" + parallel blocks — GPTNeoX/GPT-J, where mlp reads resid_pre directly). ``mlp_in_mod``/ + ``mlp_out_mod`` are the same module for container archs, (fc1, fc2) for fc-split.""" cap: dict[str, Any] = {} def grab(key: str): # type: ignore[no-untyped-def] @@ -463,8 +710,8 @@ def perturb_attn(_m: Any, _i: Any, out: Any): # type: ignore[no-untyped-def] handles = [ block.register_forward_pre_hook(grab_in("resid_pre"), with_kwargs=True), attn.register_forward_hook(grab("attn_out")), - mlp.register_forward_hook(grab("mlp_out")), - mlp.register_forward_pre_hook(grab_in("mlp_in"), with_kwargs=True), + mlp_out_mod.register_forward_hook(grab("mlp_out")), + mlp_in_mod.register_forward_pre_hook(grab_in("mlp_in"), with_kwargs=True), block.register_forward_hook(grab("resid_post")), ] model(ids) @@ -472,7 +719,7 @@ def perturb_attn(_m: Any, _i: Any, out: Any): # type: ignore[no-untyped-def] h.remove() mlp_in_clean = cap.pop("mlp_in", None) handles = [ - mlp.register_forward_pre_hook(grab_in("mlp_in"), with_kwargs=True), + mlp_in_mod.register_forward_pre_hook(grab_in("mlp_in"), with_kwargs=True), attn.register_forward_hook(perturb_attn), ] model(ids) @@ -490,7 +737,13 @@ def perturb_attn(_m: Any, _i: Any, out: Any): # type: ignore[no-untyped-def] return False if mlp_in_clean is None or mlp_in_perturbed is None: return False - if not (rp.shape == ao.shape == mo.shape == rpost.shape == mlp_in_clean.shape): + # OPT-style blocks flatten the FFN to (batch·seq, d); with the probe's batch=1 that + # is a pure reshape of the block-level (1, seq, d) — normalize before comparing. + if mo.shape != rpost.shape and mo.numel() == rpost.numel(): + mo = mo.reshape(rpost.shape) + if not (rp.shape == ao.shape == mo.shape == rpost.shape): + return False + if mlp_in_clean.shape != mlp_in_perturbed.shape or mlp_in_clean.numel() != rp.numel(): return False # (1) sub-block outputs add to the residual without intervening norm/scale. identity = (rpost - rp - ao - mo).abs().max().item() @@ -514,6 +767,33 @@ def _locate_layers(model: Any) -> Any: ) +def _locate_mlp(block: Any) -> tuple[Any, Any]: + """The modules bounding the MLP: ``(in_module, out_module)`` — the mlp_in boundary is + in_module's input, mlp_out is out_module's output. Container archs return the mlp + module twice; fc-split blocks (OPT/XGLM: fc1/fc2 directly on the block) return + ``(fc1, fc2)``. ``(None, None)`` when neither layout is found (mlp_out gated).""" + mlp = _first_attr(block, _MLP_ATTRS) + if mlp is not None: + return mlp, mlp + fc1, fc2 = (getattr(block, name, None) for name in _MLP_SPLIT_ATTRS) + if fc1 is not None and fc2 is not None: + return fc1, fc2 + return None, None + + +def _projection_host(attn: Any) -> Any: + """The module whose direct attrs are the q/k/v/out projections. Usually ``attn`` + itself; GPT-Neo-style blocks wrap the real attention (with its standard q_proj/ + out_proj) one level down at ``attn.attention``. Descend only when the located module + has neither a q- nor an out-projection — GPTNeoX's ``block.attention`` (fused + query_key_value + dense) has ``dense`` directly, so it never descends.""" + if _first_attr(attn, _Q_PROJ_ATTRS) is None and _first_attr(attn, _O_PROJ_ATTRS) is None: + inner = getattr(attn, "attention", None) + if inner is not None: + return inner + return attn + + def _first_attr(obj: Any, names: tuple[str, ...]) -> Any: for name in names: found = getattr(obj, name, None) diff --git a/transformer_lens/model_bridge/sources/vllm/driver.py b/transformer_lens/model_bridge/sources/vllm/driver.py index e78aa47ffa..91e0270dce 100644 --- a/transformer_lens/model_bridge/sources/vllm/driver.py +++ b/transformer_lens/model_bridge/sources/vllm/driver.py @@ -20,10 +20,16 @@ class VLLMDriver(DriverBase): """Driver wrapping a vLLM ``LLM``; captures via ``collective_rpc``.""" - # vLLM owns the model in a worker — no torch surface (parameters/state_dict/grads). - _supported_features = frozenset() - # Logits synthesized for the final position only (sampler bypass). - provides_sequence_logits = False + # vLLM owns the model in a worker — no torch module surface (parameters/state_dict/ + # grads). Named-weight reads ARE served: get_param() returns CPU clones via the + # tl_get_param RPC (what logit reconstruction and direct-logit-attribution use). + _supported_features = frozenset({"weight_access"}) + # Full-sequence logits reconstructed host-side (ln_final @ lm_head.weight.T); vLLM's + # sampler only hands back the final position, so the driver rebuilds the rest. + provides_sequence_logits = True + + # Post-weight final-norm capture that lm_head consumes — reconstruction reads it. + _LN_FINAL = "ln_final.hook_normalized" def __init__( self, @@ -34,11 +40,15 @@ def __init__( hf_config: Any, max_num_batched_tokens: int, enable_batching: bool = False, + enable_position_interventions: bool = False, ) -> None: super().__init__(adapter.cfg, tokenizer) self._llm = llm self._max_num_batched_tokens = max_num_batched_tokens self._enable_batching = enable_batching + # Position-scoped 'pos' interventions need (max_n, width) affine buffers, + # allocated at boot only when this is set (see plugin.patched_load_model). + self._enable_position_interventions = enable_position_interventions # Logprobs per forward = real vocab (boot's max_logprobs). d_vocab can be # padded larger, which vLLM would reject; the logits tensor stays d_vocab. self._n_logprobs = int(getattr(hf_config, "vocab_size", self.bridge_config.d_vocab)) @@ -46,9 +56,16 @@ def __init__( self.supported_hook_points = frozenset(overlay.capture_specs(hf_config).keys()) n_layers = getattr(hf_config, "num_hidden_layers", 0) + if not isinstance(n_layers, int) or n_layers <= 0: + # A raw "{i}" template would land unexpanded in non_fireable_hook_points — + # a broken config should fail at boot, not surface as a garbled hook name. + raise ValueError( + f"VLLMDriver: hf_config.num_hidden_layers={n_layers!r} — expected a " + "positive int; the config is missing or malformed." + ) nonfiring: list[str] = [] for tmpl in overlay.nonfiring_hooks(): - if "{i}" in tmpl and isinstance(n_layers, int) and n_layers > 0: + if "{i}" in tmpl: nonfiring.extend(tmpl.replace("{i}", str(i)) for i in range(n_layers)) else: nonfiring.append(tmpl) @@ -73,9 +90,11 @@ def forward( ) intervene_specs = self._validate_interventions(intervene or {}) - # Restrict the GPU→CPU read to these hooks (None = all). run_with_cache - # doesn't derive this from names_filter yet — only explicit forward(capture=). - names = list(capture) or None + # capture is authoritative — the bridge sends exactly the hooked names, so () + # means "capture nothing" and a plain forward(tokens) skips the GPU→CPU copy + # entirely. (The worker's None-means-all convention is never triggered from here; + # an empty tuple used to collapse to None and silently copy every buffer.) + names = list(capture) if self._enable_batching: return self._forward_batched(input_ids, intervene_specs, return_logits, names) @@ -88,6 +107,9 @@ def forward( f"{self._max_num_batched_tokens}; raise the boot_vllm kwarg or " "shorten the prompt." ) + # 'pos'-scoped edits target affine-buffer rows, but the compiled hook only reads + # rows [0, len(ids_list)); a pos past the prompt length would be a silent no-op. + self._reject_pos_beyond_seq(intervene_specs, len(ids_list)) from vllm import SamplingParams from vllm.inputs import TokensPrompt @@ -110,15 +132,31 @@ def forward( ) n_tokens = len(ids_list) - # collective_rpc returns one result per worker; single-rank, so [0]. - worker_captures = self._llm.collective_rpc("tl_read_captures", args=([n_tokens], names))[0] - # Add batch dim: vLLM hands back (n_tokens, width); bridge expects (1, n_tokens, width). - captured = {name: t.unsqueeze(0) for name, t in worker_captures.items()} + # Reconstructing logits needs ln_final; force it into the read even if the caller + # didn't request it (dropped from `captured` below so the surface stays as asked). + read_names = names + if return_logits and self._LN_FINAL not in names: + read_names = names + [self._LN_FINAL] + # collective_rpc returns one result per worker; single-rank, so [0]. Nothing to + # read (no captures, logits off) → skip the crossing altogether. + worker_captures = ( + self._llm.collective_rpc("tl_read_captures", args=([n_tokens], read_names))[0] + if read_names + else {} + ) logits: torch.Tensor | None = None if return_logits: - logits = self._synthesize_logits(outputs[0], n_tokens, self.bridge_config.d_vocab) + recon = self._reconstruct_logits(worker_captures.get(self._LN_FINAL)) + # Fall back to final-position log-probs if the unembedding isn't fetchable. + logits = ( + recon.unsqueeze(0) + if recon is not None + else self._synthesize_logits(outputs[0], n_tokens, self.bridge_config.d_vocab) + ) + # Add batch dim; expose only the caller's requested hooks (drop the forced ln_final). + captured = {name: t.unsqueeze(0) for name, t in worker_captures.items() if name in names} return ForwardResult(logits=logits, captured=captured, raw_output=outputs[0]) def _forward_batched( @@ -126,13 +164,13 @@ def _forward_batched( input_ids: TensorLike, intervene_specs: dict, return_logits: bool, - names: list[str] | None = None, + names: list[str], ) -> ForwardResult: """Eager batched path: per-request capture, right-padded to (B, S, W). No per-prompt length gate — chunked prefill accumulates long prompts - across forwards. Interventions are global across the batch. ``names`` - restricts the returned hooks (``None`` = all). + across forwards. Interventions are global across the batch. ``names`` is + authoritative: exactly the hooks to return (empty = none). """ from vllm import SamplingParams from vllm.inputs import TokensPrompt @@ -155,14 +193,35 @@ def _forward_batched( ), ) + # Force ln_final into the read so logits can be reconstructed (dropped below if + # the caller didn't ask for it). + read_names = names + if return_logits and self._LN_FINAL not in names: + read_names = names + [self._LN_FINAL] # Keyed by req_id (no guaranteed order) — _assemble_padded joins to slot - # k via outputs[k].request_id, not by position. - worker_captures = self._llm.collective_rpc("tl_read_batched_captures", args=(names,))[0] - captured = self._assemble_padded(outputs, worker_captures, prompt_lens) + # k via outputs[k].request_id, not by position. Empty read → skip both the + # crossing AND the join (_assemble_padded requires one worker key per request, + # so it can't run on an empty dict — mirror the single path's empty captured). + if read_names: + worker_captures = self._llm.collective_rpc( + "tl_read_batched_captures", args=(read_names,) + )[0] + captured = self._assemble_padded(outputs, worker_captures, prompt_lens) + else: + captured = {} logits: torch.Tensor | None = None if return_logits: - logits = self._synthesize_logits_batched(outputs, prompt_lens, d_vocab) + recon = self._reconstruct_logits( + captured.get(self._LN_FINAL) + ) # (batch, max_seq, d_vocab) + logits = ( + recon + if recon is not None + else self._synthesize_logits_batched(outputs, prompt_lens, d_vocab) + ) + if self._LN_FINAL not in names: + captured.pop(self._LN_FINAL, None) return ForwardResult(logits=logits, captured=captured, raw_output=outputs) @@ -245,6 +304,37 @@ def get_param(self, dotted_name: str) -> torch.Tensor | None: return None return self._llm.collective_rpc("tl_get_param", args=(dotted_name,))[0] + def _reconstruct_logits(self, ln_final: Any) -> torch.Tensor | None: + """Rebuild real logits from the captured post-weight ln_final: + ``ln_final @ lm_head.weight.T`` (+ bias, + Gemma-family tanh soft-cap). + + vLLM's ``ln_final.hook_normalized`` is the POST-weight RMSNorm value lm_head + consumes (verified empirically: it equals HF's pre-weight value times the norm + weight), so no un-fold is needed. Accepts any ``(..., d_model)`` tensor and returns + ``(..., d_vocab)`` on CPU. ``None`` if ln_final wasn't captured or no unembedding + weight is fetchable — the caller then falls back to the sampler's log-probs. + """ + if ln_final is None: + return None + weight = self.get_param("lm_head.weight") + if weight is None: # tied embeddings expose no separate lm_head + weight = self.get_param("model.embed_tokens.weight") + if weight is None: + return None + lf = ln_final.to(device=weight.device, dtype=torch.float32) + logits = lf @ weight.to(torch.float32).T + bias = self.get_param("lm_head.bias") + if bias is not None: + logits = logits + bias.to(device=logits.device, dtype=torch.float32) + cap = getattr(self.bridge_config, "output_logits_soft_cap", None) + if cap is not None and cap > 0: # Gemma-family cap; -1.0 is the "disabled" sentinel + logits = float(cap) * torch.tanh(logits / float(cap)) + d_vocab = int(self.bridge_config.d_vocab) + if logits.shape[-1] < d_vocab: # pad the padded-vocab tail (never predicted) + pad = logits.new_full((*logits.shape[:-1], d_vocab - logits.shape[-1]), float("-inf")) + logits = torch.cat([logits, pad], dim=-1) + return logits.cpu() + def close(self) -> None: # Detach hooks before dropping the LLM so they don't stay registered on # worker modules for the life of the process (long-running notebooks). @@ -334,9 +424,62 @@ def _validate_interventions(self, intervene: Mapping[str, Any]) -> dict: raise ValueError( f"Cannot intervene on {hook_name!r}: not in supported_hook_points." ) + pos = spec.get("pos") + if pos is not None: + if self._enable_batching: + # The batched/eager path applies ops to the raw tensor, not the + # (max_n, width) affine buffers, so it has no position surface. + raise NotImplementedError( + f"Intervention {hook_name!r}: per-position 'pos' is not supported on the " + "batched/eager path. Boot the compiled path (enable_batching=False) with " + "enable_position_interventions=True." + ) + if not self._enable_position_interventions: + # Default affine buffers are (width,) and broadcast across every position; + # honoring 'pos' needs the (max_n, width) buffers allocated at boot. + raise NotImplementedError( + f"Intervention {hook_name!r}: per-position 'pos' requires " + "boot_vllm(enable_position_interventions=True) (its default affine " + "buffers broadcast across all positions). Use the Inspect/HF backend for " + "position-scoped patching, or drop 'pos' for a whole-sequence edit." + ) + if not ( + isinstance(pos, int) + or (isinstance(pos, (list, tuple)) and all(isinstance(p, int) for p in pos)) + ): + raise ValueError( + f"Intervention {hook_name!r}: 'pos' must be an int or list of ints " + f"(sequence positions to patch); got {pos!r}." + ) + bad = [p for p in ([pos] if isinstance(pos, int) else pos) if p < 0] + if bad: + raise ValueError( + f"Intervention {hook_name!r}: 'pos' must be non-negative; got {bad}." + ) out[hook_name] = dict(spec) return out + @staticmethod + def _reject_pos_beyond_seq(specs: Mapping[str, Any], seq_len: int) -> None: + """Fail loud if a spec's 'pos' targets a row past the actual prompt length. + + The compiled hook applies the affine over rows ``[0, seq_len)`` only, so a ``pos`` + in ``[seq_len, max_num_batched_tokens)`` clears the driver's non-negativity check + and the worker's buffer-capacity check yet is never read — a silent no-op. Bound it + against the real sequence length (known once ``ids_list`` exists) and raise instead. + """ + for hook_name, spec in specs.items(): + pos = spec.get("pos") + if pos is None: + continue + idx = [pos] if isinstance(pos, int) else list(pos) + bad = [p for p in idx if p >= seq_len] + if bad: + raise ValueError( + f"Intervention {hook_name!r}: 'pos' {bad} is beyond the prompt length " + f"{seq_len} (positions are 0-indexed); the edit would be silently ignored." + ) + @staticmethod def _normalize_input_ids(input_ids: Any) -> list: """Coerce input_ids to a flat list[int] for ``TokensPrompt``; batch_size=1 only.""" diff --git a/transformer_lens/model_bridge/sources/vllm/plugin.py b/transformer_lens/model_bridge/sources/vllm/plugin.py index 6b54d46490..51e5d70fb5 100644 --- a/transformer_lens/model_bridge/sources/vllm/plugin.py +++ b/transformer_lens/model_bridge/sources/vllm/plugin.py @@ -49,12 +49,14 @@ def configure( max_num_batched_tokens: int, dtype: torch.dtype, enable_batching: bool = False, + enable_position_interventions: bool = False, ) -> None: """Set capture specs, buffer length, dtype, and hook flavor before ``LLM(...)``.""" _config["capture_specs"] = capture_specs _config["max_num_batched_tokens"] = max_num_batched_tokens _config["dtype"] = dtype _config["enable_batching"] = enable_batching + _config["enable_position_interventions"] = enable_position_interventions def register() -> None: @@ -84,6 +86,9 @@ def patched_load_model(self): max_n = _config["max_num_batched_tokens"] dtype = _config["dtype"] enable_batching = _config.get("enable_batching", False) + # Per-position affine buffers are (max_n, width) instead of (width,), so each + # sequence row can carry a distinct scale/bias (position-scoped patching). + per_position = _config.get("enable_position_interventions", False) device = next(self.model_runner.model.parameters()).device # Detach prior handles before reassigning — vLLM doesn't double-load @@ -124,9 +129,12 @@ def patched_load_model(self): else: capture_buf = torch.zeros(max_n, width, device=device, dtype=dtype) # Affine identity at install. Driver swaps via tl_set_interventions - # to enable suppress/scale/add/set ops between forwards. - scale_buf = torch.ones(width, device=device, dtype=dtype) - bias_buf = torch.zeros(width, device=device, dtype=dtype) + # to enable suppress/scale/add/set ops between forwards. Shape is + # (max_n, width) when position interventions are enabled so each row + # can differ; (width,) otherwise (broadcast across all positions). + affine_shape = (max_n, width) if per_position else (width,) + scale_buf = torch.ones(affine_shape, device=device, dtype=dtype) + bias_buf = torch.zeros(affine_shape, device=device, dtype=dtype) # Default closed — opened explicitly by tl_reset_capture_flags for the # next forward(s) that need to capture. capture_flag = torch.ones(1, device=device, dtype=torch.int64) @@ -142,6 +150,7 @@ def patched_load_model(self): self._tl_fire_counter, capture_flag, materialize=materialize, + per_position=per_position, ) ) self._tl_hook_handles.append(handle) @@ -158,6 +167,7 @@ def _make_capture_hook( capture_flag: torch.Tensor, *, materialize: bool = False, + per_position: bool = False, ): """GPU-only, dynamic-shape-safe affine + first-write-wins capture into pre-allocated buffers. @@ -174,18 +184,30 @@ def _make_capture_hook( steps self-copy (no overwrite). Interventions still apply on every forward regardless of the flag — the gate only affects the capture write. + ``per_position`` (compile-time constant): when set, ``scale_buf``/``bias_buf`` are + ``(max_n, width)`` and the affine is row-scoped (``buf.narrow(0, 0, n)``) so a + ``pos``-scoped intervention edits only its rows; otherwise they are ``(width,)`` + and broadcast across every position. + ``fire_counter`` is incremented per call for the fire-once check. """ + def _affine(t: torch.Tensor, n: Any) -> torch.Tensor: + # per_position is a closure constant → torch.compile specializes this branch. + # narrow(0, 0, n) keeps the SymInt dynamic shape (same trick as _gated_capture); + # the (width,) path broadcasts across all rows. + if per_position: + return t * scale_buf.narrow(0, 0, n) + bias_buf.narrow(0, 0, n) + return t * scale_buf + bias_buf + @torch.no_grad() def hook(_module, _inputs, output): fire_counter.add_(1) if materialize and isinstance(output, tuple) and len(output) == 2: hidden, residual = output if isinstance(hidden, torch.Tensor) and isinstance(residual, torch.Tensor): - t = hidden + residual - modified = t * scale_buf + bias_buf - n = t.shape[0] + n = hidden.shape[0] + modified = _affine(hidden + residual, n) _gated_capture(capture_buf, n, modified, capture_flag) # Reconstructs ``modified`` in the next layer's fused norm: exact at # identity, bounded fp16 error under intervention. @@ -201,10 +223,8 @@ def hook(_module, _inputs, output): return None # Affine transform; default scale=1 / bias=0 means identity. Driver # swaps buffer contents to enable interventions. - modified = t * scale_buf + bias_buf - # narrow() keeps the dynamic shape; [:n] gets erased under fake-tensor - # tracing and copy_ then sees the full buffer ("expand s72 -> max_n"). n = t.shape[0] + modified = _affine(t, n) _gated_capture(capture_buf, n, modified, capture_flag) # Gate on isinstance, not truthy tuple_tail — a 1-tuple has an empty tail. if isinstance(output, tuple): diff --git a/transformer_lens/model_bridge/sources/vllm/source.py b/transformer_lens/model_bridge/sources/vllm/source.py index bb6d03d501..6bb6351103 100644 --- a/transformer_lens/model_bridge/sources/vllm/source.py +++ b/transformer_lens/model_bridge/sources/vllm/source.py @@ -39,6 +39,7 @@ def boot_vllm( max_model_len: Optional[int] = None, max_num_batched_tokens: int = 2048, enable_batching: bool = False, + enable_position_interventions: bool = False, **vllm_kwargs: Any, ) -> RemoteBridge: """Boot a model via vLLM and wrap it in a :class:`RemoteBridge` via :class:`VLLMDriver`. @@ -86,7 +87,20 @@ def boot_vllm( ``batch_size > 1``) — the throughput path for SAE/probe data collection. Default ``False`` keeps the compile-validated single-prompt path. Batched caches are right-padded with zeros to the longest sequence. + + ``enable_position_interventions`` widens each hook's affine scale/bias buffers + from ``(width,)`` to ``(max_num_batched_tokens, width)`` so an intervention spec + can carry a ``pos`` field (int or list[int]) that scopes the edit to specific + sequence positions — position-scoped activation patching / tensor injection. + Costs ~2× extra resident GPU memory across all hooks (the scale and bias buffers + join the already-``(max_n, width)`` capture buffer), so it is opt-in and defaults + ``False``. Compiled-path only — incompatible with ``enable_batching``. """ + if enable_position_interventions and enable_batching: + raise ValueError( + "enable_position_interventions requires the compiled path and is incompatible " + "with enable_batching=True (the batched/eager path has no affine buffers)." + ) _reject_locked_overrides(vllm_kwargs) from transformers import AutoConfig, AutoTokenizer @@ -104,6 +118,7 @@ def boot_vllm( max_num_batched_tokens=max_num_batched_tokens, dtype=resolved_dtype, enable_batching=enable_batching, + enable_position_interventions=enable_position_interventions, ) plugin.register() @@ -177,6 +192,7 @@ def boot_vllm( hf_config=hf_config, max_num_batched_tokens=max_num_batched_tokens, enable_batching=enable_batching, + enable_position_interventions=enable_position_interventions, ) bridge = RemoteBridge(adapter=adapter, tokenizer=tokenizer, driver=driver) _log_hook_summary(model_name, architecture, driver) diff --git a/transformer_lens/model_bridge/sources/vllm/worker_extension.py b/transformer_lens/model_bridge/sources/vllm/worker_extension.py index 5873be05c7..b8cbcca048 100644 --- a/transformer_lens/model_bridge/sources/vllm/worker_extension.py +++ b/transformer_lens/model_bridge/sources/vllm/worker_extension.py @@ -170,37 +170,72 @@ def _apply_op(t: torch.Tensor, spec: Dict[str, Any]) -> torch.Tensor: return torch.zeros_like(t) + value # set +def _write_rows(buf: torch.Tensor, idx: Optional[List[int]], value: Any) -> None: + """In-place write of ``value`` (Python scalar or ``(width,)`` tensor) into ``buf``. + + ``idx is None`` writes the whole buffer — for a 2-D ``(max_n, width)`` buffer that + broadcasts a ``(width,)`` value across every row. ``idx`` (a list of row indices) + writes only those rows of a 2-D buffer. + """ + if idx is None: + if isinstance(value, torch.Tensor): + buf[...] = value # (width,) broadcasts across rows for a 2-D buffer + else: + buf.fill_(value) + else: + buf[idx] = value # advanced-index rows; scalar or (width,) broadcasts + + def _apply_intervention( scale_buf: torch.Tensor, bias_buf: torch.Tensor, spec: Dict[str, Any] ) -> None: - """Translate a spec dict to in-place buffer writes.""" + """Translate a spec dict to in-place buffer writes. + + ``spec['pos']`` (int or list[int]) scopes the edit to those sequence rows and + requires 2-D ``(max_n, width)`` affine buffers (position interventions enabled at + boot). Absent ``pos`` writes the whole buffer, applying to every position. The + caller resets both buffers to identity first, so a ``pos`` edit leaves other rows + untouched. + """ op = spec.get("op") if op not in SUPPORTED_OPS: raise ValueError(f"Unsupported intervention op: {op!r}. Supported: {sorted(SUPPORTED_OPS)}") + pos = spec.get("pos") + idx: Optional[List[int]] = None + if pos is not None: + if scale_buf.ndim != 2: + raise ValueError( + "Per-position 'pos' requires 2-D affine buffers; boot with " + "enable_position_interventions=True." + ) + idx = [pos] if isinstance(pos, int) else list(pos) + max_n = scale_buf.shape[0] + bad = [p for p in idx if p < 0 or p >= max_n] + if bad: + raise ValueError(f"Intervention 'pos' {bad} out of range [0, {max_n}).") + if op == "suppress": - scale_buf.zero_() - bias_buf.zero_() + _write_rows(scale_buf, idx, 0.0) + _write_rows(bias_buf, idx, 0.0) return if op == "scale": - scale_buf.fill_(float(spec["factor"])) - bias_buf.zero_() + _write_rows(scale_buf, idx, float(spec["factor"])) + _write_rows(bias_buf, idx, 0.0) return value = torch.as_tensor(spec["value"], device=bias_buf.device, dtype=bias_buf.dtype) + width = bias_buf.shape[-1] # 0-d broadcasts across width (e.g. "shift all dims by 0.5"); width-shaped # writes element-wise (e.g. SAE steering vector). Anything else is an error. - if value.ndim == 0: - bias_buf.fill_(value.item()) - elif value.shape == bias_buf.shape: - bias_buf.copy_(value) - else: + if value.ndim != 0 and value.shape != (width,): raise ValueError( - f"Intervention 'value' must be a scalar or shape {tuple(bias_buf.shape)}; " + f"Intervention 'value' must be a scalar or shape {(width,)}; " f"got shape {tuple(value.shape)}" ) + _write_rows(bias_buf, idx, value.item() if value.ndim == 0 else value) if op == "add": - scale_buf.fill_(1.0) + _write_rows(scale_buf, idx, 1.0) elif op == "set": - scale_buf.zero_() + _write_rows(scale_buf, idx, 0.0) else: raise RuntimeError( f"op {op!r} is in SUPPORTED_OPS but _apply_intervention has no branch for it." From ddc3980f37cec6642aeabde4254c56d8a2ede0ed Mon Sep 17 00:00:00 2001 From: Jonah Larson Date: Wed, 15 Jul 2026 09:59:32 -0500 Subject: [PATCH 10/87] vLLM & Inspect Driver Correctness Improvements (#1515) * Initial bug fixes from sweep * cleanup duplication and comments * vllm extra --- .github/workflows/checks.yml | 6 +- demos/vLLM_Bridge_Integration_Test.ipynb | 2 +- docs/source/content/drivers.md | 120 + docs/source/index.md | 1 + pyproject.toml | 9 + scripts/inspect_parity_report.py | 11 +- scripts/vllm_parity_report.py | 54 +- tests/QUARANTINES.md | 27 + tests/mocks/mock_logits_bridge.py | 77 + .../unit/model_bridge/test_driver_protocol.py | 148 + .../model_bridge/test_hook_set_enforcement.py | 117 + .../unit/model_bridge/test_inspect_driver.py | 269 ++ .../test_inspect_vllm_provider.py | 173 + tests/unit/model_bridge/test_remote_bridge.py | 121 + tests/unit/model_bridge/test_vllm_boot.py | 64 +- tests/unit/model_bridge/test_vllm_driver.py | 236 + tests/unit/test_utils.py | 5 + transformer_lens/model_bridge/bridge_core.py | 102 +- .../model_bridge/driver_protocol.py | 42 +- .../model_bridge/remote_bridge.py | 39 +- .../model_bridge/sources/AGENTS.md | 44 +- .../model_bridge/sources/_bridge_builder.py | 20 +- .../sources/inspect/_provider_base.py | 42 + .../model_bridge/sources/inspect/driver.py | 59 +- .../model_bridge/sources/inspect/eval.py | 9 +- .../model_bridge/sources/inspect/profiles.py | 8 +- .../model_bridge/sources/inspect/source.py | 15 +- .../sources/inspect/transformers_provider.py | 49 +- .../sources/inspect/vllm_provider.py | 70 +- .../sources/transformers/source.py | 12 +- .../sources/transformers_driver.py | 19 + .../model_bridge/sources/vllm/driver.py | 284 +- .../sources/vllm/intervention_specs.py | 87 +- .../sources/vllm/overlays/decoder_only.py | 21 +- .../model_bridge/sources/vllm/source.py | 128 +- .../sources/vllm/worker_extension.py | 13 +- .../supported_architectures/mamba.py | 2 +- .../model_bridge/transformer_bridge.py | 136 +- transformer_lens/utilities/tokenize_utils.py | 7 +- uv.lock | 4044 ++++++++++++++--- 40 files changed, 5655 insertions(+), 1037 deletions(-) create mode 100644 docs/source/content/drivers.md create mode 100644 tests/mocks/mock_logits_bridge.py create mode 100644 tests/unit/model_bridge/test_hook_set_enforcement.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 393f920ba1..eec59704b7 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -102,7 +102,7 @@ jobs: - name: Install dependencies run: | uv lock --check - uv sync + uv sync --extra inspect - name: Authenticate HuggingFace if: env.HF_TOKEN != '' run: uv run python -c "import os; from huggingface_hub import login; login(token=os.environ['HF_TOKEN'])" @@ -141,7 +141,7 @@ jobs: - name: Install dependencies run: | uv lock --check - uv sync + uv sync --extra inspect - name: MPS Availability Check run: | uv run python -c " @@ -283,7 +283,7 @@ jobs: - name: Install dependencies run: | uv lock --check - uv sync + uv sync --extra inspect - name: Authenticate HuggingFace if: env.HF_TOKEN != '' run: uv run python -c "import os; from huggingface_hub import login; login(token=os.environ['HF_TOKEN'])" diff --git a/demos/vLLM_Bridge_Integration_Test.ipynb b/demos/vLLM_Bridge_Integration_Test.ipynb index ade7aa1e2c..21bcd1e883 100644 --- a/demos/vLLM_Bridge_Integration_Test.ipynb +++ b/demos/vLLM_Bridge_Integration_Test.ipynb @@ -42,7 +42,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# Install vllm and TransformerLens @ feature/vllm-batched. ~3-5 minutes.\n# Branch must match this notebook: feature/vllm-batched has the hook-fire counter\n# (Step 7) and the batched capture surface (Step 10). feature/driver-system lacks\n# both, so collective_rpc raises NotImplementedError on tl_reset_counter / the\n# batched RPCs.\n#\n# vLLM pinned to 0.20.2 — the version the internal-API walks in\n# transformer_lens/model_bridge/sources/vllm/{plugin,internals}.py were validated\n# against. Installed from the CUDA-12.9 *release wheel*, NOT PyPI's default\n# `vllm==0.20.2`: the PyPI wheel is a CUDA-13 build that fails on Colab's CUDA-12.8\n# image with `libcudart.so.13: cannot open shared object file` and drags in\n# cuda-toolkit 13, which conflicts with Colab's cu12 RAPIDS stack. The cu129 wheel\n# uses libcudart.so.12 (present) and stays in the cu12 family. vLLM has no cu128\n# wheel for 0.20.2; cu129 is CUDA-12.x-compatible with the cu128 runtime.\n# Re-validate the internals walks before bumping vLLM (it moves them every 4-6 weeks).\n%pip install -q \"https://github.com/vllm-project/vllm/releases/download/v0.20.2/vllm-0.20.2%2Bcu129-cp38-abi3-manylinux_2_31_x86_64.whl\"\n%pip install -q git+https://github.com/TransformerLensOrg/TransformerLens.git@feature/vllm-batched", + "source": "# Install vllm and TransformerLens @ dev-4.x. ~3-5 minutes.\n# Branch must match this notebook: dev-4.x has the hook-fire counter (Step 7)\n# and the batched capture surface (Step 10), which collective_rpc reaches via\n# tl_reset_counter / the batched RPCs.\n#\n# vLLM pinned to 0.20.2 — the version the internal-API walks in\n# transformer_lens/model_bridge/sources/vllm/{plugin,internals}.py were validated\n# against. Installed from the CUDA-12.9 *release wheel*, NOT PyPI's default\n# `vllm==0.20.2`: the PyPI wheel is a CUDA-13 build that fails on Colab's CUDA-12.8\n# image with `libcudart.so.13: cannot open shared object file` and drags in\n# cuda-toolkit 13, which conflicts with Colab's cu12 RAPIDS stack. The cu129 wheel\n# uses libcudart.so.12 (present) and stays in the cu12 family. vLLM has no cu128\n# wheel for 0.20.2; cu129 is CUDA-12.x-compatible with the cu128 runtime.\n# Re-validate the internals walks before bumping vLLM (it moves them every 4-6 weeks).\n%pip install -q \"https://github.com/vllm-project/vllm/releases/download/v0.20.2/vllm-0.20.2%2Bcu129-cp38-abi3-manylinux_2_31_x86_64.whl\"\n%pip install -q git+https://github.com/TransformerLensOrg/TransformerLens.git@dev-4.x", "id": "4ab1eb60e6b1" }, { diff --git a/docs/source/content/drivers.md b/docs/source/content/drivers.md new file mode 100644 index 0000000000..c3b66ac2f4 --- /dev/null +++ b/docs/source/content/drivers.md @@ -0,0 +1,120 @@ +# Execution Backends (the Driver System) + +TransformerLens v4 separates *what you study* (the bridge's hook names, cache, and intervention surface) from *what runs the forward pass* (a **Driver**). Every backend — local HuggingFace `transformers`, vLLM, or an `inspect_ai` provider — satisfies the same protocol, so the same hook names work everywhere; what changes is which hooks each backend can fire and whether gradients exist at all. + +This page covers the user-facing surface. The contract lives in [`transformer_lens/model_bridge/driver_protocol.py`](https://github.com/TransformerLensOrg/TransformerLens/blob/dev-4.x/transformer_lens/model_bridge/driver_protocol.py). + +--- + +## The Driver protocol + +A driver is anything that implements: + +```python +def forward(input_ids, *, capture=(), intervene=None, + max_new_tokens=1, return_logits=True, **kwargs) -> ForwardResult +def close() -> None +def supports(feature: str) -> bool +``` + +plus two declared hook-name sets: + +- `supported_hook_points` — canonical bridge hook names this backend can fire (e.g. `blocks.0.hook_out`). +- `non_fireable_hook_points` — names the backend structurally cannot serve (fused kernels, sampler shortcuts). + +`ForwardResult` carries `logits`, a `captured` mapping of hook name → activation, and the engine's `raw_output`. Tensors are native to the driver's framework and converted at the bridge boundary. `validate_driver` checks the contract when a bridge is constructed. + +Interventions come in two dialects: + +- **Callables** (`InterventionFn`) — arbitrary Python hook functions, for drivers that can dispatch Python at the engine boundary (the transformers backend). +- **Declarative specs** (`InterventionSpec`) — plain mappings like `{"op": "suppress"}`, for drivers that can't run Python mid-forward (vLLM under `torch.compile`, remote providers). + +Capability tiers follow from this: **circuit-finding and anything gradient-based runs on the transformers backend; capture and steering scale out on vLLM; both use the same hook names**, so analyses transfer between them. + +--- + +## The three backends + +### transformers — full hooks + gradients + +The reference backend. Wraps a local HF `nn.Module`; the full HookPoint tree fires, backward hooks and gradients work, and `parameters()` / `state_dict()` / weight access are all available. + +```python +from transformer_lens.model_bridge import TransformerBridge + +bridge = TransformerBridge.boot_transformers("gpt2", device="cpu") +logits, cache = bridge.run_with_cache("Hello, world") +``` + +There is also `TransformerBridge.boot_native(config)`, which builds a small randomly-initialized TL-native model on the same driver — no HuggingFace Hub call — useful for tests and toy-model experiments. + +### vLLM — high-throughput capture + declarative interventions + +`RemoteBridge.boot_vllm` constructs a vLLM engine (PagedAttention, `torch.compile`, CUDA graphs) and installs capture hooks inside the worker *before* compilation. Activations come back over `collective_rpc` and replay through the bridge's HookPoint tree. This is the throughput path for SAE/probe data collection; unlike observation-only tools (vllm-lens), each hook also applies an affine transform `output = output * scale + bias`, so declarative interventions (`suppress` / `scale` / `add` / `set`) propagate to downstream layers. + +```python +import torch +from transformer_lens.model_bridge import RemoteBridge + +bridge = RemoteBridge.boot_vllm( + "meta-llama/Llama-3.2-1B", + dtype=torch.float16, + max_model_len=2048, # cap the KV-cache reservation +) +logits, cache = bridge.run_with_cache("Hello, world") + +# Declarative intervention: zero the embedding output for this forward only. +logits2, cache2 = bridge.run_with_cache( + "Hello, world", + intervene={"embed.hook_out": {"op": "suppress"}}, +) +``` + +Notes grounded in the source docstrings ([`sources/vllm/source.py`](https://github.com/TransformerLensOrg/TransformerLens/blob/dev-4.x/transformer_lens/model_bridge/sources/vllm/source.py)): + +- **Fireable hooks** (decoder-only overlay): `embed.hook_out`, `blocks.{i}.hook_out` / `attn.hook_out` / `mlp.hook_out`, `ln_final.hook_normalized`. +- **Returned logits are reconstructed full-sequence logits**: vLLM's sampler bypasses `lm_head`, so the driver rebuilds them host-side as `ln_final @ lm_head.weight.T` (+ bias, + Gemma soft-cap) — valid at every position, so loss works. If the unembedding weight is unreachable it falls back to final-position log-probs and the bridge rejects `return_type="loss"`. +- **Convention alignment**: vLLM materializes `ln_final` *post-weight*, but the driver un-folds the exposed capture (÷ weight, or ÷ (1 + weight) for Gemma) so `ln_final.hook_normalized` matches the pre-weight value `boot_transformers` serves. If the norm weight is unreachable it warns and serves the raw post-weight value. See [`sources/vllm/overlays/decoder_only.py`](https://github.com/TransformerLensOrg/TransformerLens/blob/dev-4.x/transformer_lens/model_bridge/sources/vllm/overlays/decoder_only.py) for which hooks diverge from HF conventions. +- `enable_batching=True` switches to the eager batched path (`batch_size > 1`, chunked prefill) for data collection; `enable_position_interventions=True` lets a spec carry a `pos` field (int or list) to scope an edit to specific sequence positions. +- Requires a CUDA GPU. Install with `pip install "transformer-lens[vllm]"` (or `uv sync --extra vllm`). The extra is Linux-only (vLLM ships no macOS/Windows wheels), pins the validated `vllm 0.20.x` band — which in turn pins its matching `torch` — and cannot co-install with the `[lit]` extra (numpy version conflict). See [`sources/vllm/internals.py`](https://github.com/TransformerLensOrg/TransformerLens/blob/dev-4.x/transformer_lens/model_bridge/sources/vllm/internals.py) before bumping the band. + +### Inspect — interp inside `inspect_ai` evals + +`RemoteBridge.boot_inspect` wraps an `inspect_ai` model provider in a bridge, so activation capture and interventions run inside the same harness as behavioral evals. Install with the `inspect` extra (`uv sync --extra inspect`, or `pip install "transformer-lens[inspect]"`). + +```python +from transformer_lens.model_bridge import RemoteBridge + +bridge = RemoteBridge.boot_inspect("HuggingFaceTB/SmolLM2-135M") # provider="tl_bridge" +logits, cache = bridge.run_with_cache("Hello, world") +``` + +From the [`boot_inspect` docstring](https://github.com/TransformerLensOrg/TransformerLens/blob/dev-4.x/transformer_lens/model_bridge/sources/inspect/source.py): + +- The default `tl_bridge` provider is HF-backed: residual/attn/mlp capture, full affine interventions, and full-sequence logits. `tl_bridge_vllm` is the vLLM-backed sibling; `provider="vllm-lens"` targets a running vllm-lens provider (residual-only, additive-steering-only). +- **Fireable hooks** (`tl_bridge`): `blocks.{i}.hook_in` (resid_pre), `ln2.hook_in` (resid_mid), `hook_out` (resid_post), `attn.hook_out`, `mlp.hook_out`, plus head-split `attn.hook_q/k/v` / `attn.hook_z` / `attn.hook_pattern` where a per-model structural self-check finds them. `embed`, `ln_final`, and `attn.hook_attn_scores` are always non-fireable — use `boot_transformers()` for those. +- For parity with `boot_transformers`, the provider loads with the same dtype (fp32 by default) and eager attention. +- For capture *during an eval*, add the `capture_activations([...])` solver from [`sources/inspect/eval.py`](https://github.com/TransformerLensOrg/TransformerLens/blob/dev-4.x/transformer_lens/model_bridge/sources/inspect/eval.py) to a Task's solver chain: full activations go to per-sample `.npz` artifacts, and a compact summary lands in the sample store for `samples_df` analysis. + +--- + +## Fundamental limits + +Serving engines are not autograd engines. On the vLLM backend (and any future serving backend): + +- **No gradients.** PagedAttention and the compiled graph have no autograd surface; backward hooks, attribution patching, and anything gradient-based need `boot_transformers`. +- **No attention patterns or scores.** The QKᵀ → softmax path is fused into the attention kernel (`attn.hook_pattern` / `attn.hook_attn_scores` are declared non-fireable, along with pre/post-RoPE Q/K). +- **Interventions are declarative only.** Arbitrary Python hook functions cannot run inside the compiled worker; the spec vocabulary (`suppress` / `scale` / `add` / `set`, optional `pos`) is the intervention surface. + +The Inspect `tl_bridge` provider is HF-backed, so its captures are numerically faithful to `boot_transformers` — but the driver surface is capture/intervene over a wire format, not a local module: no gradients or weight mutation through the bridge. + +--- + +## Verifying parity + +Two scripts diff each remote backend against `boot_transformers` on real models, comparing every hook the driver claims to serve: + +- `uv run python scripts/vllm_parity_report.py` — GPU-only; boots both backends in fp32 and diffs all fireable capture points (with a refold diagnostic that distinguishes an `ln_final` un-fold regression from a mapping error), plus an argmax agreement check. +- `uv run python scripts/inspect_parity_report.py` — validates the provider's structural self-check: every boundary it offers must match `boot_transformers`. + +`demos/vLLM_Bridge_Integration_Test.ipynb` is the end-to-end GPU validation of the vLLM capture *and mutation* path (a manual Colab run, not CI). diff --git a/docs/source/index.md b/docs/source/index.md index b10bd3a9dd..7c5253e336 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -55,6 +55,7 @@ content/tutorials content/citation content/contributing content/hook_system +content/drivers content/compatibility_mode content/debugging_numerical_divergence generated/demos/Main_Demo diff --git a/pyproject.toml b/pyproject.toml index 66a141d886..771176d1ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,12 @@ evals=["lm-eval>=0.4", "chardet<6"] inspect=["inspect_ai>=0.3"] lit=["lit-nlp>=1.3"] + # The vLLM driver's validated version band (see sources/vllm/internals.py before + # bumping). Linux-only marker: vLLM ships no macOS/Windows wheels and the driver + # is GPU-only anyway — elsewhere `[vllm]` is a clean no-op instead of a failed + # source build. Declared conflicting with `lit` in [tool.uv] (vllm's transitive + # numpy>=2 vs lit-nlp's numpy<2 cap). + vllm=["vllm>=0.20.2,<0.21; sys_platform == 'linux'"] [project.scripts] build-docs="docs.make_docs:build_docs" @@ -100,6 +106,9 @@ [tool.uv] default-groups=["dev", "docs", "jupyter", "multimodal"] + # vllm and lit cannot co-install (numpy>=2 vs numpy<2); forking the resolution + # lets each extra lock cleanly on its own. + conflicts=[[{extra="vllm"}, {extra="lit"}]] [tool.pytest] diff --git a/scripts/inspect_parity_report.py b/scripts/inspect_parity_report.py index 9a6083ca8f..89cd868b03 100644 --- a/scripts/inspect_parity_report.py +++ b/scripts/inspect_parity_report.py @@ -5,8 +5,9 @@ are correctly withheld and reported, not compared. Validates that self-check against real models: everything offered must match boot_transformers; it is not a per-PR CI job. -Run: uv run python scripts/inspect_parity_report.py -Override the model list with TL_PARITY_MODELS="id1,id2,...". +Run: uv run python scripts/inspect_parity_report.py +Env: TL_PARITY_MODELS="id1,id2,..." overrides the model list. + TL_INSPECT_ATOL / TL_INSPECT_RTOL override tolerance (defaults 1e-3). """ from __future__ import annotations @@ -57,7 +58,8 @@ ] PROMPT = "The quick brown fox" -ATOL, RTOL = 1e-3, 1e-3 +ATOL = float(os.environ.get("TL_INSPECT_ATOL", "1e-3")) +RTOL = float(os.environ.get("TL_INSPECT_RTOL", "1e-3")) # Kind -> TransformerBridge-native hook suffix: the five d_model boundaries plus the @@ -163,7 +165,8 @@ def main() -> None: passed = sorted({r["arch"] for r in rows if r["status"] == "PASS"}) print("\nPARITY-VERIFIED ARCHITECTURES (measured PASS):") print(" " + ", ".join(passed)) - sys.exit(0) + # Non-zero exit if anything actually ran and failed — lets a scripted run gate. + sys.exit(1 if any(r["status"] == "FAIL" for r in rows) else 0) if __name__ == "__main__": diff --git a/scripts/vllm_parity_report.py b/scripts/vllm_parity_report.py index bad526806d..1abf934774 100644 --- a/scripts/vllm_parity_report.py +++ b/scripts/vllm_parity_report.py @@ -7,10 +7,10 @@ - ``embed.hook_out`` and per-layer ``blocks.{i}.hook_out`` / ``attn.hook_out`` / ``mlp.hook_out`` are semantically identical to boot_transformers and compared directly. -- ``ln_final.hook_normalized`` carries vLLM's POST-weight RMSNorm value under a hook - whose HT convention is PRE-weight; it is un-folded (÷ weight, or ÷ (1+weight) for - Gemma) before comparison. The raw (un-un-folded) diff is also reported so a wrong - un-fold direction is visible rather than silent. +- ``ln_final.hook_normalized`` is un-folded to the pre-weight convention by the driver + itself (÷ weight, or ÷ (1+weight) for Gemma), so it is compared directly too; a + driver-side un-fold regression shows up as an O(1) mismatch on this hook. The + re-folded diff is also reported so a wrong un-fold direction is distinguishable. Non-fireable points (fused attention pattern/scores/rope, unembed) are withheld and reported, not compared. A final-position argmax agreement check sanity-checks logits. @@ -75,14 +75,13 @@ def _diff(a: torch.Tensor, b: torch.Tensor) -> tuple[float, bool]: return d, torch.allclose(a2, b2, atol=ATOL, rtol=RTOL) -def _unfold_lnf(vllm_val: torch.Tensor, weight: torch.Tensor, is_gemma: bool) -> torch.Tensor: - """vLLM ln_final is post-weight (x·rsqrt(var+eps)·w); recover the pre-weight value HT - exposes by dividing out the norm weight (Gemma folds 1 + weight).""" +def _refold_lnf(vllm_val: torch.Tensor, weight: torch.Tensor, is_gemma: bool) -> torch.Tensor: + """Re-apply the norm weight to the driver's (pre-weight) ln_final capture — used only + as a diagnostic so a wrong un-fold direction in the driver reads as 'refolded matches' + instead of an opaque mismatch.""" w = weight.detach().to(vllm_val.device, torch.float32) - denom = (1.0 + w) if is_gemma else w - # Guard against near-zero weight entries producing spurious blow-ups. - denom = torch.where(denom.abs() < 1e-6, torch.ones_like(denom), denom) - return vllm_val.to(torch.float32) / denom + factor = (1.0 + w) if is_gemma else w + return vllm_val.to(torch.float32) * factor def verify(model_id: str) -> dict: @@ -128,24 +127,25 @@ def verify(model_id: str) -> dict: if not ok: mism.append(f"{hk} maxdiff={d:.2e}") - # ln_final: compare un-folded; also report the raw diff so a wrong un-fold shows. + # ln_final: the driver already un-folds to the pre-weight convention — compare + # directly. The refolded diff is diagnostic: "direct FAIL + refolded ok" means + # the driver's un-fold regressed (serving raw post-weight values again). if LNF_HOOK in offered and LNF_HOOK in v_cache and LNF_HOOK in hf_cache: - raw_d, raw_ok = _diff(hf_cache[LNF_HOOK], v_cache[LNF_HOOK]) - weight = vllm._driver.get_param("model.norm.weight") - if weight is None: - notes.append(f"ln_final raw={raw_d:.2e} (no norm weight to un-fold)") - if not raw_ok: - mism.append(f"{LNF_HOOK} maxdiff={raw_d:.2e} (un-fold unavailable)") + d, ok = _diff(hf_cache[LNF_HOOK], v_cache[LNF_HOOK]) + worst = max(worst, d if d != float("inf") else worst) + if ok: + notes.append(f"ln_final direct={d:.2e}") else: - unfolded = _unfold_lnf(v_cache[LNF_HOOK], weight, is_gemma) - uf_d, uf_ok = _diff(hf_cache[LNF_HOOK], unfolded) - worst = max(worst, uf_d if uf_d != float("inf") else worst) - notes.append(f"ln_final unfold={uf_d:.2e} raw={raw_d:.2e}") - if not uf_ok: - mism.append(f"{LNF_HOOK} unfold maxdiff={uf_d:.2e} (raw={raw_d:.2e})") - - # Logit sanity: vLLM synthesizes final-position log-probs only; check top-1 agrees - # (reusing the run_with_cache logits above — no second forward). + weight = vllm._driver.get_param("model.norm.weight") + detail = f"{LNF_HOOK} maxdiff={d:.2e}" + if weight is not None: + refolded = _refold_lnf(v_cache[LNF_HOOK], weight, is_gemma) + rf_d, rf_ok = _diff(hf_cache[LNF_HOOK], refolded) + detail += f" (refolded={rf_d:.2e}{', un-fold regressed' if rf_ok else ''})" + mism.append(detail) + + # Logit sanity: the driver reconstructs full-sequence logits host-side; check the + # final-position top-1 agrees (reusing the run_with_cache logits — no second forward). try: hf_top = int(torch.as_tensor(hf_logits)[0, -1].argmax()) v_top = int(torch.as_tensor(v_logits)[0, -1].argmax()) diff --git a/tests/QUARANTINES.md b/tests/QUARANTINES.md index 5a587ea38e..47f03ca50b 100644 --- a/tests/QUARANTINES.md +++ b/tests/QUARANTINES.md @@ -19,6 +19,33 @@ Rule ([AGENTS.md §10](../AGENTS.md#10-hard-rules)): **never add `xfail` / `skip --- +## Optional dependency — gated but installed in CI + +`inspect_ai` lives in the `[inspect]` extra. The CI test jobs (`compatibility-checks`, `coverage-test`, `mps-checks`) install it via `uv sync --extra inspect` in [`checks.yml`](../.github/workflows/checks.yml), so these run on every CI push — before that change they silently skipped on every job. + +| Path | Marker | Trigger | +|---|---|---| +| [`unit/model_bridge/test_inspect_driver.py`:19](unit/model_bridge/test_inspect_driver.py) (whole file) | `importorskip("inspect_ai")` | `uv sync --extra inspect` | +| [`unit/model_bridge/test_inspect_vllm_provider.py`:19](unit/model_bridge/test_inspect_vllm_provider.py) (whole file) | `importorskip("inspect_ai")` | same | +| [`acceptance/model_bridge/test_inspect_provider.py`:19](acceptance/model_bridge/test_inspect_provider.py) (whole file) | `pytestmark skipif(inspect_ai missing)` | same | +| [`../transformer_lens/model_bridge/sources/inspect/conftest.py`](../transformer_lens/model_bridge/sources/inspect/conftest.py) | `collect_ignore_glob` (doctest-modules of the provider files) | same | + +**Un-skip:** already un-skipped in CI. A local plain `uv sync` still skips them; install the extra. + +--- + +## ⚠️ Coverage gap — no automated lane (real vLLM) + +| Path | Marker | Trigger | +|---|---|---| +| [`unit/model_bridge/test_vllm_driver.py`](unit/model_bridge/test_vllm_driver.py) (×15) | `importorskip("vllm")` per-test | `uv sync --extra vllm` on a Linux CUDA machine (validated band: `vllm 0.20.x`) | + +A `[vllm]` extra exists (Linux-only marker; declared conflicting with `[lit]` in `[tool.uv]` — vllm needs `numpy>=2` via `opencv-python-headless` while `lit-nlp` caps `numpy<2`), but CI does not install it: vllm is GPU-only and its 15 real-engine tests would not pass on CPU runners (the file's other tests mock the LLM and run everywhere). Note the extra's `vllm 0.20.x` band exact-pins `torch==2.11.0`, which is what the project lockfile resolves to. The real-engine execution path is otherwise covered only by the manual GPU run of [`demos/vLLM_Bridge_Integration_Test.ipynb`](../demos/vLLM_Bridge_Integration_Test.ipynb). + +**Un-skip:** a GPU CI lane that installs vllm, or locally on a CUDA machine with `vllm==0.20.2` installed alongside the project env. + +--- + ## Permanent — hardware requirement | Path | Marker | Required | diff --git a/tests/mocks/mock_logits_bridge.py b/tests/mocks/mock_logits_bridge.py new file mode 100644 index 0000000000..2ff8e9829e --- /dev/null +++ b/tests/mocks/mock_logits_bridge.py @@ -0,0 +1,77 @@ +"""Minimal one-block TransformerBridge over a stub torch model + logits-returning driver.""" +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import torch +from torch import nn + +from tests.mocks.architecture_adapter import MockArchitectureAdapter +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge.bridge import TransformerBridge +from transformer_lens.model_bridge.driver_protocol import ForwardResult +from transformer_lens.model_bridge.generalized_components import ( + AttentionBridge, + BlockBridge, + NormalizationBridge, +) +from transformer_lens.model_bridge.sources._driver_base import DriverBase + + +def mock_bridge_cfg() -> TransformerBridgeConfig: + return TransformerBridgeConfig( + d_model=4, + d_head=2, + n_layers=1, + n_ctx=8, + n_heads=2, + d_vocab=16, + d_mlp=8, + architecture="Mock", + ) + + +class LogitsDriver(DriverBase): + """Returns real logits so post-hook forward paths complete.""" + + def __init__(self, model, cfg, tokenizer=None): + super().__init__(cfg, tokenizer) + # forward()'s encoder-decoder probe reads bridge.original_model. + self.underlying_model = model + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult(logits=torch.randn(1, 3, 16)) + + +def build_mock_logits_bridge(driver_cls=LogitsDriver) -> TransformerBridge: + """Encoder-shaped stub bridge (ln_final + one block with ln1/attn).""" + model = nn.Module() + model.final_norm = nn.LayerNorm(10) + model.encoder = nn.Module() + model.encoder.layers = nn.ModuleList([nn.Module() for _ in range(1)]) + model.encoder.layers[0].norm1 = nn.LayerNorm(10) + model.encoder.layers[0].self_attn = nn.Module() + + adapter = MockArchitectureAdapter() + adapter.component_mapping = { + "ln_final": NormalizationBridge(name="final_norm", config={}), + "blocks": BlockBridge( + name="encoder.layers", + submodules={ + "ln1": NormalizationBridge(name="norm1", config={}), + "attn": AttentionBridge(name="self_attn", config=SimpleNamespace(n_heads=1)), + }, + ), + } + driver = driver_cls(model, mock_bridge_cfg(), tokenizer=None) + return TransformerBridge(model, adapter, tokenizer=MagicMock(), driver=driver) diff --git a/tests/unit/model_bridge/test_driver_protocol.py b/tests/unit/model_bridge/test_driver_protocol.py index e0b652b989..2f796f2f06 100644 --- a/tests/unit/model_bridge/test_driver_protocol.py +++ b/tests/unit/model_bridge/test_driver_protocol.py @@ -309,6 +309,90 @@ def forward( with pytest.raises(TypeError, match="empty supported_hook_points AND"): validate_driver(driver, after_bridge_construction=True) + def test_unknown_feature_strings_rejected(self): + """A driver declaring a feature string outside KNOWN_FEATURES is a contract bug.""" + from transformer_lens.model_bridge.sources._driver_base import DriverBase + + class MadeUpFeatures(DriverBase): + _supported_features = frozenset({"generate_streaming"}) + + def __init__(self, cfg): + super().__init__(cfg, tokenizer=None) + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult() + + driver = MadeUpFeatures(self._cfg()) + with pytest.raises(TypeError, match="unknown feature strings"): + validate_driver(driver) + + def test_non_bool_provides_sequence_logits_rejected(self): + """getattr-with-True-default would silently pick the unsafe value.""" + from transformer_lens.model_bridge.sources._driver_base import DriverBase + + class StringyFlag(DriverBase): + provides_sequence_logits = "yes" # type: ignore[assignment] + + def __init__(self, cfg): + super().__init__(cfg, tokenizer=None) + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult() + + driver = StringyFlag(self._cfg()) + with pytest.raises(TypeError, match="provides_sequence_logits must be bool"): + validate_driver(driver) + + def test_missing_provides_sequence_logits_rejected(self): + """Duck-typed drivers (no DriverBase) can't silently default to unsafe True.""" + cfg = self._cfg() + + class DuckDriver: + architecture = "Duck" + bridge_config = cfg + tokenizer = None + supported_hook_points = frozenset({"x"}) + non_fireable_hook_points = frozenset() + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult() + + def close(self): + pass + + def supports(self, feature): + return False + + with pytest.raises(TypeError, match="provides_sequence_logits"): + validate_driver(DuckDriver()) + def test_bridge_accepts_kw_only_input_ids_driver(self): """Bridge passes input_ids by keyword so kw-only drivers don't TypeError.""" from types import SimpleNamespace @@ -458,6 +542,70 @@ def test_hf_driver_passes_post_construction_check(self): validate_driver(bridge._driver, after_bridge_construction=True) +class TestTransformersDriverDialect: + """HF backend serves hooks via HookPoints, not driver.forward args — + spec-dialect args must fail loud, not silently no-op.""" + + def _driver(self): + from transformer_lens.model_bridge.sources.transformers_driver import ( + TransformersDriver, + ) + + return TransformersDriver(nn.Linear(4, 4), _stub_adapter(), tokenizer=None) + + def test_capture_rejected(self): + with pytest.raises(NotImplementedError, match="capture"): + self._driver().forward(torch.randn(1, 4), capture=("blocks.0.hook_out",)) + + def test_intervene_rejected(self): + with pytest.raises(NotImplementedError, match="intervene"): + self._driver().forward( + torch.randn(1, 4), intervene={"blocks.0.hook_out": {"op": "suppress"}} + ) + + def test_max_new_tokens_rejected(self): + with pytest.raises(NotImplementedError, match="generate"): + self._driver().forward(torch.randn(1, 4), max_new_tokens=4) + + def test_plain_forward_still_works(self): + result = self._driver().forward(torch.randn(2, 4)) + assert isinstance(result.logits, torch.Tensor) + + +class TestLossGateOnTorchBridge: + """Final-position-only drivers must refuse loss on TransformerBridge too, + not just RemoteBridge — silent NaN loss otherwise.""" + + def _build_bridge(self, provides_sequence_logits: bool): + from tests.mocks.mock_logits_bridge import ( + LogitsDriver, + build_mock_logits_bridge, + ) + + psl = provides_sequence_logits + + class FinalPositionDriver(LogitsDriver): + provides_sequence_logits = psl + + return build_mock_logits_bridge(driver_cls=FinalPositionDriver) + + def test_loss_refused_when_sequence_logits_unavailable(self): + bridge = self._build_bridge(provides_sequence_logits=False) + tokens = torch.tensor([[1, 2, 3]]) + with pytest.raises(NotImplementedError, match="final position only"): + bridge.forward(tokens, return_type="loss") + with pytest.raises(NotImplementedError, match="final position only"): + bridge.forward(tokens, return_type="both") + # Logits path is unaffected. + logits = bridge.forward(tokens, return_type="logits") + assert isinstance(logits, torch.Tensor) + + def test_loss_allowed_when_sequence_logits_available(self): + bridge = self._build_bridge(provides_sequence_logits=True) + loss = bridge.forward(torch.tensor([[1, 2, 3]]), return_type="loss") + assert isinstance(loss, torch.Tensor) and loss.dim() == 0 + + class TestBridgeConsumesCaptures: """Bridge replays ForwardResult.captured through its HookPoint tree.""" diff --git a/tests/unit/model_bridge/test_hook_set_enforcement.py b/tests/unit/model_bridge/test_hook_set_enforcement.py new file mode 100644 index 0000000000..e4d52794b3 --- /dev/null +++ b/tests/unit/model_bridge/test_hook_set_enforcement.py @@ -0,0 +1,117 @@ +"""Hook-set contract enforcement on the torch bridge: typo'd or backend-unservable +hook names must raise instead of silently running unhooked / caching nothing.""" +from __future__ import annotations + +import pytest +import torch + +from tests.mocks.mock_logits_bridge import build_mock_logits_bridge as _build_bridge + + +class TestUnknownHookNamesFailLoud: + def test_run_with_hooks_typo_raises_keyerror(self): + bridge = _build_bridge() + with pytest.raises(KeyError, match="does not exist"): + bridge.run_with_hooks( + torch.tensor([[1, 2, 3]]), + fwd_hooks=[("blocks.0.hook_definitely_typo", lambda act, hook: None)], + ) + + def test_hooks_context_typo_raises_keyerror(self): + bridge = _build_bridge() + with pytest.raises(KeyError, match="does not exist"): + with bridge.hooks(fwd_hooks=[("blocks.0.hook_definitely_typo", lambda a, hook: None)]): + pass + + def test_run_with_hooks_valid_name_still_attaches(self): + bridge = _build_bridge() + target = next(iter(bridge._hook_registry)) + seen: list = [] + bridge.run_with_hooks( + torch.tensor([[1, 2, 3]]), + fwd_hooks=[(target, lambda act, hook: seen.append(hook.name))], + ) + # The mock model never routes tensors through HookPoints, so the hook + # can't fire — the assertion is that attach succeeded without raising + # and was cleaned up. + assert len(bridge._hook_registry[target].fwd_hooks) == 0 + + +class TestRunWithCacheFilterEnforcement: + def test_string_filter_matching_nothing_raises(self): + bridge = _build_bridge() + with pytest.raises(KeyError, match="matched no hook points"): + bridge.run_with_cache(torch.tensor([[1, 2, 3]]), names_filter="no.such.hook") + + def test_list_filter_matching_nothing_raises(self): + bridge = _build_bridge() + with pytest.raises(KeyError, match="matched no hook points"): + bridge.run_with_cache( + torch.tensor([[1, 2, 3]]), names_filter=["no.such.hook", "also.not.real"] + ) + + def test_callable_filter_matching_nothing_is_allowed(self): + """Programmatic filters legitimately produce empty intersections.""" + bridge = _build_bridge() + _, cache = bridge.run_with_cache(torch.tensor([[1, 2, 3]]), names_filter=lambda name: False) + assert len(cache) == 0 + + def test_no_filter_caches_everything_without_raising(self): + bridge = _build_bridge() + out, cache = bridge.run_with_cache(torch.tensor([[1, 2, 3]])) + assert out is not None + + def test_partially_matching_list_filter_is_allowed(self): + """One valid name in a list keeps the run alive (matches HT leniency).""" + bridge = _build_bridge() + target = next(iter(bridge._hook_registry)) + _, cache = bridge.run_with_cache( + torch.tensor([[1, 2, 3]]), names_filter=[target, "no.such.hook"] + ) + assert cache is not None + + +class TestNonFireableEnforcement: + def test_add_hook_on_non_fireable_raises(self): + bridge = _build_bridge() + sacrificed = next(iter(bridge._hook_registry)) + bridge._driver.non_fireable_hook_points = frozenset({sacrificed}) + with pytest.raises(NotImplementedError, match="cannot fire"): + bridge.add_hook(sacrificed, lambda act, hook: None) + + def test_run_with_hooks_on_non_fireable_raises(self): + bridge = _build_bridge() + sacrificed = next(iter(bridge._hook_registry)) + bridge._driver.non_fireable_hook_points = frozenset({sacrificed}) + with pytest.raises(NotImplementedError, match="boot_transformers"): + bridge.run_with_hooks( + torch.tensor([[1, 2, 3]]), + fwd_hooks=[(sacrificed, lambda act, hook: None)], + ) + + def test_run_with_cache_on_non_fireable_raises(self): + bridge = _build_bridge() + sacrificed = next(iter(bridge._hook_registry)) + bridge._driver.non_fireable_hook_points = frozenset({sacrificed}) + with pytest.raises(NotImplementedError, match="cannot fire"): + bridge.run_with_cache(torch.tensor([[1, 2, 3]]), names_filter=sacrificed) + + +class TestAddHookAliasResolution: + def test_add_hook_accepts_registry_alias(self): + """Aliases visible in hook_dict must attach via add_hook, same as run_with_hooks.""" + bridge = _build_bridge() + # Find an alias: a hook_dict key whose HookPoint carries a different canonical name. + alias = next( + ( + name + for name, hp in bridge.hook_dict.items() + if hp.name is not None and hp.name != name + ), + None, + ) + if alias is None: + pytest.skip("mock bridge registered no aliases") + canonical = bridge.hook_dict[alias].name + bridge.add_hook(alias, lambda act, hook: None) + assert len(bridge._hook_registry[canonical].fwd_hooks) == 1 diff --git a/tests/unit/model_bridge/test_inspect_driver.py b/tests/unit/model_bridge/test_inspect_driver.py index 1ee943f37a..47909c6888 100644 --- a/tests/unit/model_bridge/test_inspect_driver.py +++ b/tests/unit/model_bridge/test_inspect_driver.py @@ -541,6 +541,275 @@ def test_turn_activations_batch_dim_is_rank_aware(self): assert turn["blocks.0.attn.hook_pattern"].shape == (1, 2, 3, 3) +class TestHeterogeneousLayers: + """Hybrid attn/SSM stacks: capability detection probes layer 0 only, so a layer + missing the targeted submodule must fail loud instead of installing nothing.""" + + def _provider(self): + import torch.nn as nn + + from transformer_lens.model_bridge.sources.inspect.transformers_provider import ( + TransformerLensTransformersModelAPI, + ) + + class AttnBlock(nn.Module): + def __init__(self): + super().__init__() + self.self_attn = nn.Linear(4, 4) + self.mlp = nn.Linear(4, 4) + + class SSMBlock(nn.Module): # no attention / MLP submodule + def __init__(self): + super().__init__() + self.mixer = nn.Linear(4, 4) + + api = object.__new__(TransformerLensTransformersModelAPI) + api._layers = nn.ModuleList([AttnBlock(), SSMBlock()]) + api._geometry = (2, 2, 2) + return api + + def test_capture_on_layer_missing_attn_raises(self): + with pytest.raises(RuntimeError, match="no attention submodule"): + self._provider()._install_hooks({1: {"attn_out"}}, {}, {}, "cid") + + def test_intervention_on_layer_missing_attn_raises(self): + with pytest.raises(RuntimeError, match="no attention submodule"): + self._provider()._install_hooks({}, {1: {"attn_out": {"op": "suppress"}}}, {}, "cid") + + def test_capture_on_layer_missing_mlp_raises(self): + with pytest.raises(RuntimeError, match="no MLP submodule"): + self._provider()._install_hooks({1: {"mlp_out"}}, {}, {}, "cid") + + def test_head_kind_on_layer_missing_attn_raises(self): + with pytest.raises(RuntimeError, match="no attention submodule"): + self._provider()._install_hooks({1: {"q"}}, {}, {}, "cid") + + def test_layer_with_modules_still_installs(self): + handles = self._provider()._install_hooks({0: {"attn_out", "mlp_out"}}, {}, {}, "cid") + assert len(handles) == 2 + for handle in handles: + handle.remove() + + def test_resid_kinds_on_ssm_layer_still_install(self): + # Block in/out boundaries exist on every layer; no attn/mlp needed. + handles = self._provider()._install_hooks({1: {"resid_pre", "resid_post"}}, {}, {}, "cid") + assert len(handles) == 2 + for handle in handles: + handle.remove() + + +class TestTimeoutRecovery: + def test_timeout_abandons_wedged_loop_and_rebuilds(self, monkeypatch): + """A hung provider forward blocks the loop thread; after the TimeoutError the + driver must serve later forwards on a fresh loop, not queue behind the wedge.""" + import threading + + import transformer_lens.model_bridge.sources.inspect.driver as drv_mod + + monkeypatch.setattr(drv_mod, "_PROVIDER_TIMEOUT_S", 0.2) + + release = threading.Event() + calls = {"n": 0} + metadata = {"tl_logits": wire.encode_array(np.zeros((1, D_VOCAB), np.float32))} + output = SimpleNamespace(metadata=metadata) + + async def generate(_input, config=None): + calls["n"] += 1 + if calls["n"] == 1: + release.wait(30) # sync-block the loop thread, like a hung torch forward + return output + + driver = _driver(SimpleNamespace(generate=generate)) + try: + with pytest.warns(UserWarning, match="abandon"): + with pytest.raises(TimeoutError, match="exceeded"): + driver.forward(torch.tensor([[1]])) + wedged = driver._loop is None # poisoned state cleared for rebuild + assert wedged + result = driver.forward(torch.tensor([[1]])) # fresh loop, must not time out + assert isinstance(result, ForwardResult) + assert calls["n"] == 2 + finally: + release.set() # let the abandoned daemon thread finish + + def test_ensure_loop_is_thread_safe(self): + import threading + + driver = _driver() + loops: list = [] + barrier = threading.Barrier(8) + + def worker(): + barrier.wait() + loops.append(driver._ensure_loop()) + + threads = [threading.Thread(target=worker) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + assert len({id(loop) for loop in loops}) == 1 + driver.close() + + +class TestForProvider: + def test_known_providers_resolve(self): + assert isinstance(profiles.for_provider("vllm-lens"), profiles.VLLMLensProfile) + assert isinstance(profiles.for_provider("vllm-lens-v2"), profiles.VLLMLensProfile) + assert isinstance(profiles.for_provider("tl_bridge"), profiles.TLBridgeProfile) + assert isinstance(profiles.for_provider("tl_bridge_vllm"), profiles.TLBridgeProfile) + + def test_unknown_provider_raises(self): + # Falling back to the full-capability codec would boot then NaN downstream. + with pytest.raises(ValueError, match="tl_bridge.*vllm-lens"): + profiles.for_provider("hf") + + +class TestEvalArtifactEpochs: + def test_epoch_in_artifact_filename(self, tmp_path, monkeypatch): + """Multi-epoch evals reuse sample_ids; artifacts must not overwrite across epochs.""" + import asyncio + + from transformer_lens.model_bridge.sources.inspect import eval as eval_mod + + payload = wire.encode_activations({"0:resid_post": np.ones((2, D_MODEL), np.float32)}) + output = SimpleNamespace(metadata={"activations": payload}) + + async def generate(_messages, config=None): + return output + + monkeypatch.setattr(eval_mod, "get_model", lambda: SimpleNamespace(generate=generate)) + stored: dict = {} + monkeypatch.setattr( + eval_mod, + "store", + lambda: SimpleNamespace(set=lambda k, v: stored.__setitem__(k, v)), + ) + + solve = eval_mod.capture_activations(["blocks.0.hook_out"], output_dir=str(tmp_path)) + for epoch in (1, 2): + state = SimpleNamespace(messages=[], sample_id="s1", epoch=epoch) + asyncio.run(solve(state, None)) + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "s1_epoch1.npz", + "s1_epoch2.npz", + ] + assert stored["tl_activations_path"].endswith("s1_epoch2.npz") + + +class TestHFProviderGenerateConfig: + """stop_seqs wiring + warn-once for unsupported GenerateConfig fields (HF provider).""" + + def _api(self): + from transformer_lens.model_bridge.sources.inspect.transformers_provider import ( + TransformerLensTransformersModelAPI, + ) + + api = object.__new__(TransformerLensTransformersModelAPI) + api.model_name = "fake" + api._device = "cpu" + api._eval_capture = {} + + def tokenize(text, **kw): + return SimpleNamespace(input_ids=[ord(c) % 50 for c in text]) + + tok = SimpleNamespace( + chat_template=None, pad_token_id=0, eos_token_id=1, decode=lambda ids, **kw: "x" + ) + from unittest.mock import MagicMock + + callable_tok = MagicMock(wraps=tok) + callable_tok.side_effect = tokenize + callable_tok.chat_template = None + callable_tok.pad_token_id = 0 + callable_tok.eos_token_id = 1 + callable_tok.decode = tok.decode + api._tokenizer = callable_tok + + hf = MagicMock() + hf.generate.return_value = SimpleNamespace( + sequences=torch.tensor([[5, 6, 65]]), scores=None + ) + api._hf = hf + return api + + def _config(self, **overrides): + from inspect_ai.model import GenerateConfig + + return GenerateConfig(**overrides) + + def test_stop_seqs_wired_to_hf_generate(self): + api = self._api() + api._generate_eval("hi", self._config(max_tokens=1, stop_seqs=["END"]), []) + gen_kwargs = api._hf.generate.call_args.kwargs + assert gen_kwargs["stop_strings"] == ["END"] + assert gen_kwargs["tokenizer"] is api._tokenizer + + def test_unsupported_config_field_warns_once(self, monkeypatch): + import warnings as warnings_mod + + from transformer_lens.model_bridge.sources.inspect import _provider_base + + monkeypatch.setattr(_provider_base, "_WARNED_UNSUPPORTED", set()) + api = self._api() + with pytest.warns(UserWarning, match="frequency_penalty"): + api._generate_eval("hi", self._config(max_tokens=1, frequency_penalty=0.5), []) + with warnings_mod.catch_warnings(record=True) as rec: + warnings_mod.simplefilter("always") + api._generate_eval("hi", self._config(max_tokens=1, frequency_penalty=0.5), []) + assert not [w for w in rec if "frequency_penalty" in str(w.message)] + + +class TestParityScript: + SCRIPT = pathlib.Path(__file__).resolve().parents[3] / "scripts" / "inspect_parity_report.py" + + def _load(self, monkeypatch, env=None): + import importlib.util + import uuid + + for key, value in (env or {}).items(): + monkeypatch.setenv(key, value) + spec = importlib.util.spec_from_file_location( + f"inspect_parity_{uuid.uuid4().hex}", self.SCRIPT + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + def test_tolerance_env_overrides(self, monkeypatch): + module = self._load(monkeypatch, {"TL_INSPECT_ATOL": "5e-2", "TL_INSPECT_RTOL": "7e-2"}) + assert module.ATOL == 5e-2 and module.RTOL == 7e-2 + + def test_default_tolerances(self, monkeypatch): + monkeypatch.delenv("TL_INSPECT_ATOL", raising=False) + monkeypatch.delenv("TL_INSPECT_RTOL", raising=False) + module = self._load(monkeypatch) + assert module.ATOL == 1e-3 and module.RTOL == 1e-3 + + def test_exit_code_1_on_fail(self, monkeypatch, capsys): + module = self._load(monkeypatch, {"TL_PARITY_MODELS": "m1,m2"}) + rows = { + "m1": {"model": "m1", "arch": "A", "status": "PASS", "detail": ""}, + "m2": {"model": "m2", "arch": "B", "status": "FAIL", "detail": "diff"}, + } + monkeypatch.setattr(module, "verify", lambda m: rows[m]) + with pytest.raises(SystemExit) as excinfo: + module.main() + assert excinfo.value.code == 1 + + def test_exit_code_0_on_pass_and_skip(self, monkeypatch, capsys): + module = self._load(monkeypatch, {"TL_PARITY_MODELS": "m1,m2"}) + rows = { + "m1": {"model": "m1", "arch": "A", "status": "PASS", "detail": ""}, + "m2": {"model": "m2", "arch": "B", "status": "SKIP", "detail": "gated"}, + } + monkeypatch.setattr(module, "verify", lambda m: rows[m]) + with pytest.raises(SystemExit) as excinfo: + module.main() + assert excinfo.value.code == 0 + + def test_driver_imports_no_torch(): """The acceptance gate: the driver file must not import torch (data-only boundary).""" import transformer_lens.model_bridge.sources.inspect.driver as drv diff --git a/tests/unit/model_bridge/test_inspect_vllm_provider.py b/tests/unit/model_bridge/test_inspect_vllm_provider.py index 921c8e6fd8..995d23d199 100644 --- a/tests/unit/model_bridge/test_inspect_vllm_provider.py +++ b/tests/unit/model_bridge/test_inspect_vllm_provider.py @@ -259,6 +259,35 @@ def test_generate_eval_with_logprobs(self, monkeypatch): assert lp.content[0].logprob == -0.5 assert len(lp.content[0].top_logprobs) == 2 + def test_generate_eval_stop_seqs_wired_to_sampling_params(self, monkeypatch): + llm = MagicMock() + llm.generate.return_value = [_make_request_output([65], finish_reason="stop")] + provider, _ = _make_provider(monkeypatch, llm) + asyncio.run( + provider.generate( + self._user_msg(), None, None, self._config(max_tokens=4, stop_seqs=["END"]) + ) + ) + sp_kwargs = sys.modules["vllm"].SamplingParams.call_args.kwargs + assert sp_kwargs["stop"] == ["END"] + + def test_generate_eval_unsupported_field_warns_once(self, monkeypatch): + import warnings as warnings_mod + + from transformer_lens.model_bridge.sources.inspect import _provider_base + + monkeypatch.setattr(_provider_base, "_WARNED_UNSUPPORTED", set()) + llm = MagicMock() + llm.generate.return_value = [_make_request_output([65], finish_reason="stop")] + provider, _ = _make_provider(monkeypatch, llm) + config = self._config(max_tokens=1, presence_penalty=0.5) + with pytest.warns(UserWarning, match="presence_penalty"): + asyncio.run(provider.generate(self._user_msg(), None, None, config)) + with warnings_mod.catch_warnings(record=True) as rec: + warnings_mod.simplefilter("always") + asyncio.run(provider.generate(self._user_msg(), None, None, config)) + assert not [w for w in rec if "presence_penalty" in str(w.message)] + def test_generate_eval_passes_temperature_top_p(self, monkeypatch): llm = MagicMock() llm.generate.return_value = [_make_request_output([65], finish_reason="length")] @@ -404,6 +433,66 @@ def test_capture_skips_logits_when_disabled(self, monkeypatch): ) assert "tl_logits" not in out.metadata + def test_intervention_gated_kind_raises_before_rpc(self, monkeypatch): + # resid_pre resolves to a hook name but is gated by vLLM's fused execution; + # without the served check it would reach the worker and silently no-op. + provider, llm, _ = self._make_capture_provider(monkeypatch) + with pytest.raises(ValueError, match="resid_pre|gated"): + asyncio.run( + provider.generate( + [], + None, + None, + self._config_with( + { + "input_ids": [1, 2], + "capture": ["0:resid_post"], + "interventions": {"0:resid_pre": {"op": "suppress"}}, + } + ), + ) + ) + assert llm.collective_rpc.call_count == 0 # validated before any worker RPC + + def test_intervention_capture_only_kind_raises(self, monkeypatch): + provider, _, _ = self._make_capture_provider(monkeypatch) + with pytest.raises(ValueError, match="gated|capture-only"): + asyncio.run( + provider.generate( + [], + None, + None, + self._config_with( + { + "input_ids": [1, 2], + "capture": ["0:resid_post"], + "interventions": {"0:resid_mid": {"op": "suppress"}}, + } + ), + ) + ) + + def test_intervention_pos_rejected_with_remediation(self, monkeypatch): + # 'pos' dies in the worker with an unfollowable message; reject up front and + # point at the HF provider instead. + provider, llm, _ = self._make_capture_provider(monkeypatch) + with pytest.raises(ValueError, match="tl_bridge"): + asyncio.run( + provider.generate( + [], + None, + None, + self._config_with( + { + "input_ids": [1, 2], + "capture": ["0:resid_post"], + "interventions": {"0:resid_post": {"op": "suppress", "pos": 1}}, + } + ), + ) + ) + assert llm.collective_rpc.call_count == 0 + def test_capture_gated_kind_raises(self, monkeypatch): # resid_pre (blocks.{i}.hook_in) is gated by vLLM's fused execution. provider, _, _ = self._make_capture_provider(monkeypatch) @@ -562,3 +651,87 @@ def test_eval_capture_emits_eval_completion_too(self, monkeypatch): # _make_request_output([65, 66]) ⇒ "AB" via fake tokenizer. assert out.choices[0].message.text == "AB" assert out.usage.output_tokens == 2 + + +class TestBootInspectVLLMDtype: + """boot_inspect must forward the resolved dtype to the tl_bridge_vllm provider so + bridge_config.dtype matches what the engine loads.""" + + def _boot(self, monkeypatch, **boot_kwargs): + import inspect_ai.model as inspect_ai_model + import torch + + from transformer_lens.config import TransformerBridgeConfig + from transformer_lens.model_bridge.architecture_adapter import ( + ArchitectureAdapter, + ) + from transformer_lens.model_bridge.sources.inspect import source as source_mod + + cfg = TransformerBridgeConfig( + d_model=4, + d_head=2, + n_layers=2, + n_ctx=8, + n_heads=2, + d_vocab=16, + d_mlp=8, + architecture="LlamaForCausalLM", + ) + adapter = ArchitectureAdapter(cfg) + adapter.component_mapping = {} + + captured: dict[str, Any] = {} + + def fake_get_model(name, memoize=False, **kwargs): + captured["name"] = name + captured.update(kwargs) + api = SimpleNamespace( + supported_kinds=lambda: frozenset({"resid_post"}), + capability_note=lambda: "", + provides_sequence_logits=False, + ) + return SimpleNamespace(api=api) + + from transformers import AutoConfig + + monkeypatch.setattr( + AutoConfig, + "from_pretrained", + lambda *a, **kw: SimpleNamespace(architectures=["LlamaForCausalLM"]), + ) + monkeypatch.setattr(inspect_ai_model, "get_model", fake_get_model) + monkeypatch.setattr(source_mod, "get_hf_token", lambda: None) + monkeypatch.setattr(source_mod, "build_bridge_config_from_hf", lambda *a, **kw: cfg) + monkeypatch.setattr( + source_mod, + "ArchitectureAdapterFactory", + SimpleNamespace(select_architecture_adapter=lambda c: adapter), + ) + + def _fake_configure_tokenizer(tok, cfg_): + cfg_.tokenizer_prepends_bos, cfg_.tokenizer_appends_eos = (True, False) + return tok + + monkeypatch.setattr(source_mod, "configure_tokenizer", _fake_configure_tokenizer) + + bridge = source_mod.boot_inspect( + "fake/model", + tokenizer=SimpleNamespace(), + provider="tl_bridge_vllm", + **boot_kwargs, + ) + return bridge, captured + + def test_default_dtype_forwarded_as_fp32(self, monkeypatch): + import torch + + _, captured = self._boot(monkeypatch) + assert captured["name"] == "tl_bridge_vllm/fake/model" + # Without forwarding, the provider silently defaults to the HF-config dtype. + assert captured["dtype"] is torch.float32 + + def test_explicit_dtype_forwarded(self, monkeypatch): + import torch + + _, captured = self._boot(monkeypatch, dtype=torch.float16) + assert captured["dtype"] is torch.float16 diff --git a/tests/unit/model_bridge/test_remote_bridge.py b/tests/unit/model_bridge/test_remote_bridge.py index 1d0940b937..2c41cb2225 100644 --- a/tests/unit/model_bridge/test_remote_bridge.py +++ b/tests/unit/model_bridge/test_remote_bridge.py @@ -276,6 +276,42 @@ def forward( with pytest.raises(ValueError, match="Invalid return_type"): bridge.forward(torch.tensor([[1, 2, 3]]), return_type="nonsense") + def test_forward_rejects_list_of_strings(self): + """list[str] used to be passed through as raw input_ids and crash in numpy.""" + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + with pytest.raises(TypeError, match="list of strings"): + bridge.forward(["hello", "world"]) + + def test_forward_rejects_stop_at_layer(self): + """stop_at_layer used to be silently swallowed by **kwargs.""" + import torch + + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + with pytest.raises(NotImplementedError, match="stop_at_layer"): + bridge.forward(torch.tensor([[1, 2]]), stop_at_layer=1) + + def test_run_with_hooks_rejects_stop_at_layer(self): + import torch + + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + with pytest.raises(NotImplementedError, match="stop_at_layer"): + bridge.run_with_hooks(torch.tensor([[1, 2]]), stop_at_layer=1) + + def test_run_with_cache_rejects_stop_at_layer(self): + import torch + + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + with pytest.raises(NotImplementedError, match="stop_at_layer"): + bridge.run_with_cache(torch.tensor([[1, 2]]), stop_at_layer=1) + + def test_run_with_hooks_rejects_names_filter_kwarg(self): + """The parameter existed in the signature but was never read.""" + import torch + + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + with pytest.raises(TypeError, match="names_filter"): + bridge.run_with_hooks(torch.tensor([[1, 2]]), names_filter="blocks.0.hook_resid_pre") + def test_forward_replays_captures(self): import torch @@ -310,3 +346,88 @@ def forward( bridge.forward(torch.tensor([[1, 2]])) assert len(recorded) == 1 assert tuple(recorded[0].shape) == (2, 3) + + +class TestRemoteBridgeHookSetEnforcement: + """Driver-declared hook sets are a contract, not metadata — requests + outside them must fail loud instead of yielding empty caches.""" + + def test_add_hook_resolves_ht_alias(self): + """add_hook accepts the same HT-style aliases run_with_hooks resolves.""" + driver = _stub_driver(supported_hooks=frozenset({"embed.hook_out"})) + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=driver) + bridge.add_hook("hook_embed", lambda act, hook: None) + assert len(bridge._hook_registry["embed.hook_out"].fwd_hooks) == 1 + + def test_run_with_hooks_unknown_hook_name_raises(self): + """Unknown string names used to be silently skipped — unhooked forward.""" + import torch + + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + with pytest.raises(KeyError, match="does not exist"): + with pytest.warns(UserWarning, match="read-only"): + bridge.run_with_hooks( + torch.tensor([[1, 2]]), + fwd_hooks=[("blocks.0.attn.hook_z_typo", lambda act, hook: None)], + ) + + def test_run_with_cache_unmatched_names_filter_raises(self): + """A filter outside the driver whitelist used to return (logits, {}).""" + import torch + + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + with pytest.raises(KeyError, match="matched no hook points"): + bridge.run_with_cache(torch.tensor([[1, 2]]), names_filter="blocks.0.attn.hook_z") + + def test_run_with_cache_unmatched_list_filter_raises(self): + import torch + + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + with pytest.raises(KeyError, match="matched no hook points"): + bridge.run_with_cache( + torch.tensor([[1, 2]]), names_filter=["nope.hook_a", "nope.hook_b"] + ) + + def test_run_with_cache_matching_filter_still_works(self): + import torch + + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=_stub_driver()) + _, cache = bridge.run_with_cache( + torch.tensor([[1, 2]]), names_filter="blocks.0.hook_resid_pre" + ) + assert cache is not None # no raise; driver returns no captures here + + def test_non_fireable_hook_raises_backend_message(self): + """Hooks the backend declares non-fireable must name the remedy.""" + import torch + + class PartialDriver(DriverBase): + supported_hook_points = frozenset({"blocks.0.hook_resid_pre"}) + non_fireable_hook_points = frozenset({"blocks.0.attn.hook_pattern"}) + + def __init__(self): + super().__init__(_cfg(), tokenizer=None) + + def forward( + self, + input_ids=None, + *, + capture=(), + intervene=None, + max_new_tokens=1, + return_logits=True, + **kw, + ): + return ForwardResult() + + bridge = RemoteBridge(_stub_adapter(), tokenizer=None, driver=PartialDriver()) + with pytest.raises(NotImplementedError, match="cannot fire"): + bridge.add_hook("blocks.0.attn.hook_pattern", lambda act, hook: None) + with pytest.raises(NotImplementedError, match="boot_transformers"): + bridge.run_with_cache(torch.tensor([[1, 2]]), names_filter="blocks.0.attn.hook_pattern") + with pytest.raises(NotImplementedError, match="cannot fire"): + with pytest.warns(UserWarning, match="read-only"): + bridge.run_with_hooks( + torch.tensor([[1, 2]]), + fwd_hooks=[("blocks.0.attn.hook_pattern", lambda act, hook: None)], + ) diff --git a/tests/unit/model_bridge/test_vllm_boot.py b/tests/unit/model_bridge/test_vllm_boot.py index 7d79bbc639..a54a2438b6 100644 --- a/tests/unit/model_bridge/test_vllm_boot.py +++ b/tests/unit/model_bridge/test_vllm_boot.py @@ -62,8 +62,12 @@ def mocked_boot(monkeypatch): monkeypatch.setattr("transformers.AutoConfig.from_pretrained", auto_config) monkeypatch.setattr("transformers.AutoTokenizer.from_pretrained", auto_tokenizer) + fake_llm = MagicMock(name="llm") + # The boot-time reconstruction probe fetches the unembedding via get_param, whose + # return type is beartype-enforced — the RPC must yield a real tensor. + fake_llm.collective_rpc = MagicMock(return_value=[torch.ones(16, 4)]) fake_vllm = MagicMock() - fake_vllm.LLM = MagicMock(return_value=MagicMock(name="llm")) + fake_vllm.LLM = MagicMock(return_value=fake_llm) monkeypatch.setitem(sys.modules, "vllm", fake_vllm) monkeypatch.setattr( @@ -91,11 +95,24 @@ def mocked_boot(monkeypatch): lambda: None, ) + # Real tokenizer setup/probing can't run on a MagicMock tokenizer. + def _fake_configure_tokenizer(tokenizer, cfg_): + cfg_.tokenizer_prepends_bos, cfg_.tokenizer_appends_eos = (False, True) + return tokenizer + + configure_tok = MagicMock(side_effect=_fake_configure_tokenizer) + monkeypatch.setattr( + "transformer_lens.model_bridge.sources.vllm.source.configure_tokenizer", + configure_tok, + ) + yield { "auto_config": auto_config, "auto_tokenizer": auto_tokenizer, "vllm_llm": fake_vllm.LLM, "hf_config": hf_config, + "cfg": cfg, + "configure_tok": configure_tok, } plugin._config.clear() @@ -176,6 +193,39 @@ def test_plugin_config_cleared_after_boot(mocked_boot): assert plugin._config == {} +def test_plugin_config_cleared_when_llm_construction_fails(mocked_boot): + """A failed boot (OOM, gated repo) must not leave stale specs patched in — + the next in-process vllm.LLM(...) would walk our dot-paths on a foreign model.""" + mocked_boot["vllm_llm"].side_effect = RuntimeError("CUDA out of memory") + with pytest.raises(RuntimeError, match="CUDA out of memory"): + boot_vllm("any-model") + assert plugin._config == {} + + +def test_rejects_prefix_caching_override(): + """Prefix caching breaks the row=position capture invariant — locked off.""" + with pytest.raises(ValueError, match="enable_prefix_caching"): + boot_vllm("any-model", enable_prefix_caching=True) + + +def test_bos_detection_written_to_config(mocked_boot): + """boot_vllm must probe the tokenizer like boot_transformers — the dataclass + default (prepends_bos=True) is wrong for Qwen-family tokenizers and shifts + every activation by one position.""" + bridge = boot_vllm("any-model") + assert mocked_boot["configure_tok"].called + assert mocked_boot["cfg"].tokenizer_prepends_bos is False + assert mocked_boot["cfg"].tokenizer_appends_eos is True + assert bridge is not None + + +def test_logit_reconstruction_probed_at_boot(mocked_boot): + """The unembedding is fetched once at boot, not re-cloned per forward.""" + bridge = boot_vllm("any-model") + assert bridge._driver._unembed_probed is True + assert bridge._driver._unembed is not None + + def test_llm_construction_kwargs(mocked_boot): """Pin the kwargs boot_vllm passes to vllm.LLM(...). Catches regressions like forgetting worker_extension_cls (collective_rpc methods unreachable), @@ -195,3 +245,15 @@ def test_llm_construction_kwargs(mocked_boot): assert kwargs["pipeline_parallel_size"] == 1 assert kwargs["skip_tokenizer_init"] is True assert kwargs["disable_log_stats"] is True + assert kwargs["enable_prefix_caching"] is False + # Always explicit — "auto" would downcast fp32 checkpoints under the buffers. + assert kwargs["dtype"] == "float16" + + +def test_missing_vllm_raises_actionable_import_error(monkeypatch): + """Without vllm installed, boot_vllm must name the packaging extra — and fail + before any HF network I/O or plugin state mutation.""" + monkeypatch.setitem(sys.modules, "vllm", None) # forces ImportError on import + with pytest.raises(ImportError, match=r"transformer-lens\[vllm\]"): + boot_vllm("any-model") + assert plugin._config == {} diff --git a/tests/unit/model_bridge/test_vllm_driver.py b/tests/unit/model_bridge/test_vllm_driver.py index 80140367c4..a5b174c128 100644 --- a/tests/unit/model_bridge/test_vllm_driver.py +++ b/tests/unit/model_bridge/test_vllm_driver.py @@ -637,3 +637,239 @@ def test_run_with_hooks_warns_fwd_hooks_are_read_only(self): torch.tensor([[1, 2, 3]]), fwd_hooks=[("embed.hook_out", lambda a, hook: a)], ) + + +class TestVLLMDriverSpecKeyValidation: + """Unknown spec keys and mis-shaped values must fail loud, not silently degrade.""" + + def test_unknown_spec_key_rejected(self): + """A typo'd 'position' would otherwise become a whole-sequence edit.""" + with pytest.raises(ValueError, match="unknown spec key"): + _driver(enable_position_interventions=True)._validate_interventions( + {"embed.hook_out": {"op": "add", "value": 1.0, "position": 3}} + ) + + def test_known_keys_accepted(self): + out = _driver(enable_position_interventions=True)._validate_interventions( + {"embed.hook_out": {"op": "add", "value": 1.0, "pos": 2}} + ) + assert out["embed.hook_out"]["pos"] == 2 + + def test_value_width_mismatch_rejected(self): + """embed.hook_out width is 4 (overlay spec); a 2-element value would broadcast + wrong or crash mid-forward.""" + with pytest.raises(ValueError, match="2 elements"): + _driver()._validate_interventions( + {"embed.hook_out": {"op": "add", "value": [1.0, 2.0]}} + ) + + def test_value_width_match_accepted(self): + out = _driver()._validate_interventions( + {"embed.hook_out": {"op": "set", "value": [1.0, 2.0, 3.0, 4.0]}} + ) + assert out["embed.hook_out"]["value"] == [1.0, 2.0, 3.0, 4.0] + + def test_scalar_value_accepted(self): + out = _driver()._validate_interventions({"embed.hook_out": {"op": "add", "value": 2.5}}) + assert out["embed.hook_out"]["value"] == 2.5 + + def test_pos_bool_rejected(self): + """bool is an int subclass — pos=True must not pass as position 1.""" + driver = _driver(enable_position_interventions=True) + with pytest.raises(ValueError, match="int or list of ints"): + driver._validate_interventions({"embed.hook_out": {"op": "suppress", "pos": True}}) + with pytest.raises(ValueError, match="int or list of ints"): + driver._validate_interventions({"embed.hook_out": {"op": "suppress", "pos": [True]}}) + + +class TestVLLMDriverLogitReconstruction: + """Boot-time probe caches the unembedding; reconstruction slices padded vocab.""" + + def test_probe_unavailable_downgrades_sequence_logits(self): + driver = _driver(captures={}) + driver._llm.collective_rpc = MagicMock(return_value=[None]) + assert driver.probe_logit_reconstruction() is False + assert driver._unembed is None and driver._unembed_probed + assert driver.provides_sequence_logits is False + + def test_probe_available_caches_weight(self): + driver = _driver(captures={}) + driver._llm.collective_rpc = MagicMock(return_value=[torch.ones(16, 4)]) + assert driver.probe_logit_reconstruction() is True + assert driver.provides_sequence_logits is True + weight32, _bias = driver._unembed + assert weight32.dtype == torch.float32 + # Idempotent: a second probe answers from the cache, no extra RPC. + calls_after_first = driver._llm.collective_rpc.call_count + assert driver.probe_logit_reconstruction() is True + assert driver._llm.collective_rpc.call_count == calls_after_first + + def test_reconstruct_uses_cached_weight_without_rpc(self): + """After the probe, per-forward reconstruction must not re-fetch the weight.""" + driver = _driver(captures={}) + driver._unembed = (torch.eye(4, dtype=torch.float32).repeat(4, 1), None) + driver._unembed_probed = True + driver._llm.collective_rpc = MagicMock( + side_effect=AssertionError("reconstruction must not RPC") + ) + out = driver._reconstruct_logits(torch.ones(3, 4)) + assert out is not None and out.shape == (3, 16) + + def test_reconstruct_slices_padded_vocab(self): + """vLLM pads vocab to a multiple of 64; d_vocab=16 here, weight padded to 24. + Un-sliced, the zero-filled pad columns become phantom argmax candidates when + every real logit is negative.""" + driver = _driver(captures={}) + weight = torch.zeros(24, 4, dtype=torch.float32) + weight[:16] = -1.0 # real vocab rows: all-negative logits + driver._unembed = (weight, None) + driver._unembed_probed = True + out = driver._reconstruct_logits(torch.ones(3, 4)) + assert out.shape == (3, 16) + assert int(out[0].argmax()) < 16 + + def test_reconstruct_returns_none_when_probed_unavailable(self): + driver = _driver(captures={}) + driver._unembed_probed = True # probed, nothing found + assert driver._reconstruct_logits(torch.ones(3, 4)) is None + + +class TestVLLMDriverLnFinalUnfold: + """The exposed ln_final capture honors the hook name's pre-weight convention.""" + + def test_unfold_divides_by_weight(self): + driver = _driver(captures={}) + driver._llm.collective_rpc = MagicMock(return_value=[torch.full((4,), 2.0)]) + out = driver._unfold_ln_final(torch.ones(1, 3, 4)) + assert torch.allclose(out, torch.full((1, 3, 4), 0.5)) + + def test_unfold_gemma_uses_one_plus_weight(self): + driver = _driver(captures={}) + driver.architecture = "Gemma2ForCausalLM" + driver._llm.collective_rpc = MagicMock(return_value=[torch.full((4,), 1.0)]) + out = driver._unfold_ln_final(torch.ones(1, 3, 4)) + assert torch.allclose(out, torch.full((1, 3, 4), 0.5)) + + def test_unfold_near_zero_weight_guarded(self): + driver = _driver(captures={}) + driver._llm.collective_rpc = MagicMock(return_value=[torch.zeros(4)]) + out = driver._unfold_ln_final(torch.ones(1, 3, 4)) + assert torch.isfinite(out).all() + + def test_unfold_warns_and_returns_raw_when_weight_unreachable(self): + driver = _driver(captures={}) + driver._llm.collective_rpc = MagicMock(return_value=[None]) + t = torch.ones(1, 3, 4) + with pytest.warns(UserWarning, match="POST-weight"): + out = driver._unfold_ln_final(t) + assert torch.equal(out, t) + # Negative outcome cached: no per-forward RPC retry, no repeat warning. + out2 = driver._unfold_ln_final(t) + assert torch.equal(out2, t) + assert driver._llm.collective_rpc.call_count == 1 + + def test_unfold_caches_fetched_weight(self): + driver = _driver(captures={}) + driver._llm.collective_rpc = MagicMock(return_value=[torch.full((4,), 2.0)]) + driver._unfold_ln_final(torch.ones(1, 3, 4)) + assert driver._llm.collective_rpc.call_count == 1 + driver._unfold_ln_final(torch.ones(1, 3, 4)) + assert driver._llm.collective_rpc.call_count == 1 # cached, no second RPC + + +class TestVLLMDriverCloseRefcount: + """close() only tears down vLLM's process-global distributed state when this is + the last live driver — a re-bound notebook boot must not break the new engine.""" + + def _patch_distributed(self, monkeypatch): + import sys as _sys + + dist = MagicMock() + monkeypatch.setitem(_sys.modules, "vllm", MagicMock()) + monkeypatch.setitem(_sys.modules, "vllm.distributed", MagicMock()) + monkeypatch.setitem(_sys.modules, "vllm.distributed.parallel_state", dist) + return dist + + def test_teardown_deferred_until_last_driver(self, monkeypatch): + from transformer_lens.model_bridge.sources.vllm import driver as driver_module + + dist = self._patch_distributed(monkeypatch) + monkeypatch.setattr(driver_module, "_LIVE_DRIVERS", 0) + a, b = _driver(captures={}), _driver(captures={}) + a.close() + dist.destroy_model_parallel.assert_not_called() + b.close() + dist.destroy_model_parallel.assert_called_once() + dist.destroy_distributed_environment.assert_called_once() + + def test_double_close_decrements_once(self, monkeypatch): + from transformer_lens.model_bridge.sources.vllm import driver as driver_module + + dist = self._patch_distributed(monkeypatch) + monkeypatch.setattr(driver_module, "_LIVE_DRIVERS", 0) + a, b = _driver(captures={}), _driver(captures={}) + a.close() + a.close() # idempotent: must not reach zero while b is live + dist.destroy_model_parallel.assert_not_called() + b.close() + dist.destroy_model_parallel.assert_called_once() + + def test_close_drops_weight_caches(self, monkeypatch): + self._patch_distributed(monkeypatch) + driver = _driver(captures={}) + driver._unembed = (torch.ones(16, 4), None) + driver._lnf_inv_denom = torch.ones(4) + driver.close() + assert driver._unembed is None and driver._lnf_inv_denom is None + + +class TestBatchedForwardPaddingSemantics: + """Reconstruction pads and attention masks on the batched path.""" + + def test_reconstructed_pad_positions_are_neg_inf(self): + """Zero-filled ln_final pad rows reconstruct into finite garbage (0 @ W); + they must be masked to the -inf convention the fallback path uses.""" + pytest.importorskip("vllm") + outputs = [ + _batched_request_output("req-A", top_logprobs={7: 2.0}), + _batched_request_output("req-B", top_logprobs={3: 2.0}), + ] + captures_by_req = { + "req-A": {"ln_final.hook_normalized": torch.ones(3, 4)}, + "req-B": {"ln_final.hook_normalized": torch.ones(2, 4)}, + } + driver = _batched_driver(outputs=outputs, captures_by_req=captures_by_req) + driver._unembed = (torch.ones(16, 4, dtype=torch.float32), None) + driver._unembed_probed = True + result = driver.forward([[1, 2, 3], [4, 5]]) + assert result.logits.shape == (2, 3, 16) + assert torch.isfinite(result.logits[0]).all() # 3 real rows + assert torch.isfinite(result.logits[1, :2]).all() + assert torch.isinf(result.logits[1, 2]).all() # pad row for the 2-token prompt + + def test_attention_mask_trims_padded_rows(self): + """A right-padded (B, S) batch + mask must send only real tokens to vLLM.""" + pytest.importorskip("vllm") + outputs = [ + _batched_request_output("req-A", top_logprobs={7: 2.0}), + _batched_request_output("req-B", top_logprobs={3: 2.0}), + ] + driver = _batched_driver(outputs=outputs, captures_by_req={}) + driver.forward( + torch.tensor([[1, 2, 3], [4, 5, 0]]), + attention_mask=torch.tensor([[1, 1, 1], [1, 1, 0]]), + return_logits=False, + ) + prompts = driver._llm.generate.call_args.kwargs["prompts"] + assert prompts[0]["prompt_token_ids"] == [1, 2, 3] + assert prompts[1]["prompt_token_ids"] == [4, 5] # pad token trimmed + + def test_attention_mask_batch_mismatch_raises(self): + pytest.importorskip("vllm") + driver = _batched_driver(outputs=[], captures_by_req={}) + with pytest.raises(ValueError, match="attention_mask batch dim"): + driver.forward( + torch.tensor([[1, 2], [3, 4]]), + attention_mask=torch.tensor([[1, 1]]), + return_logits=False, + ) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 86e685c691..78050e503b 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -465,6 +465,11 @@ def test_no_split_tokens_across_chunks(self): streaming=False, max_length=64, add_bos_token=False, + # Main-process: pytest's jaxtyping import hook instruments the closure, + # which datasets>=4.8 dill-ships to pool workers (even for num_proc=1) — + # instrumentation state is unpicklable. Plain-shell multiproc works; the + # chunking assertion below is what this test is for. + num_proc=None, ) # Tokenize the same text cleanly in one shot (no chunking) diff --git a/transformer_lens/model_bridge/bridge_core.py b/transformer_lens/model_bridge/bridge_core.py index 96f30b94a5..4aca75f49d 100644 --- a/transformer_lens/model_bridge/bridge_core.py +++ b/transformer_lens/model_bridge/bridge_core.py @@ -300,6 +300,7 @@ def _finalize_return( return logits if return_type is None: return None + self._check_loss_supported(return_type) if return_type == "loss": if is_audio_model: raise ValueError( @@ -349,8 +350,44 @@ def _finalize_return( return preds raise ValueError(f"Invalid return_type: {return_type}") + def _check_loss_supported(self, return_type: Optional[str]) -> None: + """Loss needs full-sequence logits; final-position-only drivers would NaN.""" + if return_type in ("loss", "both") and not getattr( + self._driver, "provides_sequence_logits", True + ): + raise NotImplementedError( + f"return_type={return_type!r} is unsupported on this driver: it " + "provides next-token logits for the final position only, so loss " + "over earlier positions is undefined. Use return_type='logits' " + "and read logits[..., -1, :]." + ) + # ---- hook lookup / mutation ---- + def _check_hook_fireable(self, *names: str) -> None: + """Fail loud when the driver declares it can't fire a requested hook — + attaching anyway would yield a silently-unhooked forward / empty cache.""" + non_fireable: frozenset = getattr(self._driver, "non_fireable_hook_points", frozenset()) + for name in names: + if name in non_fireable: + raise NotImplementedError( + f"this backend cannot fire {name!r}; use boot_transformers() " + "for full hook coverage." + ) + + def _resolve_hook_point( + self, name: str, aliases: Dict[str, str], hook_dict: Dict[str, HookPoint] + ) -> Tuple[str, HookPoint]: + """Resolve an (aliased) string hook name to its HookPoint, enforcing + fireability. A name that resolves to nothing raises — a typo'd name must + not run unhooked.""" + canonical = aliases.get(name, name) + self._check_hook_fireable(name, canonical) + hook_point = hook_dict.get(canonical) + if hook_point is None: + raise KeyError(f"Hook name {name!r} does not exist on this model.") + return canonical, hook_point + def get_hook_point(self, hook_name: str) -> Optional[HookPoint]: """Get a hook point by name from the bridge's hook system.""" if hook_name in self._hook_registry: @@ -396,8 +433,20 @@ def add_hook( hook_point.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) return - # Registry-first: works for any bridge (RemoteBridge has no component tree). + # Fast path: canonical registry names skip the alias-map build (hook_dict + + # map construction cost ~ms on large models; add_hook is often called per layer). registry_hp = self._hook_registry.get(name) + if registry_hp is not None: + self._check_hook_fireable(name) + registry_hp.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) + return + # Same alias resolution run_with_hooks uses, so HT-style names work here too. + canonical = build_alias_to_canonical_map(self.hook_dict).get(name, name) + if canonical != name: + self._check_hook_fireable(name, canonical) + else: + self._check_hook_fireable(name) + registry_hp = self._hook_registry.get(canonical) if registry_hp is not None: registry_hp.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) return @@ -489,19 +538,15 @@ def add_hook_to_point( def apply_hooks(hook_list: List[Tuple[Any, Callable]], is_fwd: bool) -> None: direction: Literal["fwd", "bwd"] = "fwd" if is_fwd else "bwd" - aliases = build_alias_to_canonical_map(self.hook_dict) + hook_dict = self.hook_dict + aliases = build_alias_to_canonical_map(hook_dict) for hook_name_or_filter, hook_fn in hook_list: if isinstance(hook_name_or_filter, str): - hook_dict = self.hook_dict - actual_hook_name = hook_name_or_filter - if hook_name_or_filter in aliases: - actual_hook_name = aliases[hook_name_or_filter] - if actual_hook_name in hook_dict: - add_hook_to_point( - hook_dict[actual_hook_name], hook_fn, actual_hook_name, direction - ) + actual_hook_name, hook_point = self._resolve_hook_point( + hook_name_or_filter, aliases, hook_dict + ) + add_hook_to_point(hook_point, hook_fn, actual_hook_name, direction) else: - hook_dict = self.hook_dict seen_hooks = set() for n, hook_point in hook_dict.items(): if hook_name_or_filter(n): @@ -533,7 +578,6 @@ def run_with_hooks( reset_hooks_end: bool = True, clear_contexts: bool = False, return_type: Optional[str] = "logits", - names_filter: Optional[Union[str, List[str], Callable[[str], bool]]] = None, stop_at_layer: Optional[int] = None, remove_batch_dim: bool = False, **kwargs: Any, @@ -544,6 +588,12 @@ def run_with_hooks( (KV cache cleaned up on stop). ``remove_batch_dim`` squeezes/unsqueezes the batch dim around hook callbacks (batch_size==1 only). """ + if "names_filter" in kwargs: + # **kwargs would silently absorb it; fail loud. + raise TypeError( + "run_with_hooks() got an unexpected keyword argument 'names_filter'; " + "use run_with_cache(names_filter=...) to scope caching." + ) added_hooks: List[Tuple[HookPoint, Literal["fwd", "bwd"]]] = [] effective_stop_layer = None if stop_at_layer is not None and hasattr(self, "blocks"): @@ -593,7 +643,8 @@ def apply_hooks( hook_list: List[Tuple[Union[str, Callable], Callable]], is_fwd: bool ) -> None: direction: Literal["fwd", "bwd"] = "fwd" if is_fwd else "bwd" - aliases = build_alias_to_canonical_map(self.hook_dict) + hook_dict = self.hook_dict + aliases = build_alias_to_canonical_map(hook_dict) for hook_name_or_filter, hook_fn in hook_list: if remove_batch_dim: original_hook_fn = hook_fn @@ -611,16 +662,11 @@ def wrapped_hook_fn(tensor, hook, _orig_fn=original_hook_fn): hook_fn = wrapped_hook_fn if isinstance(hook_name_or_filter, str): - hook_dict = self.hook_dict - actual_hook_name = hook_name_or_filter - if hook_name_or_filter in aliases: - actual_hook_name = aliases[hook_name_or_filter] - if actual_hook_name in hook_dict: - add_hook_to_point( - hook_dict[actual_hook_name], hook_fn, actual_hook_name, direction - ) + actual_hook_name, hook_point = self._resolve_hook_point( + hook_name_or_filter, aliases, hook_dict + ) + add_hook_to_point(hook_point, hook_fn, actual_hook_name, direction) else: - hook_dict = self.hook_dict seen_hooks: set = set() for n, hook_point in hook_dict.items(): if hook_name_or_filter(n): @@ -710,6 +756,10 @@ def create_names_filter_fn(filter_input): raise ValueError("names_filter must be a string, list of strings, or callable") names_filter_fn = create_names_filter_fn(names_filter) + if isinstance(names_filter, (str, list)): + requested = [names_filter] if isinstance(names_filter, str) else names_filter + for name in requested: + self._check_hook_fireable(name, aliases.get(name, name)) cache: Dict[str, torch.Tensor] = {} hooks: List[Tuple[HookPoint, str]] = [] visited: set[int] = set() @@ -745,8 +795,10 @@ def cache_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: effective_stop_layer = len(self.blocks) + stop_at_layer else: effective_stop_layer = stop_at_layer + matched_any = False for hook_name, hook in hook_dict.items(): if names_filter_fn(hook_name): + matched_any = True if effective_stop_layer is not None: if hook_name.startswith("blocks."): try: @@ -756,6 +808,12 @@ def cache_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: except (IndexError, ValueError): pass hooks.append((hook, hook_name)) + # Explicit string/list filters matching nothing must not return (logits, {}) silently. + if not matched_any and names_filter and isinstance(names_filter, (str, list)): + raise KeyError( + f"names_filter {names_filter!r} matched no hook points on this model; " + "check the name against model.hook_dict (this backend may not serve it)." + ) for hp, name in hooks: hp.add_hook(make_cache_hook(name)) processed_args = [input] diff --git a/transformer_lens/model_bridge/driver_protocol.py b/transformer_lens/model_bridge/driver_protocol.py index ffe7a4862e..71b3587415 100644 --- a/transformer_lens/model_bridge/driver_protocol.py +++ b/transformer_lens/model_bridge/driver_protocol.py @@ -48,15 +48,38 @@ class ForwardResult: raw_output: Any = None +# Feature strings ``supports()`` may be queried with. The bridge consumes +# "parameters" (input-device placement); the rest are informational capability +# declarations for callers. validate_driver rejects drivers declaring strings +# outside this set. +KNOWN_FEATURES = frozenset( + {"gradients", "parameters", "state_dict", "weight_access", "intervention_callbacks"} +) + + @runtime_checkable class Driver(Protocol): - """The forward-pass contract. Hook installation is the driver's problem.""" + """The forward-pass contract. Hook installation is the driver's problem. + + ``forward`` has two dialects: + + - **Module-replacement drivers** (TransformersDriver): hooks fire via the + bridge's HookPoint system during the real torch forward, so ``capture``/ + ``intervene``/``max_new_tokens`` are not served here — conforming drivers + raise ``NotImplementedError`` on them rather than silently ignore. + - **Spec drivers** (vLLM, Inspect): no local module, so ``capture`` names + hook points to record and ``intervene`` carries declarative edit specs; + results come back in ``ForwardResult.captured``. + """ architecture: str bridge_config: TransformerBridgeConfig tokenizer: Any supported_hook_points: frozenset[str] non_fireable_hook_points: frozenset[str] + # False when the driver returns logits for the final position only — + # the bridge then refuses return_type="loss"/"both" instead of NaN-ing. + provides_sequence_logits: bool def forward( self, @@ -74,8 +97,8 @@ def close(self) -> None: ... def supports(self, feature: str) -> bool: - """Capability flag. Known features: gradients, parameters, state_dict, - generate_streaming, weight_access, intervention_callbacks.""" + """Capability flag over :data:`KNOWN_FEATURES`. The bridge consults + "parameters"; the others are caller-facing declarations.""" ... # Note: torch-specific surface (parameters, named_parameters, state_dict, @@ -133,6 +156,18 @@ def validate_driver(driver: Any, *, after_bridge_construction: bool = False) -> raise TypeError("Driver missing required attribute: 'tokenizer'") _expect_attr_type(driver, "supported_hook_points", frozenset) _expect_attr_type(driver, "non_fireable_hook_points", frozenset) + # getattr(..., True) defaults would silently pick the UNSAFE value for + # loss gating, so the attribute is mandatory. + _expect_attr_type(driver, "provides_sequence_logits", bool) + + declared_features = getattr(driver, "_supported_features", None) + if declared_features is not None: + unknown = frozenset(declared_features) - KNOWN_FEATURES + if unknown: + raise TypeError( + f"Driver._supported_features contains unknown feature strings " + f"{sorted(unknown)}; known features: {sorted(KNOWN_FEATURES)}." + ) overlap = driver.supported_hook_points & driver.non_fireable_hook_points if overlap: @@ -183,6 +218,7 @@ def _expect_attr_type(obj: Any, name: str, expected: type) -> None: __all__ = [ "Driver", "ForwardResult", + "KNOWN_FEATURES", "Intervention", "InterventionFn", "InterventionSpec", diff --git a/transformer_lens/model_bridge/remote_bridge.py b/transformer_lens/model_bridge/remote_bridge.py index a5c4ad9a05..6b9c5e8b83 100644 --- a/transformer_lens/model_bridge/remote_bridge.py +++ b/transformer_lens/model_bridge/remote_bridge.py @@ -78,18 +78,18 @@ def forward( **kwargs: Any, ) -> Any: """Tokenize → driver.forward → replay captures → finalize per return_type.""" - if return_type in ("loss", "both") and not getattr( - self._driver, "provides_sequence_logits", True - ): - # Final-position-only logits ⇒ loss over the -inf earlier positions is nan. - raise NotImplementedError( - f"RemoteBridge does not support return_type={return_type!r} on this driver: " - "it provides next-token logits for the final position only, so loss over " - "earlier positions is undefined. Use return_type='logits' and read " - "logits[..., -1, :] (or the per-row last token in batched mode)." - ) + # Early copy of _finalize_return's gate — fail before the wasted remote forward. + self._check_loss_supported(return_type) + self._reject_stop_at_layer(kwargs.pop("stop_at_layer", None)) if isinstance(input, str): kwargs["input_ids"] = self.to_tokens(input) # BOS-aware, matches boot_transformers + elif isinstance(input, list) and any(isinstance(item, str) for item in input): + # Would otherwise be treated as raw input_ids and crash deep in numpy. + raise TypeError( + "RemoteBridge.forward received a list of strings; batched string " + "input is unsupported here. Pass a single str, or tokenize with " + "to_tokens() and pass token ids." + ) elif input is not None: kwargs["input_ids"] = input @@ -126,7 +126,6 @@ def run_with_hooks( reset_hooks_end: bool = True, clear_contexts: bool = False, return_type: Optional[str] = "logits", - names_filter: Optional[Union[str, List[str], Callable[[str], bool]]] = None, stop_at_layer: Optional[int] = None, remove_batch_dim: bool = False, **kwargs: Any, @@ -138,6 +137,7 @@ def run_with_hooks( raise NotImplementedError( "RemoteBridge has no backward pass; bwd_hooks are unsupported." ) + self._reject_stop_at_layer(stop_at_layer) if fwd_hooks: warnings.warn( "RemoteBridge fwd_hooks fire on already-captured activations (read-only): " @@ -155,12 +155,25 @@ def run_with_hooks( reset_hooks_end=reset_hooks_end, clear_contexts=clear_contexts, return_type=return_type, - names_filter=names_filter, - stop_at_layer=stop_at_layer, remove_batch_dim=remove_batch_dim, **kwargs, ) + @staticmethod + def _reject_stop_at_layer(stop_at_layer: Any) -> None: + # BridgeCore's stop hooks need a local `blocks` tree; silently ignoring + # the kwarg would lie about compute cost. + if stop_at_layer is not None: + raise NotImplementedError( + "RemoteBridge does not support stop_at_layer: the remote engine " + "always runs the full forward pass." + ) + + def run_with_cache(self, *args: Any, **kwargs: Any) -> Any: + """Cache via driver captures; stop_at_layer rejected (full remote forward).""" + self._reject_stop_at_layer(kwargs.get("stop_at_layer")) + return super().run_with_cache(*args, **kwargs) + def to_tokens(self, input: Any, prepend_bos: bool | None = None, truncate: bool = True) -> Any: """Tokenize a string with the same BOS handling as ``TransformerBridge``. diff --git a/transformer_lens/model_bridge/sources/AGENTS.md b/transformer_lens/model_bridge/sources/AGENTS.md index a75bbe210d..42d201312c 100644 --- a/transformer_lens/model_bridge/sources/AGENTS.md +++ b/transformer_lens/model_bridge/sources/AGENTS.md @@ -1,47 +1,59 @@ # Sources — AGENTS.md -Backend loaders that translate external models (HF, native) into a `TransformerBridge`. Read [the root AGENTS.md](../../../AGENTS.md) for project-wide rules. +Execution backends for the bridge. Each source boots an engine (HF, TL-native, vLLM, inspect_ai) and wraps it in a `TransformerBridge` or `RemoteBridge` via a Driver. Read [the root AGENTS.md](../../../AGENTS.md) for project-wide rules. ## File map | File | Role | |---|---| -| [`_bridge_builder.py`](_bridge_builder.py) | Loader-agnostic helpers: `build_bridge_config_from_hf`, `build_bridge_from_module`, `detect_tokenizer_bos_eos` | -| [`transformers.py`](transformers.py) | HuggingFace backend: `boot` (entry point used by `TransformerBridge.boot_transformers`), `map_default_transformer_lens_config` (HF→TL config translation), `check_model_support`, `list_supported_models` | -| [`native/model.py`](native/model.py), [`native/init.py`](native/init.py) | TL-native models built from scratch (no HF), used by `TransformerBridge.boot_native` | +| [`../driver_protocol.py`](../driver_protocol.py) | The `Driver` protocol: `forward(input_ids, capture=, intervene=, max_new_tokens=, return_logits=) → ForwardResult`, `close()`, `supports(feature)`, declared `supported_hook_points` / `non_fireable_hook_points`. Also `to_torch` (torch → DLPack → numpy ladder) and `validate_driver` (run at both bridge constructors). | +| [`_driver_base.py`](_driver_base.py) | `DriverBase` — optional ABC with defaults for the protocol's members (`supports`, `close`, `provides_sequence_logits`). Duck-typing works too. | +| [`_hf_format.py`](_hf_format.py) | HF-*format* (not HF-loading) utilities shared by every source whose backend yields an HF-shaped config/tokenizer: `map_default_transformer_lens_config`, `determine_architecture_from_hf_config`, `setup_tokenizer` | +| [`_bridge_builder.py`](_bridge_builder.py) | Loader-agnostic assembly: `build_bridge_config_from_hf`, `build_bridge_from_module`, `detect_tokenizer_bos_eos`; hosts the single `_HF_PASSTHROUGH_ATTRS` list (lines 22–58) | +| [`transformers_driver.py`](transformers_driver.py) | `TransformersDriver` — wraps an HF `nn.Module`; the reference torch driver (gradients, parameters, state_dict, weight access, intervention callbacks) | +| [`transformers/`](transformers/) | HF backend: `source.py` has `boot` (→ `TransformerBridge.boot_transformers`), `helpers.py` has `check_model_support` / `list_supported_models` / checkpoint-revision resolution; `__init__.py` re-exports the historical `sources.transformers` names | +| [`native/`](native/) | TL-native models built from scratch (no HF): `model.py` / `init.py`, booted via `TransformerBridge.boot_native` | +| [`vllm/`](vllm/) | vLLM backend: `source.py` has `boot_vllm` (→ `RemoteBridge.boot_vllm`); `plugin.py` / `worker_extension.py` install capture hooks in the worker pre-compile; `overlays/` declare per-arch capture specs; `internals.py` pins the validated vLLM version | +| [`inspect/`](inspect/) | inspect_ai backend: `source.py` has `boot_inspect` (→ `RemoteBridge.boot_inspect`); ships two providers (`transformers_provider.py` HF-backed, `vllm_provider.py`) plus `eval.py` helpers for capture inside evals. Package name shadows stdlib `inspect` — never `import inspect` bare inside it | -## ⚠ The duplicate `_HF_PASSTHROUGH_ATTRS` trap +## The Driver system -There are **two** identical `_HF_PASSTHROUGH_ATTRS` lists: +Every backend satisfies the `Driver` protocol in [`../driver_protocol.py`](../driver_protocol.py): the bridge calls `driver.forward(...)` and reads a `ForwardResult` (logits, captured activations, raw engine output); hook installation is the driver's problem. Drivers declare which canonical hook names they can fire (`supported_hook_points`) and which they structurally cannot (`non_fireable_hook_points`); `validate_driver` enforces the contract at bridge construction. -- `transformers.py:481` — inside `map_default_transformer_lens_config`, copies HF-config attrs onto `tl_config`. -- `_bridge_builder.py:18` — module-level, copies HF-config attrs onto `bridge_config` AFTER `TransformerBridgeConfig.from_dict` runs. +Implementations: -Both fire sequentially on the same `hf_config` during `boot_transformers`. **When adding a passthrough attr, add it to BOTH lists** — adding to only one half-fixes the bug. A previous regen-agent run shipped a half-fix that the regression test caught only because we also added an assertion on the canonical bridge config. +- `TransformersDriver` — local HF `nn.Module`; full hooks + gradients. Built for you by `TransformerBridge.boot_transformers` (and by `build_bridge_from_module` / `boot_native`). +- `VLLMDriver` ([`vllm/driver.py`](vllm/driver.py)) — high-throughput capture + declarative affine interventions on a vLLM engine; no gradients, no attention patterns. +- `InspectDriver` ([`inspect/driver.py`](inspect/driver.py)) — capture/intervene through an `inspect_ai` provider, for interp inside evals. -## Two-pass config-translation pipeline +Boot entry points: `TransformerBridge.boot_transformers` / `TransformerBridge.boot_native` (torch bridges), `RemoteBridge.boot_vllm` / `RemoteBridge.boot_inspect` (no local `nn.Module`; lazy imports so the heavy backend is only needed by its callers). The `boot_*` methods are attached to the bridge classes by each source's `__init__.py` via `setattr`. + +## `_HF_PASSTHROUGH_ATTRS` — one list, one copy point + +There is exactly **one** `_HF_PASSTHROUGH_ATTRS` list, at [`_bridge_builder.py:22-58`](_bridge_builder.py). Do not re-introduce a second copy elsewhere (an older layout had a duplicate in a since-deleted `transformers.py`; it's gone on purpose). + +Config-translation pipeline (`build_bridge_config_from_hf`): ``` hf_config │ - ├─ map_default_transformer_lens_config() [transformers.py] - │ ├─ explicit handlers (d_model, n_heads, head_dim → d_head, etc.) - │ └─ _HF_PASSTHROUGH_ATTRS copy → tl_config + ├─ map_default_transformer_lens_config() [_hf_format.py] + │ └─ explicit handlers (d_model, n_heads, head_dim → d_head, etc.) → tl_config │ ├─ TransformerBridgeConfig.from_dict() [filters to declared fields] │ - └─ _HF_PASSTHROUGH_ATTRS copy → bridge_config [_bridge_builder.py] + └─ _HF_PASSTHROUGH_ATTRS copy → bridge_config [_bridge_builder.py:78] ``` **Implications:** - If an attr has an explicit handler in `map_default_transformer_lens_config` (like `head_dim` → `d_head`), it's translated to a declared `TransformerBridgeConfig` field — adding it to PASSTHROUGH is **wrong** (often raises `AttributeError` from a read-only property). -- If an attr is purely runtime / adapter-specific (like Cohere's `logit_scale`), it has no explicit handler and is dropped by `from_dict` — it MUST be in both PASSTHROUGH lists, or the adapter's `getattr(cfg, "", default)` silently falls back to its default forever. +- If an attr is purely runtime / adapter-specific (like Cohere's `logit_scale`), it has no explicit handler and is dropped by `from_dict` — it MUST be in the PASSTHROUGH list, or the adapter's `getattr(cfg, "", default)` silently falls back to its default forever. - See [config/AGENTS.md](../../config/AGENTS.md) for the decision tree. ## `boot_transformers` vs `boot_native` -- `boot_transformers` is the HF path. Goes through `boot()` in `transformers.py` → `build_bridge_from_module` in `_bridge_builder.py` → adapter init. +- `boot_transformers` is the HF path. Goes through `boot()` in [`transformers/source.py`](transformers/source.py) → `build_bridge_config_from_hf` in `_bridge_builder.py` → adapter init → `TransformersDriver`. - `boot_native` builds a TL-native transformer from a `TransformerBridgeConfig` directly (no HF dependency). Uses [`native.py`](../supported_architectures/native.py) as the single adapter; the model class is in [`native/model.py`](native/model.py). Almost every contributor change goes through the `boot_transformers` path. Touch `native/` only when adding a primitive (new norm type, new positional embedding variant) that the cfg-driven dispatch in `native/model.py` doesn't already cover. diff --git a/transformer_lens/model_bridge/sources/_bridge_builder.py b/transformer_lens/model_bridge/sources/_bridge_builder.py index 040c4b6485..13faea9eb3 100644 --- a/transformer_lens/model_bridge/sources/_bridge_builder.py +++ b/transformer_lens/model_bridge/sources/_bridge_builder.py @@ -108,6 +108,19 @@ def detect_tokenizer_bos_eos(tokenizer: Any) -> tuple[bool, bool]: return prepends_bos, appends_eos +def configure_tokenizer(tokenizer: Any, cfg: Any) -> Any: + """Shared boot step: normalize the tokenizer and record its BOS/EOS behavior on cfg. + + Every source must run this — skipping it leaves the dataclass default + ``tokenizer_prepends_bos=True``, which position-shifts every activation on + non-BOS-prepending tokenizers (Qwen family).""" + tokenizer = setup_tokenizer( + tokenizer, default_padding_side=getattr(cfg, "default_padding_side", None) + ) + cfg.tokenizer_prepends_bos, cfg.tokenizer_appends_eos = detect_tokenizer_bos_eos(tokenizer) + return tokenizer + + def build_bridge_from_module( model: nn.Module, architecture: str, @@ -196,12 +209,7 @@ def build_bridge_from_module( adapter.prepare_model(model) if tokenizer is not None: - default_padding_side = getattr(adapter.cfg, "default_padding_side", None) - tokenizer = setup_tokenizer(tokenizer, default_padding_side=default_padding_side) - ( - adapter.cfg.tokenizer_prepends_bos, - adapter.cfg.tokenizer_appends_eos, - ) = detect_tokenizer_bos_eos(tokenizer) + tokenizer = configure_tokenizer(tokenizer, adapter.cfg) from transformer_lens.model_bridge.sources.transformers_driver import ( TransformersDriver, diff --git a/transformer_lens/model_bridge/sources/inspect/_provider_base.py b/transformer_lens/model_bridge/sources/inspect/_provider_base.py index 6248bcbafe..55013994c6 100644 --- a/transformer_lens/model_bridge/sources/inspect/_provider_base.py +++ b/transformer_lens/model_bridge/sources/inspect/_provider_base.py @@ -18,6 +18,7 @@ import json import re import uuid +import warnings from typing import Any, Mapping import torch @@ -79,6 +80,34 @@ def _parse_tool_calls(text: str) -> list[ToolCall] | None: return calls or None +# Generation-semantics GenerateConfig fields neither provider maps; warn (once per field +# per process) instead of silently ignoring them. +_UNSUPPORTED_GENERATE_FIELDS = ( + "frequency_penalty", + "presence_penalty", + "logit_bias", + "best_of", + "num_choices", +) +_WARNED_UNSUPPORTED: set[str] = set() + + +def _warn_unsupported_config(config: GenerateConfig, provider: str) -> None: + """Warn once per process per set-but-unsupported GenerateConfig field.""" + new = [ + field + for field in _UNSUPPORTED_GENERATE_FIELDS + if getattr(config, field, None) is not None and field not in _WARNED_UNSUPPORTED + ] + if new: + _WARNED_UNSUPPORTED.update(new) + warnings.warn( + f"{provider}: ignoring unsupported GenerateConfig field(s) {new}.", + UserWarning, + stacklevel=3, + ) + + def _require_served(kind: str, served: frozenset[str], note: str, context: str) -> None: """Raise if ``kind`` was gated by the structural self-check — without this the eval path would silently return a derivation (e.g. ``resid_mid``) the driver path excludes.""" @@ -89,6 +118,19 @@ def _require_served(kind: str, served: frozenset[str], note: str, context: str) ) +def _require_interveneable(kind: str, served: frozenset[str], note: str, context: str) -> None: + """``_require_served`` plus the capture-only gate — shared by every provider's + intervention entry point so gated/capture-only kinds fail identically.""" + from .hooks import INTERVENEABLE_KINDS + + _require_served(kind, served, note, context) + if kind not in INTERVENEABLE_KINDS: + raise ValueError( + f"{context}: kind {kind!r} is capture-only " + f"(interveneable: {sorted(INTERVENEABLE_KINDS)})." + ) + + class _InspectModelAPIBase(ModelAPI): """Shared Inspect ModelAPI scaffolding for ``tl_bridge``-style providers. diff --git a/transformer_lens/model_bridge/sources/inspect/driver.py b/transformer_lens/model_bridge/sources/inspect/driver.py index 518036b276..2460b437df 100644 --- a/transformer_lens/model_bridge/sources/inspect/driver.py +++ b/transformer_lens/model_bridge/sources/inspect/driver.py @@ -57,6 +57,7 @@ def __init__(self, model: Any, adapter: Any, tokenizer: Any, profile: Any = None # Background event loop, created lazily on first forward. self._loop: asyncio.AbstractEventLoop | None = None self._loop_thread: threading.Thread | None = None + self._loop_lock = threading.Lock() # guards lazy creation / abandonment self._warned_missing: set[str] = set() # hooks we've already warned were absent def forward( @@ -143,37 +144,63 @@ def _warn_missing(self, missing: list[str]) -> None: def close(self) -> None: log = logging.getLogger("transformer_lens.inspect") - if self._loop is not None: + with self._loop_lock: + loop, self._loop = self._loop, None + thread, self._loop_thread = self._loop_thread, None + if loop is not None: try: - self._loop.call_soon_threadsafe(self._loop.stop) - if self._loop_thread is not None: - self._loop_thread.join(timeout=5) - self._loop.close() + loop.call_soon_threadsafe(loop.stop) + if thread is not None: + thread.join(timeout=5) + loop.close() except Exception as e: log.debug("event-loop teardown failed during close(): %s", e) - self._loop = None - self._loop_thread = None self._model = None # drop the provider reference; the server owns its own lifecycle # ---- helpers ---- def _ensure_loop(self) -> asyncio.AbstractEventLoop: """A private loop on a daemon thread — works even when the caller is already - inside a running loop (Jupyter), unlike asyncio.run().""" - if self._loop is None: - self._loop = asyncio.new_event_loop() - self._loop_thread = threading.Thread( - target=self._loop.run_forever, daemon=True, name="inspect-driver-loop" - ) - self._loop_thread.start() - return self._loop + inside a running loop (Jupyter), unlike asyncio.run(). Locked: unlocked + check-then-create would leak loops under concurrent first calls.""" + loop = self._loop + if loop is not None: # GIL-safe fast path; the lock only guards create/abandon + return loop + with self._loop_lock: + if self._loop is None: + self._loop = asyncio.new_event_loop() + self._loop_thread = threading.Thread( + target=self._loop.run_forever, daemon=True, name="inspect-driver-loop" + ) + self._loop_thread.start() + return self._loop + + def _abandon_loop(self) -> None: + """After a timeout the loop thread is still occupied by the hung forward (cancel + can't interrupt sync work), so every later call would queue behind it and time out. + Abandon it — the daemon thread stops once the hung call returns — and rebuild + lazily on the next forward.""" + with self._loop_lock: + loop, self._loop, self._loop_thread = self._loop, None, None + if loop is not None: + try: + loop.call_soon_threadsafe(loop.stop) # takes effect when the hung call ends + except Exception: + pass + warnings.warn( + "InspectDriver: abandoning the wedged event-loop thread after a provider " + "timeout; a fresh loop will be created on the next forward.", + UserWarning, + stacklevel=3, + ) def _run_coro(self, coro: Any) -> Any: future = asyncio.run_coroutine_threadsafe(coro, self._ensure_loop()) try: return future.result(timeout=_PROVIDER_TIMEOUT_S) # re-raises provider errors except FutureTimeout: - future.cancel() + future.cancel() # best-effort; can't interrupt a sync forward on the loop thread + self._abandon_loop() raise TimeoutError( f"Inspect provider call exceeded {_PROVIDER_TIMEOUT_S:.0f}s " "(set TL_INSPECT_TIMEOUT_S to change) — the remote/provider forward looks " diff --git a/transformer_lens/model_bridge/sources/inspect/eval.py b/transformer_lens/model_bridge/sources/inspect/eval.py index 176d6036a4..006679daef 100644 --- a/transformer_lens/model_bridge/sources/inspect/eval.py +++ b/transformer_lens/model_bridge/sources/inspect/eval.py @@ -73,8 +73,13 @@ async def solve(state: TaskState, generate: Generate) -> TaskState: ) activations = {name_by_wire[wk]: arr for wk, arr in decoded.items()} os.makedirs(output_dir, exist_ok=True) - path = os.path.join(output_dir, f"{state.sample_id}.npz") - np.savez_compressed(path, **activations) + # Epoch in the name — multi-epoch runs reuse sample_ids and would overwrite. + epoch = getattr(state, "epoch", None) + stem = f"{state.sample_id}_epoch{epoch}" if epoch is not None else str(state.sample_id) + path = os.path.join(output_dir, f"{stem}.npz") + # Explicit allow_pickle: plain float arrays never need pickle, and it keeps + # the **activations unpack off numpy 2.3's typed keyword slot. + np.savez_compressed(path, allow_pickle=False, **activations) store().set(store_key, reduce_fn(activations)) store().set(f"{store_key}_path", path) return state diff --git a/transformer_lens/model_bridge/sources/inspect/profiles.py b/transformer_lens/model_bridge/sources/inspect/profiles.py index 225317bb7a..7be2ae40cb 100644 --- a/transformer_lens/model_bridge/sources/inspect/profiles.py +++ b/transformer_lens/model_bridge/sources/inspect/profiles.py @@ -141,4 +141,10 @@ def for_provider(provider: str) -> Any: """Pick the codec for a provider name (the part before ``/`` in get_model).""" if provider.startswith("vllm-lens"): return VLLMLensProfile() - return TLBridgeProfile() + if provider in ("tl_bridge", "tl_bridge_vllm"): + return TLBridgeProfile() + # An unknown provider would otherwise get full-capability codec and NaN downstream. + raise ValueError( + f"No Inspect codec for provider {provider!r}. Known providers: 'tl_bridge', " + "'tl_bridge_vllm', 'vllm-lens*'." + ) diff --git a/transformer_lens/model_bridge/sources/inspect/source.py b/transformer_lens/model_bridge/sources/inspect/source.py index 656d31558b..0d25338dd0 100644 --- a/transformer_lens/model_bridge/sources/inspect/source.py +++ b/transformer_lens/model_bridge/sources/inspect/source.py @@ -13,9 +13,8 @@ from transformer_lens.model_bridge.remote_bridge import RemoteBridge from transformer_lens.model_bridge.sources._bridge_builder import ( build_bridge_config_from_hf, - detect_tokenizer_bos_eos, + configure_tokenizer, ) -from transformer_lens.model_bridge.sources._hf_format import setup_tokenizer from transformer_lens.utilities.hf_utils import get_hf_token from . import profiles @@ -78,13 +77,7 @@ def boot_inspect( if tokenizer is None: tokenizer = AutoTokenizer.from_pretrained(model_name, token=hf_token) # Match boot_transformers' tokenizer setup so to_tokens(str) is token-identical. - tokenizer = setup_tokenizer( - tokenizer, default_padding_side=getattr(adapter.cfg, "default_padding_side", None) - ) - ( - adapter.cfg.tokenizer_prepends_bos, - adapter.cfg.tokenizer_appends_eos, - ) = detect_tokenizer_bos_eos(tokenizer) + tokenizer = configure_tokenizer(tokenizer, adapter.cfg) if provider == "tl_bridge": # The provider's raw HF forward must match boot_transformers' load: same dtype, @@ -93,6 +86,10 @@ def boot_inspect( inspect_kwargs["model_kwargs"] = _provider_model_kwargs( dict(inspect_kwargs.get("model_kwargs", {})), adapter, resolved_dtype, hf_token ) + elif provider == "tl_bridge_vllm": + # Otherwise the provider defaults to the HF-config dtype and bridge_config.dtype + # lies about what the engine actually loaded. + inspect_kwargs["dtype"] = resolved_dtype # memoize=False: inspect_ai caches get_model by name, which would (a) return a stale # model ignoring a changed dtype/kwargs on re-boot and (b) keep weights resident past diff --git a/transformer_lens/model_bridge/sources/inspect/transformers_provider.py b/transformer_lens/model_bridge/sources/inspect/transformers_provider.py index 74ca10b513..fef520df9e 100644 --- a/transformer_lens/model_bridge/sources/inspect/transformers_provider.py +++ b/transformer_lens/model_bridge/sources/inspect/transformers_provider.py @@ -30,7 +30,13 @@ ) from . import hooks, wire -from ._provider_base import _InspectModelAPIBase, _parse_tool_calls, _require_served +from ._provider_base import ( + _InspectModelAPIBase, + _parse_tool_calls, + _require_interveneable, + _require_served, + _warn_unsupported_config, +) # NOT "transformer_lens" — inspect_ai ships a built-in provider by that name (the # reverse direction: serving a HookedTransformer as an Inspect model for generation). @@ -126,17 +132,13 @@ def _generate_capture(self, input: Any, extra_args: Mapping[str, Any], config: G _, _, kind = key.partition(":") _require_served(kind, self._kinds, self._capability_note, f"capture {key!r}") interventions: Mapping[str, Any] = extra_args.get("interventions", {}) - # The driver validates before sending, but this direct interface (extra_args) is - # documented — validate here too so a gated/capture-only kind fails loud instead - # of silently no-op'ing when no hook installs for it. + # extra_args is a documented surface: gated/capture-only kinds must fail here, + # not silently no-op when no hook installs for them. for key in interventions: _, _, kind = key.partition(":") - _require_served(kind, self._kinds, self._capability_note, f"intervention {key!r}") - if kind not in hooks.INTERVENEABLE_KINDS: - raise ValueError( - f"intervention {key!r}: kind {kind!r} is capture-only " - f"(interveneable: {sorted(hooks.INTERVENEABLE_KINDS)})." - ) + _require_interveneable( + kind, self._kinds, self._capability_note, f"intervention {key!r}" + ) want_logits = bool(extra_args.get("return_logits", True)) capture, intervene = _plan(capture_keys, interventions) @@ -189,6 +191,7 @@ def _generate_eval(self, input: Any, config: GenerateConfig, tools: Any): """Plain Inspect generation: HF generate from the chat input (rendering ``tools`` into the template), honoring max_tokens/sampling, with optional per-token logprobs, token usage, parsed tool calls, and per-turn activation capture (agent rollouts).""" + _warn_unsupported_config(config, PROVIDER_NAME) ids = self._messages_to_ids(input, tools) prompt_len = int(ids.shape[1]) max_new = int(config.max_tokens) if config.max_tokens else 16 @@ -201,6 +204,10 @@ def _generate_eval(self, input: Any, config: GenerateConfig, tools: Any): "output_scores": True, "pad_token_id": self._tokenizer.pad_token_id or self._tokenizer.eos_token_id, } + if config.stop_seqs: + # transformers turns stop_strings into a StopStringCriteria; needs the tokenizer. + gen["stop_strings"] = list(config.stop_seqs) + gen["tokenizer"] = self._tokenizer if temperature is not None and temperature > 0: gen["temperature"] = float(temperature) if config.top_p is not None: @@ -300,6 +307,8 @@ def _install_hooks(self, capture, intervene, raw, call_id: str) -> list: continue attn = _first_attr(block, _ATTN_ATTRS) _, mlp_out_mod = _locate_mlp(block) + # Capabilities were probed on layer 0; hybrid archs can lack the module here. + _require_layer_modules(layer, attn, mlp_out_mod, cap_kinds, iv_kinds) if "resid_pre" in cap_kinds or "resid_pre" in iv_kinds: handles.append( block.register_forward_pre_hook( @@ -407,6 +416,26 @@ def _install_head_hooks(self, layer, attn, cap_kinds, iv_kinds, raw, call_id: st return handles +def _require_layer_modules( + layer: int, attn: Any, mlp_out_mod: Any, cap_kinds: Any, iv_kinds: Any +) -> None: + """Fail loud when this layer lacks a targeted submodule — detection ran on layer 0, + so hybrid attn/SSM stacks would otherwise install nothing and silently no-op.""" + wanted = set(cap_kinds) | set(iv_kinds) + if attn is None: + needs_attn = wanted & ({"attn_out"} | (hooks.HEAD_KINDS - {"pattern"})) + if needs_attn: + raise RuntimeError( + f"blocks.{layer} has no attention submodule: cannot serve " + f"{sorted(needs_attn)} at layer {layer} (heterogeneous layers)." + ) + if mlp_out_mod is None and "mlp_out" in wanted: + raise RuntimeError( + f"blocks.{layer} has no MLP submodule: cannot serve mlp_out at layer " + f"{layer} (heterogeneous layers)." + ) + + def _plan(capture_keys, interventions): """Resolve wire keys → per-layer kinds to capture (resid_mid needs pre+attn) and intervene.""" capture: dict[int, set[str]] = defaultdict(set) diff --git a/transformer_lens/model_bridge/sources/inspect/vllm_provider.py b/transformer_lens/model_bridge/sources/inspect/vllm_provider.py index f04abb15a8..43f4c59843 100644 --- a/transformer_lens/model_bridge/sources/inspect/vllm_provider.py +++ b/transformer_lens/model_bridge/sources/inspect/vllm_provider.py @@ -36,7 +36,13 @@ ) from . import hooks, wire -from ._provider_base import _InspectModelAPIBase, _parse_tool_calls, _require_served +from ._provider_base import ( + _InspectModelAPIBase, + _parse_tool_calls, + _require_interveneable, + _require_served, + _warn_unsupported_config, +) # Distinct from the HF ``tl_bridge`` provider and from inspect_ai's built-in ``vllm``. PROVIDER_NAME = "tl_bridge_vllm" @@ -96,7 +102,15 @@ def __init__( ) -> None: super().__init__(model_name, base_url, api_key, [], config) from transformers import AutoConfig, AutoTokenizer - from vllm import LLM + + try: + from vllm import LLM + except ImportError as exc: + raise ImportError( + "The tl_bridge_vllm provider requires vLLM (Linux + CUDA). Install with " + 'pip install "transformer-lens[vllm]" or uv sync --extra vllm; ' + "validated against vllm 0.20.x." + ) from exc from transformer_lens.utilities.hf_utils import get_hf_token @@ -146,25 +160,28 @@ def __init__( # isn't visible to worker subprocesses and hooks silently fail to install. os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0" - self._llm = LLM( - model=model_name, - gpu_memory_utilization=gpu_memory_utilization, - max_model_len=max_model_len, - max_num_batched_tokens=max_num_batched_tokens, - # Worker extension's tl_* methods are reachable via collective_rpc. - worker_extension_cls=_WORKER_EXTENSION_CLS, - # Full-vocab logprobs so _generate_capture can synthesize logits at the - # generated position (vLLM caps logprobs to this value; default 20 is too - # small for mech interp). - max_logprobs=int(hf_config_preview.vocab_size), - dtype=str(resolved_dtype).replace("torch.", "") if dtype is not None else "auto", - **_LOCKED_VLLM_KWARGS, - **vllm_kwargs, - ) + try: + self._llm = LLM( + model=model_name, + gpu_memory_utilization=gpu_memory_utilization, + max_model_len=max_model_len, + max_num_batched_tokens=max_num_batched_tokens, + # Worker extension's tl_* methods are reachable via collective_rpc. + worker_extension_cls=_WORKER_EXTENSION_CLS, + # Full-vocab logprobs so _generate_capture can synthesize logits at the + # generated position (vLLM caps logprobs to this value; default 20 is too + # small for mech interp). + max_logprobs=int(hf_config_preview.vocab_size), + dtype=str(resolved_dtype).replace("torch.", "") if dtype is not None else "auto", + **_LOCKED_VLLM_KWARGS, + **vllm_kwargs, + ) + finally: + # Always clear, even on a failed boot — stale specs would make the next + # in-process vllm.LLM(...) walk our dot-paths on a foreign model. + plugin._config.clear() # Capture-path validity was enforced inside patched_load_model during LLM(...). hf_config = extract_hf_config(self._llm) - # Don't leak our specs to a subsequent non-TL vllm.LLM(...) in the same process. - plugin._config.clear() # Capture-relevant constants used by _generate_capture. self._d_vocab = int(hf_config.vocab_size) @@ -218,6 +235,17 @@ def _generate_capture(self, input: Any, extra_args: Mapping[str, Any], config: G interventions: Mapping[str, Any] = extra_args.get("interventions", {}) intervention_specs: dict[str, Any] = {} for wk, spec in interventions.items(): + # extra_args is a documented surface: gated/capture-only kinds must fail + # here, not deep in the worker. + _, _, kind = wk.partition(":") + _require_interveneable(kind, self._kinds, self._capability_note, f"intervention {wk!r}") + if isinstance(spec, Mapping) and spec.get("pos") is not None: + raise ValueError( + f"intervention {wk!r}: per-position 'pos' is not supported on the " + "tl_bridge_vllm provider (its worker runs without position " + "interventions). Use boot_inspect(provider='tl_bridge') for " + "position-targeted patching." + ) name = hooks.name_from_wire_key(wk) if name is None: raise ValueError(f"intervention wire key {wk!r} is not a fireable hook.") @@ -279,6 +307,8 @@ def _generate_eval(self, input: Any, config: GenerateConfig, tools: Any): from vllm import SamplingParams from vllm.inputs import TokensPrompt + _warn_unsupported_config(config, PROVIDER_NAME) + # Worker-side intervention buffers are persistent: any prior capture-path call that # pushed specs (e.g. bridge.forward(intervene=...)) would still be applied here # without a reset. The HF provider installs hooks per-call so it's leak-immune. @@ -296,6 +326,8 @@ def _generate_eval(self, input: Any, config: GenerateConfig, tools: Any): temperature = float(config.temperature) if config.temperature is not None else 0.0 sp_kwargs: dict[str, Any] = {"max_tokens": max_new, "temperature": temperature} + if config.stop_seqs: + sp_kwargs["stop"] = list(config.stop_seqs) if temperature > 0: if config.top_p is not None: sp_kwargs["top_p"] = float(config.top_p) diff --git a/transformer_lens/model_bridge/sources/transformers/source.py b/transformer_lens/model_bridge/sources/transformers/source.py index ce0c7def5b..4b358e28a6 100644 --- a/transformer_lens/model_bridge/sources/transformers/source.py +++ b/transformer_lens/model_bridge/sources/transformers/source.py @@ -57,7 +57,17 @@ def boot( n_devices: int | None = None, max_memory: dict[str | int, str] | None = None, ) -> TransformerBridge: - """Boot a model from HuggingFace. + """Boot a model from HuggingFace (exposed as ``TransformerBridge.boot_transformers``). + + Returns raw HF weights by default — logits/activations match HF, *not* + legacy ``HookedTransformer`` (which folds LayerNorm + centers weights). + Call ``enable_compatibility_mode()`` on the result for HookedTransformer- + equivalent numerics. Generation, argmax, and CE loss are unaffected. + + Attention implementation is forced to ``"eager"`` so hooks can capture scores + and patterns. For an apples-to-apples HF comparison, load the HF model with + ``attn_implementation="eager"`` too; comparing against the default ``"sdpa"`` + shows ~1e-3 fp32 drift from kernel-level op reordering, not a bridge bug. Args: model_name: The name of the model to load. diff --git a/transformer_lens/model_bridge/sources/transformers_driver.py b/transformer_lens/model_bridge/sources/transformers_driver.py index c745788e2b..bff87ae6b9 100644 --- a/transformer_lens/model_bridge/sources/transformers_driver.py +++ b/transformer_lens/model_bridge/sources/transformers_driver.py @@ -37,6 +37,25 @@ def forward( return_logits: bool = True, **kwargs: Any, ) -> ForwardResult: + # Module-replacement dialect: these args are served by the bridge's + # HookPoints, not here — silently ignoring them would be a lie. + if capture: + raise NotImplementedError( + "TransformersDriver.forward does not serve capture=: on the HF " + "backend activations are captured through the bridge's HookPoint " + "system — use run_with_cache()/run_with_hooks() on the bridge." + ) + if intervene is not None: + raise NotImplementedError( + "TransformersDriver.forward does not serve intervene=: on the HF " + "backend interventions are torch hooks — use run_with_hooks() on " + "the bridge." + ) + if max_new_tokens != 1: + raise NotImplementedError( + "TransformersDriver.forward does not generate; use " + "bridge.generate() for multi-token decoding." + ) if input_ids is not None: raw = self._model(input_ids, **kwargs) else: diff --git a/transformer_lens/model_bridge/sources/vllm/driver.py b/transformer_lens/model_bridge/sources/vllm/driver.py index 91e0270dce..aec008f45d 100644 --- a/transformer_lens/model_bridge/sources/vllm/driver.py +++ b/transformer_lens/model_bridge/sources/vllm/driver.py @@ -3,6 +3,8 @@ import gc import logging +import threading +import warnings from typing import Any, Mapping import torch @@ -14,7 +16,13 @@ ) from transformer_lens.model_bridge.sources._driver_base import DriverBase -from .intervention_specs import SUPPORTED_OPS +from .intervention_specs import validate_spec + +# vLLM's distributed teardown operates on process-wide globals, so close() may only run +# it when no other VLLMDriver-owned engine is alive (notebook re-binding boots B before +# dropping A; A's __del__ must not destroy the process groups B is using). +_LIVE_DRIVERS = 0 +_LIVE_DRIVERS_LOCK = threading.Lock() class VLLMDriver(DriverBase): @@ -52,8 +60,22 @@ def __init__( # Logprobs per forward = real vocab (boot's max_logprobs). d_vocab can be # padded larger, which vLLM would reject; the logits tensor stays d_vocab. self._n_logprobs = int(getattr(hf_config, "vocab_size", self.bridge_config.d_vocab)) - - self.supported_hook_points = frozenset(overlay.capture_specs(hf_config).keys()) + # Unembedding cache: (weight_fp32, bias_fp32|None) once probe_logit_reconstruction + # runs — a per-forward re-fetch clones a d_vocab×d_model tensor to CPU every call. + self._unembed: tuple[torch.Tensor, Any] | None = None + self._unembed_probed = False + # fp32 reciprocal of the guarded final-norm weight; None after a failed probe. + self._lnf_inv_denom: torch.Tensor | None = None + self._lnf_probed = False + + capture_specs = overlay.capture_specs(hf_config) + self._hook_widths = {name: width for name, (_path, width) in capture_specs.items()} + self._capture_paths = {name: path for name, (path, _width) in capture_specs.items()} + self.supported_hook_points = frozenset(capture_specs.keys()) + + global _LIVE_DRIVERS + with _LIVE_DRIVERS_LOCK: + _LIVE_DRIVERS += 1 n_layers = getattr(hf_config, "num_hidden_layers", 0) if not isinstance(n_layers, int) or n_layers <= 0: @@ -89,6 +111,9 @@ def forward( "overwrite the prefill buffer; multi-step capture is multi-buffer work." ) intervene_specs = self._validate_interventions(intervene or {}) + # Pad tokens fed to vLLM are real content to it (no mask concept) — honor a + # caller-supplied mask by trimming rows to their true lengths, never swallow it. + attention_mask = kwargs.pop("attention_mask", None) # capture is authoritative — the bridge sends exactly the hooked names, so () # means "capture nothing" and a plain forward(tokens) skips the GPU→CPU copy @@ -97,9 +122,14 @@ def forward( names = list(capture) if self._enable_batching: - return self._forward_batched(input_ids, intervene_specs, return_logits, names) + return self._forward_batched( + input_ids, intervene_specs, return_logits, names, attention_mask + ) ids_list = self._normalize_input_ids(input_ids) + if attention_mask is not None: + n_real = int(torch.as_tensor(attention_mask).sum()) + ids_list = ids_list[:n_real] if len(ids_list) > self._max_num_batched_tokens: # Worker buffers silently clamp on overflow — fail loud here instead. raise ValueError( @@ -121,22 +151,17 @@ def forward( # subsequent forwards self-copy. Without this, repeated bridge.forward calls would # see the gate closed from the prior call and read stale buffers. self._llm.collective_rpc("tl_reset_capture_flags") - # Full-vocab logprobs → position -1 of the synthesized logits (see _n_logprobs). outputs = self._llm.generate( prompts=[TokensPrompt(prompt_token_ids=ids_list)], sampling_params=SamplingParams( max_tokens=int(max_new_tokens), temperature=0.0, - logprobs=self._n_logprobs if return_logits else None, + logprobs=self._sampler_logprobs(return_logits), ), ) n_tokens = len(ids_list) - # Reconstructing logits needs ln_final; force it into the read even if the caller - # didn't request it (dropped from `captured` below so the surface stays as asked). - read_names = names - if return_logits and self._LN_FINAL not in names: - read_names = names + [self._LN_FINAL] + read_names = self._read_names(names, return_logits) # collective_rpc returns one result per worker; single-rank, so [0]. Nothing to # read (no captures, logits off) → skip the crossing altogether. worker_captures = ( @@ -155,16 +180,41 @@ def forward( else self._synthesize_logits(outputs[0], n_tokens, self.bridge_config.d_vocab) ) - # Add batch dim; expose only the caller's requested hooks (drop the forced ln_final). - captured = {name: t.unsqueeze(0) for name, t in worker_captures.items() if name in names} + captured = self._expose_captured( + {name: t.unsqueeze(0) for name, t in worker_captures.items()}, names + ) return ForwardResult(logits=logits, captured=captured, raw_output=outputs[0]) + def _sampler_logprobs(self, return_logits: bool) -> int | None: + # Sampler logprobs are the reconstruction fallback only; skipping them avoids + # marshaling a d_vocab-entry Logprob dict host-side per request. + return self._n_logprobs if return_logits and self._unembed is None else None + + def _read_names(self, names: list[str], return_logits: bool) -> list[str]: + # Reconstruction needs ln_final even when uncaptured; skip once probed-unavailable. + recon_possible = self._unembed is not None or not self._unembed_probed + if return_logits and recon_possible and self._LN_FINAL not in names: + return names + [self._LN_FINAL] + return names + + def _expose_captured( + self, captured: Mapping[str, torch.Tensor], names: list[str] + ) -> dict[str, torch.Tensor]: + """Filter to the caller's requested hooks (dropping any forced ln_final) and + convert the exposed ln_final to the pre-weight convention its name promises — + reconstruction consumed the raw post-weight value already.""" + out = {name: t for name, t in captured.items() if name in names} + if self._LN_FINAL in out: + out[self._LN_FINAL] = self._unfold_ln_final(out[self._LN_FINAL]) + return out + def _forward_batched( self, input_ids: TensorLike, intervene_specs: dict, return_logits: bool, names: list[str], + attention_mask: Any = None, ) -> ForwardResult: """Eager batched path: per-request capture, right-padded to (B, S, W). @@ -176,6 +226,18 @@ def _forward_batched( from vllm.inputs import TokensPrompt prompts_ids = self._normalize_input_ids_batched(input_ids) + if attention_mask is not None: + # Right-padded tensor batches carry pad ids vLLM would treat as content; + # trim each row to its masked length so per-row final positions are real. + mask = torch.as_tensor(attention_mask) + if mask.dim() == 1: + mask = mask.unsqueeze(0) + if mask.shape[0] != len(prompts_ids): + raise ValueError( + f"attention_mask batch dim {mask.shape[0]} != number of prompts " + f"{len(prompts_ids)}." + ) + prompts_ids = [ids[: int(row.sum())] for ids, row in zip(prompts_ids, mask)] prompt_lens = [len(ids) for ids in prompts_ids] # Reset accumulators so prior-forward chunks don't leak into the cat. @@ -189,15 +251,11 @@ def _forward_batched( sampling_params=SamplingParams( max_tokens=1, temperature=0.0, - logprobs=self._n_logprobs if return_logits else None, + logprobs=self._sampler_logprobs(return_logits), ), ) - # Force ln_final into the read so logits can be reconstructed (dropped below if - # the caller didn't ask for it). - read_names = names - if return_logits and self._LN_FINAL not in names: - read_names = names + [self._LN_FINAL] + read_names = self._read_names(names, return_logits) # Keyed by req_id (no guaranteed order) — _assemble_padded joins to slot # k via outputs[k].request_id, not by position. Empty read → skip both the # crossing AND the join (_assemble_padded requires one worker key per request, @@ -215,13 +273,18 @@ def _forward_batched( recon = self._reconstruct_logits( captured.get(self._LN_FINAL) ) # (batch, max_seq, d_vocab) + if recon is not None: + # Pad rows reconstruct from zero-filled ln_final into finite garbage + # (0 @ W = plausible uniform logits) — mask them to the -inf convention + # the fallback and per-row consumers rely on. + for k, n in enumerate(prompt_lens): + recon[k, n:] = float("-inf") logits = ( recon if recon is not None else self._synthesize_logits_batched(outputs, prompt_lens, d_vocab) ) - if self._LN_FINAL not in names: - captured.pop(self._LN_FINAL, None) + captured = self._expose_captured(captured, names) return ForwardResult(logits=logits, captured=captured, raw_output=outputs) @@ -304,63 +367,126 @@ def get_param(self, dotted_name: str) -> torch.Tensor | None: return None return self._llm.collective_rpc("tl_get_param", args=(dotted_name,))[0] + def probe_logit_reconstruction(self) -> bool: + """One-time unembedding fetch + cache; downgrades ``provides_sequence_logits`` + honestly when no unembedding is reachable (the fallback path is final-position + log-probs, which cannot back a loss). Idempotent — later calls return the + cached availability.""" + if self._unembed_probed: + return self._unembed is not None + self._unembed_probed = True + weight = self.get_param("lm_head.weight") + if weight is None: # tied embeddings expose no separate lm_head + weight = self.get_param("model.embed_tokens.weight") + if weight is None: + self.provides_sequence_logits = False + return False + bias = self.get_param("lm_head.bias") + d_vocab = int(self.bridge_config.d_vocab) + # Slice vLLM's vocab-pad rows at cache time; fp32 residency (~2× checkpoint + # dtype on CPU) trades memory for skipping a full-matrix upcast per forward. + self._unembed = ( + weight.to(torch.float32)[:d_vocab], + bias.to(torch.float32)[:d_vocab] if bias is not None else None, + ) + self.provides_sequence_logits = True + return True + def _reconstruct_logits(self, ln_final: Any) -> torch.Tensor | None: """Rebuild real logits from the captured post-weight ln_final: ``ln_final @ lm_head.weight.T`` (+ bias, + Gemma-family tanh soft-cap). vLLM's ``ln_final.hook_normalized`` is the POST-weight RMSNorm value lm_head consumes (verified empirically: it equals HF's pre-weight value times the norm - weight), so no un-fold is needed. Accepts any ``(..., d_model)`` tensor and returns - ``(..., d_vocab)`` on CPU. ``None`` if ln_final wasn't captured or no unembedding - weight is fetchable — the caller then falls back to the sampler's log-probs. + weight), so no un-fold is needed here — the raw worker value feeds this directly. + Accepts any ``(..., d_model)`` tensor and returns ``(..., d_vocab)`` on CPU. + ``None`` if ln_final wasn't captured or no unembedding weight is fetchable — + the caller then falls back to the sampler's log-probs. """ - if ln_final is None: + if ln_final is None or not self.probe_logit_reconstruction(): return None - weight = self.get_param("lm_head.weight") - if weight is None: # tied embeddings expose no separate lm_head - weight = self.get_param("model.embed_tokens.weight") - if weight is None: - return None - lf = ln_final.to(device=weight.device, dtype=torch.float32) - logits = lf @ weight.to(torch.float32).T - bias = self.get_param("lm_head.bias") - if bias is not None: - logits = logits + bias.to(device=logits.device, dtype=torch.float32) + assert self._unembed is not None # probe returned True + weight32, bias32 = self._unembed + lf = ln_final.to(device=weight32.device, dtype=torch.float32) + logits = lf @ weight32.T + if bias32 is not None: + logits = logits + bias32.to(device=logits.device) + d_vocab = int(self.bridge_config.d_vocab) + if logits.shape[-1] > d_vocab: + # vLLM pads vocab embeddings to a multiple of 64; its own sampler slices to + # org_vocab_size before sampling — mirror that, or pad columns (zero-filled + # at load) become phantom argmax candidates and bias softmax denominators. + logits = logits[..., :d_vocab] cap = getattr(self.bridge_config, "output_logits_soft_cap", None) if cap is not None and cap > 0: # Gemma-family cap; -1.0 is the "disabled" sentinel logits = float(cap) * torch.tanh(logits / float(cap)) - d_vocab = int(self.bridge_config.d_vocab) if logits.shape[-1] < d_vocab: # pad the padded-vocab tail (never predicted) pad = logits.new_full((*logits.shape[:-1], d_vocab - logits.shape[-1]), float("-inf")) logits = torch.cat([logits, pad], dim=-1) return logits.cpu() + def _unfold_ln_final(self, t: torch.Tensor) -> torch.Tensor: + """Convert vLLM's post-weight RMSNorm capture to the pre-weight value the + canonical hook name promises (÷ weight; Gemma folds ``1 + weight``). Warns + once and serves the raw value when the norm weight is unreachable — loud + beats silent cross-backend mismatch.""" + if not self._lnf_probed: + self._lnf_probed = True + # The overlay's capture spec owns the module path; derive the weight from it. + path = self._capture_paths.get(self._LN_FINAL, "model.norm") + weight = self.get_param(f"{path}.weight") + if weight is None: + warnings.warn( + "ln_final.hook_normalized: norm weight unreachable — the captured " + "value stays POST-weight and will not match boot_transformers.", + UserWarning, + stacklevel=3, + ) + else: + w = weight.detach().to(torch.float32) + denom = (1.0 + w) if "gemma" in self.architecture.lower() else w + # Near-zero weight entries would blow up the division; identity beats inf. + denom = torch.where(denom.abs() < 1e-6, torch.ones_like(denom), denom) + self._lnf_inv_denom = denom.reciprocal() + if self._lnf_inv_denom is None: + return t + inv = self._lnf_inv_denom.to(device=t.device) + return (t.to(torch.float32) * inv).to(t.dtype) + def close(self) -> None: + if self._llm is None: # already closed — keep the refcount single-shot + return # Detach hooks before dropping the LLM so they don't stay registered on # worker modules for the life of the process (long-running notebooks). log = logging.getLogger("transformer_lens.vllm") - if self._llm is not None: - try: - self._llm.collective_rpc("tl_remove_hooks") - except Exception as e: - # Best-effort: engine may already be torn down or the RPC surface - # gone. Log so hook-leak debugging has a thread to pull. - log.debug("tl_remove_hooks failed during close(): %s", e) - self._llm = None - # vLLM 0.20.2 has no LLM.shutdown() — model weights and KV cache stay - # resident until process exit unless we explicitly tear down the - # distributed environment vLLM set up at construction. Both calls are - # best-effort: they're no-ops if there's no distributed state. try: - from vllm.distributed.parallel_state import ( - destroy_distributed_environment, - destroy_model_parallel, - ) - - destroy_model_parallel() - destroy_distributed_environment() + self._llm.collective_rpc("tl_remove_hooks") except Exception as e: - log.debug("vLLM distributed teardown failed during close(): %s", e) + # Best-effort: engine may already be torn down or the RPC surface + # gone. Log so hook-leak debugging has a thread to pull. + log.debug("tl_remove_hooks failed during close(): %s", e) + self._llm = None + self._unembed = None + self._lnf_inv_denom = None + + global _LIVE_DRIVERS + with _LIVE_DRIVERS_LOCK: + _LIVE_DRIVERS -= 1 + last_driver = _LIVE_DRIVERS <= 0 + # vLLM 0.20.2 has no LLM.shutdown(); the distributed teardown below hits + # process-global state, so it runs only for the last live driver (see + # _LIVE_DRIVERS). Both calls are best-effort no-ops otherwise. + if last_driver: + try: + from vllm.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + ) + + destroy_model_parallel() + destroy_distributed_environment() + except Exception as e: + log.debug("vLLM distributed teardown failed during close(): %s", e) # Free the caching allocator's blocks here so the caller doesn't have to. gc.collect() try: @@ -392,40 +518,15 @@ def _synthesize_logits(request_output: Any, n_tokens: int, d_vocab: int) -> torc return logits def _validate_interventions(self, intervene: Mapping[str, Any]) -> dict: - """Reject callables, validate spec format and hook names; return a plain dict.""" + """Shared spec validation plus driver-side gating (hook membership, pos support).""" out: dict = {} for hook_name, spec in intervene.items(): - if callable(spec): - raise NotImplementedError( - "VLLMDriver requires intervention specs (dict), not callables. " - "Supported ops: suppress, scale (factor: float), add (value: scalar or width-shaped), " - "set (value: scalar or width-shaped)." - ) - if not isinstance(spec, Mapping) or "op" not in spec: - raise ValueError( - f"Intervention spec for {hook_name!r} must be a dict with 'op' key; got {spec!r}" - ) - op = spec["op"] - if op not in SUPPORTED_OPS: - raise ValueError( - f"Unsupported intervention op {op!r} for {hook_name!r}. " - f"Supported: {sorted(SUPPORTED_OPS)}." - ) - if op == "scale" and "factor" not in spec: - raise ValueError( - f"Intervention {hook_name!r}: op='scale' requires 'factor' (float)." - ) - if op in ("add", "set") and "value" not in spec: - raise ValueError( - f"Intervention {hook_name!r}: op={op!r} requires 'value' " - "(scalar or width-shaped tensor/list)." - ) + validated = validate_spec(hook_name, spec, width=self._hook_widths.get(hook_name)) if hook_name not in self.supported_hook_points: raise ValueError( f"Cannot intervene on {hook_name!r}: not in supported_hook_points." ) - pos = spec.get("pos") - if pos is not None: + if validated.get("pos") is not None: if self._enable_batching: # The batched/eager path applies ops to the raw tensor, not the # (max_n, width) affine buffers, so it has no position surface. @@ -443,20 +544,7 @@ def _validate_interventions(self, intervene: Mapping[str, Any]) -> dict: "buffers broadcast across all positions). Use the Inspect/HF backend for " "position-scoped patching, or drop 'pos' for a whole-sequence edit." ) - if not ( - isinstance(pos, int) - or (isinstance(pos, (list, tuple)) and all(isinstance(p, int) for p in pos)) - ): - raise ValueError( - f"Intervention {hook_name!r}: 'pos' must be an int or list of ints " - f"(sequence positions to patch); got {pos!r}." - ) - bad = [p for p in ([pos] if isinstance(pos, int) else pos) if p < 0] - if bad: - raise ValueError( - f"Intervention {hook_name!r}: 'pos' must be non-negative; got {bad}." - ) - out[hook_name] = dict(spec) + out[hook_name] = validated return out @staticmethod diff --git a/transformer_lens/model_bridge/sources/vllm/intervention_specs.py b/transformer_lens/model_bridge/sources/vllm/intervention_specs.py index ab35dac72d..4181de8b75 100644 --- a/transformer_lens/model_bridge/sources/vllm/intervention_specs.py +++ b/transformer_lens/model_bridge/sources/vllm/intervention_specs.py @@ -1,9 +1,88 @@ -"""Intervention op vocabulary shared by the driver (spec validation) and the -worker extension (spec → buffer translation). +"""Intervention op vocabulary + spec validation shared by every producer path +(VLLMDriver, the worker RPCs — which also cover the Inspect vLLM provider). -Adding a new op requires (a) listing it here, (b) handling it in -``worker_extension._apply_intervention``. +Adding a new op requires (a) listing it here, (b) allowing its keys in +``validate_spec``, (c) handling it in ``worker_extension._apply_intervention``. """ from __future__ import annotations +from typing import Any, Mapping + +import torch + SUPPORTED_OPS = frozenset({"suppress", "scale", "add", "set"}) + + +def validate_spec(hook_name: str, spec: Any, *, width: int | None = None) -> dict: + """Validate one declarative intervention spec; returns a plain-dict copy. + + ``width`` enables the value-shape check where the caller knows the hook width. + """ + if callable(spec): + raise NotImplementedError( + "vLLM accepts intervention specs (dict), not callables. " + "Supported ops: suppress, scale (factor: float), add (value: scalar or " + "width-shaped), set (value: scalar or width-shaped)." + ) + if not isinstance(spec, Mapping) or "op" not in spec: + raise ValueError( + f"Intervention spec for {hook_name!r} must be a dict with 'op' key; got {spec!r}" + ) + op = spec["op"] + if op not in SUPPORTED_OPS: + raise ValueError( + f"Unsupported intervention op {op!r} for {hook_name!r}. " + f"Supported: {sorted(SUPPORTED_OPS)}." + ) + if op == "scale" and "factor" not in spec: + raise ValueError(f"Intervention {hook_name!r}: op='scale' requires 'factor' (float).") + if op in ("add", "set") and "value" not in spec: + raise ValueError( + f"Intervention {hook_name!r}: op={op!r} requires 'value' " + "(scalar or width-shaped tensor/list)." + ) + # Unknown keys must fail loud: a typo'd "position"/"positions" would otherwise + # silently become a whole-sequence edit. + allowed = {"op", "pos"} + if op == "scale": + allowed.add("factor") + if op in ("add", "set"): + allowed.add("value") + extra = set(spec) - allowed + if extra: + raise ValueError( + f"Intervention {hook_name!r}: unknown spec key(s) {sorted(extra)}; " + f"allowed for op={op!r}: {sorted(allowed)}." + ) + value = spec.get("value") + if value is not None and not isinstance(value, (int, float)): + try: + n_elements = int(torch.as_tensor(value).numel()) + except (TypeError, ValueError, RuntimeError) as exc: + raise ValueError( + f"Intervention {hook_name!r}: 'value' must be a scalar or a " + f"width-shaped tensor/list; got {type(value).__name__}." + ) from exc + if width is not None and n_elements != width: + # A mis-shaped value would otherwise surface as a broadcast error + # mid-forward — or broadcast along the wrong axis in square cases. + raise ValueError( + f"Intervention {hook_name!r}: 'value' has {n_elements} elements " + f"but the hook width is {width}." + ) + pos = spec.get("pos") + if pos is not None: + valid_pos = isinstance(pos, int) and not isinstance(pos, bool) + if not valid_pos and isinstance(pos, (list, tuple)): + valid_pos = all(isinstance(p, int) and not isinstance(p, bool) for p in pos) + if not valid_pos: + raise ValueError( + f"Intervention {hook_name!r}: 'pos' must be an int or list of ints " + f"(sequence positions to patch); got {pos!r}." + ) + negative = [p for p in ([pos] if isinstance(pos, int) else pos) if p < 0] + if negative: + raise ValueError( + f"Intervention {hook_name!r}: 'pos' must be non-negative; got {negative}." + ) + return dict(spec) diff --git a/transformer_lens/model_bridge/sources/vllm/overlays/decoder_only.py b/transformer_lens/model_bridge/sources/vllm/overlays/decoder_only.py index 0ce38efb37..50b2e80d3c 100644 --- a/transformer_lens/model_bridge/sources/vllm/overlays/decoder_only.py +++ b/transformer_lens/model_bridge/sources/vllm/overlays/decoder_only.py @@ -15,11 +15,11 @@ separately (fused-residual). The plugin's hook materializes the sum so the captured value matches HF's "post-MLP residual stream". - ``ln_final.hook_normalized``: vLLM exposes ``x * rsqrt(var+eps) * weight``; - HF/HT exposes the pre-weight value. They are NOT auto-converted — the cache - carries vLLM's post-weight value under this name, so a direct diff against - ``boot_transformers`` will mismatch here. To convert, fetch the weight via - ``bridge._driver.get_param("model.norm.weight")`` and divide the capture by it - (or by ``1 + weight`` for Gemma). + HF/HT exposes the pre-weight value. The driver un-folds the user-facing + capture (÷ weight, or ÷ (1 + weight) for Gemma) so the cache matches + ``boot_transformers``; logit reconstruction consumes the raw post-weight + value internally. If the norm weight is unreachable the driver warns and + serves the raw post-weight value. """ from __future__ import annotations @@ -58,6 +58,17 @@ def nonfiring_hooks(self) -> List[str]: "blocks.{i}.attn.hook_attn_scores", "blocks.{i}.attn.hook_rot_q", "blocks.{i}.attn.hook_rot_k", + # Head-split projections live inside the fused QKVParallelLinear / + # RowParallelLinear kernels — unobservable per-head. + "blocks.{i}.attn.hook_q", + "blocks.{i}.attn.hook_k", + "blocks.{i}.attn.hook_v", + "blocks.{i}.attn.hook_z", + # Block/MLP inputs: vLLM passes (hidden, residual) fused between layers, + # so the HF-convention pre-block/pre-MLP stream is never materialized. + "blocks.{i}.hook_in", + "blocks.{i}.mlp.hook_in", + "blocks.{i}.attn.hook_in", # vLLM's sampler bypasses lm_head.__call__ — capture-via-forward-hook # never fires; driver synthesizes argmax-matching logits from the # sampler's returned token. diff --git a/transformer_lens/model_bridge/sources/vllm/source.py b/transformer_lens/model_bridge/sources/vllm/source.py index 6bb6351103..e8661c4681 100644 --- a/transformer_lens/model_bridge/sources/vllm/source.py +++ b/transformer_lens/model_bridge/sources/vllm/source.py @@ -3,6 +3,7 @@ import logging import os +import threading import warnings from typing import Any, Dict, Optional @@ -14,6 +15,7 @@ from transformer_lens.model_bridge.remote_bridge import RemoteBridge from transformer_lens.model_bridge.sources._bridge_builder import ( build_bridge_config_from_hf, + configure_tokenizer, ) from transformer_lens.utilities.hf_utils import get_hf_token @@ -23,13 +25,22 @@ from .overlays import get_overlay # Forced LLM(...) kwargs that the capture-hook design depends on. Caller override → ValueError. +# enable_prefix_caching MUST stay off: a prefix-cache hit makes the prefill compute only the +# uncached suffix, so hooks fire for a subset of positions — captures land row-misaligned, +# interventions skip cached positions, and an intervened forward writes poisoned K/V that a +# later clean forward on the same prompt would silently reuse. _LOCKED_KWARGS = { "tensor_parallel_size": 1, "pipeline_parallel_size": 1, "skip_tokenizer_init": True, "disable_log_stats": True, + "enable_prefix_caching": False, } +# Serializes the configure() → LLM(...) → clear() handoff: the spec channel is a module +# global, so interleaved boots would cross-wire capture specs between engines. +_BOOT_LOCK = threading.Lock() + def boot_vllm( model_name: str, @@ -54,21 +65,23 @@ def boot_vllm( affine transform ``output = output * scale + bias`` (default identity), so interventions (``suppress`` / ``scale`` / ``add`` / ``set``) propagate to downstream layers. The hook's return value replaces the module output per - PyTorch ``register_forward_hook`` semantics. **The mutation path under - torch.compile + CUDA graphs is verified end-to-end by** - ``demos/vLLM_Bridge_Integration_Test.ipynb``; unit tests cover the dispatch - protocol only. + PyTorch ``register_forward_hook`` semantics. The mutation path under + torch.compile + CUDA graphs is exercised end-to-end by + ``demos/vLLM_Bridge_Integration_Test.ipynb`` (a manual GPU run, not CI); + unit tests cover the dispatch protocol only. Some captures use vLLM-native conventions that differ from HF/HT; see :mod:`transformer_lens.model_bridge.sources.vllm.overlays.decoder_only` for which hooks diverge and the conversion to apply for HT-equivalent values. - **Returned logits are log-probs, not raw logits.** vLLM bypasses ``lm_head``; - the driver fills the final position from the sampler's ``log_softmax`` output. - Correct for argmax / next-token, but absolute scale is off — temperature - scaling and logit-lens magnitude analysis will be wrong. Only the final - position is populated (earlier positions are ``-inf``); ``return_type`` in - ``{"loss", "both"}`` is rejected for that reason. + **Returned logits are reconstructed full-sequence logits.** vLLM's sampler + bypasses ``lm_head``, so the driver rebuilds real logits host-side as + ``ln_final @ lm_head.weight.T`` (+ bias, + Gemma soft-cap) from the captured + final-norm activation — valid at every position, so ``return_type`` in + ``{"loss", "both"}`` works. If the unembedding weight is unreachable the + driver falls back to the sampler's final-position log-probs (earlier + positions ``-inf``), declares ``provides_sequence_logits=False``, and the + bridge then rejects loss. GPU memory cost: each capture buffer is ``max_num_batched_tokens × width`` at the model's dtype. For Llama-3.2-1B at fp16 with ``max_num_batched_tokens=2048``, @@ -102,6 +115,16 @@ def boot_vllm( "with enable_batching=True (the batched/eager path has no affine buffers)." ) _reject_locked_overrides(vllm_kwargs) + # Import-check first: fail with an actionable message before any network I/O + # or plugin state mutation. + try: + from vllm import LLM + except ImportError as exc: + raise ImportError( + "boot_vllm requires vLLM (Linux + CUDA). Install with " + 'pip install "transformer-lens[vllm]" or uv sync --extra vllm; ' + "the driver is validated against vllm 0.20.x." + ) from exc from transformers import AutoConfig, AutoTokenizer @@ -113,14 +136,6 @@ def boot_vllm( overlay = get_overlay(architecture) resolved_dtype = dtype or _dtype_from_hf_config(hf_config_preview) - plugin.configure( - capture_specs=overlay.capture_specs(hf_config_preview), - max_num_batched_tokens=max_num_batched_tokens, - dtype=resolved_dtype, - enable_batching=enable_batching, - enable_position_interventions=enable_position_interventions, - ) - plugin.register() # The plugin's _config singleton must be visible to the worker, which only # holds with single-process execution. Multi-GPU is unsupported. Override @@ -136,43 +151,55 @@ def boot_vllm( ) os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0" - from vllm import LLM - # Batched capture reads query_start_loc from the forward context, untraceable # under torch.compile — so the batched path must run eager. eager_kwargs: Dict[str, Any] = {"enforce_eager": True} if enable_batching else {} - llm = LLM( - model=model_name, - gpu_memory_utilization=gpu_memory_utilization, - max_model_len=max_model_len, - # Critical: vLLM defaults max_num_batched_tokens to 8192 for chunked prefill. - # We size our capture buffer to this same value, so vLLM must compile for - # the matching dynamic-shape range — otherwise Dynamo's symbolic-shape - # hint exceeds the buffer dim and narrow() fails at compile time. - max_num_batched_tokens=max_num_batched_tokens, - # Register TLWorkerExtension so its tl_* methods are reachable via - # collective_rpc. Passed as a dotted path; vLLM imports the class at - # worker construction and mixes it into the Worker via multiple - # inheritance (asserts no attribute name collisions — hence the tl_ prefix). - worker_extension_cls="transformer_lens.model_bridge.sources.vllm.worker_extension.TLWorkerExtension", - # Allow full-vocab logprobs so the driver can synthesize real logits for - # the generated position (vLLM caps logprobs to this value; default 20 is - # too small for mech-interp). One ~512 KB buffer per call; negligible. - max_logprobs=hf_config_preview.vocab_size, - dtype=str(resolved_dtype).replace("torch.", "") if dtype is not None else "auto", - **eager_kwargs, - **_LOCKED_KWARGS, - **vllm_kwargs, - ) + with _BOOT_LOCK: + plugin.configure( + capture_specs=overlay.capture_specs(hf_config_preview), + max_num_batched_tokens=max_num_batched_tokens, + dtype=resolved_dtype, + enable_batching=enable_batching, + enable_position_interventions=enable_position_interventions, + ) + plugin.register() + try: + llm = LLM( + model=model_name, + gpu_memory_utilization=gpu_memory_utilization, + max_model_len=max_model_len, + # Critical: vLLM defaults max_num_batched_tokens to 8192 for chunked prefill. + # We size our capture buffer to this same value, so vLLM must compile for + # the matching dynamic-shape range — otherwise Dynamo's symbolic-shape + # hint exceeds the buffer dim and narrow() fails at compile time. + max_num_batched_tokens=max_num_batched_tokens, + # Register TLWorkerExtension so its tl_* methods are reachable via + # collective_rpc. Passed as a dotted path; vLLM imports the class at + # worker construction and mixes it into the Worker via multiple + # inheritance (asserts no attribute name collisions — hence the tl_ prefix). + worker_extension_cls="transformer_lens.model_bridge.sources.vllm.worker_extension.TLWorkerExtension", + # Allow full-vocab logprobs so the fallback path can synthesize logits when + # host-side reconstruction is unavailable (vLLM caps logprobs to this value; + # default 20 is too small for mech-interp). + max_logprobs=hf_config_preview.vocab_size, + # Always explicit — vLLM's "auto" downcasts fp32 checkpoints to fp16, which + # would leave the capture/affine buffers (allocated at resolved_dtype) at a + # different dtype than the engine's activations. + dtype=str(resolved_dtype).replace("torch.", ""), + **eager_kwargs, + **_LOCKED_KWARGS, + **vllm_kwargs, + ) + finally: + # Always clear, even on a failed boot: a stale populated _config would make the + # next in-process vllm.LLM(...) walk our dot-paths on a foreign model. + plugin._config.clear() # Capture-path validity is enforced inside patched_load_model during # LLM(...) above — any missing dot-path raises AttributeError there. By # the time we reach this line every spec has already been walked successfully. hf_config = extract_hf_config(llm) - # _config has been consumed into per-Worker state; clear so a non-TL - # vllm.LLM(...) in the same process doesn't inherit our specs. - plugin._config.clear() if tokenizer is None: tokenizer = AutoTokenizer.from_pretrained(model_name, token=hf_token) @@ -184,6 +211,9 @@ def boot_vllm( bridge_config = build_bridge_config_from_hf(hf_config, architecture, model_name, resolved_dtype) adapter = ArchitectureAdapterFactory.select_architecture_adapter(bridge_config) + # Match boot_transformers' tokenizer setup so to_tokens(str) is token-identical. + tokenizer = configure_tokenizer(tokenizer, adapter.cfg) + driver = VLLMDriver( llm=llm, adapter=adapter, @@ -194,6 +224,9 @@ def boot_vllm( enable_batching=enable_batching, enable_position_interventions=enable_position_interventions, ) + # One-time unembedding fetch: caches the weight for per-forward reconstruction and + # downgrades provides_sequence_logits honestly if no unembedding is reachable. + driver.probe_logit_reconstruction() bridge = RemoteBridge(adapter=adapter, tokenizer=tokenizer, driver=driver) _log_hook_summary(model_name, architecture, driver) return bridge @@ -204,7 +237,8 @@ def _reject_locked_overrides(vllm_kwargs: Dict[str, Any]) -> None: if key in vllm_kwargs and vllm_kwargs[key] != locked: raise ValueError( f"boot_vllm forces {key}={locked}; caller passed {key}={vllm_kwargs[key]}. " - "Multi-device / continuous batching / vLLM-owned tokenizer are unsupported." + "Multi-device, prefix caching, continuous batching, and vLLM-owned " + "tokenizers are unsupported — each breaks the row=position capture invariant." ) diff --git a/transformer_lens/model_bridge/sources/vllm/worker_extension.py b/transformer_lens/model_bridge/sources/vllm/worker_extension.py index b8cbcca048..036ee36645 100644 --- a/transformer_lens/model_bridge/sources/vllm/worker_extension.py +++ b/transformer_lens/model_bridge/sources/vllm/worker_extension.py @@ -20,7 +20,7 @@ import torch -from .intervention_specs import SUPPORTED_OPS +from .intervention_specs import SUPPORTED_OPS, validate_spec class TLWorkerExtension: @@ -69,6 +69,9 @@ def tl_set_interventions(self, specs: Dict[str, Dict[str, Any]]) -> None: for hook_name, spec in specs.items(): if hook_name not in scale_bufs: raise KeyError(f"Unknown hook for intervention: {hook_name!r}") + # Authoritative validation: producers that bypass VLLMDriver (the Inspect + # vLLM provider pushes specs over this same RPC) get identical rejection. + validate_spec(hook_name, spec, width=scale_bufs[hook_name].shape[-1]) _apply_intervention(scale_bufs[hook_name], bias_bufs[hook_name], spec) def tl_get_param(self, dotted_name: str) -> Optional[torch.Tensor]: @@ -129,12 +132,8 @@ def tl_read_batched_captures( def tl_set_batched_interventions(self, specs: Dict[str, Dict[str, Any]]) -> None: """Store the global spec dict the eager hook reads; ``{}`` clears.""" - for spec in specs.values(): - op = spec.get("op") - if op not in SUPPORTED_OPS: - raise ValueError( - f"Unsupported intervention op: {op!r}. Supported: {sorted(SUPPORTED_OPS)}" - ) + for hook_name, spec in specs.items(): + validate_spec(hook_name, spec) self._tl_intervention_specs = dict(specs) def tl_remove_hooks(self) -> None: diff --git a/transformer_lens/model_bridge/supported_architectures/mamba.py b/transformer_lens/model_bridge/supported_architectures/mamba.py index 0023bbb5c5..e08aa501f2 100644 --- a/transformer_lens/model_bridge/supported_architectures/mamba.py +++ b/transformer_lens/model_bridge/supported_architectures/mamba.py @@ -20,7 +20,7 @@ class MambaArchitectureAdapter(ArchitectureAdapter): SSM config fields (state_size, conv_kernel, expand, time_step_rank, intermediate_size) are propagated from the HF config via - ``_HF_PASSTHROUGH_ATTRS`` in sources/transformers.py. + ``_HF_PASSTHROUGH_ATTRS`` in sources/_bridge_builder.py. """ # verify_models is transformer-shaped today and would need a dedicated diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index 9038a3e1e2..96c4a88199 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -11,6 +11,7 @@ TYPE_CHECKING, Any, Callable, + ClassVar, Dict, Iterator, List, @@ -67,13 +68,6 @@ def _resolve_attr_path(obj: nn.Module, attr_path: str) -> torch.Tensor: return cast(torch.Tensor, result) -# build_alias_to_canonical_map lives in bridge_core.py; re-import for the module's -# internal use (run_with_cache, hooks() context manager in this file). -from transformer_lens.model_bridge.bridge_core import ( # noqa: E402 - build_alias_to_canonical_map, -) - - class TransformerBridge(BridgeCore, HookIntrospectionMixin, nn.Module): """Torch-backed bridge: HF, vLLM-via-torch, anything that wraps an ``nn.Module``. @@ -165,97 +159,10 @@ def __init__( validate_driver(self._driver, after_bridge_construction=True) self.processor = None - @classmethod - def boot_transformers( - cls, - model_name: str, - hf_config_overrides: Optional[dict] = None, - device: Optional[Union[str, torch.device]] = None, - dtype: torch.dtype = torch.float32, - tokenizer: Optional[Any] = None, - load_weights: bool = True, - trust_remote_code: bool = False, - model_class: Optional[type] = None, - hf_model: Optional[Any] = None, - device_map: Optional[Union[str, Dict[str, Union[str, int]]]] = None, - n_devices: Optional[int] = None, - max_memory: Optional[Dict[Union[str, int], str]] = None, - n_ctx: Optional[int] = None, - revision: Optional[str] = None, - checkpoint_index: Optional[int] = None, - checkpoint_value: Optional[int] = None, - ) -> "TransformerBridge": - """Boot a model from HuggingFace (alias for sources.transformers.boot). - - Returns raw HF weights by default — logits/activations match HF, *not* - legacy ``HookedTransformer`` (which folds LayerNorm + centers weights). - Call ``enable_compatibility_mode()`` on the result for HookedTransformer- - equivalent numerics. Generation, argmax, and CE loss are unaffected. - - Attention implementation is forced to ``"eager"`` so hooks can capture scores - and patterns. For an apples-to-apples HF comparison, load the HF model with - ``attn_implementation="eager"`` too; comparing against the default ``"sdpa"`` - shows ~1e-3 fp32 drift from kernel-level op reordering, not a bridge bug. - - Args: - model_name: The name of the model to load. - hf_config_overrides: Optional overrides applied to the HuggingFace config before model load. - device: The device to use. If None, will be determined automatically. Mutually exclusive - with ``device_map``. - dtype: The dtype to use for the model. - tokenizer: Optional pre-initialized tokenizer to use; if not provided one will be created. - load_weights: If False, load model without weights (on meta device) for config inspection only. - trust_remote_code: Whether to trust remote code for custom model architectures. - model_class: Optional HuggingFace model class to use instead of the default - auto-detected class (e.g., BertForNextSentencePrediction). - hf_model: Optional pre-loaded HuggingFace model to use instead of loading one. Useful - for models loaded with custom configurations (e.g., quantization via - BitsAndBytesConfig). When provided, load_weights is ignored. If the pre-loaded - model was built with a ``device_map``, ``cfg.device`` and ``cfg.n_devices`` are - derived from its ``hf_device_map`` automatically. - device_map: HuggingFace-style device map for multi-GPU inference. Pass ``"auto"``, - ``"balanced"``, ``"sequential"``, or an explicit ``{submodule_path: device}`` dict. - Mutually exclusive with ``device``. - n_devices: Convenience shortcut: split the model across this many CUDA devices. - Translated to a ``max_memory`` dict over devices 0..n_devices-1 and passed as - ``device_map`` to HF. Requires CUDA with at least this many visible devices. - max_memory: Optional per-device memory budget, passed through to HF's dispatcher. - Only used when ``device_map`` or ``n_devices`` is in effect. - n_ctx: Optional context length override. Writes to the appropriate HF config field - for this model automatically (callers don't need to know the field name). - Warns if larger than the model's default context length. - revision: Optional HF revision (branch, tag, or commit). Forwarded to the underlying - ``AutoConfig.from_pretrained`` and ``AutoModelForCausalLM.from_pretrained`` calls. - Mutually exclusive with ``checkpoint_index`` / ``checkpoint_value``. - checkpoint_index: Index into the available training checkpoints for the model family - (currently ``EleutherAI/pythia*`` and ``stanford-crfm/*``). Resolved to a revision - string via known per-family naming conventions. - checkpoint_value: Training step or token count of the desired checkpoint. Alternative - to ``checkpoint_index``; must match an entry in the family's checkpoint label list. - - Returns: - The bridge to the loaded model. - """ - from transformer_lens.model_bridge.sources.transformers import boot - - return boot( - model_name=model_name, - hf_config_overrides=hf_config_overrides, - device=device, - dtype=dtype, - tokenizer=tokenizer, - load_weights=load_weights, - trust_remote_code=trust_remote_code, - model_class=model_class, - hf_model=hf_model, - device_map=device_map, - n_devices=n_devices, - max_memory=max_memory, - n_ctx=n_ctx, - revision=revision, - checkpoint_index=checkpoint_index, - checkpoint_value=checkpoint_value, - ) + # boot_transformers / list_supported_models / check_model_support are + # attached by sources.transformers.__init__ (setattr on this class) so the + # source package owns its own boot entry point. + boot_transformers: ClassVar[Callable[..., "TransformerBridge"]] @property def original_model(self) -> nn.Module: @@ -696,39 +603,6 @@ def process_weights( component_mapping=self.real_components, ) - def _calculate_loss(self, logits, tokens, loss_per_token=False): - """Calculate cross-entropy loss.""" - shift_logits = logits[..., :-1, :].contiguous() - shift_labels = tokens[..., 1:].contiguous() - loss_fct = torch.nn.CrossEntropyLoss(reduction="none" if loss_per_token else "mean") - flat_logits = shift_logits.view(-1, shift_logits.size(-1)) - flat_labels = shift_labels.view(-1) - loss = loss_fct(flat_logits, flat_labels) - if loss_per_token: - return loss.view(shift_labels.shape) - else: - return loss - - def _extract_hf_weights(self): - """Extract weights from the original HuggingFace model.""" - hf_state_dict = self.state_dict() - for layer_idx in range(self.cfg.n_layers): - combined_qkv_key = f"transformer.h.{layer_idx}.attn.c_attn.weight" - combined_qkv_bias_key = f"transformer.h.{layer_idx}.attn.c_attn.bias" - if combined_qkv_key in hf_state_dict: - separate_keys_to_remove = [ - f"transformer.h.{layer_idx}.attn.q.weight", - f"transformer.h.{layer_idx}.attn.q.bias", - f"transformer.h.{layer_idx}.attn.k.weight", - f"transformer.h.{layer_idx}.attn.k.bias", - f"transformer.h.{layer_idx}.attn.v.weight", - f"transformer.h.{layer_idx}.attn.v.bias", - ] - for key_to_remove in separate_keys_to_remove: - if key_to_remove in hf_state_dict: - del hf_state_dict[key_to_remove] - return hf_state_dict - def to_tokens( self, input: Union[str, List[str]], diff --git a/transformer_lens/utilities/tokenize_utils.py b/transformer_lens/utilities/tokenize_utils.py index 22d1702b61..0ba674febc 100644 --- a/transformer_lens/utilities/tokenize_utils.py +++ b/transformer_lens/utilities/tokenize_utils.py @@ -7,7 +7,7 @@ import os from copy import deepcopy -from typing import Any +from typing import Any, Optional import einops import numpy as np @@ -27,7 +27,7 @@ def tokenize_and_concatenate( max_length: int = 1024, column_name: str = "text", add_bos_token: bool = True, - num_proc: int = 10, + num_proc: Optional[int] = 10, set_format: bool = True, ) -> Dataset | IterableDataset: """Tokenize each document, join with token-level EOS between docs, and reshape into ``(batch, sequence_length)`` rows. @@ -44,7 +44,8 @@ def tokenize_and_concatenate( max_length (int, optional): The length of the context window of the sequence. Defaults to 1024. column_name (str, optional): The name of the text column in the dataset. Defaults to 'text'. add_bos_token (bool, optional): Whether to prepend ``bos_token_id`` to each output row. Defaults to True. - num_proc (int, optional): Number of processes for parallel tokenization. Ignored when ``streaming=True``. Defaults to 10. + num_proc (int, optional): Number of processes for parallel tokenization; ``None`` runs in + the main process (``datasets`` pools even for ``num_proc=1``). Ignored when ``streaming=True``. Defaults to 10. set_format (bool, optional): If True, calls ``set_format(type="torch")`` on the result. Set False for ``IterableDataset`` (which doesn't support format setting); wrap the output in ``(torch.LongTensor(ex["tokens"]) for ex in tokenized_dataset)`` instead. Defaults to True. diff --git a/uv.lock b/uv.lock index 13a86aec5f..0ce3944b59 100644 --- a/uv.lock +++ b/uv.lock @@ -2,17 +2,37 @@ version = 1 revision = 3 requires-python = ">=3.10, <4.0" resolution-markers = [ - "python_full_version == '3.12.*' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", - "python_full_version >= '3.14' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and sys_platform == 'linux'", - "python_full_version >= '3.14' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and sys_platform != 'linux'", -] + "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.13' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", +] +conflicts = [[ + { package = "transformer-lens", extra = "lit" }, + { package = "transformer-lens", extra = "vllm" }, +]] [[package]] name = "absl-py" @@ -29,7 +49,9 @@ version = "1.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "packaging" }, { name = "psutil" }, { name = "pyyaml" }, @@ -109,88 +131,139 @@ wheels = [ [[package]] name = "aiohttp" -version = "3.12.14" +version = "3.14.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs" }, { name = "aiosignal" }, - { name = "async-timeout", marker = "python_full_version < '3.11'" }, + { name = "async-timeout", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "attrs" }, { name = "frozenlist" }, { name = "multidict" }, { name = "propcache" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e6/0b/e39ad954107ebf213a2325038a3e7a506be3d98e1435e1f82086eec4cde2/aiohttp-3.12.14.tar.gz", hash = "sha256:6e06e120e34d93100de448fd941522e11dafa78ef1a893c179901b7d66aa29f2", size = 7822921, upload-time = "2025-07-10T13:05:33.968Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/88/f161f429f9de391eee6a5c2cffa54e2ecd5b7122ae99df247f7734dfefcb/aiohttp-3.12.14-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:906d5075b5ba0dd1c66fcaaf60eb09926a9fef3ca92d912d2a0bbdbecf8b1248", size = 702641, upload-time = "2025-07-10T13:02:38.98Z" }, - { url = "https://files.pythonhosted.org/packages/fe/b5/24fa382a69a25d242e2baa3e56d5ea5227d1b68784521aaf3a1a8b34c9a4/aiohttp-3.12.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c875bf6fc2fd1a572aba0e02ef4e7a63694778c5646cdbda346ee24e630d30fb", size = 479005, upload-time = "2025-07-10T13:02:42.714Z" }, - { url = "https://files.pythonhosted.org/packages/09/67/fda1bc34adbfaa950d98d934a23900918f9d63594928c70e55045838c943/aiohttp-3.12.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fbb284d15c6a45fab030740049d03c0ecd60edad9cd23b211d7e11d3be8d56fd", size = 466781, upload-time = "2025-07-10T13:02:44.639Z" }, - { url = "https://files.pythonhosted.org/packages/36/96/3ce1ea96d3cf6928b87cfb8cdd94650367f5c2f36e686a1f5568f0f13754/aiohttp-3.12.14-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38e360381e02e1a05d36b223ecab7bc4a6e7b5ab15760022dc92589ee1d4238c", size = 1648841, upload-time = "2025-07-10T13:02:46.356Z" }, - { url = "https://files.pythonhosted.org/packages/be/04/ddea06cb4bc7d8db3745cf95e2c42f310aad485ca075bd685f0e4f0f6b65/aiohttp-3.12.14-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:aaf90137b5e5d84a53632ad95ebee5c9e3e7468f0aab92ba3f608adcb914fa95", size = 1622896, upload-time = "2025-07-10T13:02:48.422Z" }, - { url = "https://files.pythonhosted.org/packages/73/66/63942f104d33ce6ca7871ac6c1e2ebab48b88f78b2b7680c37de60f5e8cd/aiohttp-3.12.14-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e532a25e4a0a2685fa295a31acf65e027fbe2bea7a4b02cdfbbba8a064577663", size = 1695302, upload-time = "2025-07-10T13:02:50.078Z" }, - { url = "https://files.pythonhosted.org/packages/20/00/aab615742b953f04b48cb378ee72ada88555b47b860b98c21c458c030a23/aiohttp-3.12.14-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eab9762c4d1b08ae04a6c77474e6136da722e34fdc0e6d6eab5ee93ac29f35d1", size = 1737617, upload-time = "2025-07-10T13:02:52.123Z" }, - { url = "https://files.pythonhosted.org/packages/d6/4f/ef6d9f77225cf27747368c37b3d69fac1f8d6f9d3d5de2d410d155639524/aiohttp-3.12.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abe53c3812b2899889a7fca763cdfaeee725f5be68ea89905e4275476ffd7e61", size = 1642282, upload-time = "2025-07-10T13:02:53.899Z" }, - { url = "https://files.pythonhosted.org/packages/37/e1/e98a43c15aa52e9219a842f18c59cbae8bbe2d50c08d298f17e9e8bafa38/aiohttp-3.12.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5760909b7080aa2ec1d320baee90d03b21745573780a072b66ce633eb77a8656", size = 1582406, upload-time = "2025-07-10T13:02:55.515Z" }, - { url = "https://files.pythonhosted.org/packages/71/5c/29c6dfb49323bcdb0239bf3fc97ffcf0eaf86d3a60426a3287ec75d67721/aiohttp-3.12.14-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:02fcd3f69051467bbaa7f84d7ec3267478c7df18d68b2e28279116e29d18d4f3", size = 1626255, upload-time = "2025-07-10T13:02:57.343Z" }, - { url = "https://files.pythonhosted.org/packages/79/60/ec90782084090c4a6b459790cfd8d17be2c5662c9c4b2d21408b2f2dc36c/aiohttp-3.12.14-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4dcd1172cd6794884c33e504d3da3c35648b8be9bfa946942d353b939d5f1288", size = 1637041, upload-time = "2025-07-10T13:02:59.008Z" }, - { url = "https://files.pythonhosted.org/packages/22/89/205d3ad30865c32bc472ac13f94374210745b05bd0f2856996cb34d53396/aiohttp-3.12.14-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:224d0da41355b942b43ad08101b1b41ce633a654128ee07e36d75133443adcda", size = 1612494, upload-time = "2025-07-10T13:03:00.618Z" }, - { url = "https://files.pythonhosted.org/packages/48/ae/2f66edaa8bd6db2a4cba0386881eb92002cdc70834e2a93d1d5607132c7e/aiohttp-3.12.14-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e387668724f4d734e865c1776d841ed75b300ee61059aca0b05bce67061dcacc", size = 1692081, upload-time = "2025-07-10T13:03:02.154Z" }, - { url = "https://files.pythonhosted.org/packages/08/3a/fa73bfc6e21407ea57f7906a816f0dc73663d9549da703be05dbd76d2dc3/aiohttp-3.12.14-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:dec9cde5b5a24171e0b0a4ca064b1414950904053fb77c707efd876a2da525d8", size = 1715318, upload-time = "2025-07-10T13:03:04.322Z" }, - { url = "https://files.pythonhosted.org/packages/e3/b3/751124b8ceb0831c17960d06ee31a4732cb4a6a006fdbfa1153d07c52226/aiohttp-3.12.14-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bbad68a2af4877cc103cd94af9160e45676fc6f0c14abb88e6e092b945c2c8e3", size = 1643660, upload-time = "2025-07-10T13:03:06.406Z" }, - { url = "https://files.pythonhosted.org/packages/81/3c/72477a1d34edb8ab8ce8013086a41526d48b64f77e381c8908d24e1c18f5/aiohttp-3.12.14-cp310-cp310-win32.whl", hash = "sha256:ee580cb7c00bd857b3039ebca03c4448e84700dc1322f860cf7a500a6f62630c", size = 428289, upload-time = "2025-07-10T13:03:08.274Z" }, - { url = "https://files.pythonhosted.org/packages/a2/c4/8aec4ccf1b822ec78e7982bd5cf971113ecce5f773f04039c76a083116fc/aiohttp-3.12.14-cp310-cp310-win_amd64.whl", hash = "sha256:cf4f05b8cea571e2ccc3ca744e35ead24992d90a72ca2cf7ab7a2efbac6716db", size = 451328, upload-time = "2025-07-10T13:03:10.146Z" }, - { url = "https://files.pythonhosted.org/packages/53/e1/8029b29316971c5fa89cec170274582619a01b3d82dd1036872acc9bc7e8/aiohttp-3.12.14-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f4552ff7b18bcec18b60a90c6982049cdb9dac1dba48cf00b97934a06ce2e597", size = 709960, upload-time = "2025-07-10T13:03:11.936Z" }, - { url = "https://files.pythonhosted.org/packages/96/bd/4f204cf1e282041f7b7e8155f846583b19149e0872752711d0da5e9cc023/aiohttp-3.12.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8283f42181ff6ccbcf25acaae4e8ab2ff7e92b3ca4a4ced73b2c12d8cd971393", size = 482235, upload-time = "2025-07-10T13:03:14.118Z" }, - { url = "https://files.pythonhosted.org/packages/d6/0f/2a580fcdd113fe2197a3b9df30230c7e85bb10bf56f7915457c60e9addd9/aiohttp-3.12.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:040afa180ea514495aaff7ad34ec3d27826eaa5d19812730fe9e529b04bb2179", size = 470501, upload-time = "2025-07-10T13:03:16.153Z" }, - { url = "https://files.pythonhosted.org/packages/38/78/2c1089f6adca90c3dd74915bafed6d6d8a87df5e3da74200f6b3a8b8906f/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b413c12f14c1149f0ffd890f4141a7471ba4b41234fe4fd4a0ff82b1dc299dbb", size = 1740696, upload-time = "2025-07-10T13:03:18.4Z" }, - { url = "https://files.pythonhosted.org/packages/4a/c8/ce6c7a34d9c589f007cfe064da2d943b3dee5aabc64eaecd21faf927ab11/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1d6f607ce2e1a93315414e3d448b831238f1874b9968e1195b06efaa5c87e245", size = 1689365, upload-time = "2025-07-10T13:03:20.629Z" }, - { url = "https://files.pythonhosted.org/packages/18/10/431cd3d089de700756a56aa896faf3ea82bee39d22f89db7ddc957580308/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:565e70d03e924333004ed101599902bba09ebb14843c8ea39d657f037115201b", size = 1788157, upload-time = "2025-07-10T13:03:22.44Z" }, - { url = "https://files.pythonhosted.org/packages/fa/b2/26f4524184e0f7ba46671c512d4b03022633bcf7d32fa0c6f1ef49d55800/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4699979560728b168d5ab63c668a093c9570af2c7a78ea24ca5212c6cdc2b641", size = 1827203, upload-time = "2025-07-10T13:03:24.628Z" }, - { url = "https://files.pythonhosted.org/packages/e0/30/aadcdf71b510a718e3d98a7bfeaea2396ac847f218b7e8edb241b09bd99a/aiohttp-3.12.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad5fdf6af93ec6c99bf800eba3af9a43d8bfd66dce920ac905c817ef4a712afe", size = 1729664, upload-time = "2025-07-10T13:03:26.412Z" }, - { url = "https://files.pythonhosted.org/packages/67/7f/7ccf11756ae498fdedc3d689a0c36ace8fc82f9d52d3517da24adf6e9a74/aiohttp-3.12.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4ac76627c0b7ee0e80e871bde0d376a057916cb008a8f3ffc889570a838f5cc7", size = 1666741, upload-time = "2025-07-10T13:03:28.167Z" }, - { url = "https://files.pythonhosted.org/packages/6b/4d/35ebc170b1856dd020c92376dbfe4297217625ef4004d56587024dc2289c/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:798204af1180885651b77bf03adc903743a86a39c7392c472891649610844635", size = 1715013, upload-time = "2025-07-10T13:03:30.018Z" }, - { url = "https://files.pythonhosted.org/packages/7b/24/46dc0380146f33e2e4aa088b92374b598f5bdcde1718c77e8d1a0094f1a4/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:4f1205f97de92c37dd71cf2d5bcfb65fdaed3c255d246172cce729a8d849b4da", size = 1710172, upload-time = "2025-07-10T13:03:31.821Z" }, - { url = "https://files.pythonhosted.org/packages/2f/0a/46599d7d19b64f4d0fe1b57bdf96a9a40b5c125f0ae0d8899bc22e91fdce/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:76ae6f1dd041f85065d9df77c6bc9c9703da9b5c018479d20262acc3df97d419", size = 1690355, upload-time = "2025-07-10T13:03:34.754Z" }, - { url = "https://files.pythonhosted.org/packages/08/86/b21b682e33d5ca317ef96bd21294984f72379454e689d7da584df1512a19/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a194ace7bc43ce765338ca2dfb5661489317db216ea7ea700b0332878b392cab", size = 1783958, upload-time = "2025-07-10T13:03:36.53Z" }, - { url = "https://files.pythonhosted.org/packages/4f/45/f639482530b1396c365f23c5e3b1ae51c9bc02ba2b2248ca0c855a730059/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:16260e8e03744a6fe3fcb05259eeab8e08342c4c33decf96a9dad9f1187275d0", size = 1804423, upload-time = "2025-07-10T13:03:38.504Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e5/39635a9e06eed1d73671bd4079a3caf9cf09a49df08490686f45a710b80e/aiohttp-3.12.14-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8c779e5ebbf0e2e15334ea404fcce54009dc069210164a244d2eac8352a44b28", size = 1717479, upload-time = "2025-07-10T13:03:40.158Z" }, - { url = "https://files.pythonhosted.org/packages/51/e1/7f1c77515d369b7419c5b501196526dad3e72800946c0099594c1f0c20b4/aiohttp-3.12.14-cp311-cp311-win32.whl", hash = "sha256:a289f50bf1bd5be227376c067927f78079a7bdeccf8daa6a9e65c38bae14324b", size = 427907, upload-time = "2025-07-10T13:03:41.801Z" }, - { url = "https://files.pythonhosted.org/packages/06/24/a6bf915c85b7a5b07beba3d42b3282936b51e4578b64a51e8e875643c276/aiohttp-3.12.14-cp311-cp311-win_amd64.whl", hash = "sha256:0b8a69acaf06b17e9c54151a6c956339cf46db4ff72b3ac28516d0f7068f4ced", size = 452334, upload-time = "2025-07-10T13:03:43.485Z" }, - { url = "https://files.pythonhosted.org/packages/c3/0d/29026524e9336e33d9767a1e593ae2b24c2b8b09af7c2bd8193762f76b3e/aiohttp-3.12.14-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a0ecbb32fc3e69bc25efcda7d28d38e987d007096cbbeed04f14a6662d0eee22", size = 701055, upload-time = "2025-07-10T13:03:45.59Z" }, - { url = "https://files.pythonhosted.org/packages/0a/b8/a5e8e583e6c8c1056f4b012b50a03c77a669c2e9bf012b7cf33d6bc4b141/aiohttp-3.12.14-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0400f0ca9bb3e0b02f6466421f253797f6384e9845820c8b05e976398ac1d81a", size = 475670, upload-time = "2025-07-10T13:03:47.249Z" }, - { url = "https://files.pythonhosted.org/packages/29/e8/5202890c9e81a4ec2c2808dd90ffe024952e72c061729e1d49917677952f/aiohttp-3.12.14-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a56809fed4c8a830b5cae18454b7464e1529dbf66f71c4772e3cfa9cbec0a1ff", size = 468513, upload-time = "2025-07-10T13:03:49.377Z" }, - { url = "https://files.pythonhosted.org/packages/23/e5/d11db8c23d8923d3484a27468a40737d50f05b05eebbb6288bafcb467356/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27f2e373276e4755691a963e5d11756d093e346119f0627c2d6518208483fb6d", size = 1715309, upload-time = "2025-07-10T13:03:51.556Z" }, - { url = "https://files.pythonhosted.org/packages/53/44/af6879ca0eff7a16b1b650b7ea4a827301737a350a464239e58aa7c387ef/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ca39e433630e9a16281125ef57ece6817afd1d54c9f1bf32e901f38f16035869", size = 1697961, upload-time = "2025-07-10T13:03:53.511Z" }, - { url = "https://files.pythonhosted.org/packages/bb/94/18457f043399e1ec0e59ad8674c0372f925363059c276a45a1459e17f423/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9c748b3f8b14c77720132b2510a7d9907a03c20ba80f469e58d5dfd90c079a1c", size = 1753055, upload-time = "2025-07-10T13:03:55.368Z" }, - { url = "https://files.pythonhosted.org/packages/26/d9/1d3744dc588fafb50ff8a6226d58f484a2242b5dd93d8038882f55474d41/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0a568abe1b15ce69d4cc37e23020720423f0728e3cb1f9bcd3f53420ec3bfe7", size = 1799211, upload-time = "2025-07-10T13:03:57.216Z" }, - { url = "https://files.pythonhosted.org/packages/73/12/2530fb2b08773f717ab2d249ca7a982ac66e32187c62d49e2c86c9bba9b4/aiohttp-3.12.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9888e60c2c54eaf56704b17feb558c7ed6b7439bca1e07d4818ab878f2083660", size = 1718649, upload-time = "2025-07-10T13:03:59.469Z" }, - { url = "https://files.pythonhosted.org/packages/b9/34/8d6015a729f6571341a311061b578e8b8072ea3656b3d72329fa0faa2c7c/aiohttp-3.12.14-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3006a1dc579b9156de01e7916d38c63dc1ea0679b14627a37edf6151bc530088", size = 1634452, upload-time = "2025-07-10T13:04:01.698Z" }, - { url = "https://files.pythonhosted.org/packages/ff/4b/08b83ea02595a582447aeb0c1986792d0de35fe7a22fb2125d65091cbaf3/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aa8ec5c15ab80e5501a26719eb48a55f3c567da45c6ea5bb78c52c036b2655c7", size = 1695511, upload-time = "2025-07-10T13:04:04.165Z" }, - { url = "https://files.pythonhosted.org/packages/b5/66/9c7c31037a063eec13ecf1976185c65d1394ded4a5120dd5965e3473cb21/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:39b94e50959aa07844c7fe2206b9f75d63cc3ad1c648aaa755aa257f6f2498a9", size = 1716967, upload-time = "2025-07-10T13:04:06.132Z" }, - { url = "https://files.pythonhosted.org/packages/ba/02/84406e0ad1acb0fb61fd617651ab6de760b2d6a31700904bc0b33bd0894d/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:04c11907492f416dad9885d503fbfc5dcb6768d90cad8639a771922d584609d3", size = 1657620, upload-time = "2025-07-10T13:04:07.944Z" }, - { url = "https://files.pythonhosted.org/packages/07/53/da018f4013a7a179017b9a274b46b9a12cbeb387570f116964f498a6f211/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:88167bd9ab69bb46cee91bd9761db6dfd45b6e76a0438c7e884c3f8160ff21eb", size = 1737179, upload-time = "2025-07-10T13:04:10.182Z" }, - { url = "https://files.pythonhosted.org/packages/49/e8/ca01c5ccfeaafb026d85fa4f43ceb23eb80ea9c1385688db0ef322c751e9/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:791504763f25e8f9f251e4688195e8b455f8820274320204f7eafc467e609425", size = 1765156, upload-time = "2025-07-10T13:04:12.029Z" }, - { url = "https://files.pythonhosted.org/packages/22/32/5501ab525a47ba23c20613e568174d6c63aa09e2caa22cded5c6ea8e3ada/aiohttp-3.12.14-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2785b112346e435dd3a1a67f67713a3fe692d288542f1347ad255683f066d8e0", size = 1724766, upload-time = "2025-07-10T13:04:13.961Z" }, - { url = "https://files.pythonhosted.org/packages/06/af/28e24574801fcf1657945347ee10df3892311c2829b41232be6089e461e7/aiohttp-3.12.14-cp312-cp312-win32.whl", hash = "sha256:15f5f4792c9c999a31d8decf444e79fcfd98497bf98e94284bf390a7bb8c1729", size = 422641, upload-time = "2025-07-10T13:04:16.018Z" }, - { url = "https://files.pythonhosted.org/packages/98/d5/7ac2464aebd2eecac38dbe96148c9eb487679c512449ba5215d233755582/aiohttp-3.12.14-cp312-cp312-win_amd64.whl", hash = "sha256:3b66e1a182879f579b105a80d5c4bd448b91a57e8933564bf41665064796a338", size = 449316, upload-time = "2025-07-10T13:04:18.289Z" }, - { url = "https://files.pythonhosted.org/packages/06/48/e0d2fa8ac778008071e7b79b93ab31ef14ab88804d7ba71b5c964a7c844e/aiohttp-3.12.14-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:3143a7893d94dc82bc409f7308bc10d60285a3cd831a68faf1aa0836c5c3c767", size = 695471, upload-time = "2025-07-10T13:04:20.124Z" }, - { url = "https://files.pythonhosted.org/packages/8d/e7/f73206afa33100804f790b71092888f47df65fd9a4cd0e6800d7c6826441/aiohttp-3.12.14-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3d62ac3d506cef54b355bd34c2a7c230eb693880001dfcda0bf88b38f5d7af7e", size = 473128, upload-time = "2025-07-10T13:04:21.928Z" }, - { url = "https://files.pythonhosted.org/packages/df/e2/4dd00180be551a6e7ee979c20fc7c32727f4889ee3fd5b0586e0d47f30e1/aiohttp-3.12.14-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:48e43e075c6a438937c4de48ec30fa8ad8e6dfef122a038847456bfe7b947b63", size = 465426, upload-time = "2025-07-10T13:04:24.071Z" }, - { url = "https://files.pythonhosted.org/packages/de/dd/525ed198a0bb674a323e93e4d928443a680860802c44fa7922d39436b48b/aiohttp-3.12.14-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:077b4488411a9724cecc436cbc8c133e0d61e694995b8de51aaf351c7578949d", size = 1704252, upload-time = "2025-07-10T13:04:26.049Z" }, - { url = "https://files.pythonhosted.org/packages/d8/b1/01e542aed560a968f692ab4fc4323286e8bc4daae83348cd63588e4f33e3/aiohttp-3.12.14-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d8c35632575653f297dcbc9546305b2c1133391089ab925a6a3706dfa775ccab", size = 1685514, upload-time = "2025-07-10T13:04:28.186Z" }, - { url = "https://files.pythonhosted.org/packages/b3/06/93669694dc5fdabdc01338791e70452d60ce21ea0946a878715688d5a191/aiohttp-3.12.14-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b8ce87963f0035c6834b28f061df90cf525ff7c9b6283a8ac23acee6502afd4", size = 1737586, upload-time = "2025-07-10T13:04:30.195Z" }, - { url = "https://files.pythonhosted.org/packages/a5/3a/18991048ffc1407ca51efb49ba8bcc1645961f97f563a6c480cdf0286310/aiohttp-3.12.14-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0a2cf66e32a2563bb0766eb24eae7e9a269ac0dc48db0aae90b575dc9583026", size = 1786958, upload-time = "2025-07-10T13:04:32.482Z" }, - { url = "https://files.pythonhosted.org/packages/30/a8/81e237f89a32029f9b4a805af6dffc378f8459c7b9942712c809ff9e76e5/aiohttp-3.12.14-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdea089caf6d5cde975084a884c72d901e36ef9c2fd972c9f51efbbc64e96fbd", size = 1709287, upload-time = "2025-07-10T13:04:34.493Z" }, - { url = "https://files.pythonhosted.org/packages/8c/e3/bd67a11b0fe7fc12c6030473afd9e44223d456f500f7cf526dbaa259ae46/aiohttp-3.12.14-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a7865f27db67d49e81d463da64a59365ebd6b826e0e4847aa111056dcb9dc88", size = 1622990, upload-time = "2025-07-10T13:04:36.433Z" }, - { url = "https://files.pythonhosted.org/packages/83/ba/e0cc8e0f0d9ce0904e3cf2d6fa41904e379e718a013c721b781d53dcbcca/aiohttp-3.12.14-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0ab5b38a6a39781d77713ad930cb5e7feea6f253de656a5f9f281a8f5931b086", size = 1676015, upload-time = "2025-07-10T13:04:38.958Z" }, - { url = "https://files.pythonhosted.org/packages/d8/b3/1e6c960520bda094c48b56de29a3d978254637ace7168dd97ddc273d0d6c/aiohttp-3.12.14-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:9b3b15acee5c17e8848d90a4ebc27853f37077ba6aec4d8cb4dbbea56d156933", size = 1707678, upload-time = "2025-07-10T13:04:41.275Z" }, - { url = "https://files.pythonhosted.org/packages/0a/19/929a3eb8c35b7f9f076a462eaa9830b32c7f27d3395397665caa5e975614/aiohttp-3.12.14-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e4c972b0bdaac167c1e53e16a16101b17c6d0ed7eac178e653a07b9f7fad7151", size = 1650274, upload-time = "2025-07-10T13:04:43.483Z" }, - { url = "https://files.pythonhosted.org/packages/22/e5/81682a6f20dd1b18ce3d747de8eba11cbef9b270f567426ff7880b096b48/aiohttp-3.12.14-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7442488b0039257a3bdbc55f7209587911f143fca11df9869578db6c26feeeb8", size = 1726408, upload-time = "2025-07-10T13:04:45.577Z" }, - { url = "https://files.pythonhosted.org/packages/8c/17/884938dffaa4048302985483f77dfce5ac18339aad9b04ad4aaa5e32b028/aiohttp-3.12.14-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f68d3067eecb64c5e9bab4a26aa11bd676f4c70eea9ef6536b0a4e490639add3", size = 1759879, upload-time = "2025-07-10T13:04:47.663Z" }, - { url = "https://files.pythonhosted.org/packages/95/78/53b081980f50b5cf874359bde707a6eacd6c4be3f5f5c93937e48c9d0025/aiohttp-3.12.14-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f88d3704c8b3d598a08ad17d06006cb1ca52a1182291f04979e305c8be6c9758", size = 1708770, upload-time = "2025-07-10T13:04:49.944Z" }, - { url = "https://files.pythonhosted.org/packages/ed/91/228eeddb008ecbe3ffa6c77b440597fdf640307162f0c6488e72c5a2d112/aiohttp-3.12.14-cp313-cp313-win32.whl", hash = "sha256:a3c99ab19c7bf375c4ae3debd91ca5d394b98b6089a03231d4c580ef3c2ae4c5", size = 421688, upload-time = "2025-07-10T13:04:51.993Z" }, - { url = "https://files.pythonhosted.org/packages/66/5f/8427618903343402fdafe2850738f735fd1d9409d2a8f9bcaae5e630d3ba/aiohttp-3.12.14-cp313-cp313-win_amd64.whl", hash = "sha256:3f8aad695e12edc9d571f878c62bedc91adf30c760c8632f09663e5f564f4baa", size = 448098, upload-time = "2025-07-10T13:04:53.999Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/82/78/8ea7308cac6934de8c74a14f3d5f65d1c89287426688be79538d0e5c013d/aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035", size = 7955794, upload-time = "2026-06-07T21:09:35.529Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/67/58ded4b3f2e10f94972d8928050c85330e249a31dd45a0e5f3c0e9c3fa05/aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e", size = 766140, upload-time = "2026-06-07T21:05:37.471Z" }, + { url = "https://files.pythonhosted.org/packages/18/68/4ae5b4e08943f316594bb68da89957d3baf5760588fa09509594bd777e4b/aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491", size = 519430, upload-time = "2026-06-07T21:05:40.751Z" }, + { url = "https://files.pythonhosted.org/packages/cb/c1/316c8f3549dbe5245f92bfd523ec6f32dd4d98cafe21df3f6a19b1184c75/aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce", size = 514406, upload-time = "2026-06-07T21:05:42.111Z" }, + { url = "https://files.pythonhosted.org/packages/5a/ee/fb0ac28684e8d753b83c8a4eebc19a5846912aa0a4daaabb6a9936363840/aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3", size = 1703649, upload-time = "2026-06-07T21:05:43.427Z" }, + { url = "https://files.pythonhosted.org/packages/3b/57/aa2beab673331f111885db8a7b69dfe3ab0e53e446a0ace18ca694b4dc58/aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505", size = 1675126, upload-time = "2026-06-07T21:05:44.897Z" }, + { url = "https://files.pythonhosted.org/packages/47/ea/dad128abe365e79be03b16ed464198ac73e0d257e8260c6f7d6f31cbef26/aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521", size = 1771558, upload-time = "2026-06-07T21:05:46.405Z" }, + { url = "https://files.pythonhosted.org/packages/63/f3/b5b4e10327cb85d34d24232c6b71b64602f190b3ccb238a043ac6b187dac/aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd", size = 1856631, upload-time = "2026-06-07T21:05:47.844Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb", size = 1714139, upload-time = "2026-06-07T21:05:49.982Z" }, + { url = "https://files.pythonhosted.org/packages/29/c4/93067c85a0373492ce8e577435203c5947c454af074ac48ed4f3a1b9dd4a/aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42", size = 1588321, upload-time = "2026-06-07T21:05:51.431Z" }, + { url = "https://files.pythonhosted.org/packages/c4/39/9ff91aaf02af8b7b8222a987466da539f154c3e01732c22b5f5a20a8ee66/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b", size = 1670375, upload-time = "2026-06-07T21:05:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e4/77452a3676b8d99ac1375f77691d6bf65ea6e9f4b201b82ef77c916dc767/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192", size = 1690933, upload-time = "2026-06-07T21:05:54.902Z" }, + { url = "https://files.pythonhosted.org/packages/7d/84/b0059a7c7fc05ea23f3bc1596ba91c12f79588b9450564a24cac37536d0a/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05", size = 1740798, upload-time = "2026-06-07T21:05:56.458Z" }, + { url = "https://files.pythonhosted.org/packages/8f/3a/e2a513ecbfc362591caa51a7f7e011b3bfc8938b388ae44cd95560d36999/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe", size = 1576412, upload-time = "2026-06-07T21:05:57.953Z" }, + { url = "https://files.pythonhosted.org/packages/a1/10/08f1654f538f93d36dcac66310a06eefce4641cdafca83f9f0a5317be254/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d", size = 1750199, upload-time = "2026-06-07T21:05:59.488Z" }, + { url = "https://files.pythonhosted.org/packages/99/e4/d91b70c57d8b8e9611e4a2e52238ca3698d3dc1c2efe25b7a9bf594ac584/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966", size = 1699356, upload-time = "2026-06-07T21:06:01.131Z" }, + { url = "https://files.pythonhosted.org/packages/3d/f1/15340176f35ff61b95dbe34020bcf43f9e624a2d7bbac934715ff97d2033/aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6", size = 458939, upload-time = "2026-06-07T21:06:02.86Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c2/a2f1ec5b37f903109e43ae2862268cfe4a67a60c1b2cf43169fcdff5995f/aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df", size = 482583, upload-time = "2026-06-07T21:06:04.666Z" }, + { url = "https://files.pythonhosted.org/packages/d0/7a/7b56f6732ef79530afaa72aa335d41b67c8d79b946995f0b11ad72985435/aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c", size = 453470, upload-time = "2026-06-07T21:06:06.322Z" }, + { url = "https://files.pythonhosted.org/packages/26/dd/bf526e6f0a1120dd6f2df2e97bacfe4d358f13d17a0ff5847301a1375a51/aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2", size = 765225, upload-time = "2026-06-07T21:06:07.957Z" }, + { url = "https://files.pythonhosted.org/packages/8f/e1/a2872aa55495a70f61310d411541c6ee23812d9a884e000c716e1bc3edbf/aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f", size = 518743, upload-time = "2026-06-07T21:06:09.749Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e7/c60c7b209e509cc787de3cea0550a518538cfc08003e1c1e14c1c63fff71/aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8", size = 514139, upload-time = "2026-06-07T21:06:11.26Z" }, + { url = "https://files.pythonhosted.org/packages/5b/8d/614ace2f579702c9840ab1e1447fd8509e35b0b904f7196418fa2f57b25d/aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04", size = 1784088, upload-time = "2026-06-07T21:06:12.887Z" }, + { url = "https://files.pythonhosted.org/packages/49/e0/726e90f99542bf292f81a96a12cc4847deb86f3ccf62c6f4014a201f4d33/aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8", size = 1737835, upload-time = "2026-06-07T21:06:14.564Z" }, + { url = "https://files.pythonhosted.org/packages/0b/4b/d176d5c4db9d33dacf0543102ea59503bc1d528af4cfd0b719949ca49389/aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6", size = 1842801, upload-time = "2026-06-07T21:06:16.228Z" }, + { url = "https://files.pythonhosted.org/packages/dc/d6/5a99b563690ea0cbed912ae94a2ce33993a5709a651a3a4fe761e7dd973a/aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af", size = 1929992, upload-time = "2026-06-07T21:06:17.947Z" }, + { url = "https://files.pythonhosted.org/packages/76/7f/a987b14a3859094b3cea3f4825219c3e5536242564af6e3f9c2f6c994eb2/aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730", size = 1786989, upload-time = "2026-06-07T21:06:19.677Z" }, + { url = "https://files.pythonhosted.org/packages/f1/1a/420e5c85a3e73349372ed22ce0b6af86bfa6ce16a4b20a64a2e94608c781/aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621", size = 1640129, upload-time = "2026-06-07T21:06:22.558Z" }, + { url = "https://files.pythonhosted.org/packages/a7/80/18a592ed3be0a402cc03670bd72ee1f8563ddbe1d8d5542dbf868f274136/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee", size = 1756576, upload-time = "2026-06-07T21:06:24.8Z" }, + { url = "https://files.pythonhosted.org/packages/ec/0b/8b3d5713373858ff71a617daf6e3b0e81ad63e79d09a3cf2f6b6b983939c/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573", size = 1754668, upload-time = "2026-06-07T21:06:26.528Z" }, + { url = "https://files.pythonhosted.org/packages/9f/49/fd564575cf225821d7ba5a117cb8bc27213d8a7e1811162afb43ae077039/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7", size = 1817019, upload-time = "2026-06-07T21:06:28.297Z" }, + { url = "https://files.pythonhosted.org/packages/ed/1b/e850c9ae6fc91356552ae668bb6c51e93fa29c8aef13398a10b56678557f/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf", size = 1631638, upload-time = "2026-06-07T21:06:30.242Z" }, + { url = "https://files.pythonhosted.org/packages/eb/94/3c337ba72451a89806ace6f75bddc92bafc5b8d53d90115a512858024b63/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85", size = 1835660, upload-time = "2026-06-07T21:06:31.943Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9c/9c18cf367a0498212d9ba7daf990b504a5e8ae064cda4b504e2647c89c03/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3", size = 1775698, upload-time = "2026-06-07T21:06:33.72Z" }, + { url = "https://files.pythonhosted.org/packages/b5/63/a251a9d2a6cb45065b2ddc0bde2b3dd10108740a9a42f632c66405a761a2/aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126", size = 458386, upload-time = "2026-06-07T21:06:35.279Z" }, + { url = "https://files.pythonhosted.org/packages/17/ca/69274c51dcd6e8947d77b2806cf47a4a15f2c846e2cbeb1882547d3da283/aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5", size = 483406, upload-time = "2026-06-07T21:06:36.824Z" }, + { url = "https://files.pythonhosted.org/packages/2c/8a/c25904f77690c3688ec140f87591ef11a0cfe36bf3d5c0f1f38056fb62b3/aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b", size = 452987, upload-time = "2026-06-07T21:06:38.371Z" }, + { url = "https://files.pythonhosted.org/packages/1d/21/151624b51cd92553d95424daf4bf19f19ce9be9002d19253e7e7ce67197b/aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480", size = 757402, upload-time = "2026-06-07T21:06:40.311Z" }, + { url = "https://files.pythonhosted.org/packages/c2/82/280619e0bd7bf2454987e19282616e84762255dd9c8468f62382e8c191f1/aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d", size = 512310, upload-time = "2026-06-07T21:06:42.207Z" }, + { url = "https://files.pythonhosted.org/packages/55/b2/2aac325583aaa1353045f96dffa586d8a34e8322e14a7ba49cffeb103ab4/aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2", size = 512448, upload-time = "2026-06-07T21:06:43.813Z" }, + { url = "https://files.pythonhosted.org/packages/8a/72/a60607cb849faa8af8a356c9329ea2eb6f395d49e82cc82ccba1fd8deb8f/aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2", size = 1766854, upload-time = "2026-06-07T21:06:45.391Z" }, + { url = "https://files.pythonhosted.org/packages/b5/d3/d9fe1c9ec7557ab4d0d82bebaa728c6418f0b93295ec2f4ab015f7710cc7/aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a", size = 1740884, upload-time = "2026-06-07T21:06:47.413Z" }, + { url = "https://files.pythonhosted.org/packages/c1/dc/f2cecfaf9337ba3e63f181500814ff502aa3d00d9c7ec93a9d23d10a27b2/aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264", size = 1810034, upload-time = "2026-06-07T21:06:50.165Z" }, + { url = "https://files.pythonhosted.org/packages/66/d7/2ff65c5e65c0d7476daf7e15c032e0805e36811185b9623e3238ad6c763e/aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842", size = 1904054, upload-time = "2026-06-07T21:06:52.035Z" }, + { url = "https://files.pythonhosted.org/packages/20/9c/d445818389df371f56d141d881153ba23183c4735a03f7356ffb43f7757d/aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c", size = 1790278, upload-time = "2026-06-07T21:06:54.049Z" }, + { url = "https://files.pythonhosted.org/packages/4d/aa/bf04cb4d865fc6101c2229a294ad744973b72e513fdc5a6b791e6983d72a/aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95", size = 1591795, upload-time = "2026-06-07T21:06:55.911Z" }, + { url = "https://files.pythonhosted.org/packages/dc/b4/4dac0038960427ba832f6609dfb4ea5437d7fd80c72001b9e48f834f428b/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199", size = 1728397, upload-time = "2026-06-07T21:06:57.777Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/7cd4e8ad7aa3b75f17d56bb5498dd604a93d4e6eece822ba0568c413fff0/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817", size = 1766504, upload-time = "2026-06-07T21:07:00.009Z" }, + { url = "https://files.pythonhosted.org/packages/f9/df/fc01d9fcad0f73fed3f3d361f1f94f975947b50dff82919f6dc2bf4316cc/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a", size = 1777806, upload-time = "2026-06-07T21:07:02.064Z" }, + { url = "https://files.pythonhosted.org/packages/41/09/47e2d090bddcc8fb4ccb4c314aadc32d7c5d9bb55f50f6ad1c92fc15d501/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4", size = 1580707, upload-time = "2026-06-07T21:07:03.942Z" }, + { url = "https://files.pythonhosted.org/packages/3d/36/f1a4ce904ae0b6930cfe9afc96d0896f7ec1a620c400405d63783bb95a9c/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087", size = 1798121, upload-time = "2026-06-07T21:07:05.987Z" }, + { url = "https://files.pythonhosted.org/packages/70/0a/e0075ce9ca0279ee1d4f0c0b85f54fea02ebc83c3007651a72bece658fec/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3", size = 1767580, upload-time = "2026-06-07T21:07:07.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/a0c0a8f327a9c52095cdd8e312391b00d3ed64ab6c72bb5c33d8ec251cf7/aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4", size = 452771, upload-time = "2026-06-07T21:07:09.669Z" }, + { url = "https://files.pythonhosted.org/packages/df/d9/ea367c75f16ac9c6cdc8febb25e8318fa21a2b1bc8d6514d4b2d890bface/aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271", size = 479873, upload-time = "2026-06-07T21:07:11.538Z" }, + { url = "https://files.pythonhosted.org/packages/03/64/8d96784a7851156db8a4c6c3f6f91042fdf39fb15a4cc38c8b3c14833c45/aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847", size = 448073, upload-time = "2026-06-07T21:07:13.637Z" }, + { url = "https://files.pythonhosted.org/packages/bc/97/bd137012dd97e1649162b099135a80e1fd59aaa807b2430fc448d1029aff/aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178", size = 506882, upload-time = "2026-06-07T21:07:15.501Z" }, + { url = "https://files.pythonhosted.org/packages/ef/79/e5cc690e9d922a66887ceeaca53a8ffd5a7b0be3816142b7abc433742d89/aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf", size = 515270, upload-time = "2026-06-07T21:07:17.53Z" }, + { url = "https://files.pythonhosted.org/packages/fe/22/a73ccbf9dbd6e26dda0b24d5fd5db7da92ee3383a79f47677ffb834c5c5b/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd", size = 485841, upload-time = "2026-06-07T21:07:19.555Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b9/57ed8eaf596321c2ad747bd480fb1700dbd7177c60dfc9e4c187f629662e/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe", size = 492088, upload-time = "2026-06-07T21:07:21.581Z" }, + { url = "https://files.pythonhosted.org/packages/78/c0/5ebe5270a7c140d7c6f79dcb018640225f14d406c149e4eec04a7d82fe71/aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4", size = 501564, upload-time = "2026-06-07T21:07:23.388Z" }, + { url = "https://files.pythonhosted.org/packages/75/7f/8cdaa24fc7983865e0915153b96a9ac5bcdd3548d64c5a27d17cecccad2d/aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876", size = 751998, upload-time = "2026-06-07T21:07:25.046Z" }, + { url = "https://files.pythonhosted.org/packages/b2/f4/c4227aacfacc5cb0cc2d119b65301d177912a6842cd64e120c47af76064f/aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da", size = 510918, upload-time = "2026-06-07T21:07:27.28Z" }, + { url = "https://files.pythonhosted.org/packages/ab/01/a2d5f96cd4e74424864d30bc0a7e44d0a12dacdcfa91b5b2d1bd3dca6bf3/aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6", size = 508657, upload-time = "2026-06-07T21:07:29.252Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ed/3c0fb5c500fdd8e7ebc10d1889c04384fffa1a9163eac1356088ca9da1b1/aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96", size = 1757907, upload-time = "2026-06-07T21:07:31.03Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ab/d4c924d9bd5be3050c226612413ce68cb54c70d2c31b661bfc8d9a5b6a70/aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f", size = 1737565, upload-time = "2026-06-07T21:07:33.031Z" }, + { url = "https://files.pythonhosted.org/packages/19/2a/37326821ff779084020cdc33224d20b19f42f4183a500ff92022a739eda7/aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296", size = 1799018, upload-time = "2026-06-07T21:07:35.003Z" }, + { url = "https://files.pythonhosted.org/packages/b3/4f/6e947ba73e4ce09070761c05ed3a8ceb7c21f5e46798671d8b2aac0e4626/aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa", size = 1894416, upload-time = "2026-06-07T21:07:36.956Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6e/dbf1d0625dc711fb2851f4f3c3055c39ed58bae92082d8c627dbe6013736/aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451", size = 1783881, upload-time = "2026-06-07T21:07:39.063Z" }, + { url = "https://files.pythonhosted.org/packages/44/c2/5e25098a67268ed369483ae7d1a58bd0a13d03aab860d2a0e4a6eb25b046/aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c", size = 1587572, upload-time = "2026-06-07T21:07:41.058Z" }, + { url = "https://files.pythonhosted.org/packages/2a/bd/cf9cee17e140f942a3de73e658a543aa8fbf35a5fc67a9d2538d52d77f0b/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca", size = 1722137, upload-time = "2026-06-07T21:07:43.014Z" }, + { url = "https://files.pythonhosted.org/packages/89/6d/5684f8c59045c96f81a18cefbc1fbbd79d25b88f1c622f2a5c5c08fcb632/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09", size = 1755953, upload-time = "2026-06-07T21:07:45.933Z" }, + { url = "https://files.pythonhosted.org/packages/a8/40/35caf3170f8359760740a7d9aa0fff2e344bef98e1d1186f5a0f6dec17e6/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397", size = 1766479, upload-time = "2026-06-07T21:07:48.047Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a1/b0c61e7a137f0d81de49a82023a6df73c3c16d6fefb0f8e4a93d21639002/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080", size = 1580077, upload-time = "2026-06-07T21:07:50.069Z" }, + { url = "https://files.pythonhosted.org/packages/0b/41/194ea4623693009fcefebef7aef63c141754f153e9cd0d39d3b9e36c175c/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345", size = 1791688, upload-time = "2026-06-07T21:07:52.106Z" }, + { url = "https://files.pythonhosted.org/packages/ba/45/4de841f005cfe1fd63e2a2fe011262c515e2a62aa6994b15947e7d717ac9/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588", size = 1761094, upload-time = "2026-06-07T21:07:54.113Z" }, + { url = "https://files.pythonhosted.org/packages/e4/ae/dbce10533d3896d544d5053939ed75b7dc31a1b0973d959b1b5ae21028d6/aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780", size = 452662, upload-time = "2026-06-07T21:07:56.06Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d9/0bf1a19362c32f06229da5e7ddfcec91f93474d6307f7a2d3135e9c674dc/aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a", size = 479748, upload-time = "2026-06-07T21:07:58.319Z" }, + { url = "https://files.pythonhosted.org/packages/22/0a/62e7232dc9484fbec112ceb32efb6a624cc7994ec6e2b019286f17c4e8f2/aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8", size = 447723, upload-time = "2026-06-07T21:08:00.154Z" }, + { url = "https://files.pythonhosted.org/packages/c4/a1/5fafa04e1ca91ddb47608699d60649c1c6db3cf41c99e78fc4056f9513db/aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15", size = 508531, upload-time = "2026-06-07T21:08:02.093Z" }, + { url = "https://files.pythonhosted.org/packages/fa/2e/bfa02f699d87ffc86d5959270b28f1cb410add3ccaced8ed2e0b8a5238fc/aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8", size = 514718, upload-time = "2026-06-07T21:08:04.476Z" }, + { url = "https://files.pythonhosted.org/packages/85/a5/9594ad6289eebbc97d167c44213d557807f90e59115caad24de21ad2c3b1/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3", size = 487918, upload-time = "2026-06-07T21:08:06.377Z" }, + { url = "https://files.pythonhosted.org/packages/b4/61/16a32c36c3c49edec122a3dc811f2057df2f94d3b14aa107c8017d981618/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba", size = 494014, upload-time = "2026-06-07T21:08:08.263Z" }, + { url = "https://files.pythonhosted.org/packages/9b/89/3ebcf96ed99c05bec9c434aaac6963fd3cbab4a786ae739908a144d9ce44/aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397", size = 502398, upload-time = "2026-06-07T21:08:10.244Z" }, + { url = "https://files.pythonhosted.org/packages/fd/3d/b74870a0c2d40c355928cd5b96c7a11fa821b8a40fc41365e64479b151fb/aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448", size = 758018, upload-time = "2026-06-07T21:08:12.447Z" }, + { url = "https://files.pythonhosted.org/packages/d3/66/f42f5c984d99e49c6cff5f26f590750f2e2f7ef1fcfb99966ab5be1b632e/aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004", size = 512462, upload-time = "2026-06-07T21:08:14.624Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a7/248e1aebe0c7810b0271e021a0f2a5eb6e78a051885b3c9df49f42a5802d/aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983", size = 512824, upload-time = "2026-06-07T21:08:16.572Z" }, + { url = "https://files.pythonhosted.org/packages/26/97/2aa0e5ba0727dc3bd5aaebb7ccbc510f7dfb7fb961ec87497cd496635ab1/aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe", size = 1749898, upload-time = "2026-06-07T21:08:18.635Z" }, + { url = "https://files.pythonhosted.org/packages/00/8d/e97f6c96c891d457c8479d92a514ba194d0412f981d72c70341ee18488ed/aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333", size = 1710114, upload-time = "2026-06-07T21:08:20.892Z" }, + { url = "https://files.pythonhosted.org/packages/6f/e6/aa8d7e863048c8fceb5cd6ce74017311cec3ead07847387e12265fb4444e/aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0", size = 1802541, upload-time = "2026-06-07T21:08:23.044Z" }, + { url = "https://files.pythonhosted.org/packages/83/a8/72193137de57fda4ebfae4563182d082c8856e3b6e9871d0b46f028fb369/aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a", size = 1875776, upload-time = "2026-06-07T21:08:25.288Z" }, + { url = "https://files.pythonhosted.org/packages/a0/18/938441025db6769a3464596b2410af3afde0b21eb2f204c6f766f68af4bd/aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602", size = 1760329, upload-time = "2026-06-07T21:08:27.363Z" }, + { url = "https://files.pythonhosted.org/packages/60/29/bf2496b4065e76e09fe48015aaffe5ce161d8f089b06ac6982070f653076/aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca", size = 1587293, upload-time = "2026-06-07T21:08:29.805Z" }, + { url = "https://files.pythonhosted.org/packages/49/a2/2136674d52123b1354bd05dd5753c318db47dc0c927cc70b27bab3755456/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35", size = 1714756, upload-time = "2026-06-07T21:08:32.094Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b9/e5fd2e6f915503081c0f9b1e8540947037929c70c191da2e4d54b31a21a1/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844", size = 1721052, upload-time = "2026-06-07T21:08:34.167Z" }, + { url = "https://files.pythonhosted.org/packages/63/5a/2833e324a2263e104e31e2e91bc5bbee81bc499afd32203faee048a883f0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496", size = 1766888, upload-time = "2026-06-07T21:08:36.95Z" }, + { url = "https://files.pythonhosted.org/packages/57/fa/dea6511870913162f3b2e8c42a7614eb203a4540b8c2da43e0bfb0548f3c/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5", size = 1581679, upload-time = "2026-06-07T21:08:39.292Z" }, + { url = "https://files.pythonhosted.org/packages/14/bd/3cf0d55e71784b33534e9710a67d382d900598b4787fbce6cc7317f8c42a/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95", size = 1782021, upload-time = "2026-06-07T21:08:41.407Z" }, + { url = "https://files.pythonhosted.org/packages/c1/af/14bb5843eccbe234f4dfb78ab73e549d99727247e62ae5d62cbd22eaf5b0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444", size = 1742574, upload-time = "2026-06-07T21:08:43.795Z" }, + { url = "https://files.pythonhosted.org/packages/f2/1e/fbeb7af9210a67ac0f9c9bec0f8f4568497924e33137a3d5b48e1cf85f3f/aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0", size = 457773, upload-time = "2026-06-07T21:08:46.168Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2b/13e8d741a9ec5db7d900c060554cf8352ab85e44e2a4469ebb9d377bda17/aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719", size = 485001, upload-time = "2026-06-07T21:08:48.401Z" }, + { url = "https://files.pythonhosted.org/packages/df/30/491acfa2c4d6c3ff59c49a14fc1b50be3241e25bbb0c84c09e2da4d11395/aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec", size = 453809, upload-time = "2026-06-07T21:08:50.7Z" }, + { url = "https://files.pythonhosted.org/packages/34/e3/19dbe1a1f4cc6230eb9e314de7fe68053b0992f9302b27d12141a0b5db53/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2", size = 793320, upload-time = "2026-06-07T21:08:52.775Z" }, + { url = "https://files.pythonhosted.org/packages/7f/20/1b7182219ba1b108430d6e4dc53d25ae02dcfcf5a045b33af4e8c5167527/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340", size = 529077, upload-time = "2026-06-07T21:08:55Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c8/14ce60ec31a2e5f5274bb17d383a6f7a3aabca31ac04eee05585bbadab16/aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d", size = 532476, upload-time = "2026-06-07T21:08:57.176Z" }, + { url = "https://files.pythonhosted.org/packages/7e/02/9ac85e081e53da2e061b02fa7758fe0a12d17b8ce2d1f5e6c7cb76730328/aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94", size = 1922347, upload-time = "2026-06-07T21:08:59.563Z" }, + { url = "https://files.pythonhosted.org/packages/c0/3e/d3ba07a0ab38b5389e10bec4362d21e10a4f667cba2d79ba30837b3a5059/aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc", size = 1786465, upload-time = "2026-06-07T21:09:01.909Z" }, + { url = "https://files.pythonhosted.org/packages/0b/cb/e2ee978a00cfb2df829704a69528b18154eba5939f45bc1efa8f33aee4c5/aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251", size = 1909423, upload-time = "2026-06-07T21:09:04.357Z" }, + { url = "https://files.pythonhosted.org/packages/73/5d/1430334858b1022b58ae50399a918f0bd6fe8fa7fa183598d657ff61e040/aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1", size = 2001906, upload-time = "2026-06-07T21:09:06.722Z" }, + { url = "https://files.pythonhosted.org/packages/66/4e/560c7472d3d198a23aa5c8b19a5115bf6a9b77b7d3e4bb363da320430ad2/aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3", size = 1877095, upload-time = "2026-06-07T21:09:09.011Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f1/4745806578d447db4a784a8591e2dae3afdfc2bcb96f8f81271b13df6543/aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c", size = 1676222, upload-time = "2026-06-07T21:09:11.461Z" }, + { url = "https://files.pythonhosted.org/packages/6a/c9/48255813cca749a229ef0ab476004ec623728ad79a9c0840616f6c076325/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203", size = 1842922, upload-time = "2026-06-07T21:09:14.118Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c0/bbd054e2bee909f529523a5af3891052606af5143c09f5f183ec3b234676/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1", size = 1825035, upload-time = "2026-06-07T21:09:16.447Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ae/90395d4376deceb74e09ec26b6adf7d2015a6f8802d6d84446af860fef04/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665", size = 1849512, upload-time = "2026-06-07T21:09:18.742Z" }, + { url = "https://files.pythonhosted.org/packages/93/bd/fb25f3049957553d4ce0ba6ae480aa2f592a6985497fca590837d16c1be0/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1", size = 1668571, upload-time = "2026-06-07T21:09:21.458Z" }, + { url = "https://files.pythonhosted.org/packages/3f/22/7f73303d64dd567ff3addca90b556690ed1233a47b8f55d242fb90af3681/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d", size = 1881159, upload-time = "2026-06-07T21:09:23.813Z" }, + { url = "https://files.pythonhosted.org/packages/44/be/0474c5a8b5640e1e4aa1923430a91f4151be82e511373fe764189b89aef5/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c", size = 1841409, upload-time = "2026-06-07T21:09:26.207Z" }, + { url = "https://files.pythonhosted.org/packages/7b/3c/bb4a7cba26956cb3da4553cc2056cf67be5b5ff6e6d8fa4fbdff73bfb7ae/aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365", size = 494166, upload-time = "2026-06-07T21:09:28.505Z" }, + { url = "https://files.pythonhosted.org/packages/8a/84/ec80c2c1f66a952555a9f86df6b33af65108a6febfa0471b69013a12f807/aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9", size = 530255, upload-time = "2026-06-07T21:09:30.843Z" }, + { url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" }, ] [[package]] @@ -208,7 +281,7 @@ version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "frozenlist" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } wheels = [ @@ -251,21 +324,75 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/c1/dafbf82add040db10e6663da2a719eea9b2ca7a3b4dc79dc42cc130b121b/annoy-1.17.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c33a5d4d344c136c84976bfb2825760142a8bb25335165e24e11c9afbfa8c2e9", size = 57745, upload-time = "2023-06-14T16:37:32.469Z" }, ] +[[package]] +name = "anthropic" +version = "0.116.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform == 'linux'" }, + { name = "distro", marker = "sys_platform == 'linux'" }, + { name = "docstring-parser", marker = "sys_platform == 'linux'" }, + { name = "httpx", marker = "sys_platform == 'linux'" }, + { name = "jiter", marker = "sys_platform == 'linux'" }, + { name = "pydantic", marker = "sys_platform == 'linux'" }, + { name = "sniffio", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/a2/d31f14e28d49bae983a3634e38dfb4b31c50110b5e403596c5c6a20b23f8/anthropic-0.116.0.tar.gz", hash = "sha256:5fc248fbb9fe03ef686f8a774f81586bca31a043260aab88b387ea3660f4a396", size = 949149, upload-time = "2026-07-02T19:08:10.534Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/dd/2a1e81cf1b163acc340afc4ec74ed1d86f5eed1a809fabdeed3e0997b346/anthropic-0.116.0-py3-none-any.whl", hash = "sha256:6c0a7698e8d652455da3499978279bb2588c7264d0a35be3666009a4258c8256", size = 956896, upload-time = "2026-07-02T19:08:08.756Z" }, +] + [[package]] name = "anyio" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "idna" }, { name = "sniffio" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/95/7d/4c1bd541d4dffa1b52bd83fb8527089e097a106fc90b467a7313b105f840/anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028", size = 190949, upload-time = "2025-03-17T00:02:54.77Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" }, ] +[[package]] +name = "apache-tvm-ffi" +version = "0.1.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/60/1e787a0b5ebf318483235be2a689ee367173983067e441b8379564f667c0/apache_tvm_ffi-0.1.9.tar.gz", hash = "sha256:d2d402587e8906de0a07f4746aa78f3d452c7efe3625d4bb39ac2ad693bce530", size = 2513731, upload-time = "2026-02-27T19:28:06.602Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/3d/4594c14de64e92697a91eec8ac6518ad72a3f30776aff432e68c2c6d9d3d/apache_tvm_ffi-0.1.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d911cbbc83bf12a0d9ec03e5315ff1bb92d95702fe912cd7a050393274382e71", size = 2068752, upload-time = "2026-02-27T19:27:03.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/0a/827e4f9ae85e1be3037818abd59566d906ba1fe27295c6938b12cc482151/apache_tvm_ffi-0.1.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1c8dd4018420c0d14bace688594710909ce198056ff8ac2ad1cd462b30fe1bdd", size = 2231204, upload-time = "2026-02-27T19:27:04.734Z" }, + { url = "https://files.pythonhosted.org/packages/ae/b6/f1ec5c528918c4dae03885ec472663072a984431d7d7fb04ca0798a2e13c/apache_tvm_ffi-0.1.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f6bc8846d570b8ce38692fc91b530b44cd6ae092c805a844da23970e81b12c0", size = 2323684, upload-time = "2026-02-27T19:27:06.284Z" }, + { url = "https://files.pythonhosted.org/packages/28/08/818836fbc0f198da1597896f82d7e6556bf5678cd5150d633214bf14b718/apache_tvm_ffi-0.1.9-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3ec9149f207a7af3ea3531cad7a0b0d04ded06df4f51a547479d5eb489428dd", size = 2160066, upload-time = "2026-02-27T19:27:07.897Z" }, + { url = "https://files.pythonhosted.org/packages/c8/6b/2e7d73d055523c2fb31394cd3d55593969a0680619e1c939c2128c2fdd36/apache_tvm_ffi-0.1.9-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eefcd17f61bf503ff0f4ad429e03ef6c241c7d13682f58281d883218b854c9bd", size = 2307014, upload-time = "2026-02-27T19:27:10.287Z" }, + { url = "https://files.pythonhosted.org/packages/c1/9d/9b99efbeaaed4c78a2b7cfeda6b8fc7d6249616938c05ae0248aa0bf0d56/apache_tvm_ffi-0.1.9-cp310-cp310-win_amd64.whl", hash = "sha256:dd58da01331826fbe6c064d6f0c9bbc2d62883b78df8d15baa8ea21d37507e4d", size = 1993158, upload-time = "2026-02-27T19:27:11.884Z" }, + { url = "https://files.pythonhosted.org/packages/b0/44/130571cede8704b1412e48b3dd78de41b4d31b68241f954743d1a9925bd9/apache_tvm_ffi-0.1.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:932d94e29595a47109f0ef6e0b4209a934451582954ea8b426e758d6b3e307e3", size = 2070368, upload-time = "2026-02-27T19:27:13.779Z" }, + { url = "https://files.pythonhosted.org/packages/42/b1/9f2cfd6d49b03c5d4ec5c12548d911e2e01265be783f343103b4df716765/apache_tvm_ffi-0.1.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c0449fc3802987c3652bea266ffda2934a6f69c80bba791a3f55b91040656a18", size = 2231154, upload-time = "2026-02-27T19:27:15.691Z" }, + { url = "https://files.pythonhosted.org/packages/55/43/63faedea83494e99122466a993bcdccd31cf93c7e8a0d56731120e82e2b9/apache_tvm_ffi-0.1.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f16d73a82a9e68a439b7d233d48b1b929be17fe92df4bbf1ee2274e573144a3", size = 2323130, upload-time = "2026-02-27T19:27:17.259Z" }, + { url = "https://files.pythonhosted.org/packages/27/96/d735bc4c528efaf0a8a954076963c727aad2dde8577641aa9025ec4f2d52/apache_tvm_ffi-0.1.9-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:01ebb1308b2666c206aa9a4015eb48f03a5d98ea2e9cfb002bd5e2ca0b9c7ef3", size = 2159854, upload-time = "2026-02-27T19:27:18.789Z" }, + { url = "https://files.pythonhosted.org/packages/e4/3b/6cfc82a3ab5d9e501bbcee5df36eebe09da1c384461d7a55e2a17776d117/apache_tvm_ffi-0.1.9-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21365abd2a2a1a6d3b4e6e4f048309651125becfa795440c3607f3cc27d30ac7", size = 2307140, upload-time = "2026-02-27T19:27:20.222Z" }, + { url = "https://files.pythonhosted.org/packages/5f/61/3ffe1fe3190e12807a12b72ed0d291c7f66569c2e7c3571fde18175f19e1/apache_tvm_ffi-0.1.9-cp311-cp311-win_amd64.whl", hash = "sha256:9ee710a9fba3d9ff9747870bbd7e2175eb8d5b9c791f17fd645f35f6dab3f8aa", size = 1993218, upload-time = "2026-02-27T19:27:22.043Z" }, + { url = "https://files.pythonhosted.org/packages/df/f2/b8c4b151169f6d7ba8773c8af68b2e0c1013d7fb3f1bdf87573f47157ce9/apache_tvm_ffi-0.1.9-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:49e52350b0470654847de752e65603b604a4d3323e7e9f5e8a982f44acc4c143", size = 2041756, upload-time = "2026-02-27T19:27:23.931Z" }, + { url = "https://files.pythonhosted.org/packages/a7/c0/6d3d54f50012255b41bc3e24944c086f63c4707c8686c7c6780e9283eb96/apache_tvm_ffi-0.1.9-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d503029e66c43b1a1cb1a42a1e9bb428c8a28dcbdec31c28e705472ca648a3a", size = 2203712, upload-time = "2026-02-27T19:27:25.867Z" }, + { url = "https://files.pythonhosted.org/packages/c6/dd/2bab4c6cd86257dbf99e93452a1af833113f8dc3e25a25579f6e4e4c8a94/apache_tvm_ffi-0.1.9-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28241371934ea8af10d5067087ba1229ebddded7b2c02d33a258ec2a96df8c46", size = 2299704, upload-time = "2026-02-27T19:27:27.477Z" }, + { url = "https://files.pythonhosted.org/packages/7a/4a/b469bcb2e1014cb84d336d2a59f42958a058251c577a4c2680cacad346e2/apache_tvm_ffi-0.1.9-cp312-abi3-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87cacce81df55685fc6a76e1e3c5db1200e85e87bf5974b692c59d131b7bc622", size = 2130865, upload-time = "2026-02-27T19:27:29.092Z" }, + { url = "https://files.pythonhosted.org/packages/70/ef/5402da5d37f5270fd88ea0348acca78dba9be8bdbf6c2bcae0935eb03ef1/apache_tvm_ffi-0.1.9-cp312-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f45eb43499acac45ff6c93564f0ff2d3ca27b69656d540fd56ce59d51c0b4c65", size = 2278991, upload-time = "2026-02-27T19:27:30.729Z" }, + { url = "https://files.pythonhosted.org/packages/b5/23/1b7dc5f0807f83098183a57db6ee85b2c93b646d74a6e03781c9208aaeb0/apache_tvm_ffi-0.1.9-cp312-abi3-win_amd64.whl", hash = "sha256:d1dcf4c041d5ec05e3da1d545800c33cdbb95c113baa7705085ff79fa262752b", size = 1973200, upload-time = "2026-02-27T19:27:32.367Z" }, + { url = "https://files.pythonhosted.org/packages/4a/1e/991ae65e64ce132c1ba665562db6638f5696d6133f580e20c653de33b9af/apache_tvm_ffi-0.1.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c3349f72ddb8ce206472d0380a729f213017a2180707096f8d57114b81097dd1", size = 2072944, upload-time = "2026-02-27T19:27:34.261Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a7/1e0643949e683fb3cfababd87058c0cfef122d1a3bb6ce703f719051b842/apache_tvm_ffi-0.1.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d1f4d2b7ec7b1213632e9a104e9330bfc3dec48decffa62114c33aa188c9f43a", size = 2215954, upload-time = "2026-02-27T19:27:35.872Z" }, + { url = "https://files.pythonhosted.org/packages/d6/06/5016191ab61d2db4c3a7d754a3c1184e0836f575a7d08491669738c5e4b9/apache_tvm_ffi-0.1.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e4f01d16ba53fe118e363f7257253f07003797e4abe6fc9567f23b6a930dbff2", size = 2307291, upload-time = "2026-02-27T19:27:37.527Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f5/40bf0667330938efbfc0a51743cc53c79e41b4ece1a8abad3076192c9674/apache_tvm_ffi-0.1.9-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c0581dd6bfbce7b017ef85cfda08bbe38891cc4b3afbcfaa8bc2d383728e426", size = 2143850, upload-time = "2026-02-27T19:27:40.437Z" }, + { url = "https://files.pythonhosted.org/packages/72/4a/421cbd4ed32e8bad3b88af3e8fa145c1f6f493bdd05be15b6f2d9b3cb7d6/apache_tvm_ffi-0.1.9-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dfa14be2a49347791ef21222a8225ce7f99bfec17104a676cb4f1bf3a107088", size = 2289038, upload-time = "2026-02-27T19:27:41.972Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1a/c8923d819b49872a612033b90d29299c0be73a7cbed1ddb3dc78dfe5e9f1/apache_tvm_ffi-0.1.9-cp314-cp314t-win_amd64.whl", hash = "sha256:a42d7ca27dce83efbdce7ec970fe3e773a69c31d928730ee5d9badb1229d106c", size = 2039007, upload-time = "2026-02-27T19:27:43.618Z" }, +] + [[package]] name = "appnope" version = "0.1.4" @@ -321,6 +448,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/ed/e97229a566617f2ae958a6b13e7cc0f585470eac730a73e9e82c32a3cdd2/arrow-1.3.0-py3-none-any.whl", hash = "sha256:c728b120ebc00eb84e01882a6f5e7927a53960aa990ce7dd2b10f39005a67f80", size = 66419, upload-time = "2023-09-30T22:11:16.072Z" }, ] +[[package]] +name = "astor" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/21/75b771132fee241dfe601d39ade629548a9626d1d39f333fde31bc46febe/astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e", size = 35090, upload-time = "2019-12-10T01:50:35.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/88/97eef84f48fa04fbd6750e62dcceafba6c63c81b7ac1420856c8dcc0a3f9/astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5", size = 27488, upload-time = "2019-12-10T01:50:33.628Z" }, +] + [[package]] name = "asttokens" version = "3.0.1" @@ -335,7 +471,7 @@ name = "async-lru" version = "2.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e8/1f/989ecfef8e64109a489fff357450cb73fa73a865a92bd8c272170a6922c2/async_lru-2.3.0.tar.gz", hash = "sha256:89bdb258a0140d7313cf8f4031d816a042202faa61d0ab310a0a538baa1c24b6", size = 16332, upload-time = "2026-03-19T01:04:32.413Z" } wheels = [ @@ -425,6 +561,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2", size = 10182537, upload-time = "2025-02-01T15:17:37.39Z" }, ] +[[package]] +name = "backports-strenum" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/c7/2ed54c32fed313591ffb21edbd48db71e68827d43a61938e5a0bc2b6ec91/backports_strenum-1.3.1.tar.gz", hash = "sha256:77c52407342898497714f0596e86188bb7084f89063226f4ba66863482f42414", size = 7257, upload-time = "2023-12-09T14:36:40.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/50/56cf20e2ee5127b603b81d5a69580a1a325083e2b921aa8f067da83927c0/backports_strenum-1.3.1-py3-none-any.whl", hash = "sha256:cdcfe36dc897e2615dc793b7d3097f54d359918fc448754a517e6f23044ccf83", size = 8304, upload-time = "2023-12-09T14:36:39.905Z" }, +] + [[package]] name = "beartype" version = "0.14.1" @@ -461,7 +606,9 @@ name = "bitsandbytes" version = "0.49.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "packaging" }, { name = "torch" }, ] @@ -482,8 +629,8 @@ dependencies = [ { name = "packaging" }, { name = "pathspec" }, { name = "platformdirs" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fd/f4/a57cde4b60da0e249073009f4a9087e9e0a955deae78d3c2a493208d0c5c/black-23.12.1.tar.gz", hash = "sha256:4ce3ef14ebe8d9509188014d96af1c456a910d5b5cbf434a09fef7e024b3d0d5", size = 620809, upload-time = "2023-12-22T23:06:17.382Z" } wheels = [ @@ -502,6 +649,95 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/14/4da7b12a9abc43a601c215cb5a3d176734578da109f0dbf0a832ed78be09/black-23.12.1-py3-none-any.whl", hash = "sha256:78baad24af0f033958cad29731e27363183e140962595def56423e626f4bee3e", size = 194363, upload-time = "2023-12-22T23:06:14.278Z" }, ] +[[package]] +name = "blake3" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.12' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/6a/4cc5a9dd40fd8a6d283fd3761e5f59c490109571ef8e3c73245417e5a305/blake3-1.0.9.tar.gz", hash = "sha256:5fa374fa5070ca084368776c19b420157eb0f2d3f091343d6bc59189929d62e2", size = 116872, upload-time = "2026-06-22T18:02:25.366Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/2f/5398493cef29d9f216be1ff74a303e809e4958a633a44545035a98af4f60/blake3-1.0.9-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:38e61d3b0386af16b3c03a18e0db82b626d63796274637a1fef855fd1c778d82", size = 346497, upload-time = "2026-06-22T17:59:57.448Z" }, + { url = "https://files.pythonhosted.org/packages/0b/4d/8aeca9a40899258353a8f79ad164fba1184bc1554ca18607cab4671952f3/blake3-1.0.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e9e1d0392624c2f9d049d786f0dc547ce818d2f2b356bcf1c4d74b6f9cc026b4", size = 335390, upload-time = "2026-06-22T17:59:59.162Z" }, + { url = "https://files.pythonhosted.org/packages/a1/0a/74c67827a9cae097ccab7015018182da9cfec347c686a25ef33faf2f46a1/blake3-1.0.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8114fb2a1f6cba9cba5411d62cbcb283b2205b154d0076f20b77e22592eb2719", size = 378100, upload-time = "2026-06-22T18:00:00.468Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8e/cef564771169b6fb429d9c52652dd2da8c9bbadb63d2d66f232f8bf045de/blake3-1.0.9-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b985eb08db76550ec97444e03b10acd737baa03fd98aaf3b8455a1c644c8f5d6", size = 377559, upload-time = "2026-06-22T18:00:01.822Z" }, + { url = "https://files.pythonhosted.org/packages/d1/92/2df756e410d18aba6fef6392b35b835c76412709739a2cde552d246afa4b/blake3-1.0.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a517f0460007edec3767595115c520ed1f157ddd0ed23dddbf6b9d8b0082afb6", size = 451544, upload-time = "2026-06-22T18:00:03.293Z" }, + { url = "https://files.pythonhosted.org/packages/88/69/44423d63e7c6d09000ce69784dd9fb45bda93237f1d2f611099f5ffe27c7/blake3-1.0.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dad0a8a716dd201860f8e82011a340e6bdd5ee37a8eb4357b48ac64c4e6de1c2", size = 492654, upload-time = "2026-06-22T18:00:04.638Z" }, + { url = "https://files.pythonhosted.org/packages/a2/02/7ca45b504796a755bcd765e54f0c6762c16a1dac1adec3a03a45ae9c2f12/blake3-1.0.9-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bca166d0b01c00dcf2a936f790ed947bd9079b0a0a7df1b76746f201aa4f4ac4", size = 387295, upload-time = "2026-06-22T18:00:06.026Z" }, + { url = "https://files.pythonhosted.org/packages/0a/e4/c8fa46a0e24cb877fbf28f839d8ceda39418259f677ec55d680ea433b62b/blake3-1.0.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa6e5c7533c915a24d840ae4be787e9a6059be7e77944b005b3d967a0257a17d", size = 387632, upload-time = "2026-06-22T18:00:07.349Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b3/6315be017515868126e106f3dfe50223fbbb87bed67109bfbf883228f505/blake3-1.0.9-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:24acb1e6f31021fa08b7eb31433035facfcf0d82e964170d5eb85a30ce913ba9", size = 384740, upload-time = "2026-06-22T18:00:08.747Z" }, + { url = "https://files.pythonhosted.org/packages/6f/e8/fe7e40384c0f7995fe8dca57428241768897533b9e17cbc367c1614ef82f/blake3-1.0.9-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:216977b1d592a60150cd5de64d5853dc6afb0eb522cb387723ae7f78f380d947", size = 553251, upload-time = "2026-06-22T18:00:10.192Z" }, + { url = "https://files.pythonhosted.org/packages/19/e5/e9ecb843308db2b5ca29d604589a15f50d13c20df792260053bf9f014de4/blake3-1.0.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6f2dd643166dfeb7cf4ad53eb2d801f944d247212d3481950b4d5b4a20551461", size = 595209, upload-time = "2026-06-22T18:00:11.644Z" }, + { url = "https://files.pythonhosted.org/packages/da/42/201d43f8fee831693f34f7b384a65e41ab7720e6cd8d775cb57d9da69993/blake3-1.0.9-cp310-cp310-win32.whl", hash = "sha256:c755044ba7bec3d03dae44b968194112f0eb0e8c4523465f3dd9e1a87e178d89", size = 231157, upload-time = "2026-06-22T18:00:13.035Z" }, + { url = "https://files.pythonhosted.org/packages/f2/12/f23a64ba2ef270457345499f857628757fafd83f52274c1588e1b4a5b4c0/blake3-1.0.9-cp310-cp310-win_amd64.whl", hash = "sha256:8cd10c6a421a7d3c81136658e52e9ef58bfcc1df04193466664eb24981784f4c", size = 220829, upload-time = "2026-06-22T18:00:14.298Z" }, + { url = "https://files.pythonhosted.org/packages/27/12/aa8d72228b6ff61c675bd6f55ab138a91d71499c8a707cc9fb2052f1d2b5/blake3-1.0.9-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f169519c7ef25ef2c446b05e2f08e7e59fae312d569f98a3134b38d4caf7abd4", size = 346253, upload-time = "2026-06-22T18:00:15.537Z" }, + { url = "https://files.pythonhosted.org/packages/72/3a/820d2f729dfe152d5ebde16390f808c762dce3f21fb764ab033803ff2b1a/blake3-1.0.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b5e1f21b49492d01fa5a02084894c491ab9e7a1867fced107f7126c80d067c94", size = 335497, upload-time = "2026-06-22T18:00:16.942Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d6/d5462ec19a7f3d084fe327e08618fa107799ee708df04b3a2d620bd62816/blake3-1.0.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ee96daaa850700fd342a811fa10a8780fd2e8464a71b83a1779c7b6becd3dd5", size = 377621, upload-time = "2026-06-22T18:00:18.389Z" }, + { url = "https://files.pythonhosted.org/packages/92/98/dbc433f2a45be1b2344a6035d4212dfb6e6eb45046ad15103ead9c82d491/blake3-1.0.9-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:09deb024cd75cb200e7f647cd038800e6edc8f190c8188e0c69ec1c2b920e125", size = 377495, upload-time = "2026-06-22T18:00:20.067Z" }, + { url = "https://files.pythonhosted.org/packages/e0/3d/c7a699fb60d8ed31f3f28e6aec7658d29e45ec89e7054906b3040ce3ee65/blake3-1.0.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c99afb0459c82dd13e456b6b68d45c4768b539ca998dacd3ed726f1e75e91dc", size = 451158, upload-time = "2026-06-22T18:00:21.459Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a1/0b1b0dbf2dd772483e372237bb65385602b019e24b67424b1fc9e5447837/blake3-1.0.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:28528d1f29e6f3d45faf3482e1197e5e175730eef38bdc74e56ee11b68e0ad0d", size = 491988, upload-time = "2026-06-22T18:00:22.984Z" }, + { url = "https://files.pythonhosted.org/packages/ee/d1/ed319477f6d263a4f6b7e9aa465b06be5235a854923edbc9ea09508b6638/blake3-1.0.9-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65c0c20014df687694af5ccf0cec3bdb194511da8ebd50c30b0fd55c83fa4fd5", size = 386848, upload-time = "2026-06-22T18:00:24.319Z" }, + { url = "https://files.pythonhosted.org/packages/80/3e/a4cfb269f3e0955598b415a7843c358c4f79e826e3c9118dc9fb1f101ee6/blake3-1.0.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:964b642631a3c8fe117b3439c8ae64a9a0981af9444e409656d1f1e464bfa125", size = 387842, upload-time = "2026-06-22T18:00:25.589Z" }, + { url = "https://files.pythonhosted.org/packages/59/0e/d4ee3d89eece42f86eb46663aa42702000516b7ffbc53f60b918efe95b57/blake3-1.0.9-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:2fd000708662b04be211a22c1095b65fe399d7276e9f3bb2fd1ef8aacc545791", size = 384317, upload-time = "2026-06-22T18:00:26.891Z" }, + { url = "https://files.pythonhosted.org/packages/3a/aa/317106349d10de3b51332ad1e761f4864ebe887854396b75975304dcfbd1/blake3-1.0.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:82ecade6ac425fdfc39a4371d6d9232fd6e5c28748fd8d3489016ead17407014", size = 553005, upload-time = "2026-06-22T18:00:28.246Z" }, + { url = "https://files.pythonhosted.org/packages/39/cc/7fbce61a0b24bda1aac99da674bd74ac2b687b61db071c888ffdb30cb47a/blake3-1.0.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b4102ba86b86c992a931b4a88c58a632d6097461e14a1e63ebd2ecb98ff0898f", size = 595086, upload-time = "2026-06-22T18:00:29.96Z" }, + { url = "https://files.pythonhosted.org/packages/e6/91/6ddc7a8b582a0871f23d6db722f4950a8918096d5fa10f9f0f992c2aea39/blake3-1.0.9-cp311-cp311-win32.whl", hash = "sha256:2f4ce45da903f3d0a7e342fa70c7cce9c10cef6b529eadb4d6213be0ab0eaf84", size = 231230, upload-time = "2026-06-22T18:00:31.247Z" }, + { url = "https://files.pythonhosted.org/packages/23/68/ea698e6df48eeb417671544cfbb18c60f863cb689306cc52f19666dd98f8/blake3-1.0.9-cp311-cp311-win_amd64.whl", hash = "sha256:d819457dccfd82fe34684ec99e36725f747bd5761a0e17f537387fb31d121193", size = 220622, upload-time = "2026-06-22T18:00:32.495Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d2/9bdf8345c70993aaef635398f52edfb915d6e8ad2c000c801204e387c456/blake3-1.0.9-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a70c20542d5e7960983a0ff32999049a2b0e5ef1f22dbbbdfb51cf04828a4156", size = 344587, upload-time = "2026-06-22T18:00:34.244Z" }, + { url = "https://files.pythonhosted.org/packages/36/9d/be8b1f7f85b12bb45a0fade6ca7bdbf83a507d23d0b6141ba29fe69c8cea/blake3-1.0.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:72cdecf088a9d25e6ec79948a578995649b0dbee407e7a46c543a9ecc0f6f281", size = 328864, upload-time = "2026-06-22T18:00:35.59Z" }, + { url = "https://files.pythonhosted.org/packages/f2/78/66580635d744c826671fd219938caffb16281a26f62c4f856695d4233677/blake3-1.0.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42fa57bf462285ef16400601b0fd32214c248ba92505bbb94b1221ab9af5a092", size = 373795, upload-time = "2026-06-22T18:00:36.887Z" }, + { url = "https://files.pythonhosted.org/packages/b1/79/b5b17d3004bb81a5732c0b176c812703d200ed8c652b3b7713b9633bbe10/blake3-1.0.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b25ccde5a64be070f20e5c7a81da70292db40b164b6c77588cbd6230856badbb", size = 374183, upload-time = "2026-06-22T18:00:38.205Z" }, + { url = "https://files.pythonhosted.org/packages/3c/63/0d209c44b2041bbe130ced12a23c92dd995fbfe5bce7ee77fffea16f5cb0/blake3-1.0.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a800b87433955f37691b5f361ad29c7dd3ee089c9cd109adc5aea8e24bc4c1f", size = 446783, upload-time = "2026-06-22T18:00:39.493Z" }, + { url = "https://files.pythonhosted.org/packages/c5/51/efd1f9b8a9d3e9a0e235f3ced99a738529a1019fe78b3988e29d9c2fbba6/blake3-1.0.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6879739e7904b9c42afbedbcc2e8c36cebe140fb3fc3f5c492993579cf5cd516", size = 487369, upload-time = "2026-06-22T18:00:40.875Z" }, + { url = "https://files.pythonhosted.org/packages/8d/3f/a8dcaea9e0b26e419a540ca0cd6203c9fbb505e85b02b03c5a59bf9e6a45/blake3-1.0.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6edeb3d49a24c307995899b70dd47aa901d0e9ad51d2f8a79aba4f074f32d8c5", size = 383845, upload-time = "2026-06-22T18:00:42.251Z" }, + { url = "https://files.pythonhosted.org/packages/f6/10/e9907f5b86410d5071982aaf05d149ca4d4fd8acab7e77eebbc9a333c7b4/blake3-1.0.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcd56a7a972c4185070f7042ccc20166927eec3c0f98b8405f375d007b604a0b", size = 383851, upload-time = "2026-06-22T18:00:43.715Z" }, + { url = "https://files.pythonhosted.org/packages/34/cf/c7863a185550706a9624f6aa7b6d46470aaed0bb46a827c5cda2a7d03151/blake3-1.0.9-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:a288664d08dee154cc496e06e62517fc9e655ecec12b0d7db538d244ac79edf1", size = 380067, upload-time = "2026-06-22T18:00:45.249Z" }, + { url = "https://files.pythonhosted.org/packages/54/0a/e7af679c719368b400c9ba9c3460072aac2ba077ddbd4bc806fef28cda03/blake3-1.0.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:91db52a809b68b5bebe7c413ddcd230e1f759398e7fa7a873104595a4fa648b6", size = 549471, upload-time = "2026-06-22T18:00:46.793Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3c/37c1dd3539b7bd9b6d2eef019802aacdb4a3d48ab484b140603bbf9c5b5a/blake3-1.0.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cfaa671b07eb73883162ca940442193868358b0b904cfa266e4b74131ce966da", size = 591396, upload-time = "2026-06-22T18:00:48.122Z" }, + { url = "https://files.pythonhosted.org/packages/ae/55/4f0a23b72795292e74084834130900ea778c0583004519c86698dfffe1a5/blake3-1.0.9-cp312-cp312-win32.whl", hash = "sha256:ae47c3d5729ff89baa6ddf6de47fcfcc915985d39eb1bfcd6db653331f3c6fcc", size = 229271, upload-time = "2026-06-22T18:00:49.377Z" }, + { url = "https://files.pythonhosted.org/packages/12/91/7db93e4689f0f145bcb954dc62936e5f5090548a9fa20c6bbebfaeaa648a/blake3-1.0.9-cp312-cp312-win_amd64.whl", hash = "sha256:15566065ff90ab3da46ec0be1417406f00507af902b6fb0fbc6563e77f02fc42", size = 218220, upload-time = "2026-06-22T18:00:50.659Z" }, + { url = "https://files.pythonhosted.org/packages/41/1b/95b473d649f5322e69674622a307ffdb4f0b63adb0a0adcbc5cb8a8833c2/blake3-1.0.9-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:69ff5aebc7650954443aa701feff2028d7c7ea5b5e18ee265f15e2104e892328", size = 343869, upload-time = "2026-06-22T18:00:51.936Z" }, + { url = "https://files.pythonhosted.org/packages/4b/9d/adec22c719d8451af1dc9e624bf5907008ef1e0afa51aa69fd1e8c91e60e/blake3-1.0.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0cdfeff65488089ef86f7587c76055ff72b28d28d10e427b547f5711477c376d", size = 328482, upload-time = "2026-06-22T18:00:53.39Z" }, + { url = "https://files.pythonhosted.org/packages/5e/aa/0a6967ff9a6ae182419a681aed54f7338b34a1f71372e90f787a2afa42e6/blake3-1.0.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:766f1555cbe614f14f399c2fbec0983568d20edb36837ba04040807eb9e1a609", size = 373616, upload-time = "2026-06-22T18:00:54.701Z" }, + { url = "https://files.pythonhosted.org/packages/1c/51/5d4e198bf3ae902c6697ad6ec77d7210736ad8f680980e8b648dcfcd09a0/blake3-1.0.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:128a62136c9a39c7cb9fdaa5fb38471f2418853da7f5a89f31495735d0ba6f2c", size = 374149, upload-time = "2026-06-22T18:00:56.015Z" }, + { url = "https://files.pythonhosted.org/packages/7e/62/d3c7c364925b3f10828e5137376f3947f112c32188e899b42f09c2fde98a/blake3-1.0.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1ea0bf17b184b03444007646d902207d2b4d4f3e91a0cac3836552d83db74b9", size = 446151, upload-time = "2026-06-22T18:00:57.378Z" }, + { url = "https://files.pythonhosted.org/packages/b1/01/55b89389c5036c9d24b1d762d6265e91552e10b76a3c99fece3c4a7a4783/blake3-1.0.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73a48f7e9f0e047f51a445d9b0361ab1907bdc72b6857815a84dacd2e59556f8", size = 487256, upload-time = "2026-06-22T18:00:58.763Z" }, + { url = "https://files.pythonhosted.org/packages/b2/7a/a21b52253292ad3e4df63ea4a01ce11d3ee8f4a8a8d80eaf0c7ce92a62bd/blake3-1.0.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b27550ada40f839aca64c66127940e4318bb6ef3e291890ef913017f6f637448", size = 383977, upload-time = "2026-06-22T18:01:00.192Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f0/fe7188201a29ee9b042616c786a98afd864d537ca96198e64c3fe4ff13a9/blake3-1.0.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66c84dbc2a31eda88b55bbf5c5b711037bf0698eba0fd1faf06bdaf313c39048", size = 383615, upload-time = "2026-06-22T18:01:01.557Z" }, + { url = "https://files.pythonhosted.org/packages/22/08/f6a213b950e30fe9ef7d7fc061ec388e66ed62643570226882e6f7136ea3/blake3-1.0.9-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:dab59b324aa65c09e937d6c43de5de85ec9581627f4e79dcc9806d85b54a1c34", size = 380288, upload-time = "2026-06-22T18:01:03.025Z" }, + { url = "https://files.pythonhosted.org/packages/f1/fb/b171e47c1b835483bcf1545ebc289458165f8dc0f5c7f74a9176d7e9af03/blake3-1.0.9-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:eca281fedcbe5c56655bd5a4176e6036eddbbe57df96114a03838fce08b1e0ca", size = 549122, upload-time = "2026-06-22T18:01:04.486Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d8/7bf71c2c85a0951e406971f151435e0751716907e3924c6c48a2d6dae0db/blake3-1.0.9-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:3cbe7f190164896dc3908e920716ee66bc31d40f1a0fb603ed59ac53290fb9cf", size = 591183, upload-time = "2026-06-22T18:01:06.259Z" }, + { url = "https://files.pythonhosted.org/packages/20/85/34c3ea03cc90b2516628494ab3e0a98aec4ca8b04d037840ccd390e480ca/blake3-1.0.9-cp313-cp313-win32.whl", hash = "sha256:508ccaf8f9377cc47e6026c2897fdc37de61faeb1420dc023b6379cc2474eb65", size = 229053, upload-time = "2026-06-22T18:01:07.638Z" }, + { url = "https://files.pythonhosted.org/packages/db/2e/f09e8ed426f360aa2005206466ceab2f707486eb5d9db7051dbcbae056d1/blake3-1.0.9-cp313-cp313-win_amd64.whl", hash = "sha256:caded2806d2cbeed638c5e2517ed8b2a94165b3452fda35e72896142d22070e0", size = 217589, upload-time = "2026-06-22T18:01:08.922Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4b/b2dd7c25378a3b5de30ed908d38e6427bc4c644c0c12e8359361abd3a9ca/blake3-1.0.9-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:ab0c030cf6644c30e786b0e785bde4e4596013ae9ea6ce9877e39d52383e25d7", size = 345406, upload-time = "2026-06-22T18:01:10.311Z" }, + { url = "https://files.pythonhosted.org/packages/c6/dc/c0dab2963ddf04a4a938363f61716f9b75de6d3a9bc4a89e78f0854d4d31/blake3-1.0.9-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:83b4a2336105af3800f7e17ac4b943f293a3927a2d66a6308d50dba944a6953e", size = 330077, upload-time = "2026-06-22T18:01:11.926Z" }, + { url = "https://files.pythonhosted.org/packages/20/f1/d03950a86d105a6332a8c422cb87658a7d247e214f1ea8f29ed09ff04e00/blake3-1.0.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95fc3545f80901b0dcd0508d16bc40f15ae39556709fa6cf86675f742d4f3c9c", size = 375147, upload-time = "2026-06-22T18:01:13.198Z" }, + { url = "https://files.pythonhosted.org/packages/10/75/711b1842e0a90aaad6a1c9a9022e90aa16206ac1f224516118bc24482532/blake3-1.0.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1bd981dc318c05375c3160a99df493b7cc4c83fffa1a34d14b18a071b47b262b", size = 373711, upload-time = "2026-06-22T18:01:14.606Z" }, + { url = "https://files.pythonhosted.org/packages/9e/a0/f512799d1d0c0b4718fa6f0e99ccbe108e98bac7bf82c200803a62b57876/blake3-1.0.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:689a7e4069de681d9c5d9445b8b6473ee880ad04d7960a6789c60bd788980250", size = 446993, upload-time = "2026-06-22T18:01:15.924Z" }, + { url = "https://files.pythonhosted.org/packages/60/fb/6636ae8a46fc3352694188f5a5a325567782bc88fd1823b0b67be2c92184/blake3-1.0.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8adb0b0032e53919ee95b3d4f911448d3268316c28cd7df232ff2a1e7c9a4ba4", size = 488478, upload-time = "2026-06-22T18:01:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c5/a2b3c086f7e37c9db6017dc2890a76ad2a729e4a554896e855e511811e6b/blake3-1.0.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32bd4521ec2d477627ad93eb70f9ac4d01e12d1489024159bcaeff79466332f6", size = 384900, upload-time = "2026-06-22T18:01:18.802Z" }, + { url = "https://files.pythonhosted.org/packages/e1/b8/1298806dd6c464a6f807df24c9640ad3bf27ee54ff4de82b2b5a823a8aba/blake3-1.0.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f65d77eb05331495485048f6804f53885b192b998acb7e6fe1487d941bf08435", size = 384333, upload-time = "2026-06-22T18:01:20.35Z" }, + { url = "https://files.pythonhosted.org/packages/3c/cc/0c29d9404155adfd6db716e9765d36ea6cbed287060759f5d764f0d9d99e/blake3-1.0.9-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:ca7dfe8fb197ff8a3f5c915424183ccd52a99e8afb12680f51b2e1f4c9c6c97f", size = 381142, upload-time = "2026-06-22T18:01:21.744Z" }, + { url = "https://files.pythonhosted.org/packages/d6/91/9af20d563f0ced71e08a60fc0ee534146da4e265710ed6792d5d799f4c0f/blake3-1.0.9-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:f5c9d57f0dcb92243b6ae575c3065793edc9df9008d0ebd98d8245cdeb7c3f84", size = 550587, upload-time = "2026-06-22T18:01:23.381Z" }, + { url = "https://files.pythonhosted.org/packages/d0/fa/06f46fc0aa486b799d776f9a80ed0b3605e2be1570cf48007860948aa5d9/blake3-1.0.9-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:172d44245a19dfec08ab771c1b7a506b97783163cdc65f559fe020007e403c99", size = 591888, upload-time = "2026-06-22T18:01:24.805Z" }, + { url = "https://files.pythonhosted.org/packages/50/68/d6198f4069a7c4a184ed854df45b82cc3e2d4b0be476b2a3ee65ad2344cf/blake3-1.0.9-cp314-cp314-win32.whl", hash = "sha256:249e5964fa9e768924bc7cc3d4efe75a425bb5dd3fb7671c3eda8eeddfa50591", size = 229410, upload-time = "2026-06-22T18:01:26.24Z" }, + { url = "https://files.pythonhosted.org/packages/63/ab/f29af72a8312b3827b50e55491f1bf9ae2347591de5c47365c5cbd2525a9/blake3-1.0.9-cp314-cp314-win_amd64.whl", hash = "sha256:0aba416bb2e3ef0c65e74d5eba21062483c714cd78e7e303c9d03c547fc7d015", size = 218526, upload-time = "2026-06-22T18:01:27.779Z" }, + { url = "https://files.pythonhosted.org/packages/47/7e/d932fe437ccf656cfba77abc466fb3d1a0ce3c31df92e760d9e4c34932b4/blake3-1.0.9-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:5b35abe24a66a7b3db423eb4f8668ed7be1a362aa9c0024ab6483ec0b2c16058", size = 345049, upload-time = "2026-06-22T18:01:29.228Z" }, + { url = "https://files.pythonhosted.org/packages/55/1e/d92fb284fcacf86f5d1083e29d0a8c834b60432786928915238d9760f514/blake3-1.0.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1bbdff61e049297ef3180867ce1f079cea7e5b372fd76953c3183da5b8124206", size = 329367, upload-time = "2026-06-22T18:01:30.566Z" }, + { url = "https://files.pythonhosted.org/packages/9d/da/e25fa75d5bfea4527fc21024dde86a9376db798e469a084741968299f215/blake3-1.0.9-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09a69fcedf06785bb81d4d3d39f95ee65dbaf2cb246e174cfc9ff64d027f7551", size = 374203, upload-time = "2026-06-22T18:01:31.998Z" }, + { url = "https://files.pythonhosted.org/packages/4f/4d/0224916202b773dfdf08dcbe4ed1ad1018d4ddcd4df7a7e2978d28f89b74/blake3-1.0.9-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5d5bf0f68cd77108a942c95db98e960d9c3d5643b95172f783822ce22667759", size = 373713, upload-time = "2026-06-22T18:01:33.387Z" }, + { url = "https://files.pythonhosted.org/packages/b8/e5/4ba968831b7afaec431c588c826cef76a96d6d6976188ed07d932072e673/blake3-1.0.9-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9767f16199b99aa022b61ff825ac4dbd39864bf637ae712605a2ce1f8b6a55e0", size = 446574, upload-time = "2026-06-22T18:01:34.687Z" }, + { url = "https://files.pythonhosted.org/packages/ac/f5/08a9099c7177f282d2563abe4f7cc626c636642f7979cf58f2ab7ded2096/blake3-1.0.9-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4865a8cfb2b3d7c0baf5267f2fa6816a3384e836cd1bd0caf359f406cb1e8fba", size = 487232, upload-time = "2026-06-22T18:01:36.131Z" }, + { url = "https://files.pythonhosted.org/packages/76/16/9392bf1ebc81b5b09ce58b94613fa2d37308e825ff2dc7b54d00ee622c77/blake3-1.0.9-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:42609e4adc4b2d7423137f2cb35135bca598b925c5af09d2bc0a2c368b25aeb1", size = 384751, upload-time = "2026-06-22T18:01:37.512Z" }, + { url = "https://files.pythonhosted.org/packages/84/fc/b6e9aef02ca14ef62fa47783b9eeeb5b2d3f73fdf698d8bb94c36f5dd69f/blake3-1.0.9-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7f648fa425138452d1e585ac625c7aefddb946d9765906c4c12d564a1523cd8", size = 384546, upload-time = "2026-06-22T18:01:38.868Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cb/452e92dba9402b36a953aa8b9b06253445ccce43dcd0bcf521c5e3c3e15d/blake3-1.0.9-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:9cef6d4d07a7de0c44f5ba17f6383d55276d9efc8d601f75113538fcaa35008b", size = 380596, upload-time = "2026-06-22T18:01:40.412Z" }, + { url = "https://files.pythonhosted.org/packages/b2/01/7a84a7e10c5d14e6ed8a4403bd7f64c1e01f8ebabea0d6fe5f093b894cbd/blake3-1.0.9-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:28404301de485e9546365d01b30f65eaa835520c4211d6ef61242975b6722b60", size = 550032, upload-time = "2026-06-22T18:01:41.955Z" }, + { url = "https://files.pythonhosted.org/packages/58/7d/7aea0222f59cf84044ec52e2bfdaa0e3c355d221292b0ea1b722cf1edd6c/blake3-1.0.9-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:8a99f896e7718050ed033a888245098aab3d6a5338f91cc9450c563b53f90ad5", size = 592244, upload-time = "2026-06-22T18:01:43.426Z" }, + { url = "https://files.pythonhosted.org/packages/c0/e5/b44c230108745ff9c70c7bbafe22563772bc0c22322a8d15c10455f6ca02/blake3-1.0.9-cp314-cp314t-win32.whl", hash = "sha256:021309d760b390706fecf13498f9a25aa8f689bbb65a0896029b8fa223aae18b", size = 229481, upload-time = "2026-06-22T18:01:45.307Z" }, + { url = "https://files.pythonhosted.org/packages/ec/a6/ac03f37dc9aeebf398d42089720648b3bc8438e733d3e522196c5d12ab39/blake3-1.0.9-cp314-cp314t-win_amd64.whl", hash = "sha256:5ea0c60dd9c1e3d05610606579e4bf80f562854c46ed55f9ee8545e18987a480", size = 217979, upload-time = "2026-06-22T18:01:46.629Z" }, +] + [[package]] name = "bleach" version = "6.2.0" @@ -605,6 +841,71 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f5/10/56978295c14794b2c12007b07f3e41ba26acda9257457d7085b0bb3bb90c/brotli-1.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:e7c0af964e0b4e3412a0ebf341ea26ec767fa0b4cf81abb5e897c9338b5ad6a3", size = 375639, upload-time = "2025-11-05T18:38:55.67Z" }, ] +[[package]] +name = "cachetools" +version = "7.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/8b/0d3945a13955303b81272f759a0331e54c5c793da455e6f5706b89d2639c/cachetools-7.1.4.tar.gz", hash = "sha256:437f55a4e0c1b01a4f3077cc470e6991d47430970e36fbcb77e2be0df4fc1cd6", size = 40085, upload-time = "2026-05-21T22:40:43.376Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/7b/1fc1c09cc0756cf25861a3be10565915953876da48bb228fb9a672b20a42/cachetools-7.1.4-py3-none-any.whl", hash = "sha256:323dc4127934744db5b54eb4924482d7edafbf9554e820d1531c2e08c0e4ef54", size = 16761, upload-time = "2026-05-21T22:40:41.845Z" }, +] + +[[package]] +name = "cbor2" +version = "6.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/6f/07b4af8da8bd27f640362b1ac8271d80895407f2ede0c2bcc9433c06e1ca/cbor2-6.1.3.tar.gz", hash = "sha256:8d70680acb55c04ea5b5ad86da094f9612b53d5a8a65d0f5b3aafc3ce917ecbb", size = 89503, upload-time = "2026-07-04T10:36:48.793Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/65/ae/753c952af03e0a39c7907209a13d3147b856459d2101d1abc1911629dd11/cbor2-6.1.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:20205c698f9ea4918d1714c459f58f46464c9f0d1e467679da8bca9e4eb17d1c", size = 422565, upload-time = "2026-07-04T10:35:43.165Z" }, + { url = "https://files.pythonhosted.org/packages/65/ca/21c301d4dc617ef2bc6629f5a2322c09a6abfa3ad3502e67458407991016/cbor2-6.1.3-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:4e2e1c8ba6651e12fcf35188f24cbd2344ce0acb087cb7739e5bfb8cec6e12ea", size = 465650, upload-time = "2026-07-04T10:35:44.864Z" }, + { url = "https://files.pythonhosted.org/packages/b6/14/7d8dba29b4103629a7c4b5f9697fac70d55d47f539da095674e3bb9f4fb1/cbor2-6.1.3-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a7bcc957049da506d6e59b4edb5594c68d81a1fa43b56f826839cb15e9a82572", size = 473041, upload-time = "2026-07-04T10:35:46.221Z" }, + { url = "https://files.pythonhosted.org/packages/ff/09/0a0393e40a450833e63ca6cfd1a99432eebe40a43f2a9ef1ba4b6c6f4507/cbor2-6.1.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d9971a2052802422efbbe11fc918c0f784de7280d2e0f45a6e9a8a0dc44e8f53", size = 530869, upload-time = "2026-07-04T10:35:47.701Z" }, + { url = "https://files.pythonhosted.org/packages/ab/76/419ad2e318533a5a7bb9835ddf7e4adce2aa6e4d7559fc836b8fad19518a/cbor2-6.1.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:165ce348632a4b502d0eb5ff187138d3b34986224273faf2233f842a9331c3ff", size = 540295, upload-time = "2026-07-04T10:35:48.999Z" }, + { url = "https://files.pythonhosted.org/packages/26/dd/70064846549f42fd2a4ed4e2e4f28111756b009d62b10d665aea17d1ac1f/cbor2-6.1.3-cp310-cp310-win32.whl", hash = "sha256:9084077c4cd7e905ebe47677934c9c6580942c9d1294f99524c0369b10829e78", size = 283423, upload-time = "2026-07-04T10:35:50.415Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8c/b95cb31a48062a27141759bc76d8d8c5e2e18c959cfbc38057d9b21a6b1e/cbor2-6.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:72b23df796d083ff53d561fd86e397aa5c19010192e85ea3838d8ef21c09db6d", size = 304417, upload-time = "2026-07-04T10:35:51.786Z" }, + { url = "https://files.pythonhosted.org/packages/0d/a5/b66695c2f8543b741b7178a5b7aeea3808524b65cd4f9918d1fba93debb7/cbor2-6.1.3-cp310-cp310-win_arm64.whl", hash = "sha256:fcec149218bacf1f98caf44225b67b4908e54e0440ca446ea488aac1ddc85a3b", size = 297762, upload-time = "2026-07-04T10:35:53.034Z" }, + { url = "https://files.pythonhosted.org/packages/f8/2e/013b2c478c41585bf5c8f9659328412eda4fe8ed30ffeb8e4fde87f8b9a3/cbor2-6.1.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:84edab2df31c981d258d652a82a3e30eb7368d86d9d7284216282f65403a1e00", size = 421187, upload-time = "2026-07-04T10:35:54.334Z" }, + { url = "https://files.pythonhosted.org/packages/96/78/840809f265a4537fde0ba646d92d61c500435fd811961d70776fc021b7ab/cbor2-6.1.3-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:932e0894476fad36b186c0da6e9b1433358bea564a60ae4799e51182568ff29f", size = 463784, upload-time = "2026-07-04T10:35:55.771Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c5/4dad6125eea17b35ca5580a4f7308226c8a4511dfb91b94c329b167b6218/cbor2-6.1.3-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:5f14159423a984c387982901f67313d6582251b6733c23e8bd925d73173691bf", size = 472119, upload-time = "2026-07-04T10:35:57.122Z" }, + { url = "https://files.pythonhosted.org/packages/b2/f7/5f0387ee7b5601c6af5277051612b8163f82ccbddbae807bbc1326754e3e/cbor2-6.1.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:622ec874664b4db54bc6df40f82832ad30fa5c875ad85cde84392ac62bb33d15", size = 528659, upload-time = "2026-07-04T10:35:58.496Z" }, + { url = "https://files.pythonhosted.org/packages/d7/7f/371ce0c200955a8a999e0a34398834ba88ef2fecda8437c3264c0673aa33/cbor2-6.1.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c9144756fa7c9298d5882d1f7ad379c4a0059803a4c70329965a000bc79bf02d", size = 538983, upload-time = "2026-07-04T10:35:59.828Z" }, + { url = "https://files.pythonhosted.org/packages/48/d4/3cb4d40ce9bbfb41098656a0be5a8d01f24fa54cdc6d2665cbbaefbb8ba0/cbor2-6.1.3-cp311-cp311-win32.whl", hash = "sha256:144f8cfd2e9149389c34026243aeb646184cd78a2c657822be9bc9e7a2c5f3f5", size = 282264, upload-time = "2026-07-04T10:36:01.131Z" }, + { url = "https://files.pythonhosted.org/packages/36/1f/e9d123a071ee67ebca70b37401a03b80641d86953a72e8ea41194f99095a/cbor2-6.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:187fd06befc59e6cafafc2709e5f1f3df8afe8bca5646f9cb5b70fc7e6ab1783", size = 303941, upload-time = "2026-07-04T10:36:02.45Z" }, + { url = "https://files.pythonhosted.org/packages/32/4f/efb2ed376421641e372bfebe9fd98f11c9de3bbac1da9f2b8be5c96eb335/cbor2-6.1.3-cp311-cp311-win_arm64.whl", hash = "sha256:43f0f694f47958de50fc84e6268a3015cc2a7fce88b231456c053bc5a1c6c828", size = 296378, upload-time = "2026-07-04T10:36:03.77Z" }, + { url = "https://files.pythonhosted.org/packages/31/16/cff14259c3d19a7f0ae88b6996fe4c85f6ff1764dad889ac8a39e843e39c/cbor2-6.1.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3d939f55097c21e032f5a2d67592fcc57298986281f219356e2f519e4466f4ea", size = 412779, upload-time = "2026-07-04T10:36:04.975Z" }, + { url = "https://files.pythonhosted.org/packages/50/6c/f3641d19b7b85a63cb2756c10164131489c2cb46b379ec51ae22283fefb9/cbor2-6.1.3-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b025009478d644dab407164fd60e3ef4381af284f5af6966df94c663756d949e", size = 457781, upload-time = "2026-07-04T10:36:06.349Z" }, + { url = "https://files.pythonhosted.org/packages/55/85/0c55a66f3037056bfb8e1c7184168085fdea67ae5830404498bcf466233b/cbor2-6.1.3-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:2226d32e102e375737656ad5d141ad8c6ae3e705e04e263f24756f0eb379c6c1", size = 468373, upload-time = "2026-07-04T10:36:07.769Z" }, + { url = "https://files.pythonhosted.org/packages/46/74/40f7db3e0d880560193916a5c9b744fcf299558bed7113f77c28237c7c29/cbor2-6.1.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e61d465244d66ffed36492eef3b44d43795d76a2bba0663a2f15c186af7f7513", size = 523844, upload-time = "2026-07-04T10:36:09.404Z" }, + { url = "https://files.pythonhosted.org/packages/b5/a1/b5e07d6a08441c3a552fe2ae48ccb7e9dfc5065b9f6a3bae9879b4f0fbc0/cbor2-6.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:87fe7be8fab6ec4796aa127c1a52e09e79dbafd2aa31caf809cf04b8080a5975", size = 536238, upload-time = "2026-07-04T10:36:10.914Z" }, + { url = "https://files.pythonhosted.org/packages/c9/99/e166be0fd74bf3a91f5a0d103e34883efbc438d970f72cc8200e274787e5/cbor2-6.1.3-cp312-cp312-win32.whl", hash = "sha256:da25d345f01e6a40b2e5c57ef96b4dcff7be69394fb62f0f70e07f437f2376a9", size = 279858, upload-time = "2026-07-04T10:36:12.247Z" }, + { url = "https://files.pythonhosted.org/packages/d2/1b/90b4a121e40aba189c55a5822dd3c698eaf487e1d4a780ab18c804a5ef1c/cbor2-6.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:d5514f693db6fa6f433b4096e9b604e6a7bf151c9ef1d2db86d0858e4c5e768f", size = 300929, upload-time = "2026-07-04T10:36:13.564Z" }, + { url = "https://files.pythonhosted.org/packages/19/db/52c58a8d33464927389dde8103997b3fa51b081ce29b347ac2cc4fd0dfbf/cbor2-6.1.3-cp312-cp312-win_arm64.whl", hash = "sha256:3d43183d7beb3d3cd198d69b31bd2ee487ed704a1150c75cb0a66d6ad63d8c1a", size = 290908, upload-time = "2026-07-04T10:36:14.857Z" }, + { url = "https://files.pythonhosted.org/packages/1c/8c/5024d623dcf3f2057ec8c991f584b939ba5f9025a5ce8c31f6fac067137a/cbor2-6.1.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:21c74b8ab67977c8b87b727247eeb730145b0068ad6d47f71e9f80f6b48c65f8", size = 412334, upload-time = "2026-07-04T10:36:16.299Z" }, + { url = "https://files.pythonhosted.org/packages/61/f3/e50654203c3b746166a96bea680eb6463b20c2c160cc14dfbe43f215ef6c/cbor2-6.1.3-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f291a0ae4c1ed96eadb0afa9752568c7424f7d6fa818676d5e33005fcd22ddd9", size = 457125, upload-time = "2026-07-04T10:36:17.684Z" }, + { url = "https://files.pythonhosted.org/packages/44/86/6ef007f0d4f7afba90a80cb1657984de542e7474d2afaa7e920ac9860df3/cbor2-6.1.3-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:dc8e44c7bf172687195dcd428157885bc00ea06efc0ea30fb371163b92bef733", size = 467651, upload-time = "2026-07-04T10:36:19.007Z" }, + { url = "https://files.pythonhosted.org/packages/b2/f4/b5aa27813c02f37e03eb86bd908163562edd6fc7f99665bc7bbb25ef5e6c/cbor2-6.1.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf4d263983d830dd429d2b01f27be58ac02ba7c790c45d861f767eb63963e5", size = 523296, upload-time = "2026-07-04T10:36:20.504Z" }, + { url = "https://files.pythonhosted.org/packages/5b/46/e17b2bce2efdc26bfa045b4f6168f02923ac5f0e79732a1b3c42ce9ca9de/cbor2-6.1.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8719a7a2a2a82168844533389957b8f617a139f5f40e4d0ad7ed905fd3abebd1", size = 535537, upload-time = "2026-07-04T10:36:21.761Z" }, + { url = "https://files.pythonhosted.org/packages/54/bc/add350acf37f367ae429f997f2e047b042f2d5ef9ca62461c923fbb0f3c3/cbor2-6.1.3-cp313-cp313-win32.whl", hash = "sha256:c73b54ce09dd8d522f3c1540426e36172ba0f34abf3d89eb93909a5e14590003", size = 279233, upload-time = "2026-07-04T10:36:23.071Z" }, + { url = "https://files.pythonhosted.org/packages/e8/6f/1bbfce3b3131e4e03e8a86966a38ff92ebb72215fcf36aeecea1547f3e4b/cbor2-6.1.3-cp313-cp313-win_amd64.whl", hash = "sha256:b77df56c462c10eb3444db8ef78d8c3e71d9ef8d021ea92e97c1f9e3aa918690", size = 300585, upload-time = "2026-07-04T10:36:24.563Z" }, + { url = "https://files.pythonhosted.org/packages/e7/65/3945702dd84b6e5b7800c9c7f1ada038d33d12d0042de10e38164cc03dfd/cbor2-6.1.3-cp313-cp313-win_arm64.whl", hash = "sha256:b144be2ab3e9584ee7b6359d2a92fee0a5bec1d00dbd34c215ccc2040ac0b2ab", size = 290357, upload-time = "2026-07-04T10:36:25.983Z" }, + { url = "https://files.pythonhosted.org/packages/8f/40/04ad7d34182b27487a1824422b361b32d2607727ef20e563056eba62d12a/cbor2-6.1.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3f3167ca9920b90db4ff72652db109dfd93b56ee0d583aca12f3a5d9a7019477", size = 414615, upload-time = "2026-07-04T10:36:27.299Z" }, + { url = "https://files.pythonhosted.org/packages/8e/29/94238c61f90653a606535e7509a2af312089fe10dafcb4cf82d6905a7a1a/cbor2-6.1.3-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:48c677971e4b71e685491e1a267d9924dc205e7ddbe3f34fd2562f16c0f6bfca", size = 459084, upload-time = "2026-07-04T10:36:28.717Z" }, + { url = "https://files.pythonhosted.org/packages/c8/cb/6bd33461e8be8ded7ebb0fa38994a63752aefae2b4fcd1b2cc71ee3c06f1/cbor2-6.1.3-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ad4f3c6dfc6b83331eb04c6975efb2839ab65a3aa81502bc2b3f7945d4c4aa44", size = 469310, upload-time = "2026-07-04T10:36:30.136Z" }, + { url = "https://files.pythonhosted.org/packages/8e/d1/94195bcd8fcc1030ecaf22a7a825faa08891b5bb2d3553e465e50fe115f5/cbor2-6.1.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8bcd609eab4a39745123bfb28e73311abba3d14975a87f5906e0e8b910d918ac", size = 524287, upload-time = "2026-07-04T10:36:31.453Z" }, + { url = "https://files.pythonhosted.org/packages/06/55/57178fbf2d1206af5299c688f9c917b83f30636694c8f932dc89c6652545/cbor2-6.1.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:672727ecb27d7fb3ca0bf8a58fc489d5374ab1fee680ed3d0348a11d9d3ca78f", size = 537031, upload-time = "2026-07-04T10:36:32.769Z" }, + { url = "https://files.pythonhosted.org/packages/fb/01/beefe26258d66ce39a9b057b679bc67dba81698d5a63e0ada5b4752a5c87/cbor2-6.1.3-cp314-cp314-win32.whl", hash = "sha256:1413cef2aa7f478a38298cd3492a055e8e8e45d17fb53bbe103e79ca15c33f3c", size = 286256, upload-time = "2026-07-04T10:36:34.078Z" }, + { url = "https://files.pythonhosted.org/packages/1b/2d/79eb513a2586a2053a6f18b33693beda65e2c766820d99676a306573fed5/cbor2-6.1.3-cp314-cp314-win_amd64.whl", hash = "sha256:59df264d4a508ba61daaa0bf3c2f92d63275509549a0875c1fa38176f651e4f8", size = 313744, upload-time = "2026-07-04T10:36:35.58Z" }, + { url = "https://files.pythonhosted.org/packages/a1/c7/ab9828e4efc26badf89f1397f866f3ef03ef65cdcd55518254b84bdaf86e/cbor2-6.1.3-cp314-cp314-win_arm64.whl", hash = "sha256:b62b5d80a0eb4305cd5f0217faa4d7747bd64fe0dff9b88415e7be3782f8249b", size = 304280, upload-time = "2026-07-04T10:36:36.865Z" }, + { url = "https://files.pythonhosted.org/packages/33/cf/54a497ad1026833c1c92d482edbda0bdebb48314b51563d2fcea24ab89b4/cbor2-6.1.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:856ab525bfc599588b8d2a45babb7c3400c693ea0bb574d818467d997102fe24", size = 409638, upload-time = "2026-07-04T10:36:38.089Z" }, + { url = "https://files.pythonhosted.org/packages/3b/03/efdecd0848b9c9e43242537f6dd8ac5d441a077d362e5e6954c7775b866a/cbor2-6.1.3-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a076f35abf1dd0a4de6e2f7f4d4932abafc951a26275b6aa4a3b370c2fd3bbf4", size = 452193, upload-time = "2026-07-04T10:36:39.56Z" }, + { url = "https://files.pythonhosted.org/packages/ef/bf/b5f43c75dc5f0ca3127919d3273d8671917932aa3e90767da63867e0f06f/cbor2-6.1.3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:37ccab1d0bd3f57ff536a41e44165d0c99cb166ac6e5ffb8b93c42304b56e48d", size = 466614, upload-time = "2026-07-04T10:36:40.936Z" }, + { url = "https://files.pythonhosted.org/packages/23/ee/e85b2ddd46b3b43e39a986704353b433efab0881234e1b4d824229fc2a75/cbor2-6.1.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:76d257ce797e651fa430b0269e8e8c43549c54ce8b0d12860569b3709bf1326f", size = 518503, upload-time = "2026-07-04T10:36:42.289Z" }, + { url = "https://files.pythonhosted.org/packages/60/13/c740c0002f127dc3e9e87b61a5d1285dd3b71aa11d8e0f6a408fc1f36173/cbor2-6.1.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4af35baadb66f7c9cbb3998eab469767c04641552416c1c69dd4d3d183797119", size = 534236, upload-time = "2026-07-04T10:36:43.641Z" }, + { url = "https://files.pythonhosted.org/packages/73/cd/a57d97177f3777c96f3406efb0ad60794fdbf249d497ec24559bfd1d0328/cbor2-6.1.3-cp314-cp314t-win32.whl", hash = "sha256:61c92661665bccfed4ffa69d1fe10097f2c820d262f56a8cf909a5ebf9f6d8c6", size = 282446, upload-time = "2026-07-04T10:36:44.888Z" }, + { url = "https://files.pythonhosted.org/packages/11/c8/dd54878589df22c863d526cb82e1bb20e953d40223436449a52481c49804/cbor2-6.1.3-cp314-cp314t-win_amd64.whl", hash = "sha256:dc6bcd030bf5043662b84b0ca0f0ca942491bf509105db30cedca6e2ce82d158", size = 310300, upload-time = "2026-07-04T10:36:46.212Z" }, + { url = "https://files.pythonhosted.org/packages/05/d0/b0780a396d145e3356bfdc48578d021ade1818a6c7d7842a3d6bc6f16fbb/cbor2-6.1.3-cp314-cp314t-win_arm64.whl", hash = "sha256:da98a5e0ae9487bed497ac74e2c850b49975a3b7b5314b76c3843e2c83a6c8c4", size = 299391, upload-time = "2026-07-04T10:36:47.629Z" }, +] + [[package]] name = "certifi" version = "2025.7.14" @@ -619,7 +920,7 @@ name = "cffi" version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy'" }, + { name = "pycparser", marker = "implementation_name != 'PyPy' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ @@ -772,7 +1073,9 @@ version = "1.43.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "importlib-metadata" }, - { name = "numpy", marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and python_full_version < '3.13' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "torch" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7d/55/0a0870e1c34b96013229286bb30b7a314ba52a6d676231ab16bb9b8d4176/circuitsvis-1.43.3.tar.gz", hash = "sha256:89c6be3c9d0c60eb932269a023a99f644c8b056af6cca354de7f17797a1f04cd", size = 1752475, upload-time = "2024-12-18T17:07:48.57Z" } @@ -785,7 +1088,7 @@ name = "click" version = "8.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/60/6c/8ca2efa64cf75a977a0d7fac081354553ebe483345c734fb6b6515d96bbc/click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202", size = 286342, upload-time = "2025-05-20T23:19:49.832Z" } wheels = [ @@ -819,16 +1122,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, ] +[[package]] +name = "compressed-tensors" +version = "0.15.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "loguru", marker = "sys_platform == 'linux'" }, + { name = "pydantic", marker = "sys_platform == 'linux'" }, + { name = "torch", marker = "sys_platform == 'linux'" }, + { name = "transformers", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/41/1b/c3c4a98ec5f2727656336f07a0c35862195c310d8eb0b2fa5b4be6848680/compressed_tensors-0.15.0.1.tar.gz", hash = "sha256:a8e93054e8a5ec49c980b09ed36c4c1249b4a8ee167920a8e461c4da26e78d99", size = 229412, upload-time = "2026-04-10T14:23:54.708Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/52/93833dc1610e017ac5b7dcd59b8304d8ef67d1114c2d124e728a2cbbea12/compressed_tensors-0.15.0.1-py3-none-any.whl", hash = "sha256:e1b1f322e82e475715e242bad46925a304ea8e5c98b5055a15b8eb22fb6bfea9", size = 194260, upload-time = "2026-04-10T14:23:53.098Z" }, +] + [[package]] name = "contourpy" version = "1.3.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", + "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", ] dependencies = [ - { name = "numpy", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/54/eb9bfc647b19f2009dd5c7f5ec51c4e6ca831725f1aea7a993034f483147/contourpy-1.3.2.tar.gz", hash = "sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54", size = 13466130, upload-time = "2025-04-15T17:47:53.79Z" } wheels = [ @@ -895,17 +1217,31 @@ name = "contourpy" version = "1.3.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.12.*' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", - "python_full_version >= '3.14' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and sys_platform == 'linux'", - "python_full_version >= '3.14' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and sys_platform != 'linux'", -] -dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.11'" }, + "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.13' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", +] +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } wheels = [ @@ -1069,7 +1405,7 @@ wheels = [ [package.optional-dependencies] toml = [ - { name = "tomli", marker = "python_full_version <= '3.11'" }, + { name = "tomli", marker = "python_full_version <= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] [[package]] @@ -1077,8 +1413,8 @@ name = "cryptography" version = "46.0.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "cffi", marker = "(platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (platform_python_implementation != 'PyPy' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "typing-extensions", marker = "(python_full_version < '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/47/93/ac8f3d5ff04d54bc814e961a43ae5b0b146154c89c61b47bb07557679b18/cryptography-46.0.7.tar.gz", hash = "sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5", size = 750652, upload-time = "2026-04-08T01:57:54.692Z" } wheels = [ @@ -1134,19 +1470,61 @@ wheels = [ [[package]] name = "cuda-bindings" -version = "12.9.4" +version = "13.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder", marker = "sys_platform == 'linux'" }, + { name = "cuda-pathfinder", marker = "sys_platform == 'linux' or extra == 'extra-16-transformer-lens-lit' or extra != 'extra-16-transformer-lens-vllm'" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/d8/b546104b8da3f562c1ff8ab36d130c8fe1dd6a045ced80b4f6ad74f7d4e1/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d3c842c2a4303b2a580fe955018e31aea30278be19795ae05226235268032e5", size = 12148218, upload-time = "2025-10-21T14:51:28.855Z" }, - { url = "https://files.pythonhosted.org/packages/45/e7/b47792cc2d01c7e1d37c32402182524774dadd2d26339bd224e0e913832e/cuda_bindings-12.9.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c912a3d9e6b6651853eed8eed96d6800d69c08e94052c292fec3f282c5a817c9", size = 12210593, upload-time = "2025-10-21T14:51:36.574Z" }, - { url = "https://files.pythonhosted.org/packages/a9/c1/dabe88f52c3e3760d861401bb994df08f672ec893b8f7592dc91626adcf3/cuda_bindings-12.9.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fda147a344e8eaeca0c6ff113d2851ffca8f7dfc0a6c932374ee5c47caa649c8", size = 12151019, upload-time = "2025-10-21T14:51:43.167Z" }, - { url = "https://files.pythonhosted.org/packages/63/56/e465c31dc9111be3441a9ba7df1941fe98f4aa6e71e8788a3fb4534ce24d/cuda_bindings-12.9.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:32bdc5a76906be4c61eb98f546a6786c5773a881f3b166486449b5d141e4a39f", size = 11906628, upload-time = "2025-10-21T14:51:49.905Z" }, - { url = "https://files.pythonhosted.org/packages/a3/84/1e6be415e37478070aeeee5884c2022713c1ecc735e6d82d744de0252eee/cuda_bindings-12.9.4-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56e0043c457a99ac473ddc926fe0dc4046694d99caef633e92601ab52cbe17eb", size = 11925991, upload-time = "2025-10-21T14:51:56.535Z" }, - { url = "https://files.pythonhosted.org/packages/d1/af/6dfd8f2ed90b1d4719bc053ff8940e494640fe4212dc3dd72f383e4992da/cuda_bindings-12.9.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8b72ee72a9cc1b531db31eebaaee5c69a8ec3500e32c6933f2d3b15297b53686", size = 11922703, upload-time = "2025-10-21T14:52:03.585Z" }, - { url = "https://files.pythonhosted.org/packages/6c/19/90ac264acc00f6df8a49378eedec9fd2db3061bf9263bf9f39fd3d8377c3/cuda_bindings-12.9.4-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d80bffc357df9988dca279734bc9674c3934a654cab10cadeed27ce17d8635ee", size = 11924658, upload-time = "2025-10-21T14:52:10.411Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/8464d133752951c154feafb3b65c297e7d80f301183d220bec4c830f1441/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86", size = 6073403, upload-time = "2026-05-29T23:11:36.22Z" }, + { url = "https://files.pythonhosted.org/packages/a8/1f/5ef51f5fbaa5d4d3201bb3d7555af028ec1aa4416275ccbf73c9e34e3d2d/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0", size = 6675244, upload-time = "2026-05-29T23:11:38.664Z" }, + { url = "https://files.pythonhosted.org/packages/fc/64/bb17e4d168569ef7be05c44474fe3dc19278d60a69ba228e45a431c86444/cuda_bindings-13.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:c0c4b1a995098c46695c24257a342dc97d6e6d3f3050b944c9f43bd26d734051", size = 5625597, upload-time = "2026-05-29T23:11:40.808Z" }, + { url = "https://files.pythonhosted.org/packages/51/6b/457ca12dad3ee9bfcc9a545cfd6b64b359ba49de40f776f6e028e678f262/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474", size = 6053539, upload-time = "2026-05-29T23:11:43.19Z" }, + { url = "https://files.pythonhosted.org/packages/95/7a/c5e3c34a409b148f5c0f5a4ea374158f95d488862c1dffedf9aa5c639df9/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708", size = 6674166, upload-time = "2026-05-29T23:11:45.478Z" }, + { url = "https://files.pythonhosted.org/packages/93/f7/0e35987a21914f84068061dcf4b61466ccbce1c62ddc9727596d5ed0c26f/cuda_bindings-13.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:507b0e19e7f934c5e30f30f0244ad70a75812619a7d3a0d742543caae1bd50f1", size = 5664286, upload-time = "2026-05-29T23:11:47.719Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/5e7dba1ba576dd73da5dee894ca076ca5e959450dfff66d6d510a255d1f7/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49", size = 6025351, upload-time = "2026-05-29T23:11:49.685Z" }, + { url = "https://files.pythonhosted.org/packages/39/2a/6d2e9047d1fb243dbaa364b01e0297534b9ed7fd27dba1c9f361519cf69b/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a", size = 6657965, upload-time = "2026-05-29T23:11:52.227Z" }, + { url = "https://files.pythonhosted.org/packages/7c/95/872a0392122f1fb43fcb06869790ef3171f37beee9f7db8f441739113570/cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff", size = 5875099, upload-time = "2026-05-29T23:11:54.635Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6e/2394f8163360f8391f8f1b7e72d300a82724edb81a7b7084c799fbd4c91f/cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf", size = 5920504, upload-time = "2026-05-29T23:11:56.883Z" }, + { url = "https://files.pythonhosted.org/packages/34/c2/ef9b6a63f7dc432712a462c816662e662e00d38caa9b861c8c2588195d03/cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7", size = 6476660, upload-time = "2026-05-29T23:11:59.188Z" }, + { url = "https://files.pythonhosted.org/packages/0c/2f/6a0dd496550c6fafbf6aeb1bf40242eeabb2fd138a43892aabb4be8224c2/cuda_bindings-13.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:18c8c167c8907b8f02531ca810534315c458dabef31f7965095619bf647b9202", size = 5830027, upload-time = "2026-05-29T23:12:01.205Z" }, + { url = "https://files.pythonhosted.org/packages/b1/81/bff68ce829999c1e4209c761bbf903b1c06ec570416ddb25020864ad5907/cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8", size = 6013639, upload-time = "2026-05-29T23:12:03.509Z" }, + { url = "https://files.pythonhosted.org/packages/d4/e0/c8a1f0c8f9ffdea4f5fe6dbab89b326cef4d85caf489dad39e209da89416/cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80", size = 6534419, upload-time = "2026-05-29T23:12:05.633Z" }, + { url = "https://files.pythonhosted.org/packages/48/45/2734be44dbc80ac082ec23a86b41c8294992dcb90033645ed1bc50aafe4c/cuda_bindings-13.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:8de12ef60bf40756852cb62bbb40460609269f6ece522903d1cc93d73a3ececb", size = 5961055, upload-time = "2026-05-29T23:12:07.971Z" }, + { url = "https://files.pythonhosted.org/packages/52/b8/83b1f563925b290f2d11a01a77a84013ba56052fe3653a5bef3ccfbb43d6/cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76", size = 5809771, upload-time = "2026-05-29T23:12:10.422Z" }, + { url = "https://files.pythonhosted.org/packages/12/20/e79b4bfe98f075195afb6343d41c498f9dbd2d161d7021d4d28bceb83581/cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9", size = 6358584, upload-time = "2026-05-29T23:12:12.767Z" }, + { url = "https://files.pythonhosted.org/packages/27/2a/b59bcac016ab9985d6b48a5d05b0d698461a159ca03ee11c4abd54da2ac4/cuda_bindings-13.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61120b5e4f4a63f67efd7e7396914cb9ef871bb1f0021e990fb70277be240a4d", size = 6740329, upload-time = "2026-05-29T23:12:15.153Z" }, +] + +[[package]] +name = "cuda-core" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backports-strenum", marker = "python_full_version < '3.11' and sys_platform == 'linux'" }, + { name = "cuda-pathfinder", marker = "sys_platform == 'linux'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/21/ef85f3e15d394c9ca41fe116d78cd9e28533b9d7ead842f9241b332acf01/cuda_core-1.0.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9632db74eceb1cd72a7c95b61a5e4cfb9cc2291de0503e170334d936cab3316", size = 4788165, upload-time = "2026-05-12T20:11:17.116Z" }, + { url = "https://files.pythonhosted.org/packages/e0/41/c2c07b313c6cbb5d93010200c62b01ddb9f6c6f43a096a75c7b902c42ad6/cuda_core-1.0.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46690b0864417a5f2f9a7d10408e2570cbacae195c890a41286701eefb01ba79", size = 5061723, upload-time = "2026-05-12T20:11:19.767Z" }, + { url = "https://files.pythonhosted.org/packages/d1/2d/b16b0af698a1bf4db337345daa7a44cd372fef107a3b692ffe1e0e6c5cc5/cuda_core-1.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:1693fae113604cf9c114bfe3da15a15981f9d44ae78ecceb0b23e24c628ad19b", size = 4742003, upload-time = "2026-05-12T20:11:21.943Z" }, + { url = "https://files.pythonhosted.org/packages/41/4b/4ac1d0639241da756c634add606f93a7f3a39bef12f70e1fb4b40cc53c21/cuda_core-1.0.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3effd11283bc46fd06348c2fd18a0941ba7718a6f447343858c944c1a93a6dab", size = 4784340, upload-time = "2026-05-12T20:11:23.961Z" }, + { url = "https://files.pythonhosted.org/packages/01/55/bb3e701f4af504e5e39e837135dc80022ec4c84858b2886ad577fe696a77/cuda_core-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1934517ff8a9dcd21b3f4a28e15e12643164b7d3ec187a4ee7560e22fd2dfc17", size = 5059041, upload-time = "2026-05-12T20:11:26.045Z" }, + { url = "https://files.pythonhosted.org/packages/a2/e3/3ffaca2eabc71d0f9d29368fabc8ffb309353f05f418ea4c7eb5f223cf09/cuda_core-1.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:95c91d434a9baca066646cefa577227385104670a02fbe8e3defaadda84becf5", size = 4746198, upload-time = "2026-05-12T20:11:28.405Z" }, + { url = "https://files.pythonhosted.org/packages/0d/a0/1daeae599cadd612689dbbf70d7da1c01883964fc2fbc7386f3c630a68cf/cuda_core-1.0.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6816dc020aee6103d8071bc02d8e4e1d91f2b49596f666896d608d92224d79d1", size = 4789856, upload-time = "2026-05-12T20:11:30.862Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4d/603557ab3cb171cc2a61d3678a39cb4dae3fd21275078bfbd1c0b0b5230b/cuda_core-1.0.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be7b65311bf78964b7905adbf3c0f8f717d432f2854dc45169277729bf60f1e2", size = 5106023, upload-time = "2026-05-12T20:11:33.509Z" }, + { url = "https://files.pythonhosted.org/packages/c2/1a/ae079963c9df7f4274227eb63cf8f6083a532a6443adb340d951fd21c626/cuda_core-1.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:1a5c1aa3b738a7599ea289498d038fe625d259fd7ab795394541eee58a8e29bc", size = 4663076, upload-time = "2026-05-12T20:11:35.784Z" }, + { url = "https://files.pythonhosted.org/packages/57/f9/a6676b1fa555fad5748a945f4b530b51b898b4771a1e5d9f3520d3f415ea/cuda_core-1.0.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c427e5025096d96fcd5092fdc85d5d5e4ac3dea007914e90472ed52f27220446", size = 4749800, upload-time = "2026-05-12T20:11:38.012Z" }, + { url = "https://files.pythonhosted.org/packages/9c/9d/4534a9564a812ee95b43db7324f9b25cbffda001bb348bb5b3f90dad50b9/cuda_core-1.0.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b392178202c652368883dbe3773cee14f3e1ed6b8bf45d1a1bcdd37c73604e06", size = 5078597, upload-time = "2026-05-12T20:11:40.836Z" }, + { url = "https://files.pythonhosted.org/packages/c6/7c/2f68b0bdeb7dd36204f752468254d6b4487c6d82e9e442cfbe815a656eac/cuda_core-1.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:b99e3ca9bf3bd2c7d3028e5dc541b00e432e21373816889cdf2722b675bd9be8", size = 4647545, upload-time = "2026-05-12T20:11:43.494Z" }, + { url = "https://files.pythonhosted.org/packages/c2/45/55b07d643c87f1234b3cbc9d8383c0962b368ba1d6686a1919d6f6001af4/cuda_core-1.0.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9b0f115a68f2f84c0f6d9b7e863a29517f6dfe5f7b7d07d1d9da8904754e9a2", size = 4816184, upload-time = "2026-05-12T20:11:46.114Z" }, + { url = "https://files.pythonhosted.org/packages/51/08/1aeffc9a529a7f94c9cee9bfd3a991743398b5f90aab30f06f2a4bc8205e/cuda_core-1.0.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:af2db9e50e81d73e4f0b72ad279d0a9c789372393938fe75c17236b9ed974d7d", size = 5104496, upload-time = "2026-05-12T20:11:48.518Z" }, + { url = "https://files.pythonhosted.org/packages/66/66/965330a44bcfa548793cf13244083528a597da2a18ff42d00fe8ef91ba03/cuda_core-1.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:29783ba03d36960b6612b15ff97123e88cfadfb7b1884f3581839f6bfba4b29f", size = 4765708, upload-time = "2026-05-12T20:11:51.107Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ab/db09228d5a8c124a93514726d2e18f31824f66d3a6769ee4e51721dd64cf/cuda_core-1.0.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4410bf1ef15c2ec23dccc302da76893c9354b530dee422e3277b116231bf5fe1", size = 4996094, upload-time = "2026-05-12T20:11:53.575Z" }, + { url = "https://files.pythonhosted.org/packages/29/e7/8ced56d6c6fa32b7385a8dccefd1424e3c1201bfee3385d9710609a43d16/cuda_core-1.0.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e0f1324486bb90be6bde28bd0afbaf78a38827948d37f93f90318a01da8a3f8c", size = 5212461, upload-time = "2026-05-12T20:11:56.238Z" }, + { url = "https://files.pythonhosted.org/packages/da/06/36236c44ed4025a6cbf5b6450364fc29e0f3d35ef4becb13b168fcd271f4/cuda_core-1.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:180bc808166483b0d6658a7c0d3f1312083b220f301de49476c1bc459cc46cf8", size = 5551965, upload-time = "2026-05-12T20:11:58.84Z" }, ] [[package]] @@ -1157,6 +1535,90 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/11/c8/26f2e4aae92f11522a96043892ba39a90eac610d5242523aa863212bc1c7/cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689", size = 51671, upload-time = "2026-05-27T01:21:25.413Z" }, ] +[[package]] +name = "cuda-python" +version = "13.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, + { name = "cuda-core", marker = "sys_platform == 'linux'" }, + { name = "cuda-pathfinder", marker = "sys_platform == 'linux'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/31/7ff3f7768eded7535c621abc2fecb9d181a34ea4cae3afe682feb796f242/cuda_python-13.3.1-py3-none-any.whl", hash = "sha256:280b014139ab447b6dd70a377db1596f310d6e887d9d342e6651b919ec145fb3", size = 8295, upload-time = "2026-05-29T23:28:47.012Z" }, +] + +[[package]] +name = "cuda-tile" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/1d/03499651b6957b31ab70c875d91f23220da5f8f84cbad1fb7cba2d7dd435/cuda_tile-1.5.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:27da6113d3469de0b2f75dd269904e730b6d2bc81336addbd299c593d6a125c4", size = 322958, upload-time = "2026-07-08T01:49:22.702Z" }, + { url = "https://files.pythonhosted.org/packages/d1/aa/8af16bde9b0c41cda286791749a246195716a15eea7f0dc79a99d37d8686/cuda_tile-1.5.0-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:cc390ac00d2ecd5f7c2e26198b410787fa39baf3737c6abceac9bda3ca9fccef", size = 324757, upload-time = "2026-07-08T01:49:17.497Z" }, + { url = "https://files.pythonhosted.org/packages/50/ed/e98669f59bdea9d5f698cb22aab3bbc6ed10eadef640a79bf2d7e5a3c4cc/cuda_tile-1.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:d808355e4cb6a850c7e3e82a5cd656c0062c33225fce69a8fd40290bc5b0bcaa", size = 305184, upload-time = "2026-07-08T01:49:28.779Z" }, + { url = "https://files.pythonhosted.org/packages/5e/4d/e07fd65640c26c1f990ee621af11f073672e8d96501663026c7e1978f5b8/cuda_tile-1.5.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:81b8a93e757258260bd05dbbe6eec8bb50655ee1a88d5ebbc8412c526d3c0ed4", size = 322684, upload-time = "2026-07-08T01:49:21.318Z" }, + { url = "https://files.pythonhosted.org/packages/eb/a8/981c2eb351f15a4b75b5913e35d2ef16cee32a45c950d0e16f85e626dd05/cuda_tile-1.5.0-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:9494170237d34bbbce83f2ada005d2dabb704f1f8c6a0af59088c180bd1bf028", size = 324278, upload-time = "2026-07-08T01:49:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/95/b2/a863d460f6b70431869be5a18f82b1a1af91b5d7629e914c204e3545cac2/cuda_tile-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:bda5f1721a6daa45124033f031685d1b30439ea9dbb07e0d46ec2355cfde0657", size = 304926, upload-time = "2026-07-08T01:49:16.158Z" }, + { url = "https://files.pythonhosted.org/packages/7c/6d/cc2fb5a25689a501564a2eced4acf654f307e801a2c1506be97c0d100491/cuda_tile-1.5.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:87652483baa9c81a9a24e4450f016e4ee78fd205d8422dad8996571bd1f2622e", size = 322641, upload-time = "2026-07-08T01:49:23.388Z" }, + { url = "https://files.pythonhosted.org/packages/1c/f5/b4ba9d0fc71198d939ebf9a090228179995d8411ee9def8f638a0e3ccdc5/cuda_tile-1.5.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:cef6d30acc37557643ece0de3770fc4c33497c4af40209e424f72fbfcbe6ea5a", size = 324990, upload-time = "2026-07-08T01:49:17.739Z" }, + { url = "https://files.pythonhosted.org/packages/a5/6e/7a60f317c503580ab7946dbb7fd080438fe953d0ddfdc81904beb9a1fab7/cuda_tile-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:16d97a60ed1d33388abbca85ea08cdae6325cc700476b3135f190d0fb50329f4", size = 304817, upload-time = "2026-07-08T01:49:38.853Z" }, + { url = "https://files.pythonhosted.org/packages/00/46/60aea981ee7cc0b159eb08c42b795e8e95ae8a7fb451e4565cad0f43cca0/cuda_tile-1.5.0-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:cfa4a5ef920d9c1fee702611b25bde449915f12bf929c1b8a3faee0c9b03f750", size = 322643, upload-time = "2026-07-08T01:49:26.107Z" }, + { url = "https://files.pythonhosted.org/packages/26/d5/ae03d2b70ed8d6c21ca809ddc98227ad07988e7fe67e7e41d888c0b13d32/cuda_tile-1.5.0-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:e7cb56186b0cc98166b72c7e5a3764c236151aa8d53e0b37a153766b79ea005a", size = 324995, upload-time = "2026-07-08T01:49:19.931Z" }, + { url = "https://files.pythonhosted.org/packages/da/a6/e3f6e9aefcc94d548e5d69f01a02ba5da7c5e200702eba3eb7a4f572a883/cuda_tile-1.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:c400918faa8492d84c4da3da81ce01ed30d81ed780f9ddbc1a9bdd588058b776", size = 304825, upload-time = "2026-07-08T01:49:37.749Z" }, + { url = "https://files.pythonhosted.org/packages/82/f2/861000a1cc2204be90295c980c488670600e89c638138192c61bf79949f1/cuda_tile-1.5.0-cp314-cp314-manylinux2014_aarch64.whl", hash = "sha256:b88d7a3ea0cb30a962c0ff9cb01e2b2b87c6ef816fd53dd92ead3bcff3449e37", size = 322775, upload-time = "2026-07-08T01:49:25.897Z" }, + { url = "https://files.pythonhosted.org/packages/65/38/165499cbfb7c1ada110592fa9224e20851125b337bbe2e656b5d56c676f0/cuda_tile-1.5.0-cp314-cp314-manylinux2014_x86_64.whl", hash = "sha256:d4054dc12b6d35d69cb07fcc224183b84f3ad1a0e85ec4414cf660144b8467b0", size = 325052, upload-time = "2026-07-08T01:49:20.082Z" }, + { url = "https://files.pythonhosted.org/packages/fd/76/4f0b7bd3ac2d8383b3cc166ce67c528653b51a761ebc0fc6c132e57ef19b/cuda_tile-1.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:b3c5e0a5af1347b6326fe7a49e44b15ea01ae2d8ed138617c8331b6aba311d16", size = 305719, upload-time = "2026-07-08T01:50:05.793Z" }, + { url = "https://files.pythonhosted.org/packages/2c/53/e5947ebd44774183f72c317907d803648d8cd87e8a571bb0a4e89a578309/cuda_tile-1.5.0-cp314-cp314t-manylinux2014_aarch64.whl", hash = "sha256:a4ede5529bd5e13318ec9bbf3f79abaf23b6368c58d247eaf2e5eb5ccae3fa73", size = 324979, upload-time = "2026-07-08T01:49:29.759Z" }, + { url = "https://files.pythonhosted.org/packages/d7/b8/6260d8089287dd8e8f5e456a60391537520b20cd90287eacaac4c71fd140/cuda_tile-1.5.0-cp314-cp314t-manylinux2014_x86_64.whl", hash = "sha256:1088a3ebb5622c24ec32034d0d451bab9c1b85ae67fcdc647464951ec1d1b6ad", size = 326849, upload-time = "2026-07-08T01:49:23.211Z" }, + { url = "https://files.pythonhosted.org/packages/df/ea/104cd115a15768ed3e6d58ce658744784e3376dcf9ca5f1402c6ddab61f5/cuda_tile-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:70ecc0e4be063317b5216dc620085be8021f78092a81613268129693cf63e179", size = 313357, upload-time = "2026-07-08T01:50:02.713Z" }, +] + +[[package]] +name = "cuda-toolkit" +version = "13.0.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364, upload-time = "2025-12-19T23:24:07.328Z" }, +] + +[package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas", marker = "sys_platform == 'linux' or (sys_platform == 'win32' and extra == 'extra-16-transformer-lens-lit') or (sys_platform == 'win32' and extra != 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +cudart = [ + { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or (sys_platform == 'win32' and extra == 'extra-16-transformer-lens-lit') or (sys_platform == 'win32' and extra != 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +cufft = [ + { name = "nvidia-cufft", marker = "sys_platform == 'linux' or (sys_platform == 'win32' and extra == 'extra-16-transformer-lens-lit') or (sys_platform == 'win32' and extra != 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +cufile = [ + { name = "nvidia-cufile", marker = "sys_platform == 'linux' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +cupti = [ + { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or (sys_platform == 'win32' and extra == 'extra-16-transformer-lens-lit') or (sys_platform == 'win32' and extra != 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +curand = [ + { name = "nvidia-curand", marker = "sys_platform == 'linux' or (sys_platform == 'win32' and extra == 'extra-16-transformer-lens-lit') or (sys_platform == 'win32' and extra != 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +cusolver = [ + { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or (sys_platform == 'win32' and extra == 'extra-16-transformer-lens-lit') or (sys_platform == 'win32' and extra != 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +cusparse = [ + { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or (sys_platform == 'win32' and extra == 'extra-16-transformer-lens-lit') or (sys_platform == 'win32' and extra != 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +nvjitlink = [ + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or (sys_platform == 'win32' and extra == 'extra-16-transformer-lens-lit') or (sys_platform == 'win32' and extra != 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +nvrtc = [ + { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or (sys_platform == 'win32' and extra == 'extra-16-transformer-lens-lit') or (sys_platform == 'win32' and extra != 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +nvtx = [ + { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or (sys_platform == 'win32' and extra == 'extra-16-transformer-lens-lit') or (sys_platform == 'win32' and extra != 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] + [[package]] name = "cycler" version = "0.12.1" @@ -1181,15 +1643,18 @@ wheels = [ [[package]] name = "datasets" -version = "4.0.0" +version = "4.8.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dill" }, { name = "filelock" }, { name = "fsspec", extra = ["http"] }, + { name = "httpx" }, { name = "huggingface-hub" }, { name = "multiprocess" }, - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "packaging" }, { name = "pandas" }, { name = "pyarrow" }, @@ -1198,9 +1663,9 @@ dependencies = [ { name = "tqdm" }, { name = "xxhash" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e3/9d/348ed92110ba5f9b70b51ca1078d4809767a835aa2b7ce7e74ad2b98323d/datasets-4.0.0.tar.gz", hash = "sha256:9657e7140a9050db13443ba21cb5de185af8af944479b00e7ff1e00a61c8dbf1", size = 569566, upload-time = "2025-07-09T14:35:52.431Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/34/14cd8e76f907f7d4dca2334cfeec9f81d30fd15c25a015f99aaea694eaed/datasets-4.8.5.tar.gz", hash = "sha256:0f0c1c3d56ffff2c93b2f4c63c95bac94f3d7e8621aea2a2a576275233bba772", size = 605649, upload-time = "2026-04-27T15:43:57.384Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/62/eb8157afb21bd229c864521c1ab4fa8e9b4f1b06bafdd8c4668a7a31b5dd/datasets-4.0.0-py3-none-any.whl", hash = "sha256:7ef95e62025fd122882dbce6cb904c8cd3fbc829de6669a5eb939c77d50e203d", size = 494825, upload-time = "2025-07-09T14:35:50.658Z" }, + { url = "https://files.pythonhosted.org/packages/65/99/00f3196036501b53032c4b1ab8337a0b978dee832ed276dae3815df4e8b5/datasets-4.8.5-py3-none-any.whl", hash = "sha256:5079900781719c0e063a8efdd2cd95a31ad0c63209178669cd23cf1b926149ff", size = 528973, upload-time = "2026-04-27T15:43:53.702Z" }, ] [[package]] @@ -1246,6 +1711,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] +[[package]] +name = "depyf" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astor", marker = "sys_platform == 'linux'" }, + { name = "dill", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/35/83fb0178212279aa0af031031905804c6de5618435d229f41ed21bb9ad2c/depyf-0.20.0.tar.gz", hash = "sha256:fb7683bd72c44f67b56029df2c47721e9a02ffa4d7b19095f1c54c4ebf797a98", size = 6168761, upload-time = "2025-10-13T12:33:38.589Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/65/4df6936130b56e1429114e663e7c1576cf845f3aef1b2dd200c0a5d19dba/depyf-0.20.0-py3-none-any.whl", hash = "sha256:d31effad4261cebecb58955d832e448ace88f432328f95f82fd99c30fd9308d4", size = 39381, upload-time = "2025-10-13T12:33:33.647Z" }, +] + +[[package]] +name = "detect-installer" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/ce/6897d812825e9d4c53e3c7112726e800cc5231b013b2223bf64f653ff362/detect_installer-0.1.0.tar.gz", hash = "sha256:00ad7ba0a36e3cf7d08a40d3643011746dbc112597c7d475cc91c416710ca4e7", size = 3049, upload-time = "2026-02-23T10:40:22.567Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/34/8cc73273414405086c58852916e4031812a6a30fe04c057e37ad99397b7f/detect_installer-0.1.0-py3-none-any.whl", hash = "sha256:034fb20fd665c36e6ba52b8821525ea07fb4f7f938cac459df889fb33801528a", size = 4539, upload-time = "2026-02-23T10:40:23.807Z" }, +] + [[package]] name = "dill" version = "0.3.8" @@ -1255,6 +1742,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c9/7a/cef76fd8438a42f96db64ddaa85280485a9c395e7df3db8158cfec1eee34/dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7", size = 116252, upload-time = "2024-01-27T23:42:14.239Z" }, ] +[[package]] +name = "diskcache" +version = "5.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/21/1c1ffc1a039ddcc459db43cc108658f32c57d271d7289a2794e401d0fdb6/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916, upload-time = "2023-08-31T06:12:00.316Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" }, +] + +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + +[[package]] +name = "dnspython" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, +] + [[package]] name = "docstring-parser" version = "0.18.0" @@ -1282,6 +1796,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/87/62/9773de14fe6c45c23649e98b83231fffd7b9892b6cf863251dc2afa73643/einops-0.8.1-py3-none-any.whl", hash = "sha256:919387eb55330f5757c6bea9165c5ff5cfe63a642682ea788a6d472576d81737", size = 64359, upload-time = "2025-02-09T03:17:01.998Z" }, ] +[[package]] +name = "email-validator" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dnspython", marker = "sys_platform == 'linux'" }, + { name = "idna", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/22/900cb125c76b7aaa450ce02fd727f452243f2e91a61af068b40adba60ea9/email_validator-2.3.0.tar.gz", hash = "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426", size = 51238, upload-time = "2025-08-26T13:09:06.831Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, +] + [[package]] name = "entrypoints" version = "0.4" @@ -1296,8 +1823,7 @@ name = "etils" version = "1.13.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", + "python_full_version < '3.11'", ] sdist = { url = "https://files.pythonhosted.org/packages/9b/a0/522bbff0f3cdd37968f90dd7f26c7aa801ed87f5ba335f156de7f2b88a48/etils-1.13.0.tar.gz", hash = "sha256:a5b60c71f95bcd2d43d4e9fb3dc3879120c1f60472bb5ce19f7a860b1d44f607", size = 106368, upload-time = "2025-07-15T10:29:10.563Z" } wheels = [ @@ -1317,14 +1843,10 @@ name = "etils" version = "1.14.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.12.*' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", - "python_full_version >= '3.14' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and sys_platform == 'linux'", - "python_full_version >= '3.14' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and sys_platform != 'linux'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/26/ce/6e067242fde898841922ac6fc82b0bb2fe35c38e995880bdffdfbe30182a/etils-1.14.0.tar.gz", hash = "sha256:8136e7f4c4173cd0af0ca5481c4475152f0b8686192951eefa60ee8711e1ede4", size = 108127, upload-time = "2026-03-04T17:41:36.291Z" } wheels = [ @@ -1348,7 +1870,9 @@ dependencies = [ { name = "fsspec", extra = ["http"] }, { name = "huggingface-hub" }, { name = "multiprocess" }, - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "packaging" }, { name = "pandas" }, { name = "requests" }, @@ -1365,7 +1889,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -1415,6 +1939,176 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/26/a3/0bd5f0cdb0bbc92650e8dc457e9250358411ee5d1b65e42b6632387daf81/fastapi-0.136.0-py3-none-any.whl", hash = "sha256:8793d44ec7378e2be07f8a013cf7f7aa47d6327d0dfe9804862688ec4541a6b4", size = 117556, upload-time = "2026-04-16T11:47:11.922Z" }, ] +[package.optional-dependencies] +standard = [ + { name = "email-validator", marker = "sys_platform == 'linux'" }, + { name = "fastapi-cli", extra = ["standard"], marker = "(sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "fastar", marker = "sys_platform == 'linux'" }, + { name = "httpx", marker = "sys_platform == 'linux'" }, + { name = "jinja2", marker = "sys_platform == 'linux'" }, + { name = "pydantic-extra-types", marker = "sys_platform == 'linux'" }, + { name = "pydantic-settings", marker = "sys_platform == 'linux'" }, + { name = "python-multipart", marker = "sys_platform == 'linux'" }, + { name = "uvicorn", extra = ["standard"], marker = "(sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] + +[[package]] +name = "fastapi-cli" +version = "0.0.29" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "rich-toolkit", marker = "sys_platform == 'linux'" }, + { name = "tomli", marker = "python_full_version < '3.11' and sys_platform == 'linux'" }, + { name = "typer", marker = "sys_platform == 'linux'" }, + { name = "uvicorn", extra = ["standard"], marker = "(sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/82/986d498e3b4c41b043bd14ece562fc1a766b56aa1bfd980ae10717c9bc46/fastapi_cli-0.0.29.tar.gz", hash = "sha256:d1140852664a91754da6db4db1e750ace4059f1a21adcf9b161ad4310271a621", size = 23948, upload-time = "2026-07-08T12:45:03.19Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/13/0786e5265018238f9a014270ff89d84b02065c2303df9bfac6857ab54bab/fastapi_cli-0.0.29-py3-none-any.whl", hash = "sha256:05bf08e0e527e3649a50c44bd1e0a2c13575c6cf9a939ff70013f288afc074de", size = 13191, upload-time = "2026-07-08T12:45:02.378Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "fastapi-cloud-cli", marker = "sys_platform == 'linux'" }, + { name = "uvicorn", extra = ["standard"], marker = "(sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] + +[[package]] +name = "fastapi-cloud-cli" +version = "0.22.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "detect-installer", marker = "sys_platform == 'linux'" }, + { name = "fastar", marker = "sys_platform == 'linux'" }, + { name = "httpx", marker = "sys_platform == 'linux'" }, + { name = "pydantic", extra = ["email"], marker = "(sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "rich-toolkit", marker = "sys_platform == 'linux'" }, + { name = "rignore", marker = "sys_platform == 'linux'" }, + { name = "sentry-sdk", marker = "sys_platform == 'linux'" }, + { name = "typer", marker = "sys_platform == 'linux'" }, + { name = "uvicorn", extra = ["standard"], marker = "(sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/f2/36bfe990baa656de89a2b98a77a15dcd018474f7245c8e4a10cada0553c5/fastapi_cloud_cli-0.22.2.tar.gz", hash = "sha256:7ec78c1fed58f578af5eb1fb54ec4b456eba4dc1eaca1c3a93cf499a7cbc7ab3", size = 94480, upload-time = "2026-07-14T09:27:01.349Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/25/e01631a63a5213fc783e5b84e8a27eca800f21549aa414644a0a29181045/fastapi_cloud_cli-0.22.2-py3-none-any.whl", hash = "sha256:37c6b05adb94a4c9f59916a559d041c565db36b5f3dddeed420bf0a51182c363", size = 77744, upload-time = "2026-07-14T09:27:02.571Z" }, +] + +[[package]] +name = "fastar" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/0f/0aeb3fc50046617702acc0078b277b58367fd62eb727b9ec733ae0e8bbcc/fastar-0.11.0.tar.gz", hash = "sha256:aa7f100f7313c03fdb20f1385927ba95671071ba308ad0c1763fef295e1895ce", size = 70238, upload-time = "2026-04-13T17:11:17.143Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/4a/0d79fe52243a4130aa41d0a3a9eea22e00427db761e1a6782ee817c50222/fastar-0.11.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:e7c906ad371ca365591ebcb7630009923f3eceb20956814494d15591a78e9e46", size = 709786, upload-time = "2026-04-13T17:09:53.974Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e4/77c94eaafc035e39f5ce5176e32743da4e3fe890f28790e708e53d8f75cd/fastar-0.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6919497b35fa5bd978d2c26ee117cf1771b90ee5073f7518e44b9bc364b57715", size = 632127, upload-time = "2026-04-13T17:09:39.023Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f6/97658dd992f4e45747d35adb24c0b100f6b6d451490685ae3fe8a3a2ee1b/fastar-0.11.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:56b50206aeedd99e22b83289e6fb3ff8f7d7da4407d2419902e4716b4f90585a", size = 869608, upload-time = "2026-04-13T17:09:08.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/fc/81c1ec4d8146a437399e7b95631b51be312f323a9ce64569f932db6c3914/fastar-0.11.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a1811a69ae81d469720df0c8af3f84f834a93b5e4f8be0e0e8bde6a52fa11f2", size = 762925, upload-time = "2026-04-13T17:07:52.788Z" }, + { url = "https://files.pythonhosted.org/packages/b9/35/49baf480ecb197aea7ce2515c503a2f25061958dd3b4c98e98a3a11cdcc7/fastar-0.11.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:10486238c55589a3947c38f9cfb88a67d8a608eb8dddc722038237d0278a41d7", size = 759913, upload-time = "2026-04-13T17:08:07.324Z" }, + { url = "https://files.pythonhosted.org/packages/94/eb/946f1980267f2824efb7d7c518d47a49b89c0e9cd7c449301f5a7531558a/fastar-0.11.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1555ef9992d368a6ec39092276990cef8d329c39a1d86ebd847eaa3b10efd472", size = 926054, upload-time = "2026-04-13T17:08:22.196Z" }, + { url = "https://files.pythonhosted.org/packages/0c/19/d5eb611085ce054382570d8d4e24a5e2ff23cd6d2404528a6643841d6059/fastar-0.11.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b1f4aca0a9620b76988bbf6225cdea6678a392902444ca18bb8a51495b165a89", size = 818594, upload-time = "2026-04-13T17:08:52.366Z" }, + { url = "https://files.pythonhosted.org/packages/4a/52/18e8d55c0d3d917713f381cb2d0cb793da00c209c802e011d8dc72018cd5/fastar-0.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75beeecac7d11a666a6c4a0b7f7e80842ae5cf523f2f890b99c78fc82b403545", size = 823005, upload-time = "2026-04-13T17:09:23.051Z" }, + { url = "https://files.pythonhosted.org/packages/2c/b4/0fecdcf33e5aaffe777b96a1c10a3204fe0b05bf18e971033a0bfedafc1c/fastar-0.11.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:a08cdf5d16daa401c65c9c7493a18db7dc515c52155a17071ec7098bb07da9d3", size = 887115, upload-time = "2026-04-13T17:08:37.385Z" }, + { url = "https://files.pythonhosted.org/packages/08/f8/2a6ad1c2523eb72a4595a9331162fc67ce0f0aee3348728598026c516986/fastar-0.11.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6e210375e5a7ba53586cbd6017aa417d2d2ceacbe8671682470281bd0a15e8ef", size = 973595, upload-time = "2026-04-13T17:10:09.258Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a6/2aa48843228673feacc2b80876b8924e63ea9c5f5f607bd7a72416b86bae/fastar-0.11.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a2988eb2604b8e15670f355425e8c800e4dcd4edfbcbfe194397f8f17b7eb19e", size = 1036988, upload-time = "2026-04-13T17:10:26.133Z" }, + { url = "https://files.pythonhosted.org/packages/92/ac/3dd14b21c323e8484f47c910110d1d93139ba44621ac2c4c597dbe9fcdb7/fastar-0.11.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:34abc857b46068fdf91d157bd0203bfd6791dc7a432d1ed180f5af6c2f5bcce9", size = 1078267, upload-time = "2026-04-13T17:10:43.645Z" }, + { url = "https://files.pythonhosted.org/packages/de/a1/3f89e58d6fa99160c9e7e17220c8ab5040b5cc017c4fac2356c6ed18453d/fastar-0.11.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0d884be84e37a01053776395441fc960031974e0265801ce574efc3d05e0cdaf", size = 1032551, upload-time = "2026-04-13T17:11:00.667Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ea/24dd3cfc2096933d7d2a80c926e79602cff1fa481124ed2165b60c1dd9ef/fastar-0.11.0-cp310-cp310-win32.whl", hash = "sha256:c721c1ad758e3e4c2c1fd9e96911a0fa58c0a6be5668f1bcfd0b741e72c7cb63", size = 456022, upload-time = "2026-04-13T17:11:41.859Z" }, + { url = "https://files.pythonhosted.org/packages/82/ef/6eb39ee9cdd59822d1c7337c4d28fdc948885bdf455af9e70efa9879e06f/fastar-0.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:ba4180b7c3080f55f9035fdd7d8c39fe0e1485087a68ff615bb4784a10b8106b", size = 488392, upload-time = "2026-04-13T17:11:27.486Z" }, + { url = "https://files.pythonhosted.org/packages/11/7a/fb367bdaf4efa2c7952a45aeab2e87a564293ecffe150af673ec8edfda46/fastar-0.11.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:b82fd6f996e65a86f67a6bd64dd22ef3e8ae2dcaed0ae3b550e71f7e1bbb1df5", size = 709869, upload-time = "2026-04-13T17:09:55.62Z" }, + { url = "https://files.pythonhosted.org/packages/80/ff/b87efb0dcfd081c62c7c7601d7681dabe63103cd51fc16f8d57a1ab45961/fastar-0.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27eed386fd0558e6daa29211111bbd7b740f7c7e881197f8a00ac7c0f3cdb1d7", size = 631668, upload-time = "2026-04-13T17:09:40.537Z" }, + { url = "https://files.pythonhosted.org/packages/24/7c/0ed6dd38b9adc04b3a8ec3b7045908e7c2170ba0ff6e6d2c51bc9fc770f3/fastar-0.11.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a6931bebc1d8e95ddeef55732c195449e6b44ef33aa31b325505097ed3b4d6aa", size = 869663, upload-time = "2026-04-13T17:09:09.78Z" }, + { url = "https://files.pythonhosted.org/packages/58/ce/8b7fb3f23855accebaaf2d2637eac7f261a7a5d936f861a172079f1ef511/fastar-0.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:891f72ce42a5e28a74fbd4d5fbf1a3ac1a1163d13cbc200cbd005fb0fabc54bd", size = 762938, upload-time = "2026-04-13T17:07:54.51Z" }, + { url = "https://files.pythonhosted.org/packages/07/cc/5491e2b677bb841f768e3aba052d0344338a5c78aa5d4c18b443831a8e8d/fastar-0.11.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5b83c1f61f7017d6e1498568038f8745440cfc16ca2f697ec81bac83050108f6", size = 759232, upload-time = "2026-04-13T17:08:08.864Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b7/643630bdbd179e41e9fae31c03b4cf6061dbf4d6fbbae8425d16eb12545d/fastar-0.11.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db73a9b765a516e73983b25341e7b5e0189733878279e278b2295131b0e3a21e", size = 926271, upload-time = "2026-04-13T17:08:23.68Z" }, + { url = "https://files.pythonhosted.org/packages/09/5d/37ade50003b4540e0a53ef100f6692d7ab2ac1122d5acf39920cc09a3e8b/fastar-0.11.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:625827d52eb4e8fec942e0233f125ff8010fcf6a67c0a974a8e5f4666b771e3c", size = 818634, upload-time = "2026-04-13T17:08:54.268Z" }, + { url = "https://files.pythonhosted.org/packages/c3/ff/135d177de32cc1e837c99019e4643e6e79352bde49544d4ece5b5eebf56b/fastar-0.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7f5fd8fa21ec0a88296a38dc5d7fc35efd3b26d46a17b8b7c73c5563925ca15", size = 822755, upload-time = "2026-04-13T17:09:25.01Z" }, + { url = "https://files.pythonhosted.org/packages/27/cb/b835dbe76ceac7fa6105851468c259ffd06830eb9c029402e499d0ec153b/fastar-0.11.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:8c15af91b8cd87ddf23ea55355ae513c1de3ab67178f26dad017c9e9c0af6096", size = 887101, upload-time = "2026-04-13T17:08:39.248Z" }, + { url = "https://files.pythonhosted.org/packages/9e/54/aa8289eb57fc550535470397cb051f5a58a7c89ca4de31d5502b916dd894/fastar-0.11.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:03a112395a8b0bff251423bd1564c012f0cc058ad8b6bd8fba96f3d7fc117e44", size = 973606, upload-time = "2026-04-13T17:10:10.98Z" }, + { url = "https://files.pythonhosted.org/packages/1f/fd/776d50a0897c01dc6bfd0926772ee913436fdae91b9affaf0a0cbd09f0a1/fastar-0.11.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f2994bb8f5f8c11eb12beae1e6e77a907173c9819236b8a4c8f0573652ceccce", size = 1036696, upload-time = "2026-04-13T17:10:28.502Z" }, + { url = "https://files.pythonhosted.org/packages/c8/f1/cf0f9b499fb37ac065c8a01ec642f96a3c5eb849c38ae983b59f3b3245e0/fastar-0.11.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:dcf99e4b5973d842c7f19c776c3a83cdc0977d505edce6206438505c0456b517", size = 1078182, upload-time = "2026-04-13T17:10:45.318Z" }, + { url = "https://files.pythonhosted.org/packages/f8/9e/21e4701aec4a1123d4dc4d31578dc18875582b5710e4725f7ceb752a248b/fastar-0.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29c9c386dc0d5dda78845a8e6b1480d26ab861c1e0b68f42ae5735cb70ca07f1", size = 1032336, upload-time = "2026-04-13T17:11:02.364Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e2/5872b28c72c27ec1a00760eace6ff35f714f41ebbd5208cf016b12e29250/fastar-0.11.0-cp311-cp311-win32.whl", hash = "sha256:030b2580fc394f2c9b7890b6735810404e9b9ed5e0344db150b945965b5482b7", size = 457368, upload-time = "2026-04-13T17:11:43.528Z" }, + { url = "https://files.pythonhosted.org/packages/fd/6e/ce6832a16193eb4466f4108be8809c249b51cb1f89dd7894545700d079d5/fastar-0.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:83ab57ae067969cd0b483ac3b6dccc4b595fc77f5c820760998648d4c42822b5", size = 488605, upload-time = "2026-04-13T17:11:29.161Z" }, + { url = "https://files.pythonhosted.org/packages/15/5a/9cfb80661cf38fd7b0889224beb7d2746784d4ade2a931ed9775a18d8602/fastar-0.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:27b1a4cee2298b704de8151d310462ee7335ed036011ca9aa6e784b30b6c73a9", size = 464580, upload-time = "2026-04-13T17:11:18.583Z" }, + { url = "https://files.pythonhosted.org/packages/0f/06/a5773706afc8bd496769786590bbc56d2d0ee419a299cc12ea3f5717fcf3/fastar-0.11.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3c51f1c2cdddbd1420d2897ace7738e36c65e17f6ae84e0bfe763f8d1068bb97", size = 708394, upload-time = "2026-04-13T17:09:57.269Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a6/d5e2a4e48495616440a21eed07558219ca90243ad00b0502586f95bd4833/fastar-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0d9d6b052baf5380baea866675dab6ccd04ec2460d12b1c46f10ce3f4ee6a820", size = 628417, upload-time = "2026-04-13T17:09:42.145Z" }, + { url = "https://files.pythonhosted.org/packages/ab/69/9816d69ac8265c9e50456637a487ccfb7a9c566efd9dbcd673df9c2558c2/fastar-0.11.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:bd2f05666d4df7e14885b5c38fefd92a785917387513d33d837ff42ec143a22f", size = 863950, upload-time = "2026-04-13T17:09:11.506Z" }, + { url = "https://files.pythonhosted.org/packages/5b/0d/f88daad53aff2e754b6b5ff2a7113f72447a34f6ef17cc23ca99988117b7/fastar-0.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1e6e74aba1ae77ca4aedcaf1697cd413319f4c88a5ccbe5b42c709517c5097e", size = 760737, upload-time = "2026-04-13T17:07:55.958Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a6/82ef4ecd969d50d92ed3ed9dbd8fe77faa24be5e5736f716edc9f4ce8d62/fastar-0.11.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:38ef77fe940bbc9b37a98bd838727f844b11731cd39358a2640ff864fb385086", size = 757603, upload-time = "2026-04-13T17:08:10.623Z" }, + { url = "https://files.pythonhosted.org/packages/03/35/50249f0d827251f8ac511495e2eacccebda80a00a0ad73e9615b8113b84f/fastar-0.11.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8955e61b32d6aff82c983217abf80933fd823b0e727586fc72f08043d996fd59", size = 923952, upload-time = "2026-04-13T17:08:25.526Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d8/faee41659e9c379d906d24eaee6d6833ac8cfef0a5df480e5c2a8d3efb33/fastar-0.11.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:483532442cdb08fbff0169510224eae0836f2f672cea6aacb52847d90fefdc46", size = 816574, upload-time = "2026-04-13T17:08:56.076Z" }, + { url = "https://files.pythonhosted.org/packages/22/47/0448ea7992b997dad2bf004bfd98eca74b5858630eae080b50c7b17d9ddc/fastar-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef5a6071121e05d8287fc75bccb054bcbac8bb0501200a0c0a8feeace5303ea4", size = 819382, upload-time = "2026-04-13T17:09:26.66Z" }, + { url = "https://files.pythonhosted.org/packages/33/ef/0d63eb43586831b7a6f8b22c4d77125a7c594423af1f4f090fa9541b9b40/fastar-0.11.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:e45e598af5afe8412197d4786efd6cf29be02e7d3d4f6a3461149eae5d7e94f1", size = 885254, upload-time = "2026-04-13T17:08:40.9Z" }, + { url = "https://files.pythonhosted.org/packages/01/25/edd584675d69e49a165052c3ee886df1c5d574f3e7d813c990306387c623/fastar-0.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2e160919b1c47ddb8538e7e8eb4cd527281b40f0bf75110a75993838ef61f286", size = 971239, upload-time = "2026-04-13T17:10:12.997Z" }, + { url = "https://files.pythonhosted.org/packages/a5/37/e8bb24f506ba2b08fbaf36c5800e843bd4d542954e9331f00418e2d23349/fastar-0.11.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4bb4dc0fc8f7a6807febcebce8a2f3626ba4955a9263d81ecc630aad83be84c0", size = 1035185, upload-time = "2026-04-13T17:10:30.207Z" }, + { url = "https://files.pythonhosted.org/packages/9a/bf/be753736296338149ee4cb3e92e2b5423d6ba17c7b951d15218fd7e99bbf/fastar-0.11.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4ec95af56aa173f6e320e1183001bf108ba59beaf13edd1fc8200648db203588", size = 1072191, upload-time = "2026-04-13T17:10:47.072Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cd/a81c1aaafb5a22ce57c98ae22f39c89413ed53e4ee6e1b1444b0bd666a6c/fastar-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:136cf342735464091c39dc3708168f9fdeb9ebea40b1ead937c61afaf46143d9", size = 1028054, upload-time = "2026-04-13T17:11:04.293Z" }, + { url = "https://files.pythonhosted.org/packages/ec/88/1ce4eed3d70627c95f49ca017f6bbbf2ddcc4b0c601d293259de7689bc20/fastar-0.11.0-cp312-cp312-win32.whl", hash = "sha256:35f23c11b556cc4d3704587faacbc0037f7bdf6c4525cd1d09c70bda4b1c6809", size = 454198, upload-time = "2026-04-13T17:11:45.168Z" }, + { url = "https://files.pythonhosted.org/packages/8f/1d/26ce92f4331cd61a69840db9ca6115829805eec24f285481a854f578e917/fastar-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:920bc56c3c0b8a8ca492904941d1883c1c947c858cd93343356c29122a38f44c", size = 486697, upload-time = "2026-04-13T17:11:31.084Z" }, + { url = "https://files.pythonhosted.org/packages/ed/96/e6eda4480559c69b05d466e7b5ea9170e81fef3795a73e059959a3258319/fastar-0.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:395248faf89e8a6bd5dc1fd544c8465113b627cb6d7c8b296796b60ebea33593", size = 462591, upload-time = "2026-04-13T17:11:20.577Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d6/3be260037e86fb694e88d47f583bac3a0188c99cee1a6b257ac26cb6b53c/fastar-0.11.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:33f544b08b4541b678e53749b4552a44720d96761fb79c172b005b1089c443ed", size = 707975, upload-time = "2026-04-13T17:09:58.866Z" }, + { url = "https://files.pythonhosted.org/packages/e1/cd/7867aefb1784662554a335f2952c75a50f0c70585ed0d2210d6cc15e5627/fastar-0.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:91c1c792447e4a642745f347ff9847c52af39633071c57ee67ed53c157fc3506", size = 628460, upload-time = "2026-04-13T17:09:43.776Z" }, + { url = "https://files.pythonhosted.org/packages/e5/2b/d11d84bdd5e0e377771b955755771e3460b290da5809cb78c1b735ee2228/fastar-0.11.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:881247e6b6eaea59fc6569f9b61447aa6b9fc2ee864e048b4643d69c52745805", size = 863054, upload-time = "2026-04-13T17:09:13.048Z" }, + { url = "https://files.pythonhosted.org/packages/25/39/d3f428b318fa940b1b6e785b8d54fc895dfb5d5b945ef8d5442ffa904fb2/fastar-0.11.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:863b7929845c9fec92ef6c8d59579cf46af5136655e5342f8df5cebe46cab06c", size = 760247, upload-time = "2026-04-13T17:07:57.396Z" }, + { url = "https://files.pythonhosted.org/packages/9e/04/03949aee82aabb8ede06ac5a4a5579ffaf98a8fe59ce958494508ff15513/fastar-0.11.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:96b4a57df12bf3211662627a3ea29d62ecb314a2434a0d0843f9fc23e47536e5", size = 756512, upload-time = "2026-04-13T17:08:12.415Z" }, + { url = "https://files.pythonhosted.org/packages/3f/0c/2ca1ae0a3828ca51047962d932b80daca2522db73e8cb9d040cb6ebe28d5/fastar-0.11.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ceef1c2c4df7b7b8ebd3f5d718bbf457b9bbdf25ce0bd07870211ec4fbd9aff4", size = 922183, upload-time = "2026-04-13T17:08:27.187Z" }, + { url = "https://files.pythonhosted.org/packages/65/68/7fe808b1f73a68e686f25434f538c6dc10ef4dfb3db0ace22cd861744bf8/fastar-0.11.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8e545918441910a779659d4759ad0eef349e935fbdb4668a666d3681567eb05", size = 816394, upload-time = "2026-04-13T17:08:57.657Z" }, + { url = "https://files.pythonhosted.org/packages/1f/17/07d086080f8a83b8d7966955e29bcdbd6a060f5bd949dc9d5abd3658cead/fastar-0.11.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28095bb8f821e85fc2764e1a55f03e5e2876dee2abe7cd0ee9420d929905d643", size = 818983, upload-time = "2026-04-13T17:09:28.46Z" }, + { url = "https://files.pythonhosted.org/packages/fb/e2/2c4edf0910af2e814ff6d65b77a91196d472ca8a9fb2033bd983f6856caa/fastar-0.11.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0fafb95ecbe70f666a5e9b35dd63974ccdc9bb3d99ccdbd4014a823ec3e659b5", size = 884689, upload-time = "2026-04-13T17:08:42.763Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/04fdcbd6558e60de4ced3b55230fac47675d181252582b2fcec3c74608e5/fastar-0.11.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:af48fed039b94016629dcdad1c95c90c486326dd068de2b0a4df419ee09b6821", size = 970677, upload-time = "2026-04-13T17:10:15.124Z" }, + { url = "https://files.pythonhosted.org/packages/df/b3/2b860a9658550167dbd5824c85e88d0b4b912bf493e42a6322544d6e483d/fastar-0.11.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:74cd96163f39b8638ab4e8d49708ca887959672a22871d8170d01f067319533b", size = 1034026, upload-time = "2026-04-13T17:10:32.318Z" }, + { url = "https://files.pythonhosted.org/packages/b7/9b/fa42ea1188b144bac4b1b60753dfd449974a4d5eda132029ee7711569f94/fastar-0.11.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4e8b993cb5613bab495ed482810bedc0986633fcb9a3b55c37ec88e0d6714f6a", size = 1071147, upload-time = "2026-04-13T17:10:48.833Z" }, + { url = "https://files.pythonhosted.org/packages/95/c8/d2e501556dca9f1fbc9246111a31792fb49ad908fa4927f34938a97a3604/fastar-0.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfe39d91fc28e37e06162d94afe01050220edb7df554acb5b702b5503e564816", size = 1028377, upload-time = "2026-04-13T17:11:06.374Z" }, + { url = "https://files.pythonhosted.org/packages/db/33/5f11f23eca0a569cd052507bc45dda2e5468697f8665728d25be44120f7d/fastar-0.11.0-cp313-cp313-win32.whl", hash = "sha256:c5f63d4d99ff4bfb37c659982ec413358bdee747005348756cc50a04d412d989", size = 454089, upload-time = "2026-04-13T17:11:46.821Z" }, + { url = "https://files.pythonhosted.org/packages/da/2f/35ff03c939cba7a255a9132367873fec6c355fd06a7f84fedcbaf4c8129f/fastar-0.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:8690ed1928d31ded3ada308e1086525fb3871f5fa81e1b69601a3f7774004583", size = 486312, upload-time = "2026-04-13T17:11:32.86Z" }, + { url = "https://files.pythonhosted.org/packages/ef/71/ee9246cbfcbfd4144558f35e7e9a306ffe0a7564730a5188c45f21d2dab8/fastar-0.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:d977ded9d98a0719a305e0a4d5ee811f1d3e856d853a50acb8ae833c3cd6d5d2", size = 461975, upload-time = "2026-04-13T17:11:22.589Z" }, + { url = "https://files.pythonhosted.org/packages/7a/cd/3644c48ecac456f928c12d47ec3bed36c36555b17c3859856f1ff860265d/fastar-0.11.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:71375bd6f03c2a43eb47bd949ea38ff45434917f9cdac79675c5b9f60de4fa73", size = 707860, upload-time = "2026-04-13T17:10:00.371Z" }, + { url = "https://files.pythonhosted.org/packages/69/ca/dee04476ae3626b2b040a60ad84628f77e1ffd8444232f2426b0ca1e0d7e/fastar-0.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:eddfd9cab16e19ae247fe44bf992cb403ccfe27d3931d6de29a4695d95ad386c", size = 628216, upload-time = "2026-04-13T17:09:45.355Z" }, + { url = "https://files.pythonhosted.org/packages/dc/5e/9395c7353d079cb4f5be0f7982ce0dc9f2e7dec5fd175eef466729d6023a/fastar-0.11.0-cp314-cp314-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:7c371f1d4386c699018bb64eb2fa785feacf32785559049d2bb72fe4af023f53", size = 864378, upload-time = "2026-04-13T17:09:14.611Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/1e4f67148223ff219612b6281a6000357abbcc2417964fa5c83f11d68fce/fastar-0.11.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cad7fa41e3e66554387481c1a09365e4638becd322904932674159d5f4046728", size = 760921, upload-time = "2026-04-13T17:07:59.138Z" }, + { url = "https://files.pythonhosted.org/packages/0f/82/09d11fb6d12f17993ffaf32ffd30c3c121a11e2966e84f19fb6f66430118/fastar-0.11.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf36652fa71b83761717c9899b98732498f8a2cb6327ff16bbf07f6be85c3437", size = 757012, upload-time = "2026-04-13T17:08:14.186Z" }, + { url = "https://files.pythonhosted.org/packages/52/1f/5aeeacc4cb65615e2c9292cd9c5b0cd6fb6d2e6ee472ca6adc6c1b1b22ef/fastar-0.11.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f68ff8c17833053da4841720e95edde80ce45bb994b6b7d51418dddaac70ee47", size = 924510, upload-time = "2026-04-13T17:08:28.741Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1a/1e5bdabbeaf2e856928956292609f2ff6a650f94480fb8afaca30229e483/fastar-0.11.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4563ed37a12ea1cdc398af8571258d24b988bf342b7b3bf5451bd5891243280c", size = 816602, upload-time = "2026-04-13T17:08:59.461Z" }, + { url = "https://files.pythonhosted.org/packages/87/24/f960147910da3bed41a3adfcb026e17d5f50f4cf467a3324237a7088f61a/fastar-0.11.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cee63c9875cba3b70dc44338c560facc5d6e763047dcc4a30501f9a68cf5f890", size = 819452, upload-time = "2026-04-13T17:09:29.926Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f4/3e77d7901d5707fd7f8a352e153c8ae09ea974e6fabad0b7c4eb9944b8d4/fastar-0.11.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:bd76bfffae6d0a91f4ac4a612f721e7aec108db97dccdd120ae063cd66959f27", size = 885254, upload-time = "2026-04-13T17:08:44.285Z" }, + { url = "https://files.pythonhosted.org/packages/47/01/1585edd5ec47782ae93cd94edf05828e0ab02ef00aec00aea4194a600464/fastar-0.11.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8f5b707501ec01c1bc0518f741f01d322e50c9adc19a451aa24f67a2316e9397", size = 971496, upload-time = "2026-04-13T17:10:17.024Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e9/6874c9d1236ded565a0bed54b320ac9f165f287b1d89490fb70f9f323c81/fastar-0.11.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:37c0b5a88a657839aad98b0a6c9e4ac4c2c15d6b49c44ee3935c6b08e9d3e479", size = 1034685, upload-time = "2026-04-13T17:10:34.063Z" }, + { url = "https://files.pythonhosted.org/packages/14/d8/4ab20613ce2983427aee958e39be878dba874aa227c530a845e32429c4f6/fastar-0.11.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:6c55f536c62a6efb180c1af0d5182948bff576bbfe6276e8e1359c9c7d2215d8", size = 1072675, upload-time = "2026-04-13T17:10:50.53Z" }, + { url = "https://files.pythonhosted.org/packages/1f/ae/5ac3b7c20ce4b08f011dd2b979f96caabe64f9b10b157f211ea91bdfadca/fastar-0.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3082eeca59e189b9039335862f4c2780c0c8871d656bfdf559db4414a105b251", size = 1029330, upload-time = "2026-04-13T17:11:08.138Z" }, + { url = "https://files.pythonhosted.org/packages/8a/e7/37cd6a1d4e288292170b64e19d79ecce2a7de8bb76790323399a2abc4619/fastar-0.11.0-cp314-cp314-win32.whl", hash = "sha256:b201a0a4e29f9fec2a177e13154b8725ec65ab9f83bd6415483efaa2aa18344b", size = 453940, upload-time = "2026-04-13T17:11:48.713Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1c/795c878b1ee29d79021cf8ed81f18f2b25ccde58453b0d34b9bdc7e025ea/fastar-0.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:868fddb26072a43e870a8819134b9f80ee602931be5a76e6fb873e04da343637", size = 486334, upload-time = "2026-04-13T17:11:34.882Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a4/113f104301df8bddcc0b3775b611a30cb7610baa3add933c7ccac9386467/fastar-0.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:3db39c9cc42abb0c780a26b299f24dfbc8be455985e969e15336d70d7b2f833b", size = 461534, upload-time = "2026-04-13T17:11:24.329Z" }, + { url = "https://files.pythonhosted.org/packages/5a/a6/5c5f2c2c8e0c63e56a5636ebc7721589c889e94c0092cec7eb28ae7207e6/fastar-0.11.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:49c3299dec5e125e7ebaa27545714da9c7391777366015427e0ae62d548b442b", size = 707156, upload-time = "2026-04-13T17:10:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/df/f7/982c01b61f0fc135ad2b16d01e6d0ee53cf8791e68827f5f7c5a65b2e5b1/fastar-0.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3328ed1ed56d31f5198350b17dd60449b8d6b9d47abb4688bab6aef4450a165b", size = 627032, upload-time = "2026-04-13T17:09:46.978Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c3/38f1dac77ae0c71c37b176277c96d830796b8ce2fe69705f917829b53829/fastar-0.11.0-cp314-cp314t-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:bd3eca3bbfec84a614bcb4143b4ad4f784d0895babc26cfc88436af88ca23c7a", size = 864403, upload-time = "2026-04-13T17:09:16.58Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f0/e69c363bdb3e5a5848e937b662b5469581ee6682c51bc1c0556494773929/fastar-0.11.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff86a967acb0d621dd24063dda090daa67bf4993b9570e97fe156de88a9006ca", size = 759480, upload-time = "2026-04-13T17:08:00.599Z" }, + { url = "https://files.pythonhosted.org/packages/3b/29/4d8737590c2a6357d614d7cc7288e8f68e7e449680b8922997cc4349e65e/fastar-0.11.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:86eaf7c0e985d93a7734168be2fb232b2a8cca53e41431c2782d7c12b12c03b1", size = 756219, upload-time = "2026-04-13T17:08:15.699Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ec/400de7b3b7d48801908f19cf5462177104395799472671b3e8152b2b04ca/fastar-0.11.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91f07b0b8eb67e2f177733a1f884edad7dfb9f8977ffef15927b20cb9604027d", size = 923669, upload-time = "2026-04-13T17:08:30.574Z" }, + { url = "https://files.pythonhosted.org/packages/5d/01/8926c53da923fed7ab4b96e7fbf7f73b663beb4f02095b654d6fab46f9ad/fastar-0.11.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f85c896885eb4abf1a635d54dea22cac6ae48d04fc2ea26ae652fcf1febe1220", size = 815729, upload-time = "2026-04-13T17:09:01.204Z" }, + { url = "https://files.pythonhosted.org/packages/89/f0/5fef4c7946e352651b504b1a4235dac3505e7cfd24020788ab50552e84bf/fastar-0.11.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:075c07095c8de4b774ba8f28b9c0a02b1a2cd254da50cbe464dd3bb2432e9158", size = 819812, upload-time = "2026-04-13T17:09:31.907Z" }, + { url = "https://files.pythonhosted.org/packages/b3/c8/0ebc3298b4a45e7bddc50b169ae6a6f5b80c939394d4befe6e60de535ee7/fastar-0.11.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:07f028933820c65750baf3383b807ecce1cd9385cf00ce192b79d263ad6b856c", size = 884074, upload-time = "2026-04-13T17:08:45.802Z" }, + { url = "https://files.pythonhosted.org/packages/ae/9f/7baa4cdff8d6fbca41fa5c764b48a941fed8a9ec6c4cc92de65895a28299/fastar-0.11.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:039f875efa0f01fa43c20bf4e2fc7305489c61d0ac76eda991acfba7820a0e63", size = 969450, upload-time = "2026-04-13T17:10:18.667Z" }, + { url = "https://files.pythonhosted.org/packages/d4/dc/1ebbfb58a47056ba866494f19efbcdd2ba2897096b94f36e796594b4d05b/fastar-0.11.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:fff12452a9a5c6814a012445f26365541cc3d99dcca61f09762e6a389f7a32ea", size = 1033775, upload-time = "2026-04-13T17:10:36.165Z" }, + { url = "https://files.pythonhosted.org/packages/c2/5f/ce4e3914066f08c99eb8c32952cc07c1a013e81b1db1b0f598130bf6b974/fastar-0.11.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:2bf733e09f942b6fa876efe30a90508d1f4caef5630c00fb2a84fba355873712", size = 1072158, upload-time = "2026-04-13T17:10:52.497Z" }, + { url = "https://files.pythonhosted.org/packages/03/2a/6bca72992c84151c387cc6558f3867f5ebe5fb3684ee6fa9b76280ba4b8e/fastar-0.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d1531fa848fdd3677d2dce0a4b436ea64d9ae38fb8babe2ddbc180dd153cb7a3", size = 1028577, upload-time = "2026-04-13T17:11:09.934Z" }, + { url = "https://files.pythonhosted.org/packages/83/18/7a7c15657a3da5569b26fc51cde6a80f8d84cb54b3b1aea6d74a103db4ad/fastar-0.11.0-cp314-cp314t-win32.whl", hash = "sha256:5744551bc67c6fc6581cbd0e34a0fd6e2cd0bd30b43e94b1c3119cf35064b162", size = 453601, upload-time = "2026-04-13T17:11:53.726Z" }, + { url = "https://files.pythonhosted.org/packages/6d/d8/331b59a6de279f3ad75c10c02c40a12f21d64a437d9c3d6f1af2dcbd7a76/fastar-0.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f4ce44e3b56c47cf38244b98d29f269b259740a580c47a2552efa5b96a5458fb", size = 486436, upload-time = "2026-04-13T17:11:40.089Z" }, + { url = "https://files.pythonhosted.org/packages/6b/fd/5390ec4f49100f3ecb9968a392f9e6d039f1e3fe0ecd28443716ff01e589/fastar-0.11.0-cp314-cp314t-win_arm64.whl", hash = "sha256:76c1359314355eafbc6989f20fb1ad565a3d10200117923b9da765a17e2f6f11", size = 461049, upload-time = "2026-04-13T17:11:25.918Z" }, + { url = "https://files.pythonhosted.org/packages/cc/5c/9bbeffbf1905391446dd98aa520422ce7affde5c9a7c22d757cc5d7c1397/fastar-0.11.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1266d6a004f427b0d61bd6c7b544d84cc964691b2232c2f4d635a1b75f2f6d5e", size = 711644, upload-time = "2026-04-13T17:10:07.663Z" }, + { url = "https://files.pythonhosted.org/packages/7e/af/ae5cf39d4fb82d0c592705f5ec6db1b065be5265c151b108f86126ee8773/fastar-0.11.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:298a827ec04ade43733f6ca960d0faec38706aa1494175869ea7ea17f5bad5d3", size = 634371, upload-time = "2026-04-13T17:09:52.083Z" }, + { url = "https://files.pythonhosted.org/packages/7e/36/8d4569e26473c72ccb02d1c5df3ed710073f1c06eca09c26d52ea79fd815/fastar-0.11.0-pp311-pypy311_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8800e2387e463a0e5799416a1cbe72dd0fde7270a20e4bde684145e7878f6516", size = 870850, upload-time = "2026-04-13T17:09:21.439Z" }, + { url = "https://files.pythonhosted.org/packages/bf/46/724dc796e1756d3977970f820d30d59bb8cab8e3671b285f1d82ab513aec/fastar-0.11.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7496def0a2befd82d429cb004ef7ca831585cc887947bd6b9abb68a5ef852b0b", size = 764469, upload-time = "2026-04-13T17:08:05.638Z" }, + { url = "https://files.pythonhosted.org/packages/99/e3/74d6859e632e8fb9339a14f652fb9f800c2bd6aa53071e311c0be3fbab8b/fastar-0.11.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:878eaf15463eb572e3538af7ca3a8534e5e279cf8196db902d24e5725c4af86e", size = 761375, upload-time = "2026-04-13T17:08:20.669Z" }, + { url = "https://files.pythonhosted.org/packages/a3/e7/cc70e2be5ef8731a7525552b1c35c1448cf9eae6a62cb3a56f12c1bf27ea/fastar-0.11.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0324ed1d1ef0186e1bbd843b17807d6d837d0906899d4c99378b02c5d86bdd9c", size = 928189, upload-time = "2026-04-13T17:08:35.663Z" }, + { url = "https://files.pythonhosted.org/packages/3c/33/c9a969e78dca323547276a6fee5f4f9588f7cd5ab45acec3778c67399589/fastar-0.11.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bdf9bd863205590beaf8ef6e66f315310196632180dceaf674985d01a876cac3", size = 820864, upload-time = "2026-04-13T17:09:06.366Z" }, + { url = "https://files.pythonhosted.org/packages/84/bd/6b9434b541fe55c125b5f2e017a565596a2d215aa09207e4555e4585064f/fastar-0.11.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59af8dbb683b24b90fb5b506de080faeab0a17a908e6c2a5d93a97260ed75d7b", size = 824060, upload-time = "2026-04-13T17:09:37.377Z" }, + { url = "https://files.pythonhosted.org/packages/24/8d/871d5f8cf4c6f13987119fb0a9ae8be131e34f2756c2524e9974adf33824/fastar-0.11.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:9f3df73a3c4292cfe15696cdf59cdb6c309ab59d30b34c733be13c6e32d9a264", size = 889217, upload-time = "2026-04-13T17:08:50.884Z" }, + { url = "https://files.pythonhosted.org/packages/d0/26/cca0fd2704f3ed20165e5613ed911549aef3aaf3b0b5b02fee0e8e23e6cc/fastar-0.11.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:aa3762cbb16e41a76b61f4a6914937a71aab3a7b6c2d82ca233bc686ebaf756b", size = 975418, upload-time = "2026-04-13T17:10:24.307Z" }, + { url = "https://files.pythonhosted.org/packages/99/94/8bbb0b13f5b6cbe2492f0b7cbba5103e6163976a3331466d010e781fa189/fastar-0.11.0-pp311-pypy311_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:a8c7bc8ac74cb359bb546b199288c83236372d094b402e557c197e85527495cd", size = 1038492, upload-time = "2026-04-13T17:10:41.939Z" }, + { url = "https://files.pythonhosted.org/packages/ed/d3/5b7df222a30eac2822ffd00f82fd4c2ce84fba4b369d1e1a03732fd177fc/fastar-0.11.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:587cbd060a2699c5f66281081395bb4657b2b1e0eef5c206b1aabf740019d670", size = 1080210, upload-time = "2026-04-13T17:10:58.462Z" }, + { url = "https://files.pythonhosted.org/packages/ec/6d/56ef943ea524784598c035ccbd42e564e937da0438ae3f55f0e76cb95571/fastar-0.11.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6a1c56957ac82408be37a3f63594bc83e0919e8760492a4475e542f9f1828778", size = 1034886, upload-time = "2026-04-13T17:11:15.617Z" }, +] + [[package]] name = "fastjsonschema" version = "2.21.1" @@ -1424,6 +2118,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/90/2b/0817a2b257fe88725c25589d89aec060581aabf668707a8d03b2e9e0cb2a/fastjsonschema-2.21.1-py3-none-any.whl", hash = "sha256:c9e5b7e908310918cf494a434eeb31384dd84a98b57a30bcb1f535015b554667", size = 23924, upload-time = "2024-12-02T10:55:07.599Z" }, ] +[[package]] +name = "fastsafetensors" +version = "0.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typer", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/4c/f17bd54c933fd23648ce1e4272adf27d8d7e99b788c8859544bbd39f02e7/fastsafetensors-0.3.3.tar.gz", hash = "sha256:ba4fb59be8a6adbc91723848c3c6f57a9a9a5d2247d9768f84511380d01e554c", size = 77817, upload-time = "2026-07-07T07:21:47.121Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/06/dce623c5bc294dc72e4f8eb48d55a5db829b392b3e717a5156226be807b4/fastsafetensors-0.3.3-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:db542d1dfa60dc3c610d9750e3d90107e2ef76feed34958ba8b81814969ea168", size = 1840879, upload-time = "2026-07-07T07:21:26.572Z" }, + { url = "https://files.pythonhosted.org/packages/14/28/cd0fe7ffe75a84ffe2d7da3e86bdb052f01f4b7442dde44e2837aa34e4ed/fastsafetensors-0.3.3-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50d8137ce2ca072d1717564c3810d7ccf649eb9f27179f26470258465682bf42", size = 1871721, upload-time = "2026-07-07T07:21:28.102Z" }, + { url = "https://files.pythonhosted.org/packages/65/e1/ed09e65cff92d93bf8b7ec52bd7912395df22b649105d906c5cf34be1c84/fastsafetensors-0.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:2f15b3d293468f2a6aae54b77781521a5d4f0db49bc6cd655442cca2207b3e78", size = 422851, upload-time = "2026-07-07T07:21:29.275Z" }, + { url = "https://files.pythonhosted.org/packages/64/51/40bdc05f922251c54518232b2ef47fd925094ee699c93a45c8792628a10e/fastsafetensors-0.3.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df2f641647b79093c5e3f4bb0b9c7d771f22b61279714c3059d5c71a7cf708da", size = 1858970, upload-time = "2026-07-07T07:21:30.457Z" }, + { url = "https://files.pythonhosted.org/packages/2f/f5/79f187385b4b9093742fec94e7889d22634ab6eba8403707797246fb2418/fastsafetensors-0.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92f8cf8e6617cffe24522023c726c5911c1ca5b7e3249da054abb0fac8d27040", size = 1891339, upload-time = "2026-07-07T07:21:32.022Z" }, + { url = "https://files.pythonhosted.org/packages/1f/36/e4e2f6bfaac6120e214624ead73ae12e304ad50e5dee7dbb249d9000f01f/fastsafetensors-0.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:be753c669b2e12b744f757303482d95079d48cd7fcce992680b74284b12e6ffa", size = 424283, upload-time = "2026-07-07T07:21:33.335Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/f95f7fc099ac1fc4c22aa46257d159eac88e29dd0765d21c4fc91caedb01/fastsafetensors-0.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c2f788a936ffd17938484360339645812e14cf1b2bdf4c18c035c713218e5a9", size = 1887326, upload-time = "2026-07-07T07:21:34.624Z" }, + { url = "https://files.pythonhosted.org/packages/92/8c/e3347b2a44a8ab9aced94fa450df4f309baa21f7f2981a8a7bd6a977f4d3/fastsafetensors-0.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3587bc66b8dec560ad903becf9540889013d4d47f0e10cf45f19bedc7b7bffa7", size = 1915478, upload-time = "2026-07-07T07:21:35.901Z" }, + { url = "https://files.pythonhosted.org/packages/80/65/388a55e6b2b3023fb732843335de14803f16a6d92f0cd47f1125d28b77ac/fastsafetensors-0.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:39c252a5528fa8653366f979d2ab8fa81159db4456b7c72cb5c288adb7699079", size = 424934, upload-time = "2026-07-07T07:21:37.216Z" }, + { url = "https://files.pythonhosted.org/packages/16/a2/fd30fe6ed5825cb4642bdd102d3c5a32d2d82c91bdb45ae61a9571278cf8/fastsafetensors-0.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f969b2c818942748ff1bc619e6d575a7a7156a7d488d4c41779fba8c4e1891dc", size = 1887148, upload-time = "2026-07-07T07:21:38.414Z" }, + { url = "https://files.pythonhosted.org/packages/d6/9d/d3db53dd5b5069b44f2b8f6fc5a943f4101178a6f74b48a7fc72dc68f142/fastsafetensors-0.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bbf9fbc5f74d6a3f333777eb222aa63453795dacd7ae7f559316a5398d4942ef", size = 1915579, upload-time = "2026-07-07T07:21:40.047Z" }, + { url = "https://files.pythonhosted.org/packages/3d/d3/4193c0ae305edb94a45f4b0789cf705106b87a1546ca5dafb211c8ab274a/fastsafetensors-0.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:b61a5f6e0e0b0da591f2790d3fbc6eef7f6e8597087977b946069e7bfbb2ec7a", size = 424967, upload-time = "2026-07-07T07:21:41.243Z" }, + { url = "https://files.pythonhosted.org/packages/24/80/aeb98bc1575cbd759466568ac6dcbad0a572622cdee0a604fb9604577ecf/fastsafetensors-0.3.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3386834e1595834124cfbb23e05a732cb1b4fb5f98562375c1806f14fe1d677", size = 1885958, upload-time = "2026-07-07T07:21:42.812Z" }, + { url = "https://files.pythonhosted.org/packages/1a/16/dc09db973bdc763d5ff64a36acc9e847d0b6c150591d9917ed65f304b722/fastsafetensors-0.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7db5f3ee98433cc8e5dcaaef2517f47c34a51ea8229350406b24b2f84af209d3", size = 1913491, upload-time = "2026-07-07T07:21:44.32Z" }, + { url = "https://files.pythonhosted.org/packages/7d/b8/6064fd3eeecdb7d8e624ae87773b1a8a8f81111dbbca27aca683e8f9b0c0/fastsafetensors-0.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:94bd5f003bd9a0d7bc0d262c823ed2ccd74ba0396772a8a327fe1c16f86b6e4c", size = 435707, upload-time = "2026-07-07T07:21:46.105Z" }, +] + [[package]] name = "filelock" version = "3.18.0" @@ -1433,6 +2153,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl", hash = "sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de", size = 16215, upload-time = "2025-03-14T07:11:39.145Z" }, ] +[[package]] +name = "flashinfer-cubin" +version = "0.6.8.post1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/b7/5e3b1a8c67031b421a8bd29c2bc29b900a550bb3392e8bda18bb15b5e476/flashinfer_cubin-0.6.8.post1-py3-none-any.whl", hash = "sha256:43636d4cd39e694a83d76a89f87fefcdf4cecb4c4f7dd22dac25ec368c1e901f", size = 295154113, upload-time = "2026-04-18T18:28:21.738Z" }, +] + +[[package]] +name = "flashinfer-python" +version = "0.6.8.post1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apache-tvm-ffi", marker = "sys_platform == 'linux'" }, + { name = "click", marker = "sys_platform == 'linux'" }, + { name = "cuda-tile", marker = "sys_platform == 'linux'" }, + { name = "einops", marker = "sys_platform == 'linux'" }, + { name = "ninja", marker = "sys_platform == 'linux'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "nvidia-cudnn-frontend", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cutlass-dsl", marker = "sys_platform == 'linux'" }, + { name = "nvidia-ml-py", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "requests", marker = "sys_platform == 'linux'" }, + { name = "tabulate", marker = "sys_platform == 'linux'" }, + { name = "torch", marker = "sys_platform == 'linux'" }, + { name = "tqdm", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/1e/2760fef9e74abc4480961048e5790b4c9e955872fb4d7d97900cfddced5a/flashinfer_python-0.6.8.post1.tar.gz", hash = "sha256:b18e4121baf9b93fa9a9f368ba9b981a0342895f50ab9dddc224aeb964ed346f", size = 6675885, upload-time = "2026-04-18T18:28:13.299Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/6d/1e8a8533913e33a50a486332ce0673f4fdb860f6eb9ed450327c5c1762cb/flashinfer_python-0.6.8.post1-py3-none-any.whl", hash = "sha256:818f9b8cc2fe66c42a1f6264be4841ac8821ada703685a02cfccb2b5124a710b", size = 9385316, upload-time = "2026-04-18T18:28:10.285Z" }, +] + [[package]] name = "fonttools" version = "4.62.1" @@ -1622,6 +2376,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/17/b7/fd357a7961875637930d138aa2667e6fd08bd888a5c173d4b7b2667f7cb9/furo-2023.9.10-py3-none-any.whl", hash = "sha256:513092538537dc5c596691da06e3c370714ec99bc438680edc1debffb73e5bfc", size = 324431, upload-time = "2023-09-10T14:58:02.964Z" }, ] +[[package]] +name = "gguf" +version = "0.19.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "requests", marker = "sys_platform == 'linux'" }, + { name = "tqdm", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/ae/17f1308ae45cd7b08ebb521747d5b23f4efc4d172038a4e228dd5106c3ff/gguf-0.19.0.tar.gz", hash = "sha256:dbadcd6cc7ccd44256f2229fe7c2dff5e8aa5cf0612ab987fd2b1a57e428923f", size = 111220, upload-time = "2026-05-06T13:04:03.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/bb/d71d6da82763528c2c2ed6b59a9d6142c6595545a4c448e2085d155e88c2/gguf-0.19.0-py3-none-any.whl", hash = "sha256:70bcd10edfe697fb2dad6e40af2234b9d8ece9a41a99761405121ebda1c3c1cd", size = 118475, upload-time = "2026-05-06T13:04:02.588Z" }, +] + [[package]] name = "gitdb" version = "4.0.12" @@ -1664,8 +2434,10 @@ wheels = [ [package.optional-dependencies] grpc = [ - { name = "grpcio" }, - { name = "grpcio-status" }, + { name = "grpcio", version = "1.80.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.14' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "grpcio", version = "1.82.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "grpcio-status", version = "1.62.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.14' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "grpcio-status", version = "1.82.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] [[package]] @@ -1699,11 +2471,12 @@ name = "google-cloud-translate" version = "3.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "google-api-core", extra = ["grpc"] }, + { name = "google-api-core", extra = ["grpc"], marker = "extra == 'extra-16-transformer-lens-lit'" }, { name = "google-auth" }, { name = "google-cloud-core" }, { name = "grpc-google-iam-v1" }, - { name = "grpcio" }, + { name = "grpcio", version = "1.80.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.14' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "grpcio", version = "1.82.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "proto-plus" }, { name = "protobuf" }, ] @@ -1717,7 +2490,7 @@ name = "googleapis-common-protos" version = "1.74.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "protobuf" }, + { name = "protobuf", marker = "(sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or extra == 'extra-16-transformer-lens-lit'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/20/18/a746c8344152d368a5aac738d4c857012f2c5d1fd2eac7e17b647a7861bd/googleapis_common_protos-1.74.0.tar.gz", hash = "sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1", size = 151254, upload-time = "2026-04-02T21:23:26.679Z" } wheels = [ @@ -1726,7 +2499,8 @@ wheels = [ [package.optional-dependencies] grpc = [ - { name = "grpcio" }, + { name = "grpcio", version = "1.80.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.14' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "grpcio", version = "1.82.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] [[package]] @@ -1735,7 +2509,7 @@ version = "6.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, - { name = "audioop-lts", marker = "python_full_version >= '3.13'" }, + { name = "audioop-lts", marker = "python_full_version >= '3.13' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "brotli" }, { name = "fastapi" }, { name = "gradio-client" }, @@ -1745,7 +2519,9 @@ dependencies = [ { name = "huggingface-hub" }, { name = "jinja2" }, { name = "markupsafe" }, - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "orjson" }, { name = "packaging" }, { name = "pandas" }, @@ -1798,8 +2574,9 @@ name = "grpc-google-iam-v1" version = "0.14.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "googleapis-common-protos", extra = ["grpc"] }, - { name = "grpcio" }, + { name = "googleapis-common-protos", extra = ["grpc"], marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "grpcio", version = "1.80.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.14' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "grpcio", version = "1.82.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "protobuf" }, ] sdist = { url = "https://files.pythonhosted.org/packages/44/4f/d098419ad0bfc06c9ce440575f05aa22d8973b6c276e86ac7890093d3c37/grpc_google_iam_v1-0.14.4.tar.gz", hash = "sha256:392b3796947ed6334e61171d9ab06bf7eb357f554e5fc7556ad7aab6d0e17038", size = 23706, upload-time = "2026-04-01T01:57:49.813Z" } @@ -1811,8 +2588,18 @@ wheels = [ name = "grpcio" version = "1.80.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", +] dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "(python_full_version < '3.14' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version < '3.14' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b7/48/af6173dbca4454f4637a4678b67f52ca7e0c1ed7d5894d89d434fecede05/grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257", size = 12978905, upload-time = "2026-03-30T08:49:10.502Z" } wheels = [ @@ -1868,18 +2655,106 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/8c/bbe6baf2557262834f2070cf668515fa308b2d38a4bbf771f8f7872a7036/grpcio-1.80.0-cp314-cp314-win_amd64.whl", hash = "sha256:3b01e1f5464c583d2f567b2e46ff0d516ef979978f72091fd81f5ab7fa6e2e7f", size = 5019457, upload-time = "2026-03-30T08:48:37.308Z" }, ] +[[package]] +name = "grpcio" +version = "1.82.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", +] +dependencies = [ + { name = "typing-extensions", marker = "(python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version >= '3.14' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/bc/656b89387d6f4ed7e0686c7b64c2ae7e554a759aa58122c8e5fb99392c32/grpcio-1.82.1.tar.gz", hash = "sha256:707b24abd90fcb1e45bcc080577da1dbf9971d107490589b9539af8e1e77b4b5", size = 13187300, upload-time = "2026-07-08T12:36:16.588Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/14/5d05bfd85c101cbe44a12d7c1cea9c40698e0438cddf3a70019f735b5a27/grpcio-1.82.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:91859d1cac5f47caec5fc40e9f827500cdb54ce5b36450dc9a65616b5af49c17", size = 6177087, upload-time = "2026-07-08T12:34:06.825Z" }, + { url = "https://files.pythonhosted.org/packages/19/2e/c906f8e6d0b54c0137885fff6f7b5883c6bbc381b44a0ba5ea07d7d1579b/grpcio-1.82.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:c80c9741dcef192f669876a81957cf7713b441c2f0c43631350d75fa49321d31", size = 11960907, upload-time = "2026-07-08T12:34:10.583Z" }, + { url = "https://files.pythonhosted.org/packages/de/be/ec4aa76cdf25539b9e960cbb9d5739f892ea6cde58078b5293860c1159d3/grpcio-1.82.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b89cff456796d2f0581783726ad017a2c70aff2d27b0f05504c34e2e417f7560", size = 6754802, upload-time = "2026-07-08T12:34:13.082Z" }, + { url = "https://files.pythonhosted.org/packages/e6/dd/47519c2a8fd9db47ec4493f44bd9f5b0175307e07089b1132e54b7b5b19c/grpcio-1.82.1-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:d6e8a08f7038ba7a77f71e250804e4aba84fe91d22cfc54ff43c07b7529c4728", size = 7484535, upload-time = "2026-07-08T12:34:15.164Z" }, + { url = "https://files.pythonhosted.org/packages/63/99/659711e9689c4dd553bcd4eacff9cb9f458f34b60edf7afb3bbc1b0a58a2/grpcio-1.82.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:50fd2fe83426b1b1c6cdc4d72d555223b7dddf8ce07c5bac218b13fc6d684c6f", size = 6919066, upload-time = "2026-07-08T12:34:17.367Z" }, + { url = "https://files.pythonhosted.org/packages/29/39/f2b772356b4f593ffe439795509fcbf675b0ff98211ae8ce2a180f2e559f/grpcio-1.82.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b758540a24d5394a9c578bf9f6126389f474b106ac3d9df1d53de56cb14c9fd9", size = 7525855, upload-time = "2026-07-08T12:34:19.479Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/b28cfffb989a84d8272593498bddd2d68148cce1813ad55189c469b0f1f8/grpcio-1.82.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c4ba4aac238f685743575d9d700003ac16537cce26e7c774993134f530652464", size = 8565122, upload-time = "2026-07-08T12:34:21.951Z" }, + { url = "https://files.pythonhosted.org/packages/97/f9/54956cb0c701190cbc9d7e535c3f84acf0285c6b9ed198a902766e17c3cd/grpcio-1.82.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ed6fc621d6f366c88a60f0b971d5afd21d441d9aa561ee688de5b7acdb2cf901", size = 7933872, upload-time = "2026-07-08T12:34:24.539Z" }, + { url = "https://files.pythonhosted.org/packages/76/85/5f9cd1f965bbe4329556a212f178ae0c072b18b446cae05ed32fa8847c53/grpcio-1.82.1-cp310-cp310-win32.whl", hash = "sha256:bd2f45e46fff5b91c10997d0743a987517a7dde67c64c592835c2dcaac66f587", size = 4257373, upload-time = "2026-07-08T12:34:26.566Z" }, + { url = "https://files.pythonhosted.org/packages/93/b0/c4f42f7c69c53d27ed41643421b55908bcbe885b68f5a208135c72917c98/grpcio-1.82.1-cp310-cp310-win_amd64.whl", hash = "sha256:5e171d5f0d6a0af78ea7512783f170a44f80c165259d8773e3a354a7f991f2b5", size = 5006571, upload-time = "2026-07-08T12:34:28.778Z" }, + { url = "https://files.pythonhosted.org/packages/26/5b/e5092af97fa671ca279b3e373251af4bf87d5fbda7dc85f6a616899562a7/grpcio-1.82.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:0ddb18a9a9e1f46692b3567ae4abb3f8d117ce6afea48650f8eca06d8ab5d06f", size = 6181472, upload-time = "2026-07-08T12:34:31.009Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/18053a3a2ca03d0c2a1b8cc7271e705007a16aa5dae84bac00935c5b1a7f/grpcio-1.82.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:cf855b1af246720f567b0ce5d0724d45dfa4188eecc3296a2a69257b11b9e94b", size = 11970995, upload-time = "2026-07-08T12:34:33.603Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7e/21b1acb052876ad00959ec4d1b05fe08607d650bcfa282073bb164c2703c/grpcio-1.82.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddb30cb13e25bc13cea70ffc69d6d90c49d36ea6c1d4549e6912f70177834cac", size = 6760127, upload-time = "2026-07-08T12:34:36.122Z" }, + { url = "https://files.pythonhosted.org/packages/3e/12/25eef9c245c54f0061317d13a302357fe8ea03bac240b2b02ececcf54da4/grpcio-1.82.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1e822b2774f719c017cbe700b6e47173b6ae290fb84906f52a5a3c2c60b62e1e", size = 7484377, upload-time = "2026-07-08T12:34:38.368Z" }, + { url = "https://files.pythonhosted.org/packages/a0/41/1a348767eb9d9bd7765dc4fa8a01723d3bb386d67f981ee5c6f9c02b8b1c/grpcio-1.82.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5dafb1ece8ed45dee7c738f166ec82e19673221ed5ab8967f72858a4685345b2", size = 6924269, upload-time = "2026-07-08T12:34:40.583Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b9/3aae7a03d34c86ea27988db859a6087c186f6c3f53f9b551e07afd989bfa/grpcio-1.82.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e06503106e7271e0a49fd5a1ac04747f1e47e87d900476db6fe45bc87ee411f4", size = 7531848, upload-time = "2026-07-08T12:34:43.277Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/3c4afa625d0dac9090707966916284c035fc5b2fb3e2c51e156accee6735/grpcio-1.82.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ff99bc8cafb6a952201c37b995f425e641c93ffa6e072258525feab57290141d", size = 8568217, upload-time = "2026-07-08T12:34:45.502Z" }, + { url = "https://files.pythonhosted.org/packages/20/9c/d8489c628e73e20a3d034e7f66912de7b1acb405f01d388f056a88e47924/grpcio-1.82.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:644ae1b94266ac785330f4590a69e52b6a7eb73029043a02209db81c81397d69", size = 7938771, upload-time = "2026-07-08T12:34:48.323Z" }, + { url = "https://files.pythonhosted.org/packages/4b/b7/0a92cfd1658f3a896d4aa12d4efeb7dd4ddfc723725ae22741a5241ea710/grpcio-1.82.1-cp311-cp311-win32.whl", hash = "sha256:e203d2e19d471630084a16c815616f8211dff21c268ab3c5f5bf38417832e074", size = 4256432, upload-time = "2026-07-08T12:34:50.432Z" }, + { url = "https://files.pythonhosted.org/packages/c7/6a/2872c761b025d9ec74386f22a4a7d59c5a5b00ebf718761b33739ffc45de/grpcio-1.82.1-cp311-cp311-win_amd64.whl", hash = "sha256:0d8299c285fe6cc6a1f56badf8d3bc5078c8d20273ee64bafa3783b4bc29a769", size = 5009633, upload-time = "2026-07-08T12:34:52.67Z" }, + { url = "https://files.pythonhosted.org/packages/dc/88/d1350bf3343a2ed87d801584e40609f6c6bd3087926eeca03de50348cf4a/grpcio-1.82.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:c09bd5fa0d5b1fbd773ec349fe61441c3e4ebf168c229aa7538a820bdfad6a58", size = 6144689, upload-time = "2026-07-08T12:34:55.567Z" }, + { url = "https://files.pythonhosted.org/packages/e6/33/71875cdecd27c24ac1385d4783a09853f01b84a825a36aec2a2bc7d0d080/grpcio-1.82.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1eae24810720734598e3e6a1a528d5de0f265fe3fc86575e9ecce424b9ec7379", size = 11952034, upload-time = "2026-07-08T12:34:58.128Z" }, + { url = "https://files.pythonhosted.org/packages/82/b2/d9125df3d8a140dec12cc82c05b7deafedeababcff6496f28b2fd5634d10/grpcio-1.82.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a6bd5daf5bde7b24d7ad2cbaf8bf9eac620d96222016bb5e7ddde930dec0673f", size = 6710772, upload-time = "2026-07-08T12:35:01.33Z" }, + { url = "https://files.pythonhosted.org/packages/88/9b/69e2d1627398b964f34437dc476a5aff5a2cc8e7f247d26272b5674b5faf/grpcio-1.82.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1ecfde669cb687ac020d31ff76debe5dc7a62213335f02262eb6625628da1c03", size = 7450677, upload-time = "2026-07-08T12:35:03.926Z" }, + { url = "https://files.pythonhosted.org/packages/e4/e7/8f855ca29c294956122a2a73023655b9b02602d5111dad2b9b00e7631c68/grpcio-1.82.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:011c8badee95734dee8bf05ce3464756a0ac3ebb8d443afd20c0e2b5e4640ad9", size = 6886855, upload-time = "2026-07-08T12:35:06.174Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f0/fa87e85f49925f44c479d07e58b051e69bcfef6b6d5fbc6749d140f6730a/grpcio-1.82.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b85f4564926fb23114d239392bdcae200db1e6179629edd7d7ab0ab89c96a197", size = 7501323, upload-time = "2026-07-08T12:35:08.49Z" }, + { url = "https://files.pythonhosted.org/packages/67/55/2e0b10ae1d3ef9dcc480b91dc2158f4931fc4675d3af0a2836e39b2a744f/grpcio-1.82.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2c0c8270833395644c3fe6b6a806397955a2bc0538000a19a78b90c05a6c16e0", size = 8536899, upload-time = "2026-07-08T12:35:10.975Z" }, + { url = "https://files.pythonhosted.org/packages/bd/95/a3d8b0431fa221efc51ee39d73595ede74ba82a43b7c4313192e580face2/grpcio-1.82.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2ba199205ff46c7778290fe1673c91ac8e7e45678dd5c86e9e56fa33ec8788f6", size = 7913892, upload-time = "2026-07-08T12:35:13.944Z" }, + { url = "https://files.pythonhosted.org/packages/b8/92/f2651ec704d9852a56faef394775038afba435b50ce82ab2404d119c3355/grpcio-1.82.1-cp312-cp312-win32.whl", hash = "sha256:06127691866e295c14e84a1fb86356dd962254f6abd0da4ca4b001eea9e89438", size = 4240985, upload-time = "2026-07-08T12:35:16.048Z" }, + { url = "https://files.pythonhosted.org/packages/96/4f/a5fe8bf0d0a1b24855f370293075c931f27de4eb55f0f158786095bf3c11/grpcio-1.82.1-cp312-cp312-win_amd64.whl", hash = "sha256:1fa3223a3a2e1db74f4c2b255189eb7ea875dfba56e221d252ee3fc7b204778e", size = 5001580, upload-time = "2026-07-08T12:35:18.689Z" }, + { url = "https://files.pythonhosted.org/packages/1b/3e/496992d08c0aaa11272eb6228dc8ab947da01fe835de243cd00521bce4c4/grpcio-1.82.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:b454a2d97bfab7565683a02345f86bd182ab69fd7c2bdb7414171e7538f266b1", size = 6146068, upload-time = "2026-07-08T12:35:21.365Z" }, + { url = "https://files.pythonhosted.org/packages/e7/8f/f263d6f14fdba6b56cfadd91fd3e158a52682b72c6016d1f8723d435659f/grpcio-1.82.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:3dde70abfc80b3be11de53ba0d601c439e7fb2afd3583ad1788d1146bec92fdc", size = 11948600, upload-time = "2026-07-08T12:35:24.312Z" }, + { url = "https://files.pythonhosted.org/packages/8c/14/3a02e6ee49c2d85bc15eaae321e0e11ab3542cad3c5b2de121ecce0c4296/grpcio-1.82.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5523099c98c292ea1ae08e617249db760c56a78f8deae879027fe7d1ffbcbf6", size = 6714591, upload-time = "2026-07-08T12:35:27.027Z" }, + { url = "https://files.pythonhosted.org/packages/69/80/58e3738696f48ab7645347b98d8a7f93d10e00e6218388fbfcd6c9310e3d/grpcio-1.82.1-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5e5c4dc0a59b0f8490a6bdfd6fc8395b9d8ad8a8407c7d67ca7b5bba15c0877f", size = 7454995, upload-time = "2026-07-08T12:35:29.599Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6c/2557c1a889363072fbf2285ecd0e8c44860d4dbd60f017a32537c5b863e2/grpcio-1.82.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c40d94ba820329cc191981bc22fa6f6eed0799c6d921f3c6709521d59d4a2fd7", size = 6888621, upload-time = "2026-07-08T12:35:32.38Z" }, + { url = "https://files.pythonhosted.org/packages/d2/66/907706ccaff1223f1e10fd5b37fc16faead43392fccb4e786e7e390ac141/grpcio-1.82.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c816180e31e273caaec6f8bd86a8392499d5bbb26f41da44e3dce48bde69095", size = 7505069, upload-time = "2026-07-08T12:35:35.072Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7c/ff97b0d0f635987ee5ec80dfedafa1aad629303745d48e8637d10eec5b80/grpcio-1.82.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e31fd780b261830720cb70b0fd8f0aa51d49e75a66d7464ad2e31d4b765f2580", size = 8535384, upload-time = "2026-07-08T12:35:37.954Z" }, + { url = "https://files.pythonhosted.org/packages/62/9e/a97fddd970a8d1588cade06eca20443761c1858b0ad6590a5c835aa18062/grpcio-1.82.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d76152d7c31d7210d4a106e5d8b64da5bba5d6abf11be30e2f7b0a0c59bbcbf", size = 7910707, upload-time = "2026-07-08T12:35:40.797Z" }, + { url = "https://files.pythonhosted.org/packages/20/e4/eaba1517888af483a88d449eb7566f0f7f63446d46f339c5891798435875/grpcio-1.82.1-cp313-cp313-win32.whl", hash = "sha256:38e9dcb5258226fb3282630b31b16a968df52c8c6ad514af540646e0a4578f8a", size = 4240363, upload-time = "2026-07-08T12:35:43.298Z" }, + { url = "https://files.pythonhosted.org/packages/b0/42/66a98d47732e35290bef722f6149fed3709cd4cf61166f6f53a12f417302/grpcio-1.82.1-cp313-cp313-win_amd64.whl", hash = "sha256:3dbfb52c36d9511ac2b8e6c94fdde837b393ae520cc321f52a333a2deedf5a90", size = 5000980, upload-time = "2026-07-08T12:35:46.262Z" }, + { url = "https://files.pythonhosted.org/packages/b4/cb/cf9ae9e164c6e6dc8a494faa9771763df9da150eefe19671009624d1559f/grpcio-1.82.1-cp314-cp314-linux_armv7l.whl", hash = "sha256:35f990f7784c8fd2872644f07f96ebb4d9e48e145a190ab80d0280af91a1bfb2", size = 6146901, upload-time = "2026-07-08T12:35:49.261Z" }, + { url = "https://files.pythonhosted.org/packages/3b/2a/eccf26dbcfb7f7cab8027c5490a16c8937c5aa7a2ec20a3eab2cf7a43165/grpcio-1.82.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:46536a4a1f4434df3c851b9254ff6fc7df5705b273681a15ca277d5921c178a0", size = 11954756, upload-time = "2026-07-08T12:35:52.196Z" }, + { url = "https://files.pythonhosted.org/packages/ad/75/3b3b4a3cc9f084b026af96e1d3e539b1af29ec7f41ed0dfff3cb99cc8626/grpcio-1.82.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d6650a7c1ebb7921c70e12a385439a8118efb99e669fa9ed31cf25db1843937c", size = 6723087, upload-time = "2026-07-08T12:35:54.973Z" }, + { url = "https://files.pythonhosted.org/packages/9c/8b/b0f0c9b1400a99a4da4c09b114f101b192f8f11192e76f620b8962f5d90b/grpcio-1.82.1-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b8e110c66df5204c0506d6c8787b35d48b8b699ef5aa366d6c4d67325c67fe9a", size = 7454542, upload-time = "2026-07-08T12:35:57.586Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bd/428e38868382aa193697a5aa53973f29c58e58ba4268aa0c86a2715ee58b/grpcio-1.82.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f853eae07235a51a27bb5d6a9a175a59ca55dc9b99edc6ce2f76f07332d333ae", size = 6889588, upload-time = "2026-07-08T12:36:00.012Z" }, + { url = "https://files.pythonhosted.org/packages/49/ce/03e01d5e10259bf5c08ee50570cc94724e79c956f61fd2f09b341af0956c/grpcio-1.82.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:60b0f2c95337694fc094b77d9f60f50566c84b5677393e342eb98daeee242d98", size = 7514166, upload-time = "2026-07-08T12:36:02.693Z" }, + { url = "https://files.pythonhosted.org/packages/ff/59/278b4b600329e2ba3849f3c1ea3c820b3a01b38a7ad184ba09595e8d2733/grpcio-1.82.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:b064fc444812bdaa9825d33c26f8d732d63ee6a5d78557c1faf92c98687fed27", size = 8536166, upload-time = "2026-07-08T12:36:05.349Z" }, + { url = "https://files.pythonhosted.org/packages/44/27/7ccf2ef00f27a8e47a79d641c8ceaf7d3028c7a03d9a97b4c8a9a783c086/grpcio-1.82.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7d7ede11d747b4e1bd05e3bc0260e155b65a88735a895a10f6521f19b889511e", size = 7912572, upload-time = "2026-07-08T12:36:08.393Z" }, + { url = "https://files.pythonhosted.org/packages/0d/be/33742482d2753f2d3a1b7641664b6622262d44f2f3b609f13425dd86d36f/grpcio-1.82.1-cp314-cp314-win32.whl", hash = "sha256:3d21f19838dc255ecbb79321b15ae9b98fbddff4c3d4aedb0a81bdd7f4ab572a", size = 4321856, upload-time = "2026-07-08T12:36:10.899Z" }, + { url = "https://files.pythonhosted.org/packages/cc/67/03329c847172c78ddeb1eb9be6b444fdbc12775a84c958b27e427e7b926d/grpcio-1.82.1-cp314-cp314-win_amd64.whl", hash = "sha256:e20f1edbb15f99e3128ec86433f9785fd5a451d8f115e74fe0056134f092a9d5", size = 5141114, upload-time = "2026-07-08T12:36:13.595Z" }, +] + [[package]] name = "grpcio-status" -version = "1.80.0" +version = "1.62.3" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", + "python_full_version < '3.11'", + "python_full_version == '3.13.*'", +] dependencies = [ - { name = "googleapis-common-protos" }, - { name = "grpcio" }, - { name = "protobuf" }, + { name = "googleapis-common-protos", marker = "python_full_version < '3.14'" }, + { name = "grpcio", version = "1.80.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" }, + { name = "protobuf", marker = "python_full_version < '3.14'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7c/d7/013ef01c5a1c2fd0932c27c904934162f69f41ca0f28396d3ffe4d386123/grpcio-status-1.62.3.tar.gz", hash = "sha256:289bdd7b2459794a12cf95dc0cb727bd4a1742c37bd823f760236c937e53a485", size = 13063, upload-time = "2024-08-06T00:37:08.003Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/40/972271de05f9315c0d69f9f7ebbcadd83bc85322f538637d11bb8c67803d/grpcio_status-1.62.3-py3-none-any.whl", hash = "sha256:f9049b762ba8de6b1086789d8315846e094edac2c50beaf462338b301a8fd4b8", size = 14448, upload-time = "2024-08-06T00:30:15.702Z" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b1/ed/105f619bdd00cb47a49aa2feea6232ea2bbb04199d52a22cc6a7d603b5cb/grpcio_status-1.80.0.tar.gz", hash = "sha256:df73802a4c89a3ea88aa2aff971e886fccce162bc2e6511408b3d67a144381cd", size = 13901, upload-time = "2026-03-30T08:54:34.784Z" } + +[[package]] +name = "grpcio-status" +version = "1.82.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14'", +] +dependencies = [ + { name = "googleapis-common-protos", marker = "python_full_version >= '3.14'" }, + { name = "grpcio", version = "1.82.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" }, + { name = "protobuf", marker = "python_full_version >= '3.14'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/29/4d/3037f220cea14be7e77bb52e7dec18bdc90554e218642c8ebde620de37e3/grpcio_status-1.82.1.tar.gz", hash = "sha256:d9de8ac34763cd468130fdd2923294af7c3d28d09426f6c45221d27c25931130", size = 13906, upload-time = "2026-07-08T12:39:41.943Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/80/58cd2dfc19a07d022abe44bde7c365627f6c7cb6f692ada6c65ca437d09a/grpcio_status-1.80.0-py3-none-any.whl", hash = "sha256:4b56990363af50dbf2c2ebb80f1967185c07d87aa25aa2bea45ddb75fc181dbe", size = 14638, upload-time = "2026-03-30T08:54:01.569Z" }, + { url = "https://files.pythonhosted.org/packages/46/5c/2f6c7e24b99dbaf5f8d7e5b1413fc9fc23360cdeb7f290b49a1c87b49560/grpcio_status-1.82.1-py3-none-any.whl", hash = "sha256:71c7f2bea725c0027fa396b77a55d4e9d90591bab90de4c1c03d4df9a56552f0", size = 14636, upload-time = "2026-07-08T12:39:23.113Z" }, ] [[package]] @@ -1948,6 +2823,56 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httptools" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/e5/d471fcb0e14523fe1c3f4ba58ca52480e7bd70ad7109a3846bc75892f7fb/httptools-0.8.0.tar.gz", hash = "sha256:6b2a32f18d97e16e90827d7a819ffa8dbd8cc245fc4e1fa9d1095b54ef4bd999", size = 271342, upload-time = "2026-05-25T22:17:48.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/b9/be66eb0decd730d89b9c94f930e4b8d87787b05724bb84af98bfd825f72c/httptools-0.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bf3b6f807c8541503cecfbb8a8dffb385640d0d96102f3d112aa8740f9b7c826", size = 208805, upload-time = "2026-05-25T22:16:50.434Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f7/b4d41eaae2869d31356bc4bbf546f44fae83ff298af0a043ca0625b06773/httptools-0.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da684f2e1aa2ee9bdcb083f3f3a68c5956750b375bc5df864d3a5f0c42a40b77", size = 113527, upload-time = "2026-05-25T22:16:51.672Z" }, + { url = "https://files.pythonhosted.org/packages/e6/e4/77487e14fc7be47180fd0eb4267c7486d0cc59b74031839a3daf8650136b/httptools-0.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6f21e2a3b0067bbe7f67e34cfd16276af556e5e52f4c7503be0cb5f90e905e4", size = 450035, upload-time = "2026-05-25T22:16:53.313Z" }, + { url = "https://files.pythonhosted.org/packages/da/72/5a8f787e323f56fbd86c32a4be92a86776e4cfe8b4317db999f452028362/httptools-0.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea897f0c729581ebf72131a438a7932d9b14efef72d75ada966700cac3caaeb", size = 451101, upload-time = "2026-05-25T22:16:54.696Z" }, + { url = "https://files.pythonhosted.org/packages/ed/41/b44a25560955197674b6744cb903664300e239235a5eaa69df0890d87054/httptools-0.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c0d726cc107fceb7d45f978483b4b70dd8caa836f5914d3434bb18628eb73813", size = 436140, upload-time = "2026-05-25T22:16:56.239Z" }, + { url = "https://files.pythonhosted.org/packages/74/b0/054aac84c03d7e097bf4c605fb7e74eec3d65c0276adf64ee97f3a103ff5/httptools-0.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9878eb2785ba5eb70631ad269b37976f73d647955e26c91d490eb8a4edfda4ba", size = 437041, upload-time = "2026-05-25T22:16:57.716Z" }, + { url = "https://files.pythonhosted.org/packages/bb/e8/86b85bbc0ac7892232f1a99ab96a9aa71936984fa06adfc0afc83ca7789e/httptools-0.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:b205e5f5523fa039679da0dfe5a10132b2a4abeae6a86fdd1ddc035f7f836557", size = 90454, upload-time = "2026-05-25T22:16:58.871Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d2/c3eedaef57de65c3cc5f8dc244cf12d09c84ad258a479055aad6db23206c/httptools-0.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ed377e64805bdba4943c82717333f8f8603a13b09aff9cead2717c6c817fb168", size = 208428, upload-time = "2026-05-25T22:16:59.717Z" }, + { url = "https://files.pythonhosted.org/packages/f1/94/dfe435d90d0ef61ec0f2cc3d480eef78c59727c6c2ce039f433882f6131a/httptools-0.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9518c406d7b310f05adb1a37f80acabac40504a575d7c0da6d3e365c695ac20d", size = 113366, upload-time = "2026-05-25T22:17:00.795Z" }, + { url = "https://files.pythonhosted.org/packages/cc/d4/13025f1a56e615dcb331e0bbe2d9a1143212b58c263385fc5d2e558f5bac/httptools-0.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:57278e6fa0424c42a8a3e454828ab4f0aff27b40cddf9679579b98c6dce6a376", size = 464676, upload-time = "2026-05-25T22:17:02.014Z" }, + { url = "https://files.pythonhosted.org/packages/bf/95/4c1c26c0b985f8a3331682d802598f14e32dc41bf7509266eb2c04ad4801/httptools-0.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbb8caadb2b742d293169d2b458b5c001ef70e3158704aa3d3ef9597624c5d1d", size = 464235, upload-time = "2026-05-25T22:17:03.109Z" }, + { url = "https://files.pythonhosted.org/packages/a2/82/6735be2b0ca527718c431cdb8e5f70c3862c0844a687df0f572c51e11497/httptools-0.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:52dd695b865fe96d9d2b16b64a895f3f57bf3cb064e8383cd3b5713a069e8085", size = 449809, upload-time = "2026-05-25T22:17:04.443Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f9/5811c74f37a758c8a4aa3dc430375119d335947e883efc4664d8f3559a41/httptools-0.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:20b4aac66ff65f7db06a375808b78f42a94970aa22e826b3cb2b43eb09174124", size = 452174, upload-time = "2026-05-25T22:17:05.476Z" }, + { url = "https://files.pythonhosted.org/packages/cc/94/97b75870dea07b71e3ec535cebe525b08d723152e4c7d13fa887e51f4de2/httptools-0.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:a1b4c8e7a489a0d750d91894e9a8cdc295838f1924c0ca903ae993456fddec07", size = 90991, upload-time = "2026-05-25T22:17:06.75Z" }, + { url = "https://files.pythonhosted.org/packages/14/88/1d21a36da8f5cb0fa49eafd4b169eba5608d57e75bbcf61845cbc6243216/httptools-0.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:880490234c10f70a9830743097e8958d6e4b9f5a0ffc24515023afeef984054d", size = 208247, upload-time = "2026-05-25T22:17:07.843Z" }, + { url = "https://files.pythonhosted.org/packages/a5/42/cc4feea2945cb3051038f090c9b36bd5b8a9d7f5a894a506a8983e33fd1c/httptools-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5931891fb7b441b8a3853cf1b85c82c903defce084dd5f6771ca46e31bf862c5", size = 113064, upload-time = "2026-05-25T22:17:09.136Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a6/febbb8b8db0f58b38e44ad6cb946e6a255ae49b55f2e8543408fb7501ccd/httptools-0.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b15fc622b0f869d19207c4089a501d9bcc63ca5e071ffdd2f03f922df882dcb2", size = 523851, upload-time = "2026-05-25T22:17:10.106Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e4/f90a0df0b83beff265b7e3b65f2a4cefd95792d4be0ac3e16049f2acd3c2/httptools-0.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:425f83884fd6343828d8c565f046cb72b6d19063f6924093e11bcd8e1548cd09", size = 518842, upload-time = "2026-05-25T22:17:11.218Z" }, + { url = "https://files.pythonhosted.org/packages/9e/2d/0c9ac76dd2c893841fbf6498d6acec4f2442e1b7067f6e3e316a80e494e8/httptools-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7c3c97f4311c7be57e2986629df89d49cb434dbff78eafcd48c2bff986b15a", size = 501238, upload-time = "2026-05-25T22:17:12.728Z" }, + { url = "https://files.pythonhosted.org/packages/ca/42/906adc91ae3a5fa9c59c0a2f21c139725bd7e5b41ae6acd485cd14123ebf/httptools-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a1afd7c9fbff0d9f5d489c4ce2768bd09c84a46ddefc7161e6aa82ae35c85745", size = 509567, upload-time = "2026-05-25T22:17:13.842Z" }, + { url = "https://files.pythonhosted.org/packages/05/0b/4240efeb672751ee5b9b380cb0e3fdc050bc05f68adc7a8aefc4fcd9a69a/httptools-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd96f29b4bab1d42fa6e3d008711c75e0f79e94e06827330160e3a304227f150", size = 90918, upload-time = "2026-05-25T22:17:15.155Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e5/8cfcabc5546e8022f168be28bcdaa128a240a0befdd03b59d558b4f18bd6/httptools-0.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:614ceea8ea606848bece2338ac03b3ce5324bcb4be8dc7d377ed708012fa4db8", size = 205148, upload-time = "2026-05-25T22:17:16.333Z" }, + { url = "https://files.pythonhosted.org/packages/2a/0e/0fb14848c19a686c8062ff9067c1a48793e3224b47bc5b201535b6036fce/httptools-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2d689918c15a013c65ef52d9fd495d766893ab831a2c8d89f2ac5940a5df847c", size = 111368, upload-time = "2026-05-25T22:17:17.586Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/46f1cecf06b9bbde8e4b8c88034ac7908989e5ff7a3a388ef38392949c1f/httptools-0.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eb3028cca2fc0a6d720e52ef61d8ebb62fcbfeb1de56874546d858d3f25a26b7", size = 486447, upload-time = "2026-05-25T22:17:18.564Z" }, + { url = "https://files.pythonhosted.org/packages/77/00/258bfc0837221f81d9725c45f9b948a6a6b2994a147a4fb66e85100c668f/httptools-0.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:88bdd940f2b5d487b4d032c6afa5489a7dc4694410d43de3c38c4fb3af0dc45d", size = 482448, upload-time = "2026-05-25T22:17:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/04/ab/d1cef3b5523f4d272a70f42a776c3169a2dddfe3a54de4b2ce4a36341528/httptools-0.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6a43c9dd399758ccc0531acb0a3c4a6c299ee893ee9400e9c893b7bdcfae0681", size = 464460, upload-time = "2026-05-25T22:17:20.882Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/5d1d072442277bb2b3434e0e60690b8e8c23840ef7de8b6ea54040a536d3/httptools-0.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683", size = 471312, upload-time = "2026-05-25T22:17:22.085Z" }, + { url = "https://files.pythonhosted.org/packages/0d/66/b96623b27e51a68199ef4efdda0613cced9233fe3062ac74e50749c5ad37/httptools-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:7685df791fad561384bfb139e77fde27a1ffd93134e016f95a0db424ffbf77b1", size = 90117, upload-time = "2026-05-25T22:17:23.074Z" }, + { url = "https://files.pythonhosted.org/packages/1a/12/fa3fbf5f9517b273edea2dc982aa82a8c634091e67c590792b729017bc6f/httptools-0.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:de242a49b5d18e0a8776e654e9f6bf6d89f3875a5c35b425a0e7ce940feb3fd6", size = 206183, upload-time = "2026-05-25T22:17:24.004Z" }, + { url = "https://files.pythonhosted.org/packages/30/fc/5e7c4cb443370f2090a3aba0453a07384d29ff66b7435bb90e77e1037599/httptools-0.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:159e9ab5f701ccd42e555a12f1ad8ff69702910fc1c996cf2bb66e5fcb7a231b", size = 112079, upload-time = "2026-05-25T22:17:25.216Z" }, + { url = "https://files.pythonhosted.org/packages/ba/53/771bd891eb0f236f32145d6a1775777ec85745f3cc983a1f23d1a3b8ddfe/httptools-0.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c4a9f1707e4823d54dfec6c33fa3697d302aed536ed352a7ebb5a061ddb869d0", size = 481596, upload-time = "2026-05-25T22:17:26.186Z" }, + { url = "https://files.pythonhosted.org/packages/62/42/94e15bc68ce3d423243c45d7f1b0c7561f13844f97dc52ae23182fb65628/httptools-0.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d76ad7b951387e3632c8716a9bb03ac5b45c5f16119aa409db0459520887944e", size = 480865, upload-time = "2026-05-25T22:17:27.542Z" }, + { url = "https://files.pythonhosted.org/packages/1c/7c/fe2980fc03723272e30f135b62360b075f513dfe7cc73aef36c7f04012bd/httptools-0.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a3b7387147361c3fd47a0bde763c5c91b5b4cd4dc9989b8ece84ff436c99843b", size = 463189, upload-time = "2026-05-25T22:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/15/1b/47fc5fff68acd1bfa20b4734059c9a06cadb88119dcd5258b5b0d21d91c8/httptools-0.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f256d6ce930c52ca1cb2a960b7da03548c454e7d28b06059ad41bfe789036ce0", size = 466610, upload-time = "2026-05-25T22:17:29.816Z" }, + { url = "https://files.pythonhosted.org/packages/60/bd/07b13c93ffd9bec9546e0d43f8e19378dd696dbd278511406bc07371ef1f/httptools-0.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:19d1ee275bb59ba2643ba9a3a1e51cc0c788caf2b8df506368e03f56fdd08527", size = 92705, upload-time = "2026-05-25T22:17:31.133Z" }, + { url = "https://files.pythonhosted.org/packages/fd/c4/121648f68ce066d7bd762d6b6d97e620847642d38d54f3d90ff11d947629/httptools-0.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:de1ed58a974e75d56560acc7e7fed01a454994429456f65209789992e41f2568", size = 215023, upload-time = "2026-05-25T22:17:32.401Z" }, + { url = "https://files.pythonhosted.org/packages/b9/b0/312a062ae741ae3e8baa8c8bf20be81b2e67337b259ab4349bebc7b6142e/httptools-0.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e93c227b595c6926c1acee96891dd9da4be338cfbe82e5cd3bb9d8dd7dc4ac0b", size = 117405, upload-time = "2026-05-25T22:17:33.742Z" }, + { url = "https://files.pythonhosted.org/packages/fc/37/fccd705f795386bb05bf413012fecff2a33e5aa8c2f069096de3e9fd8702/httptools-0.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2a021c3a8e65cc125390d72f59b968afca3bdcaff25bd67965e0a055a14946ca", size = 558497, upload-time = "2026-05-25T22:17:34.732Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/f172e8003576de35f5ba77ff417cf0e34429d35dc014deef15afa337a72c/httptools-0.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48774d39cbb70e2b1f71f88852a3087ae1d3a1eb80482bb48c13067ab080c14f", size = 571585, upload-time = "2026-05-25T22:17:35.813Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b9/f5564760af99f3dbbf3f9104dc00e5da27e96cf433c6bdcf77617f70bf3f/httptools-0.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:88eead8ec8680a9f146c655bc88445a325bd7921cfd8194c7337e9467282427d", size = 543297, upload-time = "2026-05-25T22:17:37.08Z" }, + { url = "https://files.pythonhosted.org/packages/99/67/8d9f2c313618e161b82f3873188e7196126da1d6e29688df40eb3997c77a/httptools-0.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2c032fa028f46871ec7e1fc59fc15e8023eab3e6bbe6ece786a1611719a5d081", size = 539535, upload-time = "2026-05-25T22:17:38.032Z" }, + { url = "https://files.pythonhosted.org/packages/48/63/b906c01e53f50d432c0defe43ce52764a111dc1bdd028bafbeb54dcfd008/httptools-0.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:384c17174464c8e873398b7af24f0b1f44d992c820328413951a625323155d77", size = 108209, upload-time = "2026-05-25T22:17:39.473Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -1963,6 +2888,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx-sse" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/4c/751061ffa58615a32c31b2d82e8482be8dd4a89154f003147acee90f2be9/httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d", size = 15943, upload-time = "2025-10-10T21:48:22.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, +] + [[package]] name = "huggingface-hub" version = "1.15.0" @@ -1970,7 +2904,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "httpx" }, { name = "packaging" }, { name = "pyyaml" }, @@ -2088,7 +3022,7 @@ name = "imageio" version = "2.37.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } }, { name = "pillow" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/84/93bcd1300216ea50811cee96873b84a1bebf8d0489ffaf7f2a3756bab866/imageio-2.37.3.tar.gz", hash = "sha256:bbb37efbfc4c400fcd534b367b91fcd66d5da639aaa138034431a1c5e0a41451", size = 389673, upload-time = "2026-03-09T11:31:12.573Z" } @@ -2148,7 +3082,7 @@ dependencies = [ { name = "click" }, { name = "debugpy" }, { name = "docstring-parser" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "fastapi" }, { name = "fsspec" }, { name = "httpx" }, @@ -2160,7 +3094,9 @@ dependencies = [ { name = "jsonschema" }, { name = "mmh3" }, { name = "nest-asyncio2" }, - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "platformdirs" }, { name = "psutil" }, { name = "pydantic" }, @@ -2177,7 +3113,7 @@ dependencies = [ { name = "typing-extensions" }, { name = "universal-pathlib" }, { name = "uvicorn" }, - { name = "zipfile-zstd", marker = "python_full_version < '3.14'" }, + { name = "zipfile-zstd", marker = "python_full_version < '3.14' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "zipp" }, { name = "zstandard" }, ] @@ -2186,16 +3122,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/33/068c58116f053f56efe5d85db9505398dd5e91cb09bf2bc40ef9263d540e/inspect_ai-0.3.241-py3-none-any.whl", hash = "sha256:3ef2708d5f4854475f8f7bc4bc42b55079f00ba3eba2b4f5733fab66e3f53fd1", size = 36159399, upload-time = "2026-06-22T14:39:34.325Z" }, ] +[[package]] +name = "interegular" +version = "0.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/9d/8b6dde58a028a3962ce17e84d5fe73758df61378e00ef8ac3d85da34b0ff/interegular-0.3.3.tar.gz", hash = "sha256:d9b697b21b34884711399ba0f0376914b81899ce670032486d0d048344a76600", size = 24705, upload-time = "2024-01-06T23:01:22.372Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/01/72d6472f80651673716d1deda2a5bbb633e563ecf94f4479da5519d69d25/interegular-0.3.3-py37-none-any.whl", hash = "sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c", size = 23635, upload-time = "2024-01-06T23:01:20.829Z" }, +] + [[package]] name = "ipykernel" version = "6.29.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "appnope", marker = "sys_platform == 'darwin'" }, + { name = "appnope", marker = "sys_platform == 'darwin' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "comm" }, { name = "debugpy" }, - { name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "ipython", version = "9.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "ipython", version = "9.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "jupyter-client" }, { name = "jupyter-core" }, { name = "matplotlib-inline" }, @@ -2216,21 +3161,24 @@ name = "ipython" version = "8.38.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", -] -dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version < '3.11'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "jedi", marker = "python_full_version < '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, - { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "stack-data", marker = "python_full_version < '3.11'" }, - { name = "traitlets", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", +] +dependencies = [ + { name = "colorama", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or (python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'win32' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "decorator", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "jedi", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "matplotlib-inline", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "pexpect", marker = "(python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32') or (python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform == 'emscripten' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform == 'win32' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "prompt-toolkit", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "pygments", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "stack-data", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "traitlets", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e5/61/1810830e8b93c72dcd3c0f150c80a00c3deb229562d9423807ec92c3a539/ipython-8.38.0.tar.gz", hash = "sha256:9cfea8c903ce0867cc2f23199ed8545eb741f3a69420bfcf3743ad1cec856d39", size = 5513996, upload-time = "2026-01-05T10:59:06.901Z" } wheels = [ @@ -2242,27 +3190,40 @@ name = "ipython" version = "9.10.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.12.*' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", - "python_full_version >= '3.14' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and sys_platform == 'linux'", - "python_full_version >= '3.14' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and sys_platform != 'linux'", -] -dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version >= '3.11'" }, - { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, - { name = "jedi", marker = "python_full_version >= '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, - { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "stack-data", marker = "python_full_version >= '3.11'" }, - { name = "traitlets", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, + "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.13' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", +] +dependencies = [ + { name = "colorama", marker = "(python_full_version >= '3.11' and sys_platform == 'win32') or (python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'win32' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "decorator", marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "jedi", marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "matplotlib-inline", marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "pexpect", marker = "(python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32') or (python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform == 'emscripten' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform == 'win32' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "prompt-toolkit", marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "pygments", marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "stack-data", marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "traitlets", marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "typing-extensions", marker = "python_full_version == '3.11.*' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a6/60/2111715ea11f39b1535bed6024b7dec7918b71e5e5d30855a5b503056b50/ipython-9.10.0.tar.gz", hash = "sha256:cd9e656be97618a0676d058134cd44e6dc7012c0e5cb36a9ce96a8c904adaf77", size = 4426526, upload-time = "2026-02-02T10:00:33.594Z" } wheels = [ @@ -2283,7 +3244,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "pygments", marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -2296,8 +3257,8 @@ version = "8.1.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "comm" }, - { name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "ipython", version = "9.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "ipython", version = "9.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "jupyterlab-widgets" }, { name = "traitlets" }, { name = "widgetsnbextension" }, @@ -2333,7 +3294,9 @@ name = "jaxtyping" version = "0.2.19" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "typeguard" }, { name = "typing-extensions" }, ] @@ -2366,6 +3329,105 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jiter" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/1f/10936e16d8860c70698a1aa939a46aa0224813b782bce4e000e637da0b2d/jiter-0.16.0.tar.gz", hash = "sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c", size = 176431, upload-time = "2026-06-29T13:05:13.657Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/d8/b959609e44012a42b1f3e5ba98ea3b33c7e41e6d4b77cd8f00fd19b1d3ad/jiter-0.16.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c5fc4f8def331036a7b8e981b4347ebe409981edbc8308a5ea842b8c3614fa6c", size = 310082, upload-time = "2026-06-29T13:02:31.356Z" }, + { url = "https://files.pythonhosted.org/packages/c6/3d/4d7f5667ea0e0548534ba880b84bb3d12924fd133aa83ad6c6c80fca3d76/jiter-0.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5a71d0d2014c3275043e1170bf3d4e771493cb0dcf07be54c567155f4d8ee64b", size = 315643, upload-time = "2026-06-29T13:02:33.204Z" }, + { url = "https://files.pythonhosted.org/packages/9b/83/bed2dcb5c9f3e1ccfcbc67dda48265fe7d5ad0c9cadda5fe95f6e3b87f94/jiter-0.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:741eed508c233a76313a1c7b001f8f21b82f14327e9196ae8bd29a2cc164ae84", size = 341363, upload-time = "2026-06-29T13:02:34.853Z" }, + { url = "https://files.pythonhosted.org/packages/f4/2f/6bb3c3dda668ebc0445689c81a2b0f26a82b10843d67ed9c9b2c3edc177f/jiter-0.16.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3fb7bc819187b56dc48aa5c833aaf92257da8e07efdb9306156667bd2eeb491c", size = 365483, upload-time = "2026-06-29T13:02:36.295Z" }, + { url = "https://files.pythonhosted.org/packages/92/35/8a045ccb39164e70dcdae696413b661771f148b68b12b175c3a04d901937/jiter-0.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c9610fd25ebccb43fca584136f5c2fbb26802447eccd430dfdbab95a0fd5126", size = 461219, upload-time = "2026-06-29T13:02:38.116Z" }, + { url = "https://files.pythonhosted.org/packages/e7/99/22292dbbf0ed0c610cfe5ddc7f3bd67237a412f121318f865196e62a07bd/jiter-0.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4a1d68ff7ca1d3b5dee20a97a3decda7d5f15003823bf6d140c81f8561d3bc5c", size = 374905, upload-time = "2026-06-29T13:02:40.357Z" }, + { url = "https://files.pythonhosted.org/packages/29/ac/2f55ccb1f0eeafa6d89d24caf52f6f0944a59290ee199e9ade62177dca42/jiter-0.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb08c276dd02dac3a284acdd02cacc630d2e3cd6572a4b85519f35cbd133c3de", size = 348320, upload-time = "2026-06-29T13:02:41.923Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/7d88b9174c40064fabc07c84a9b62e6b10f5644562ec0e0a29392edbe978/jiter-0.16.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:8fc4d94713c4697347e38faf7d6ef91547c142219bdcfc7220c4870879974244", size = 356519, upload-time = "2026-06-29T13:02:43.436Z" }, + { url = "https://files.pythonhosted.org/packages/27/57/c4a33aeef513a9d5e26e31534e0bcc752d6ea0e54c94ddb7b68bade669c2/jiter-0.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a0f05e229edb29e68cdd0ccb83cea13b64263416120cf943767a6fd72e6787f", size = 394204, upload-time = "2026-06-29T13:02:44.987Z" }, + { url = "https://files.pythonhosted.org/packages/9d/70/c6c23e76ebb3766b111bc399437bbc9f870a76e2a92e10b2a5f561d57372/jiter-0.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2c842cbf374a8daf50b2c04212995bee34ca2ac2cdc29a901b4cdb072c9c4131", size = 521477, upload-time = "2026-06-29T13:02:46.724Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d3/0001c8c0c5976af2625bb1cfb1895e8ec693b6589fe4574b8e6fc2c85501/jiter-0.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5ed466aee31294d7cdcd4d37dfe5c42c97bc29d9a5f00eacf24504358309cb9b", size = 552187, upload-time = "2026-06-29T13:02:48.144Z" }, + { url = "https://files.pythonhosted.org/packages/f6/76/311b718e07e85740e48619c0632b36f7e0b8d113984499e436452ed13a9a/jiter-0.16.0-cp310-cp310-win32.whl", hash = "sha256:b42e9ff5376819c053da25809a8d4b6fa6e473b4856ebe42e298ac958be3d7f9", size = 206513, upload-time = "2026-06-29T13:02:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/db/7f/ac680eeb0777dc0eb7dc824800ba27880d7f6bc712e362d34ad8ee559f36/jiter-0.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:10438939205546132189c8e74a2d536a707841f3a25cd7c74ee91fe503407a26", size = 199505, upload-time = "2026-06-29T13:02:50.829Z" }, + { url = "https://files.pythonhosted.org/packages/4e/3f/fae6cc967d120ec89e31c5418a51176d8278b3087fbb384a9176754f353c/jiter-0.16.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:67fddeda1688f0cce2d2ae83ccf8a80f79936f2d2997d6cc2261f82fdb54a4d3", size = 309289, upload-time = "2026-06-29T13:02:52.301Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e3/97c6c3562c077f6247d6e6ce5c82562500b6316c0d928e97e106b7a1321a/jiter-0.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c90c0f63df322be920eda6ce622e3083d8906ba267f8220fe7873213b8b4430e", size = 315181, upload-time = "2026-06-29T13:02:53.964Z" }, + { url = "https://files.pythonhosted.org/packages/7b/89/d8d073f8aa2667e46c6c0873f86fe4a512bba4293cc730f626a076211a62/jiter-0.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64c0203212098470032aabcde9356fc168f377aade3e43def61dfe17e92f2037", size = 340939, upload-time = "2026-06-29T13:02:55.412Z" }, + { url = "https://files.pythonhosted.org/packages/87/c9/db4fda3ed73fb864139305e935e5b8b38a5a24692a5a9dd356c22f1b9c8d/jiter-0.16.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12288303c9844e61e1651d02a9a6f6633e47d39f897d6991d1427161ce6b746e", size = 364932, upload-time = "2026-06-29T13:02:57.28Z" }, + { url = "https://files.pythonhosted.org/packages/a2/74/52b5e86241057f52ddd7c9a580f90effb51f9d06239f6fc612279b91a838/jiter-0.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cf109d010b4b05a105afb3d43be36a21322d345ad3111e13d15f680afef0e5b", size = 461132, upload-time = "2026-06-29T13:02:58.994Z" }, + { url = "https://files.pythonhosted.org/packages/a9/87/544a700f7447c1f31c5d7833821a4daa5683165c2d5a094fbf5b5800c3dc/jiter-0.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62c1b7fe1f77925acf5af68b6140b8810fa87dfd4dc0a9c8568ec2fa2a10429c", size = 374857, upload-time = "2026-06-29T13:03:00.455Z" }, + { url = "https://files.pythonhosted.org/packages/40/cd/0fcc3f7d39183674d5bfa9ec640faaeb506c60be7c8f94625dfba366e37c/jiter-0.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8597d23c87f59294f83bcb6229b9ed1fccee13dbba967b46930d2f1759466fee", size = 347053, upload-time = "2026-06-29T13:03:02.045Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ae/c7e64e7932ad597fa395b61440b249ada6366716e25c6e08dd2afbd021e6/jiter-0.16.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:3126a5dbad56401989ac769aca0cb56005bfb3e2366eea0ca99d1a91c3c1ee03", size = 356153, upload-time = "2026-06-29T13:03:03.706Z" }, + { url = "https://files.pythonhosted.org/packages/d4/1c/1c719044f14da814e1a060191ab19b96f3e99207bc5b4bfc6d6be34b3f80/jiter-0.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c4b4717bdb35ae456f831a6b08d01880fff399887a6bbc526a583a406e484eea", size = 393956, upload-time = "2026-06-29T13:03:05.165Z" }, + { url = "https://files.pythonhosted.org/packages/3b/dc/7b2f303a2847207e265503853a2d964a55354cffd62a5f2936c155486798/jiter-0.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:adff21bc78edfe086c15eb495b900306076de378dc2337c132401fc39bd79c91", size = 521081, upload-time = "2026-06-29T13:03:06.886Z" }, + { url = "https://files.pythonhosted.org/packages/c2/5f/501cf6e1e09caeb420195179ffc6f62aca603f1220ec53fd80d0d70b3e56/jiter-0.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dab907db06fc593645e73109acf4581ba5b548897d28b9348dc41ddc8343b2d3", size = 552085, upload-time = "2026-06-29T13:03:08.339Z" }, + { url = "https://files.pythonhosted.org/packages/79/54/aa5be86520113b79455c3877f3d1f07a348098df4083ba3688e9537e52dd/jiter-0.16.0-cp311-cp311-win32.whl", hash = "sha256:560b2cf3fb03240cd34f27409a238547488708f05b7c3924f571a60422251ec7", size = 206755, upload-time = "2026-06-29T13:03:09.653Z" }, + { url = "https://files.pythonhosted.org/packages/64/ec/2feb893eb330bd69b413866f4d5daada33c3962f1c6f270c91ca2d87fdf9/jiter-0.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:e431cfc9caf44c1d5459ff77d4e64cbf85fddb6a35dad836a15c6a9ec23087c1", size = 199155, upload-time = "2026-06-29T13:03:10.979Z" }, + { url = "https://files.pythonhosted.org/packages/b9/9c/ca040d94415048a3666fc237774df8151c96f8d2b661cbe3b184acc95876/jiter-0.16.0-cp311-cp311-win_arm64.whl", hash = "sha256:2a8e9e39cf083016137aa5cadafe3188adc2ba6ba1fbf1e5d18889ad3e9ad056", size = 194403, upload-time = "2026-06-29T13:03:12.341Z" }, + { url = "https://files.pythonhosted.org/packages/83/2b/52ace16ed031354f0539749a49e4bf33797d82bea5137910835fa4b09793/jiter-0.16.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:67c3bc1760f8c99d805dcab4e644027142a53b1d5d861f18780ebdbd5d40b72a", size = 306943, upload-time = "2026-06-29T13:03:14.035Z" }, + { url = "https://files.pythonhosted.org/packages/94/2e/34957c2c1b661c252ba9bcc60ae0bddc27e0f7202c6073326a13c5390eec/jiter-0.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5af7780e4a26bd7d0d989592bf9ef12ebf806b74ab709223ecca37c749872ea9", size = 307779, upload-time = "2026-06-29T13:03:15.418Z" }, + { url = "https://files.pythonhosted.org/packages/88/6c/59bd309cab4460c54cf1079f3eb7fe7af6a4c895c5c957a53378693bad2b/jiter-0.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5bf78d0e05e45cfdd66558893938d59afe3d1b1a824a202039b20e607d25a72", size = 335826, upload-time = "2026-06-29T13:03:17.11Z" }, + { url = "https://files.pythonhosted.org/packages/3b/8c/f5ef7b65f0df47afa16596969defb281ebb86e96df346d62be6fd853d620/jiter-0.16.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e", size = 362573, upload-time = "2026-06-29T13:03:18.781Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0b/ace4354da061ee38844a0c27dc2c21eecd27aea119e8da324bea987522d0/jiter-0.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a23f0e4f957e1be65752d2dfac9a5a06b1917af8dc85deb639c3b9d02e31290", size = 457979, upload-time = "2026-06-29T13:03:20.293Z" }, + { url = "https://files.pythonhosted.org/packages/55/40/c0253d3772eb9dcd8e6606ee9b2d53ec8e5b814589c47f140aa585f21eaa/jiter-0.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c22a488f7b9218e245a0025a9ba6b100e2e54700831cf4cf16833a27fba3ad01", size = 372302, upload-time = "2026-06-29T13:03:21.739Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d2/4839422241aa12860ce597b20068727094ba0bc480723c74924ca5bad483/jiter-0.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46add52f4ad47a08bfb1219f3e673da972191489a33016edefdb5ea55bfa8c48", size = 343805, upload-time = "2026-06-29T13:03:23.384Z" }, + { url = "https://files.pythonhosted.org/packages/e2/59/e196888a05befdda7dbe299b722d56f2f6eec65402bc34c0a3306d595feb/jiter-0.16.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9c8a956fd72c2cf1e730d01ea080341f13aa0a97a4a33b51abebe725b7ae9ca9", size = 351107, upload-time = "2026-06-29T13:03:24.815Z" }, + { url = "https://files.pythonhosted.org/packages/ec/74/4cd9e0fca65232136400354b630fbfcd2de634e22ccbb96567725981b548/jiter-0.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:561926e0573ffe4a32498420a76d64b16c513e1ab413b9d28158a8764ac701e5", size = 388441, upload-time = "2026-06-29T13:03:26.266Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8c/554691e48bc711299c0a293dd8a6179e24b2d66a54dc295421fcf64569c0/jiter-0.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:44d019fa8cdaf89bf29c71b39e3712143fdd0ac76725c6ef954f9957a5ea8730", size = 516354, upload-time = "2026-06-29T13:03:28.02Z" }, + { url = "https://files.pythonhosted.org/packages/a4/cb/01e9d69dc2cc6759d4f91e230b34489c4fdb2518992650633f9e20bece89/jiter-0.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0df91907609837f33341b8e6fe73b95991fdaa57caf1a0fbd343dffe826f386f", size = 547880, upload-time = "2026-06-29T13:03:29.534Z" }, + { url = "https://files.pythonhosted.org/packages/79/70/2953195f1c6ad00f49fa67e13df7e60acb3dd4f387101bc15abccddd905e/jiter-0.16.0-cp312-cp312-win32.whl", hash = "sha256:51d7b836acb0108d7c77df1742332cac2a1fa04a74d6dacec46e7091f0e91274", size = 203473, upload-time = "2026-06-29T13:03:31.025Z" }, + { url = "https://files.pythonhosted.org/packages/2d/05/2909a8b10699a4d560f8c502b6b2c5f3991b682b1922c1eedda242b225bd/jiter-0.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:1878349266f8ee36ecb1375cc5ba2f115f35fd9f0a1a4119e725e379126647f7", size = 196905, upload-time = "2026-06-29T13:03:32.472Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a9/6b82bb1c8d7790d602489b967b982a909e5d092875a6c2ade96444c8dfc5/jiter-0.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:2ed5738ae4af18271a51a528b8811b0cbfa4a1858de9d83359e4169855d6a331", size = 190618, upload-time = "2026-06-29T13:03:34.672Z" }, + { url = "https://files.pythonhosted.org/packages/91/c0/555fc60473d30d66894ba825e63615e3be7524fac23858356afa7a38906c/jiter-0.16.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:41977aa5654023948c2dae2a81cbf9c43343954bef1cd59a154dd15a4d84c195", size = 306203, upload-time = "2026-06-29T13:03:36.243Z" }, + { url = "https://files.pythonhosted.org/packages/d0/2b/c3eaf16f5d7c9bad66ea32f40a95bd169b29a91217fcc7f081375157e99c/jiter-0.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d28bb3c26762358dadf3e5bf0bccd29ae987d65e6988d2e6f49829c76b003c09", size = 306489, upload-time = "2026-06-29T13:03:37.846Z" }, + { url = "https://files.pythonhosted.org/packages/96/3f/02fdfc6705cad96127d883af5c34e4867f554f29ec7705ec1a46156400a9/jiter-0.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0542a7189c26920778658fc8fcf2af8bae05bae9924577f71804acef37996536", size = 335453, upload-time = "2026-06-29T13:03:39.221Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a6/e4bda5920d4b0d7c5dfb7174ce4a6b2e4d3e11c9162c452ef0eab4cdbdbd/jiter-0.16.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8fb8de1e23a0cb2a7f53c335049c7b72b6db41aa6227cdcc0972a1de5cb39450", size = 361625, upload-time = "2026-06-29T13:03:40.597Z" }, + { url = "https://files.pythonhosted.org/packages/b7/97/4e6b59b2c6e55cbb3e183595f81ad65dcfb21c915fee5e19e335df21bc55/jiter-0.16.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b72d0b2990ca754a9102779ac98d8597b7cb31678958562214a007f909eab78e", size = 456958, upload-time = "2026-06-29T13:03:42.074Z" }, + { url = "https://files.pythonhosted.org/packages/15/e0/97e9557686d2f94f4b93786eccb7eed28e9228ad132ea8237f44727314a7/jiter-0.16.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5f91b1c27fc22a57993d5a5cb8a627cb8ed4b10502716fac1ffbfe1d19d84e8", size = 372017, upload-time = "2026-06-29T13:03:43.658Z" }, + { url = "https://files.pythonhosted.org/packages/0f/94/db768b6938e0df35c86beeba3dfbbb025c9ee5c19e1aa271f2396e50864d/jiter-0.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c682bea068a90b764577bdb78a60a4c1d1606daf9cd4c893832a37c7cc9d9026", size = 343320, upload-time = "2026-06-29T13:03:45.226Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d6/5a59d938244a30735fe62d9433fd325f9021ea29d89780ea4596ea93bc89/jiter-0.16.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:8d031aabecc4f1b6276adfb42e3aabb77c89d468bf616600e8d3a11328929053", size = 350520, upload-time = "2026-06-29T13:03:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/c4a857f49c9af125f6bbcac7e3eee7f7978ed89682833062e2dbf62576b1/jiter-0.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eab2cd170150e70153de16896a1774e3a1dca80154c56b54d7a812c479a7165e", size = 387550, upload-time = "2026-06-29T13:03:48.361Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d6/5fbc2f7d6b67b754caa61a993a2e626e815dec47ffc2f9e35f01adfebec7/jiter-0.16.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6edb63a46e65a82c26800a868e49b2cac30dd5a4218b88d74bc2c848c8ad60bb", size = 515424, upload-time = "2026-06-29T13:03:49.881Z" }, + { url = "https://files.pythonhosted.org/packages/ed/54/284f0164b64a5fed915fea6ba7e9ba9b3d8d37c67d59cf2e3bb99d45cdfe/jiter-0.16.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:659039cc50b5addcc35fcc87ae2c1833b7c0a8e5326ef631a75e4478447bcf84", size = 546981, upload-time = "2026-06-29T13:03:51.363Z" }, + { url = "https://files.pythonhosted.org/packages/13/c5/2a467585a576594384e1d2c43e1224deaafc085f24e243529cf98beef8e1/jiter-0.16.0-cp313-cp313-win32.whl", hash = "sha256:c9c53be232c2e206ef9cdbad81a48bfa74c3d3f08bcf8124630a8a748aad993e", size = 202853, upload-time = "2026-06-29T13:03:53.015Z" }, + { url = "https://files.pythonhosted.org/packages/88/6a/de61d04b9eec69c71719968d2f716532a3bc121170c44a39e14979c6be81/jiter-0.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:baad945ed47f163ad833314f8e3288c396118934f94e7bbb9e243ce4b341a4fd", size = 196160, upload-time = "2026-06-29T13:03:54.447Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/b390ed59bafb3f31d008d1218578f10327714484b334439947f7e5b11e7f/jiter-0.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:3c1fd2dbe1b0af19e987f03fe66c5f5bd105a2229c1aff4ab14890b24f41d21a", size = 189862, upload-time = "2026-06-29T13:03:55.754Z" }, + { url = "https://files.pythonhosted.org/packages/a7/89/bc4f1b57d5da938fd344a466396541e586d161320d70bffd929aaafcd8f4/jiter-0.16.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b2c61484666ad42726029af0c00ef4541f0f3b5cdc550221f56c2343208018ee", size = 308239, upload-time = "2026-06-29T13:03:57.205Z" }, + { url = "https://files.pythonhosted.org/packages/65/7a/c415453e5213001bf3b411ff65dec3d303b0e76a4a2cfea9768cd4960994/jiter-0.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:63efadc657488f45db1c676d81e704cac2abf3fdb892def1faea61db053127e2", size = 308928, upload-time = "2026-06-29T13:03:58.643Z" }, + { url = "https://files.pythonhosted.org/packages/11/fc/1f4fb7ebf9a724c7741994f4aae18fba1e2f3133df14521a79194952c34a/jiter-0.16.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf0d73f50e7b6935677854f6e8e31d499ca7064dd24734f703e060f5b237d883", size = 336998, upload-time = "2026-06-29T13:04:00.071Z" }, + { url = "https://files.pythonhosted.org/packages/a0/8d/72cadaac05ccfa7cc3a0a2232862e6c72443ca40cf300ba8b57f9f18b69b/jiter-0.16.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3ea07d9bc8e7d03a9fbc051295462e6dbc295b894fd72457c3136e3e43d898", size = 362112, upload-time = "2026-06-29T13:04:01.52Z" }, + { url = "https://files.pythonhosted.org/packages/58/4a/c4b0d5f651fda90a24ffce9f8d56cde462a2e09d31ae3de3c68cef34c04e/jiter-0.16.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26798522707abb47d767db536e4148ceac1b14446bf028ee85e579a2e043cfe5", size = 459807, upload-time = "2026-06-29T13:04:03.214Z" }, + { url = "https://files.pythonhosted.org/packages/80/58/ef77879ea9aa56b50824edc5a445e226422c7a8d211f3fd2a56bcb9493cf/jiter-0.16.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bc837c1b9631be10abfe0191537fe8009838204cec7e44827401ace390ddb567", size = 373181, upload-time = "2026-06-29T13:04:04.629Z" }, + { url = "https://files.pythonhosted.org/packages/49/2e/ffbc3f254e4d8a66da3062c624a7df4b7c2b2cf9e1fe43cf394b3e104041/jiter-0.16.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49060fd70737fad59d33ba9dcc0d83247dc9e77187de26053a19c16c9f32bd69", size = 344927, upload-time = "2026-06-29T13:04:06.067Z" }, + { url = "https://files.pythonhosted.org/packages/9a/f6/0be5dc6d64a89f80aa8fec984f94dedb2973e251edcae55841d60786d578/jiter-0.16.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:adbb8edeadd431bc4477879d5d371ece7cb1334486584e0f252656dd7ffada29", size = 352754, upload-time = "2026-06-29T13:04:07.477Z" }, + { url = "https://files.pythonhosted.org/packages/da/6e/7d31243b3b91cd261dd19e9d3557fc3251a80883d3d8049c86174e7ab7af/jiter-0.16.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:31aaee5b80f672c1dc21272bcfb9cbdcfc1ea04ff50f00ed5af500b80c44fa93", size = 390553, upload-time = "2026-06-29T13:04:08.92Z" }, + { url = "https://files.pythonhosted.org/packages/25/33/51ae371fde3c88897520f62b4d5f8b27ad7103e2bb10812ff52195609853/jiter-0.16.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:6722bcef4ffc86c835574b1b2fac6b33b9fb4a889c781e67950e891591f3c55a", size = 516900, upload-time = "2026-06-29T13:04:10.407Z" }, + { url = "https://files.pythonhosted.org/packages/a0/45/6449b3d123ea439ba79507c657288f461d55049e7bcbdc2cf8eb8210f491/jiter-0.16.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:5ab4f50ff971b611d656554ea10b75f80097392c827bc32923c6eeb6386c8b00", size = 548754, upload-time = "2026-06-29T13:04:12.046Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e7/fd2fb11ae3e2649333da3aa170d04d7b3000bbdc3b270f6513382fdf4e04/jiter-0.16.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:710cc51d4ebdcd3c1f70b232c1db1ea1344a075770422bbd4bede5708335acbe", size = 122381, upload-time = "2026-06-29T13:04:13.413Z" }, + { url = "https://files.pythonhosted.org/packages/26/80/f0b147a62c315a164ed2168908286ca302310824c218d3aae52b06c0c9a9/jiter-0.16.0-cp314-cp314-win32.whl", hash = "sha256:57b37fc887a32d44798e4d8ebfa7c9683ff3da1d5bf38f08d1bb3573ccb39106", size = 204578, upload-time = "2026-06-29T13:04:14.813Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e6/4758a14304b4523a6f5adb2419340086aa3593bd4327c2b25b5948a90548/jiter-0.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:cbd18dd5e2df96b580487b5745adf57ef64ad89ba2d9662fc3c19386acce7db8", size = 198154, upload-time = "2026-06-29T13:04:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/26/be/41fa54a2e7ea41d6c99f1dc5b1f0fd4cb474680304b5d268dd518e81da3a/jiter-0.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:a32d2027a9fa67f109ff245a3252ece3ccc32cc56703e1deab6cc846a59e0585", size = 191458, upload-time = "2026-06-29T13:04:17.707Z" }, + { url = "https://files.pythonhosted.org/packages/81/6b/59127338b86d9fe4d99418f5a15118bea778103ee0fe9d9dd7e0af174e95/jiter-0.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2577196f4474ef3fc4779a088a23b0897bbf86f9ea3679c372d45b8383b43207", size = 316739, upload-time = "2026-06-29T13:04:19.663Z" }, + { url = "https://files.pythonhosted.org/packages/2d/95/49461034d5388196d3dabf98748935f017b7785d8f3f5349f834bcc4ed0d/jiter-0.16.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e89e008a93c01104161c75b4988e58716b01d62307ebfe161e52a56d2a818", size = 340911, upload-time = "2026-06-29T13:04:21.257Z" }, + { url = "https://files.pythonhosted.org/packages/cd/97/a4369f2fb82cb3dda13b98622f31249b2e014b223fe64ee534413ad72294/jiter-0.16.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e2e9efbe042210df657bade597f66d6d75723e3d8f45a12ea6d8167ff8bbce3", size = 361747, upload-time = "2026-06-29T13:04:22.677Z" }, + { url = "https://files.pythonhosted.org/packages/28/51/49b6ed456261646e1906016a6760367a28aacd3c24805e4e5fe64116c1db/jiter-0.16.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f4d9e473a5ce7d27fef8b848df4dc16e283893d3f53b4a585e72c9595f3c284", size = 460225, upload-time = "2026-06-29T13:04:24.441Z" }, + { url = "https://files.pythonhosted.org/packages/33/b5/5689aff4f66c5b60be63106e591dbfcba2190df97d2c9c7cf052361ddb98/jiter-0.16.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d30a4a1c87713060c8d1cc59a7b6c8fb6b8ef0a6900368014c76c87922a2929", size = 373169, upload-time = "2026-06-29T13:04:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/a2/96/3ae1b85ee0d6d6cab254fb7f8da018272b932bbf2d69b07e98aa2a96c746/jiter-0.16.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae96332410f866e5900d809298b1ed82735932986c672495f9701daacd80620", size = 350332, upload-time = "2026-06-29T13:04:27.302Z" }, + { url = "https://files.pythonhosted.org/packages/15/32/c99d7bafd78986556c95bf60ce84c6cc98786eac56066c12d7f828bb6747/jiter-0.16.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:da3d7ec75dc83bb18bca888b5edfae0656a26849056c59e05a7728badd17e7af", size = 353377, upload-time = "2026-06-29T13:04:28.731Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/f99a8e571287c3dec766bcc18528bbe8e8fb5365522ab5e6d64c93e87066/jiter-0.16.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ee6162b77d49a9939229df666dfa8af3e656b6701b54c4c84966d740e189264e", size = 387746, upload-time = "2026-06-29T13:04:30.319Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/c78a5b3f71040e34eb5917df26fb7ae9a2174cad1ccbf277512507c53a6e/jiter-0.16.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:63ffdbdae7d4499f4cda14eadc12ddcabef0fc0c081191bdc2247489cb698077", size = 517292, upload-time = "2026-06-29T13:04:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f7/095b38eda4c70d03651c403f29a5590f16d12ddc5d544aac9f9cddf72277/jiter-0.16.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a111256a7193bea0759267b10385e5870949c239ed7b6ddbaaf57573edb38734", size = 549259, upload-time = "2026-06-29T13:04:33.721Z" }, + { url = "https://files.pythonhosted.org/packages/2e/c5/6a0207d90e5f656d95af98ebd0934f382d37674416f215aeda2ff8063e51/jiter-0.16.0-cp314-cp314t-win32.whl", hash = "sha256:de5ba8763e56b793561f43bed197c9ea55776daa5e9a6b91eed68a909bc9cdbf", size = 206523, upload-time = "2026-06-29T13:04:35.068Z" }, + { url = "https://files.pythonhosted.org/packages/a5/31/c757d5f30a8980fd945ce7b98be10be9e4ff59c7c42f5fd86804c2e87db8/jiter-0.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b8a3f9a6008048fe9def7bf465180564a6e458047d2ce499149cfbe73c3ae9db", size = 200366, upload-time = "2026-06-29T13:04:36.61Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a2/d88de6d313d734a544a7901353ad5db67cb38dcfcd91713b7979dafc345d/jiter-0.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0fa25b09b13075c46f5bc174f2690525a925a4fc2f7c82969a2bbabff22386ce", size = 190516, upload-time = "2026-06-29T13:04:38.004Z" }, + { url = "https://files.pythonhosted.org/packages/06/d3/8e278946d43eeca2585b4dd0834a887cd71136329b837f3a16ed86a8b4b0/jiter-0.16.0-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:850ccb1d7eedb4200f4014b1c0e8a577de114fc3cd88faad646dcc9bc4bb12ad", size = 304518, upload-time = "2026-06-29T13:05:00.172Z" }, + { url = "https://files.pythonhosted.org/packages/72/43/28d4ef495028bf0506a413d4db3f4eb3e7288a382e0f065f306a17bbeb5e/jiter-0.16.0-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:e34e97bda77eb63242a410243c071e28ac7e0d8c0948c5ee658498690a4b2f2f", size = 310207, upload-time = "2026-06-29T13:05:02.123Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ca/c366b1012da1d640de975d9683acd44e4d150d9068845d0ca2610435253f/jiter-0.16.0-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7dc85ea77d4abbae8bad0d3538678aedee75bceec4e2f6c8dfb1c74772e5aa5", size = 342771, upload-time = "2026-06-29T13:05:03.55Z" }, + { url = "https://files.pythonhosted.org/packages/16/52/50cc4056fc1ae02e7154704e7ecc89df0afb8300222cfe8a52d3f67e4730/jiter-0.16.0-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17ca7fae79f6d99cd9a042b75f917eaada7b895cfc7dd2ee3a16089dcaec7a85", size = 346468, upload-time = "2026-06-29T13:05:05.452Z" }, + { url = "https://files.pythonhosted.org/packages/98/ab/664fd8c4be028b2bedd3d2ff08769c4ede23d0dbc87a77c62384a0515b5d/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:f17d61a28b4b3e0e3e2ba98490c70501403b4d196f78732439160e7fd3678127", size = 303106, upload-time = "2026-06-29T13:05:07.118Z" }, + { url = "https://files.pythonhosted.org/packages/1a/07/421f1d5b65493a76e16027b848aba6a7d28073ae75944fa4289cc914d39f/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:96e38eea538c8ddf853a35727c7be0741c76c13f04148ac5c116222f50ece3b3", size = 304658, upload-time = "2026-06-29T13:05:08.708Z" }, + { url = "https://files.pythonhosted.org/packages/0a/db/bba1155f01a01c3c37a89425d571da751bbedf5c54247b831a04cb971798/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d284fb8d94d5855d60c44fefcab4bf966f1da6fada73992b01f6f0c9bc0c6702", size = 339719, upload-time = "2026-06-29T13:05:10.41Z" }, + { url = "https://files.pythonhosted.org/packages/78/f7/18a1afcd64f35314b68c1f23afcd9994d0bc13e65cc77517afff4e83986d/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d613743df53199b1aa256a7d328340da6d7078aac7705a7db9d7a791e9cfd2", size = 343885, upload-time = "2026-06-29T13:05:12.087Z" }, +] + [[package]] name = "jmespath" version = "1.1.0" @@ -2507,7 +3569,7 @@ version = "1.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jupyter-core" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fb/45/d0df8b43c10a61529c0f4a8af5e19ebe108f0c3af8f57e0fc358969907af/jupyter_builder-1.0.2.tar.gz", hash = "sha256:6155d78a5325010532a6419ffcba89eac643fd1aa56ea83115e661924d6f6aab", size = 968638, upload-time = "2026-06-12T02:33:25.767Z" } @@ -2539,8 +3601,8 @@ version = "6.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ipykernel" }, - { name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "ipython", version = "9.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "ipython", version = "9.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "jupyter-client" }, { name = "jupyter-core" }, { name = "prompt-toolkit" }, @@ -2559,7 +3621,7 @@ version = "5.8.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "platformdirs" }, - { name = "pywin32", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'win32'" }, + { name = "pywin32", marker = "(platform_python_implementation != 'PyPy' and sys_platform == 'win32') or (platform_python_implementation == 'PyPy' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'win32' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/99/1b/72906d554acfeb588332eaaa6f61577705e9ec752ddb486f302dafa292d9/jupyter_core-5.8.1.tar.gz", hash = "sha256:0a5f9706f70e64786b75acba995988915ebd4601c8a52e534a40b51c95f59941", size = 88923, upload-time = "2025-05-27T07:38:16.655Z" } @@ -2612,10 +3674,10 @@ dependencies = [ { name = "jupyter-server-terminals" }, { name = "nbconvert" }, { name = "nbformat" }, - { name = "overrides", marker = "python_full_version < '3.12'" }, + { name = "overrides", marker = "python_full_version < '3.12' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "packaging" }, { name = "prometheus-client" }, - { name = "pywinpty", marker = "os_name == 'nt' and sys_platform != 'linux'" }, + { name = "pywinpty", marker = "(os_name == 'nt' and sys_platform != 'darwin' and sys_platform != 'linux') or (os_name == 'nt' and sys_platform == 'darwin' and extra == 'extra-16-transformer-lens-lit') or (os_name == 'nt' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-vllm') or (os_name == 'nt' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-lit') or (os_name == 'nt' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-vllm') or (os_name != 'nt' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "pyzmq" }, { name = "send2trash" }, { name = "terminado" }, @@ -2633,7 +3695,7 @@ name = "jupyter-server-terminals" version = "0.5.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pywinpty", marker = "os_name == 'nt' and sys_platform != 'linux'" }, + { name = "pywinpty", marker = "(os_name == 'nt' and sys_platform != 'darwin' and sys_platform != 'linux') or (os_name == 'nt' and sys_platform == 'darwin' and extra == 'extra-16-transformer-lens-lit') or (os_name == 'nt' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-vllm') or (os_name == 'nt' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-lit') or (os_name == 'nt' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-vllm') or (os_name != 'nt' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "terminado" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fc/d5/562469734f476159e99a55426d697cbf8e7eb5efe89fb0e0b4f83a3d3459/jupyter_server_terminals-0.5.3.tar.gz", hash = "sha256:5ae0295167220e9ace0edcfdb212afd2b01ee8d179fe6f23c899590e9b8a5269", size = 31430, upload-time = "2024-03-12T14:37:03.049Z" } @@ -2657,7 +3719,7 @@ dependencies = [ { name = "jupyterlab-server" }, { name = "notebook-shim" }, { name = "packaging" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "tornado" }, { name = "traitlets" }, ] @@ -2945,8 +4007,8 @@ name = "libcst" version = "1.8.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyyaml", marker = "python_full_version < '3.13'" }, - { name = "pyyaml-ft", marker = "python_full_version >= '3.13'" }, + { name = "pyyaml", marker = "python_full_version < '3.13' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "pyyaml-ft", marker = "python_full_version >= '3.13' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/89/aa/b52d195b167958fe1bd106a260f64cc80ec384f6ac2a9cda874d8803df06/libcst-1.8.2.tar.gz", hash = "sha256:66e82cedba95a6176194a817be4232c720312f8be6d2c8f3847f3317d95a0c7f", size = 881534, upload-time = "2025-06-13T20:56:37.915Z" } wheels = [ @@ -3022,27 +4084,27 @@ dependencies = [ { name = "absl-py" }, { name = "annoy" }, { name = "attrs" }, - { name = "etils", version = "1.13.0", source = { registry = "https://pypi.org/simple" }, extra = ["epath"], marker = "python_full_version < '3.11'" }, - { name = "etils", version = "1.14.0", source = { registry = "https://pypi.org/simple" }, extra = ["epath"], marker = "python_full_version >= '3.11'" }, + { name = "etils", version = "1.13.0", source = { registry = "https://pypi.org/simple" }, extra = ["epath"], marker = "(python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "etils", version = "1.14.0", source = { registry = "https://pypi.org/simple" }, extra = ["epath"], marker = "(python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "filelock" }, { name = "google-cloud-translate" }, - { name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "ipython", version = "9.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "ipython", version = "9.10.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "levenshtein" }, { name = "matplotlib" }, { name = "ml-collections" }, - { name = "numpy" }, - { name = "pandas", extra = ["output-formatting"] }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } }, + { name = "pandas", extra = ["output-formatting"], marker = "extra == 'extra-16-transformer-lens-lit'" }, { name = "pillow" }, { name = "portpicker" }, { name = "requests" }, { name = "rouge-score" }, { name = "sacrebleu" }, { name = "saliency" }, - { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "shap" }, { name = "six" }, { name = "termcolor" }, @@ -3066,6 +4128,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/3e/de54dc7f199e85e6ca37e2e5dae2ec3bce2151e9e28f8eb9076d71e83d56/livereload-2.7.1-py3-none-any.whl", hash = "sha256:5201740078c1b9433f4b2ba22cd2729a39b9d0ec0a2cc6b4d3df257df5ad0564", size = 22657, upload-time = "2024-12-18T13:41:56.35Z" }, ] +[[package]] +name = "llguidance" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/48/3f7a9d3ff1b36bba92b5107a3a21286821227afe9ea464736133994d61fb/llguidance-1.3.0.tar.gz", hash = "sha256:861249afd51dc325646834462ea827e57a5c2b2042e108e6aae7059fdad9104d", size = 1070460, upload-time = "2025-10-20T19:58:44.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/33/be5acb85cd8cdc4afde33d9c234eece9f318e087920255af3c05864cd3e7/llguidance-1.3.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f7685222660a762e481ac633d49cc559c64980fe2ee59c8f932a5bb5cbc0c2c2", size = 3220647, upload-time = "2025-10-20T19:58:42.542Z" }, + { url = "https://files.pythonhosted.org/packages/82/e6/b48bda5b15efeaeb62bd0dba8fc6a01d4ae5457a85dbb5d18632385fe15c/llguidance-1.3.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:098030ff0687261a3f1bd54cf21fe951fc861d56d37a0671250dd36677eaf224", size = 3099830, upload-time = "2025-10-20T19:58:40.826Z" }, + { url = "https://files.pythonhosted.org/packages/aa/11/44389d3d1526d7a5c38ffd587a5ebc61d7bee443ac1dea95f2089ad58f5f/llguidance-1.3.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f6caca5d78db7f76e1fbb0fff8607b861c32d47fa3d5dee2fc49de27ee269df", size = 2835242, upload-time = "2025-10-20T19:58:34.518Z" }, + { url = "https://files.pythonhosted.org/packages/e7/ca/53ea256396405e4dee70d5a4a35e18543408e18bb16b251d6ca6b5d80310/llguidance-1.3.0-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0612bb3f034d2487b6e8f9561f02a94a6039d88273bf0c5c539a3bd3895e47d2", size = 3297480, upload-time = "2025-10-20T19:58:37.033Z" }, + { url = "https://files.pythonhosted.org/packages/83/a8/1ff2bedb8f9acb46a2d2d603415d272bb622c142ea86f5b95445cc6e366c/llguidance-1.3.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc17e9dd602c3879bf91664a64bf72f54c74dbfbeb24ccfab6a5fe435b12f7aa", size = 3033133, upload-time = "2025-10-20T19:58:38.721Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a7/9b8086c0cfdddf3f6d47b173a404fa7ac46272f7affbee082c36740f4f1c/llguidance-1.3.0-cp39-abi3-win32.whl", hash = "sha256:2f6f558485a43e273fc5c6c974a9a3ace5d5e170076db9b40e0560e41c3ff18f", size = 2598109, upload-time = "2025-10-20T19:58:47.656Z" }, + { url = "https://files.pythonhosted.org/packages/5a/7e/809349638231f469b9056c0e1bfd924d5ef5558b3b3ec72d093b6fad33b1/llguidance-1.3.0-cp39-abi3-win_amd64.whl", hash = "sha256:1d1cd1c8618d1a13605d3e057c978651e551c8c469b481ee4041f1d6c436002d", size = 2789946, upload-time = "2025-10-20T19:58:45.958Z" }, +] + [[package]] name = "llvmlite" version = "0.47.0" @@ -3109,12 +4186,14 @@ dependencies = [ { name = "jinja2" }, { name = "jsonlines" }, { name = "more-itertools" }, - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "pytablewriter" }, { name = "rouge-score" }, { name = "sacrebleu" }, - { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "sqlitedict" }, { name = "typing-extensions" }, { name = "word2number" }, @@ -3125,6 +4204,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b3/8c2923ad4c4d911307e20ab9c7d7869d3811b5a76fe6dbe53678aafbeb04/lm_eval-0.4.11-py3-none-any.whl", hash = "sha256:9c9945475a715558649a38ffcb90f46e7bd23a849524a5e838f249161b030517", size = 8744826, upload-time = "2026-02-13T20:22:56.317Z" }, ] +[[package]] +name = "lm-format-enforcer" +version = "0.11.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "interegular", marker = "sys_platform == 'linux'" }, + { name = "packaging", marker = "sys_platform == 'linux'" }, + { name = "pydantic", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/d5/41cd417ba7dfdbbcfe46cebf81fb3dfd7c591b89897560ad05bb410a465d/lm_format_enforcer-0.11.3.tar.gz", hash = "sha256:e68081c108719cce284a9bcc889709b26ffb085a1945b5eba3a12cfa96d528da", size = 40258, upload-time = "2025-08-24T19:37:47.527Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/ef/11292bb0b85cf4c93447cab5a29f64576ed14d3ab4280e35ddd23486594a/lm_format_enforcer-0.11.3-py3-none-any.whl", hash = "sha256:cf586350875def1ae7a8fba84fcbbfc8371424b6c9d05c1fcba70aa233fbf06f", size = 45418, upload-time = "2025-08-24T19:37:46.325Z" }, +] + +[[package]] +name = "loguru" +version = "0.7.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, +] + [[package]] name = "lxml" version = "6.0.4" @@ -3326,12 +4429,14 @@ name = "matplotlib" version = "3.10.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "contourpy", version = "1.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "contourpy", version = "1.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "contourpy", version = "1.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "contourpy", version = "1.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "cycler" }, { name = "fonttools" }, { name = "kiwisolver" }, - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "packaging" }, { name = "pillow" }, { name = "pyparsing" }, @@ -3419,6 +4524,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/eb/711270faab7b7df702339a2c68b31fd3ed4fffc68b0e99e5bdf49b1e87e4/mbstrdecoder-1.1.5-py3-none-any.whl", hash = "sha256:4a50fe113d4abecfd86e8f716b2e413cce03d63af83ec3c7cdbe81dec0e519ed", size = 7966, upload-time = "2026-05-05T04:17:56.78Z" }, ] +[[package]] +name = "mcp" +version = "1.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform == 'linux'" }, + { name = "httpx", marker = "sys_platform == 'linux'" }, + { name = "httpx-sse", marker = "sys_platform == 'linux'" }, + { name = "jsonschema", marker = "sys_platform == 'linux'" }, + { name = "pydantic", marker = "sys_platform == 'linux'" }, + { name = "pydantic-settings", marker = "sys_platform == 'linux'" }, + { name = "pyjwt", extra = ["crypto"], marker = "(sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "python-multipart", marker = "sys_platform == 'linux'" }, + { name = "sse-starlette", marker = "sys_platform == 'linux'" }, + { name = "starlette", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "typing-inspection", marker = "sys_platform == 'linux'" }, + { name = "uvicorn", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6e/77/9450b8f251a13affb6281997d0523c4615f8a8b35d0b21ff30db3a5aac9d/mcp-1.28.1.tar.gz", hash = "sha256:d51e36a5f5644faea4f85ea649bfffa6bc6c26770d42798ad6a3de3d2ba69683", size = 638501, upload-time = "2026-06-26T12:57:29.093Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/5e/d118fce19f87a2e7d8101c35c8ae0ec289098a4df0ff244cec23e415aca0/mcp-1.28.1-py3-none-any.whl", hash = "sha256:2726bca5e7193f61c5dde8b12500a6de2d9acf6d1a1c0be9e8c2e706437991df", size = 222620, upload-time = "2026-06-26T12:57:27.218Z" }, +] + [[package]] name = "mdit-py-plugins" version = "0.5.0" @@ -3440,12 +4569,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "mistral-common" +version = "1.11.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonschema", marker = "sys_platform == 'linux'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "pillow", marker = "sys_platform == 'linux'" }, + { name = "pydantic", marker = "sys_platform == 'linux'" }, + { name = "pydantic-extra-types", extra = ["pycountry"], marker = "(sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "requests", marker = "sys_platform == 'linux'" }, + { name = "tiktoken", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e3/35/1e6e07189a7277be308fac5f83893cbf6784b76351d6f4b4da155b7ef4f9/mistral_common-1.11.5.tar.gz", hash = "sha256:ef8c03ad8359fa1386d66ee08d534d8f4a65a6955c9b24bd5caa2e005066cdec", size = 6383849, upload-time = "2026-06-26T12:45:39.469Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/86/de5ad2ab2e3d140f33e4dc615f9fe2b4ae2136c9d2d75306625f9735a71e/mistral_common-1.11.5-py3-none-any.whl", hash = "sha256:7c1b09f43a589027315840bfd6f3528d5abf520eed701f8d8b7a922d6e5b855e", size = 6551345, upload-time = "2026-06-26T12:45:36.997Z" }, +] + +[package.optional-dependencies] +image = [ + { name = "opencv-python-headless", marker = "sys_platform == 'linux'" }, +] + [[package]] name = "mistune" version = "3.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c4/79/bda47f7dd7c3c55770478d6d02c9960c430b0cf1773b72366ff89126ea31/mistune-3.1.3.tar.gz", hash = "sha256:a7035c21782b2becb6be62f8f25d3df81ccb4d6fa477a6525b15af06539f02a0", size = 94347, upload-time = "2025-03-19T14:27:24.955Z" } wheels = [ @@ -3465,6 +4619,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/8a/18d4ff2c7bd83f30d6924bd4ad97abf418488c3f908dea228d6f0961ad68/ml_collections-1.1.0-py3-none-any.whl", hash = "sha256:23b6fa4772aac1ae745a96044b925a5746145a70734f087eaca6626e92c05cbc", size = 76707, upload-time = "2025-04-17T08:24:59.038Z" }, ] +[[package]] +name = "ml-dtypes" +version = "0.5.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/c27b42ed9b1c7d13d9ba8b6905dece787d6259152f2309338aed29b2447b/ml_dtypes-0.5.4.tar.gz", hash = "sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453", size = 692314, upload-time = "2025-11-17T22:32:31.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/3a/c5b855752a70267ff729c349e650263adb3c206c29d28cc8ea7ace30a1d5/ml_dtypes-0.5.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b95e97e470fe60ed493fd9ae3911d8da4ebac16bd21f87ffa2b7c588bf22ea2c", size = 679735, upload-time = "2025-11-17T22:31:31.367Z" }, + { url = "https://files.pythonhosted.org/packages/41/79/7433f30ee04bd4faa303844048f55e1eb939131c8e5195a00a96a0939b64/ml_dtypes-0.5.4-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b4b801ebe0b477be666696bda493a9be8356f1f0057a57f1e35cd26928823e5a", size = 5051883, upload-time = "2025-11-17T22:31:33.658Z" }, + { url = "https://files.pythonhosted.org/packages/10/b1/8938e8830b0ee2e167fc75a094dea766a1152bde46752cd9bfc57ee78a82/ml_dtypes-0.5.4-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:388d399a2152dd79a3f0456a952284a99ee5c93d3e2f8dfe25977511e0515270", size = 5030369, upload-time = "2025-11-17T22:31:35.595Z" }, + { url = "https://files.pythonhosted.org/packages/c7/a3/51886727bd16e2f47587997b802dd56398692ce8c6c03c2e5bb32ecafe26/ml_dtypes-0.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:4ff7f3e7ca2972e7de850e7b8fcbb355304271e2933dd90814c1cb847414d6e2", size = 210738, upload-time = "2025-11-17T22:31:37.43Z" }, + { url = "https://files.pythonhosted.org/packages/c6/5e/712092cfe7e5eb667b8ad9ca7c54442f21ed7ca8979745f1000e24cf8737/ml_dtypes-0.5.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6c7ecb74c4bd71db68a6bea1edf8da8c34f3d9fe218f038814fd1d310ac76c90", size = 679734, upload-time = "2025-11-17T22:31:39.223Z" }, + { url = "https://files.pythonhosted.org/packages/4f/cf/912146dfd4b5c0eea956836c01dcd2fce6c9c844b2691f5152aca196ce4f/ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc11d7e8c44a65115d05e2ab9989d1e045125d7be8e05a071a48bc76eb6d6040", size = 5056165, upload-time = "2025-11-17T22:31:41.071Z" }, + { url = "https://files.pythonhosted.org/packages/a9/80/19189ea605017473660e43762dc853d2797984b3c7bf30ce656099add30c/ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19b9a53598f21e453ea2fbda8aa783c20faff8e1eeb0d7ab899309a0053f1483", size = 5034975, upload-time = "2025-11-17T22:31:42.758Z" }, + { url = "https://files.pythonhosted.org/packages/b4/24/70bd59276883fdd91600ca20040b41efd4902a923283c4d6edcb1de128d2/ml_dtypes-0.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:7c23c54a00ae43edf48d44066a7ec31e05fdc2eee0be2b8b50dd1903a1db94bb", size = 210742, upload-time = "2025-11-17T22:31:44.068Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c9/64230ef14e40aa3f1cb254ef623bf812735e6bec7772848d19131111ac0d/ml_dtypes-0.5.4-cp311-cp311-win_arm64.whl", hash = "sha256:557a31a390b7e9439056644cb80ed0735a6e3e3bb09d67fd5687e4b04238d1de", size = 160709, upload-time = "2025-11-17T22:31:46.557Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b8/3c70881695e056f8a32f8b941126cf78775d9a4d7feba8abcb52cb7b04f2/ml_dtypes-0.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac", size = 676927, upload-time = "2025-11-17T22:31:48.182Z" }, + { url = "https://files.pythonhosted.org/packages/54/0f/428ef6881782e5ebb7eca459689448c0394fa0a80bea3aa9262cba5445ea/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900", size = 5028464, upload-time = "2025-11-17T22:31:50.135Z" }, + { url = "https://files.pythonhosted.org/packages/3a/cb/28ce52eb94390dda42599c98ea0204d74799e4d8047a0eb559b6fd648056/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff", size = 5009002, upload-time = "2025-11-17T22:31:52.001Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f0/0cfadd537c5470378b1b32bd859cf2824972174b51b873c9d95cfd7475a5/ml_dtypes-0.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:c1a953995cccb9e25a4ae19e34316671e4e2edaebe4cf538229b1fc7109087b7", size = 212222, upload-time = "2025-11-17T22:31:53.742Z" }, + { url = "https://files.pythonhosted.org/packages/16/2e/9acc86985bfad8f2c2d30291b27cd2bb4c74cea08695bd540906ed744249/ml_dtypes-0.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:9bad06436568442575beb2d03389aa7456c690a5b05892c471215bfd8cf39460", size = 160793, upload-time = "2025-11-17T22:31:55.358Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a1/4008f14bbc616cfb1ac5b39ea485f9c63031c4634ab3f4cf72e7541f816a/ml_dtypes-0.5.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c760d85a2f82e2bed75867079188c9d18dae2ee77c25a54d60e9cc79be1bc48", size = 676888, upload-time = "2025-11-17T22:31:56.907Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b7/dff378afc2b0d5a7d6cd9d3209b60474d9819d1189d347521e1688a60a53/ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b", size = 5036993, upload-time = "2025-11-17T22:31:58.497Z" }, + { url = "https://files.pythonhosted.org/packages/eb/33/40cd74219417e78b97c47802037cf2d87b91973e18bb968a7da48a96ea44/ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:533ce891ba774eabf607172254f2e7260ba5f57bdd64030c9a4fcfbd99815d0d", size = 5010956, upload-time = "2025-11-17T22:31:59.931Z" }, + { url = "https://files.pythonhosted.org/packages/e1/8b/200088c6859d8221454825959df35b5244fa9bdf263fd0249ac5fb75e281/ml_dtypes-0.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:f21c9219ef48ca5ee78402d5cc831bd58ea27ce89beda894428bc67a52da5328", size = 212224, upload-time = "2025-11-17T22:32:01.349Z" }, + { url = "https://files.pythonhosted.org/packages/8f/75/dfc3775cb36367816e678f69a7843f6f03bd4e2bcd79941e01ea960a068e/ml_dtypes-0.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:35f29491a3e478407f7047b8a4834e4640a77d2737e0b294d049746507af5175", size = 160798, upload-time = "2025-11-17T22:32:02.864Z" }, + { url = "https://files.pythonhosted.org/packages/4f/74/e9ddb35fd1dd43b1106c20ced3f53c2e8e7fc7598c15638e9f80677f81d4/ml_dtypes-0.5.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:304ad47faa395415b9ccbcc06a0350800bc50eda70f0e45326796e27c62f18b6", size = 702083, upload-time = "2025-11-17T22:32:04.08Z" }, + { url = "https://files.pythonhosted.org/packages/74/f5/667060b0aed1aa63166b22897fdf16dca9eb704e6b4bbf86848d5a181aa7/ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a0df4223b514d799b8a1629c65ddc351b3efa833ccf7f8ea0cf654a61d1e35d", size = 5354111, upload-time = "2025-11-17T22:32:05.546Z" }, + { url = "https://files.pythonhosted.org/packages/40/49/0f8c498a28c0efa5f5c95a9e374c83ec1385ca41d0e85e7cf40e5d519a21/ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:531eff30e4d368cb6255bc2328d070e35836aa4f282a0fb5f3a0cd7260257298", size = 5366453, upload-time = "2025-11-17T22:32:07.115Z" }, + { url = "https://files.pythonhosted.org/packages/8c/27/12607423d0a9c6bbbcc780ad19f1f6baa2b68b18ce4bddcdc122c4c68dc9/ml_dtypes-0.5.4-cp313-cp313t-win_amd64.whl", hash = "sha256:cb73dccfc991691c444acc8c0012bee8f2470da826a92e3a20bb333b1a7894e6", size = 225612, upload-time = "2025-11-17T22:32:08.615Z" }, + { url = "https://files.pythonhosted.org/packages/e5/80/5a5929e92c72936d5b19872c5fb8fc09327c1da67b3b68c6a13139e77e20/ml_dtypes-0.5.4-cp313-cp313t-win_arm64.whl", hash = "sha256:3bbbe120b915090d9dd1375e4684dd17a20a2491ef25d640a908281da85e73f1", size = 164145, upload-time = "2025-11-17T22:32:09.782Z" }, + { url = "https://files.pythonhosted.org/packages/72/4e/1339dc6e2557a344f5ba5590872e80346f76f6cb2ac3dd16e4666e88818c/ml_dtypes-0.5.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:2b857d3af6ac0d39db1de7c706e69c7f9791627209c3d6dedbfca8c7e5faec22", size = 673781, upload-time = "2025-11-17T22:32:11.364Z" }, + { url = "https://files.pythonhosted.org/packages/04/f9/067b84365c7e83bda15bba2b06c6ca250ce27b20630b1128c435fb7a09aa/ml_dtypes-0.5.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:805cef3a38f4eafae3a5bf9ebdcdb741d0bcfd9e1bd90eb54abd24f928cd2465", size = 5036145, upload-time = "2025-11-17T22:32:12.783Z" }, + { url = "https://files.pythonhosted.org/packages/c6/bb/82c7dcf38070b46172a517e2334e665c5bf374a262f99a283ea454bece7c/ml_dtypes-0.5.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14a4fd3228af936461db66faccef6e4f41c1d82fcc30e9f8d58a08916b1d811f", size = 5010230, upload-time = "2025-11-17T22:32:14.38Z" }, + { url = "https://files.pythonhosted.org/packages/e9/93/2bfed22d2498c468f6bcd0d9f56b033eaa19f33320389314c19ef6766413/ml_dtypes-0.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:8c6a2dcebd6f3903e05d51960a8058d6e131fe69f952a5397e5dbabc841b6d56", size = 221032, upload-time = "2025-11-17T22:32:15.763Z" }, + { url = "https://files.pythonhosted.org/packages/76/a3/9c912fe6ea747bb10fe2f8f54d027eb265db05dfb0c6335e3e063e74e6e8/ml_dtypes-0.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:5a0f68ca8fd8d16583dfa7793973feb86f2fbb56ce3966daf9c9f748f52a2049", size = 163353, upload-time = "2025-11-17T22:32:16.932Z" }, + { url = "https://files.pythonhosted.org/packages/cd/02/48aa7d84cc30ab4ee37624a2fd98c56c02326785750cd212bc0826c2f15b/ml_dtypes-0.5.4-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:bfc534409c5d4b0bf945af29e5d0ab075eae9eecbb549ff8a29280db822f34f9", size = 702085, upload-time = "2025-11-17T22:32:18.175Z" }, + { url = "https://files.pythonhosted.org/packages/5a/e7/85cb99fe80a7a5513253ec7faa88a65306be071163485e9a626fce1b6e84/ml_dtypes-0.5.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2314892cdc3fcf05e373d76d72aaa15fda9fb98625effa73c1d646f331fcecb7", size = 5355358, upload-time = "2025-11-17T22:32:19.7Z" }, + { url = "https://files.pythonhosted.org/packages/79/2b/a826ba18d2179a56e144aef69e57fb2ab7c464ef0b2111940ee8a3a223a2/ml_dtypes-0.5.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d2ffd05a2575b1519dc928c0b93c06339eb67173ff53acb00724502cda231cf", size = 5366332, upload-time = "2025-11-17T22:32:21.193Z" }, + { url = "https://files.pythonhosted.org/packages/84/44/f4d18446eacb20ea11e82f133ea8f86e2bf2891785b67d9da8d0ab0ef525/ml_dtypes-0.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:4381fe2f2452a2d7589689693d3162e876b3ddb0a832cde7a414f8e1adf7eab1", size = 236612, upload-time = "2025-11-17T22:32:22.579Z" }, + { url = "https://files.pythonhosted.org/packages/ad/3f/3d42e9a78fe5edf792a83c074b13b9b770092a4fbf3462872f4303135f09/ml_dtypes-0.5.4-cp314-cp314t-win_arm64.whl", hash = "sha256:11942cbf2cf92157db91e5022633c0d9474d4dfd813a909383bd23ce828a4b7d", size = 168825, upload-time = "2025-11-17T22:32:23.766Z" }, +] + [[package]] name = "mmh3" version = "5.2.1" @@ -3579,6 +4779,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/0f/59204bf136d1201f8d7884cfbaf7498c5b4674e87a4c693f9bde63741ce1/mmh3-5.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:dfd51b4c56b673dfbc43d7d27ef857dd91124801e2806c69bb45585ce0fa019b", size = 40391, upload-time = "2026-03-05T15:55:56.697Z" }, ] +[[package]] +name = "model-hosting-container-standards" +version = "0.1.16" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fastapi", marker = "sys_platform == 'linux'" }, + { name = "httpx", marker = "sys_platform == 'linux'" }, + { name = "jmespath", marker = "sys_platform == 'linux'" }, + { name = "pydantic", marker = "sys_platform == 'linux'" }, + { name = "setuptools", marker = "sys_platform == 'linux'" }, + { name = "starlette", marker = "sys_platform == 'linux'" }, + { name = "supervisor", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2d/5f/bc0d0fce1bd0a35378696aa13b21feffa18d9cda837f4e1be124e45ee090/model_hosting_container_standards-0.1.16.tar.gz", hash = "sha256:d34589633900e53c3ee5f7c78280a7cf7e4f6532c35e763341a262fc85cbe84a", size = 94130, upload-time = "2026-06-15T21:29:34.771Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/ef/6eabeb251d2a0598cb5f9a274159e05ae07a1e3fe6a1473bf6035793252a/model_hosting_container_standards-0.1.16-py3-none-any.whl", hash = "sha256:47f4f65713120bc3a69feb022981a38db9e557aedf88dbd72077f20588caa12b", size = 125666, upload-time = "2026-06-15T21:29:33.415Z" }, +] + [[package]] name = "more-itertools" version = "11.0.2" @@ -3597,12 +4815,68 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, ] +[[package]] +name = "msgspec" +version = "0.21.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/60/f79b9b013a16fa3a58350c9295ddc6789f2e335f36ea61ed10a21b215364/msgspec-0.21.1.tar.gz", hash = "sha256:2313508e394b0d208f8f56892ca9b2799e2561329de9763b19619595a6c0f72c", size = 319193, upload-time = "2026-04-12T21:44:50.394Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/38/d591d9f66d43d897ecbd249f2833665823d19c8b043f16619bc8343e23df/msgspec-0.21.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72d9cd03241b8b2edb2e12dcc66c500fa480d8cbd71a8bac105809d468882064", size = 195172, upload-time = "2026-04-12T21:43:45.062Z" }, + { url = "https://files.pythonhosted.org/packages/69/1a/6899188b5982ec1324e0c629b7801eed2db987f6634fab58abd9fc82d317/msgspec-0.21.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed2ab278200e743a1d2610a4e0c8fc74f6cecb8548544cdec43f927bd9265238", size = 188316, upload-time = "2026-04-12T21:43:46.641Z" }, + { url = "https://files.pythonhosted.org/packages/9e/95/7e591b4fa11fdbbf9891164473c23420a8c781ef553295abe416bf335f42/msgspec-0.21.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd677e3001fdfed9186de72eab434da2976303cd5eb9550921d3d0c3e3e168ce", size = 216565, upload-time = "2026-04-12T21:43:48.081Z" }, + { url = "https://files.pythonhosted.org/packages/19/86/714feeaf3b84cf2027235681725593840153dedd2868578f9f2715e296bb/msgspec-0.21.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f667b90b37fad734a91671abd68e0d7f4d066862771b87e91c53996dcb7a9027", size = 222689, upload-time = "2026-04-12T21:43:49.385Z" }, + { url = "https://files.pythonhosted.org/packages/7d/b9/4384243e814f2579e5205e17d170b9c1a30121afd1393298d904817a7fa7/msgspec-0.21.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:49880fd20fdbcfe1b793f07dd83f12572bab679c9800352c8b2240289aa46a06", size = 222343, upload-time = "2026-04-12T21:43:50.612Z" }, + { url = "https://files.pythonhosted.org/packages/04/01/4b227d9c4057346271043632bad41979cf8c3dca372e41bb1f7d546395b2/msgspec-0.21.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ae0162e22849a5e91eaad907766525107523b0daea3df267a9fcb5ba4e0936ae", size = 225607, upload-time = "2026-04-12T21:43:52.129Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ce/27021d1c3e5da837743092a7b7a5e8818397e1f4c05ee8b068bd7d1fd78a/msgspec-0.21.1-cp310-cp310-win_amd64.whl", hash = "sha256:f041a2279f31e3a53319005e4d60ba77c085cfcbe394cdc7ce803c2d01fe9449", size = 188392, upload-time = "2026-04-12T21:43:53.384Z" }, + { url = "https://files.pythonhosted.org/packages/80/2b/daf7a8d6d7cf00e0dcd0439178b284ade701234abdcadf3385601da04fbd/msgspec-0.21.1-cp310-cp310-win_arm64.whl", hash = "sha256:1bf17cbd7b28a5dffc7e764c654eed8ccde5e0f1de7970628608304640d4ce4e", size = 174191, upload-time = "2026-04-12T21:43:54.6Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7f/bbc4e74cd33d316b75541149e4d35b163b63bce066530ae185a2ec3b5bfc/msgspec-0.21.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b504b6e7f7a22a24b27232b73034421692147865162daaec9f3bf62439007c87", size = 193131, upload-time = "2026-04-12T21:43:56.094Z" }, + { url = "https://files.pythonhosted.org/packages/c1/60/504886af1aaf854112663b842d5eea9a15d9588f9bf7d0d2df736424b84d/msgspec-0.21.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4692b7c1609155708c4418f88e92f63c13fdf08aa095c84bae82bad75b53389b", size = 186597, upload-time = "2026-04-12T21:43:57.242Z" }, + { url = "https://files.pythonhosted.org/packages/fa/54/d24ddeaa65b5278c9e67f48ce3c17a9831e8f3722f3c8322ee120aca22ef/msgspec-0.21.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3124010b3815451494c85ff345e693cb9fe5889cfcbbef39ed8622e0e72319c", size = 215158, upload-time = "2026-04-12T21:43:58.442Z" }, + { url = "https://files.pythonhosted.org/packages/9f/75/bb79c8b89a93ae23cd33c0d802373f16feaf9633f05d8af77091350dda0a/msgspec-0.21.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6badc03b9725352219cca017bfe71c61f2fbd0fb5982b410ac17c97c213deb30", size = 219856, upload-time = "2026-04-12T21:44:00.015Z" }, + { url = "https://files.pythonhosted.org/packages/b4/9c/c5ca26b46f0ebbd3a6683695ef89396712cb9e4199fd1f0bc1dd968216b1/msgspec-0.21.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5d2d4116ebe3035a78d9ec76e99a9d64e5fa6d44fe61a9c5de7fd1acf54bcc69", size = 220314, upload-time = "2026-04-12T21:44:01.548Z" }, + { url = "https://files.pythonhosted.org/packages/c8/31/645a351c4285dce40ed6755c3dcc0aa648e26dacb20a98018fe2cce5e87b/msgspec-0.21.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0d1009f6715f5bff3b54d4ff5c7428ad96197e0534e1645b8e9b955890c84664", size = 223215, upload-time = "2026-04-12T21:44:02.884Z" }, + { url = "https://files.pythonhosted.org/packages/09/af/8bf15736a6dd3cb4f90c5467f6dc39197d2daaf10754490cdc0aa17b7312/msgspec-0.21.1-cp311-cp311-win_amd64.whl", hash = "sha256:c6faffe5bb644ec884052679af4dfd776d4b5ca90e4a7ec7e7e319e4e6b93a6e", size = 188554, upload-time = "2026-04-12T21:44:04.151Z" }, + { url = "https://files.pythonhosted.org/packages/ef/29/cc7db3a165b62d16e64a83f82eccb79655055cb5bc1f60459a6f9d7c82f2/msgspec-0.21.1-cp311-cp311-win_arm64.whl", hash = "sha256:ee9e3f11fa94603f7d673bf795cfa31b549c4a2c723bc39b45beb1e7f5a3fb99", size = 174517, upload-time = "2026-04-12T21:44:05.66Z" }, + { url = "https://files.pythonhosted.org/packages/6e/cf/317224852c00248c620a9bcf4b26e2e4ab8afd752f18d2a6ef73ebd423b6/msgspec-0.21.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4248cf0b6129b7d230eacd493c17cc2d4f3989f3bb7f633a928a85b7dcfa251", size = 196188, upload-time = "2026-04-12T21:44:07.181Z" }, + { url = "https://files.pythonhosted.org/packages/6d/81/074612945c0666078f7366f40000013de9f6ba687491d450df699bceebc9/msgspec-0.21.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5102c7e9b3acff82178449b85006d96310e690291bb1ea0142f1b24bcb8aabcb", size = 188473, upload-time = "2026-04-12T21:44:08.736Z" }, + { url = "https://files.pythonhosted.org/packages/8a/37/655101799590bcc5fddb2bd3fe0e6194e816c2d1da7c361725f5eb89a910/msgspec-0.21.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:846758412e9518252b2ac9bffd6f0e54d9ff614f5f9488df7749f81ff5c80920", size = 218871, upload-time = "2026-04-12T21:44:09.917Z" }, + { url = "https://files.pythonhosted.org/packages/b5/d1/d4cd9fe89c7d400d7a18f86ccc94daa3f0927f53558846fcb60791dce5d6/msgspec-0.21.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21995e74b5c598c2e004110ad66ec7f1b8c20bf2bcf3b2de8fd9a3094422d3ff", size = 225025, upload-time = "2026-04-12T21:44:11.191Z" }, + { url = "https://files.pythonhosted.org/packages/24/bf/e20549e602b9edccadeeff98760345a416f9cce846a657e8b18e3396b212/msgspec-0.21.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6129f0cca52992e898fd5344187f7c8127b63d810b2fd73e36fca73b4c6475ee", size = 222672, upload-time = "2026-04-12T21:44:12.481Z" }, + { url = "https://files.pythonhosted.org/packages/b4/68/04d7a8f0f786545cf9b8c280c57aa6befb5977af6e884b8b54191cbe44b3/msgspec-0.21.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ef3ec2296248d1f8b9231acb051b6d471dfde8f21819e86c9adaaa9f42918521", size = 227303, upload-time = "2026-04-12T21:44:13.709Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4d/619866af2840875be408047bf9e70ceafbae6ab50660de7134ed1b25eb86/msgspec-0.21.1-cp312-cp312-win_amd64.whl", hash = "sha256:d4ab834a054c6f0cbeef6df9e7e1b33d5f1bc7b86dea1d2fd7cad003873e783d", size = 190017, upload-time = "2026-04-12T21:44:14.977Z" }, + { url = "https://files.pythonhosted.org/packages/5e/2e/a8f9eca8fd00e097d7a9e99ba8a4685db994494448e3d4f0b7f6e9a3c0f7/msgspec-0.21.1-cp312-cp312-win_arm64.whl", hash = "sha256:628aaa35c74950a8c59da330d7e98917e1c7188f983745782027748ee4ca573e", size = 175345, upload-time = "2026-04-12T21:44:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/7e/74/f11ede02839b19ff459f88e3145df5d711626ca84da4e23520cebf819367/msgspec-0.21.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:764173717a01743f007e9f74520ed281f24672c604514f7d76c1c3a10e8edb66", size = 196176, upload-time = "2026-04-12T21:44:17.613Z" }, + { url = "https://files.pythonhosted.org/packages/bb/40/4476c1bd341418a046c4955aff632ec769315d1e3cb94e6acf86d461f9ed/msgspec-0.21.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:344c7cd0eaed1fb81d7959f99100ef71ec9b536881a376f11b9a6c4803365697", size = 188524, upload-time = "2026-04-12T21:44:18.815Z" }, + { url = "https://files.pythonhosted.org/packages/ca/d9/9e9d7d7e5061b47540d03d640fab9b3965ba7ae49c1b2154861c8f007518/msgspec-0.21.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48943e278b3854c2f89f955ddc6f9f430d3f0784b16e47d10604ee0463cd21f5", size = 218880, upload-time = "2026-04-12T21:44:20.028Z" }, + { url = "https://files.pythonhosted.org/packages/74/66/2bb344f34abb4b57e60c7c9c761994e0417b9718ec1460bf00c296f2a7ea/msgspec-0.21.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9aa659ebb0101b1cbc31461212b87e341d961f0ab0772aaf068a99e001ec4aa", size = 225050, upload-time = "2026-04-12T21:44:21.577Z" }, + { url = "https://files.pythonhosted.org/packages/1a/84/7c1e412f76092277bf760cef12b7979d03314d259ab5b5cafde5d0c1722d/msgspec-0.21.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7b27d1a8ead2b6f5b0c4f2d07b8be1ccfcc041c8a0e704781edebe3ae13c484", size = 222713, upload-time = "2026-04-12T21:44:22.83Z" }, + { url = "https://files.pythonhosted.org/packages/4e/27/0bba04b2b4ef05f3d068429410bc71d2cea925f1596a8f41152cccd5edb8/msgspec-0.21.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:38fe93e86b61328fe544cb7fd871fad5a27c8734bfda90f65e5dbe288ae50f61", size = 227259, upload-time = "2026-04-12T21:44:24.11Z" }, + { url = "https://files.pythonhosted.org/packages/b0/2d/09574b0eea02fed2c2c1383dbaae2c7f79dc16dcd6487a886000afb5d7c4/msgspec-0.21.1-cp313-cp313-win_amd64.whl", hash = "sha256:8bc666331c35fcce05a7cd2d6221adbe0f6058f8e750711413d22793c080ac6a", size = 189857, upload-time = "2026-04-12T21:44:25.359Z" }, + { url = "https://files.pythonhosted.org/packages/46/34/105b1576ad182879914f0c821f17ee1d13abb165cb060448f96fe2aff078/msgspec-0.21.1-cp313-cp313-win_arm64.whl", hash = "sha256:42bb1241e0750c1a4346f2aa84db26c5ffd99a4eb3a954927d9f149ff2f42898", size = 175403, upload-time = "2026-04-12T21:44:26.608Z" }, + { url = "https://files.pythonhosted.org/packages/5a/ad/86954e987d1d6a5c579e2c2e7832b65e0fff194179fdac4f581536086024/msgspec-0.21.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fab48eb45fdbfbdb2c0edfec00ffc53b6b6085beefc6b50b61e01659f9f8757f", size = 196261, upload-time = "2026-04-12T21:44:27.807Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a1/c5e46c3e42b866199365e35d11dddfd1fbd8bba4fdb3c52f965b1607ce94/msgspec-0.21.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3cb779ea0c35bc807ff941d415875c1f69ca0be91a2e907ab99a171811d86a9a", size = 188729, upload-time = "2026-04-12T21:44:28.99Z" }, + { url = "https://files.pythonhosted.org/packages/85/7d/1e29a319d678d6cb962ae5bdf32a6858ebdf38f73bc654c0e9c742a0c2c8/msgspec-0.21.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68604db36b3b4dd9bf160e436e12798a4738848144cea1aca1cb984011eb160f", size = 219866, upload-time = "2026-04-12T21:44:31.104Z" }, + { url = "https://files.pythonhosted.org/packages/25/1f/cca084ca2572810fff12ea9dbdcbe39eac048f40daf4a9077b49fcbe8cee/msgspec-0.21.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d6b9dc50948eaf65df54d2fd0ff66e6d8c32f116037209ee861810eb9b676cb", size = 224993, upload-time = "2026-04-12T21:44:32.649Z" }, + { url = "https://files.pythonhosted.org/packages/71/94/d2120fc9d419a89a3a7c13e5b7078798c4b392a96a02a6e2b3ce43a8766c/msgspec-0.21.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:52c5e21930942302394429c5a582ce7e6b62c7f983b3760834c2ce107e0dd6df", size = 223535, upload-time = "2026-04-12T21:44:33.839Z" }, + { url = "https://files.pythonhosted.org/packages/75/17/42418b66a3ad972a89bab73dd78b79cc6282bb488a25e73c853cee7443b9/msgspec-0.21.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:abbb39d65681fa24ed394e01af3d59d869068324f900c61d06062b7fb9980f2f", size = 227222, upload-time = "2026-04-12T21:44:35.093Z" }, + { url = "https://files.pythonhosted.org/packages/c4/33/265c894268cca88ff67b144ca2b4c522fc8b9a6f1966a3640c70516e78e1/msgspec-0.21.1-cp314-cp314-win_amd64.whl", hash = "sha256:5666b1b560b97b6ec2eb3fca8a502298ebac56e13bbca1f88523538ce83d01ea", size = 193810, upload-time = "2026-04-12T21:44:36.612Z" }, + { url = "https://files.pythonhosted.org/packages/3b/8f/a6d35f25bf1fc63c492fdd88fdce01ba0875ead48c2b91f90f33653b4131/msgspec-0.21.1-cp314-cp314-win_arm64.whl", hash = "sha256:d8b8578e4c83b14ceea4cef0d0b747e31d9330fe4b03b2b2ad4063866a178f93", size = 179125, upload-time = "2026-04-12T21:44:38.198Z" }, + { url = "https://files.pythonhosted.org/packages/c6/39/74839641e64b99d87da55af0fc472854d42b46e2183b9e2a67fe1bb2a512/msgspec-0.21.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:15f523d51c00ebad412213bfe9f06f0a50ec2b93e0c19e824a2d267cabb48ea2", size = 200171, upload-time = "2026-04-12T21:44:39.414Z" }, + { url = "https://files.pythonhosted.org/packages/70/9b/ce0cca6d2d87fcd4b6ff97600790494e64f26a2c55d61507cd2755c16193/msgspec-0.21.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e47390360583ba3d5c6cb44cf0a9f61b0a06a899d3c2c00627cedebb2e2884b", size = 192879, upload-time = "2026-04-12T21:44:40.882Z" }, + { url = "https://files.pythonhosted.org/packages/a7/08/673a7bb05e5702dc787ddd3011195b509f9867927970da59052211929987/msgspec-0.21.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f60800e6299b798142dc40b0644da77ceac5ea0568be58228417eae14135c847", size = 226281, upload-time = "2026-04-12T21:44:42.181Z" }, + { url = "https://files.pythonhosted.org/packages/7d/45/86508cf57283e9070b3c447e3ab25b792a7a0855a3ea4e0c6d111ac34c97/msgspec-0.21.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f8e9dfcd98419cf7568808470c4317a3fb30bef0e3715b568730a2b272a20d7", size = 229863, upload-time = "2026-04-12T21:44:43.442Z" }, + { url = "https://files.pythonhosted.org/packages/2c/62/e7c9367cd08d590559faacd711edbae36840342843e669440363f33c7d36/msgspec-0.21.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:92d89dfad13bd1ea640dc3e37e724ed380da1030b272bdf5ecafb983c3ad7c75", size = 230445, upload-time = "2026-04-12T21:44:44.806Z" }, + { url = "https://files.pythonhosted.org/packages/42/b4/c0f54632103846b658a10930025f4de41c8724b5e4805a5f3b395586cb7e/msgspec-0.21.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0d03867786e5d7ba25d666df4b11320c27170f4aeafcb8e3a8b0a50a4fb742ca", size = 231822, upload-time = "2026-04-12T21:44:46.343Z" }, + { url = "https://files.pythonhosted.org/packages/ea/1d/0d85cc79d0ccf5508e9c846cc66552a6a16bf92abd1dbd8362617f7b35cd/msgspec-0.21.1-cp314-cp314t-win_amd64.whl", hash = "sha256:740fbf1c9d59992ca3537d6fbe9ebbf9eaf726a65fbf31448e0ecbc710697a63", size = 206650, upload-time = "2026-04-12T21:44:47.601Z" }, + { url = "https://files.pythonhosted.org/packages/90/91/56c5d560f20e6c20e9e4f55bd0e458f7f162aa689ee350346c04c48eac0b/msgspec-0.21.1-cp314-cp314t-win_arm64.whl", hash = "sha256:0d2cc73df6058d811a126ac3a8ad63a4dfa210c82f9cf5a004802eaf4712de90", size = 183149, upload-time = "2026-04-12T21:44:48.833Z" }, +] + [[package]] name = "multidict" version = "6.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3d/2c/5dad12e82fbdf7470f29bff2171484bf07cb3b16ada60a6589af8f376440/multidict-6.6.3.tar.gz", hash = "sha256:798a9eb12dab0a6c2e29c1de6f3468af5cb2da6053a20dfa3344907eed0937cc", size = 101006, upload-time = "2025-06-30T15:53:46.929Z" } wheels = [ @@ -3733,7 +5007,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mypy-extensions" }, { name = "pathspec" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1e/e3/034322d5a779685218ed69286c32faa505247f1f096251ef66c8fd203b08/mypy-1.17.0.tar.gz", hash = "sha256:e5d7ccc08ba089c06e2f5629c660388ef1fee708444f1dee0b9203fa031dee03", size = 3352114, upload-time = "2025-07-14T20:34:30.181Z" } @@ -3926,8 +5200,11 @@ name = "networkx" version = "3.4.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", + "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", ] sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368, upload-time = "2024-10-21T12:39:38.695Z" } wheels = [ @@ -3939,14 +5216,27 @@ name = "networkx" version = "3.5" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.12.*' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", - "python_full_version >= '3.14' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and sys_platform == 'linux'", - "python_full_version >= '3.14' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.13' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", ] sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } wheels = [ @@ -3980,259 +5270,718 @@ wheels = [ ] [[package]] -name = "nltk" -version = "3.9.4" +name = "nltk" +version = "3.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "joblib" }, + { name = "regex" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/a1/b3b4adf15585a5bc4c357adde150c01ebeeb642173ded4d871e89468767c/nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0", size = 2946864, upload-time = "2026-03-24T06:13:40.641Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f", size = 1552087, upload-time = "2026-03-24T06:13:38.47Z" }, +] + +[[package]] +name = "notebook" +version = "6.5.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi" }, + { name = "ipykernel" }, + { name = "ipython-genutils" }, + { name = "jinja2" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "nbclassic" }, + { name = "nbconvert" }, + { name = "nbformat" }, + { name = "nest-asyncio" }, + { name = "prometheus-client" }, + { name = "pyzmq" }, + { name = "send2trash" }, + { name = "terminado" }, + { name = "tornado" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/05/bc/b025bac523640c13b0c1150466475eac3d79acbf187ef6c1967596c41c43/notebook-6.5.7.tar.gz", hash = "sha256:04eb9011dfac634fbd4442adaf0a8c27cd26beef831fe1d19faf930c327768e4", size = 5786526, upload-time = "2024-05-01T17:42:26.956Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/9c/0620631da9d7013e95a8f985043cad229a0d8fb537a7e3f8ff8467565a8c/notebook-6.5.7-py3-none-any.whl", hash = "sha256:a6afa9a4ff4d149a0771ff8b8c881a7a73b3835f9add0606696d6e9d98ac1cd0", size = 529829, upload-time = "2024-05-01T17:42:22.403Z" }, +] + +[[package]] +name = "notebook-shim" +version = "0.2.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-server" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/54/d2/92fa3243712b9a3e8bafaf60aac366da1cada3639ca767ff4b5b3654ec28/notebook_shim-0.2.4.tar.gz", hash = "sha256:b4b2cfa1b65d98307ca24361f5b30fe785b53c3fd07b7a47e89acb5e6ac638cb", size = 13167, upload-time = "2024-02-14T23:35:18.353Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/33/bd5b9137445ea4b680023eb0469b2bb969d61303dedb2aac6560ff3d14a1/notebook_shim-0.2.4-py3-none-any.whl", hash = "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef", size = 13307, upload-time = "2024-02-14T23:35:16.286Z" }, +] + +[[package]] +name = "numba" +version = "0.65.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llvmlite", marker = "(sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/61/7299643b9c18d669e04be7c5bcb64d985070d07553274817b45b049e7bfe/numba-0.65.0.tar.gz", hash = "sha256:edad0d9f6682e93624c00125a471ae4df186175d71fd604c983c377cdc03e68b", size = 2764131, upload-time = "2026-04-01T03:52:01.946Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/9b/e8453d93d5cb3f53cc956f135024be09d52f4f99643acaf8fdca090a8f3c/numba-0.65.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:dff9fd5fbc9a35c517359c5823ea705d9b65f01fb46e42e35a2eabe5a52c2e96", size = 2680537, upload-time = "2026-04-01T03:51:17.325Z" }, + { url = "https://files.pythonhosted.org/packages/07/95/d6a2f0625e1092624228301eea11cdaff21ddcaf917ef3d631846a38b2f4/numba-0.65.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4c894c94afa5ffd627c7e3b693df10cb0d905bd5eb06de3dfc31775140cf4f89", size = 3739444, upload-time = "2026-04-01T03:51:19.629Z" }, + { url = "https://files.pythonhosted.org/packages/49/ed/fe518c97af035e4ec670c2edc3f0ff7a518cbed2f0b5053124d7c979bd8a/numba-0.65.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7325b1aab88f0339057288ee32f39dc660e14f93872a6fda14fa6eb9f95b047", size = 3446390, upload-time = "2026-04-01T03:51:21.55Z" }, + { url = "https://files.pythonhosted.org/packages/d0/06/5010939854249c290c6217e3fb7404914f4ed953f9923e340c3e166bcaf0/numba-0.65.0-cp310-cp310-win_amd64.whl", hash = "sha256:71e72e9ca2f619df4768f9c3962bfec60191a5a26fe2b6a8c6a07532b6146169", size = 2747200, upload-time = "2026-04-01T03:51:23.674Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ce/d67c499703eb5479ce02420e8ccd65c5753d87d2e16d563f152d71405346/numba-0.65.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:28e547d0b18024f19cbaf9de02fc5c145790213d9be8a2c95b43f93ec162b9e4", size = 2680228, upload-time = "2026-04-01T03:51:25.401Z" }, + { url = "https://files.pythonhosted.org/packages/c1/a7/11e2b24251d57cf41fc9ad83f378d890d61a890e3f8eb6338b39833f67a4/numba-0.65.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:032b0b8e879512cd424d79eed6d772a1399c6387ded184c2cf3cc22c08d750a6", size = 3744674, upload-time = "2026-04-01T03:51:27.311Z" }, + { url = "https://files.pythonhosted.org/packages/fe/0b/7c63eb742859a6243f42288441f65ac9dac96ea59f409e43b713aafbe867/numba-0.65.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af143d823624033a128b5950c0aaf9ffc2386dfe954eb757119cf0432335534c", size = 3450620, upload-time = "2026-04-01T03:51:29.092Z" }, + { url = "https://files.pythonhosted.org/packages/53/ff/1371cbbe955be340a46093a10b61462437e0fadc7a63290473a0e584cb03/numba-0.65.0-cp311-cp311-win_amd64.whl", hash = "sha256:15d159578e59a39df246b83480f78d7794b0fca40153b5684d3849a99c48a0fb", size = 2747081, upload-time = "2026-04-01T03:51:30.785Z" }, + { url = "https://files.pythonhosted.org/packages/6c/2f/8bd31a1ea43c01ac215283d83aa5f8d5acbe7a36c85b82f1757bfe9ccb31/numba-0.65.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:b27ee4847e1bfb17e9604d100417ee7c1d10f15a6711c6213404b3da13a0b2aa", size = 2680705, upload-time = "2026-04-01T03:51:32.597Z" }, + { url = "https://files.pythonhosted.org/packages/73/36/88406bd58600cc696417b8e5dd6a056478da808f3eaf48d18e2421e0c2d9/numba-0.65.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a52d92ffd297c10364bce60cd1fcb88f99284ab5df085f2c6bcd1cb33b529a6f", size = 3801411, upload-time = "2026-04-01T03:51:34.321Z" }, + { url = "https://files.pythonhosted.org/packages/0c/61/ce753a1d7646dd477e16d15e89473703faebb8995d2f71d7ad69a540b565/numba-0.65.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da8e371e328c06d0010c3d8b44b21858652831b85bcfba78cb22c042e22dbd8e", size = 3501622, upload-time = "2026-04-01T03:51:36.348Z" }, + { url = "https://files.pythonhosted.org/packages/7d/86/db87a5393f1b1fabef53ac3ba4e6b938bb27e40a04ad7cc512098fcae032/numba-0.65.0-cp312-cp312-win_amd64.whl", hash = "sha256:59bb9f2bb9f1238dfd8e927ba50645c18ae769fef4f3d58ea0ea22a2683b91f5", size = 2749979, upload-time = "2026-04-01T03:51:37.88Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f8/eee0f1ff456218db036bfc9023995ec1f85a9dc8f2422f1594f6a87829e0/numba-0.65.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:c6334094563a456a695c812e6846288376ca02327cf246cdcc83e1bb27862367", size = 2680679, upload-time = "2026-04-01T03:51:39.491Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8f/3d116e4b8e92f6abace431afa4b2b944f4d65bdee83af886f5c4b263df95/numba-0.65.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b8a9008411615c69d083d1dcf477f75a5aa727b30beb16e139799e2be945cdfd", size = 3809537, upload-time = "2026-04-01T03:51:41.42Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2c/6a3ca4128e253cb67affe06deb47688f51ce968f5111e2a06d010e6f1fa6/numba-0.65.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af96c0cba53664efcb361528b8c75e011a6556c859c7e08424c2715201c6cf7a", size = 3508615, upload-time = "2026-04-01T03:51:43.444Z" }, + { url = "https://files.pythonhosted.org/packages/96/0e/267f9a36fb282c104a971d7eecb685b411c47dce2a740fe69cf5fc2945d9/numba-0.65.0-cp313-cp313-win_amd64.whl", hash = "sha256:6254e73b9c929dc736a1fbd3d6f5680789709a5067cae1fa7198707385129c04", size = 2749938, upload-time = "2026-04-01T03:51:45.218Z" }, + { url = "https://files.pythonhosted.org/packages/56/a4/90edb01e9176053578e343d7a7276bc28356741ee67059aed8ed2c1a4e59/numba-0.65.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:ee336b398a6fca51b1f626034de99f50cb1bd87d537a166275158a3cee744b82", size = 2680878, upload-time = "2026-04-01T03:51:46.91Z" }, + { url = "https://files.pythonhosted.org/packages/24/8d/e12d6ff4b9119db3cbf7b2db1ce257576441bd3c76388c786dea74f20b02/numba-0.65.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:05c0a9fdf75d85f57dee47b719e8d6415707b80aae45d75f63f9dc1b935c29f7", size = 3778456, upload-time = "2026-04-01T03:51:48.552Z" }, + { url = "https://files.pythonhosted.org/packages/17/89/abcd83e76f6a773276fe76244140671bcc5bf820f6e2ae1a15362ae4c8c9/numba-0.65.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:583680e0e8faf124d362df23b4b593f3221a8996341a63d1b664c122401bec2f", size = 3478464, upload-time = "2026-04-01T03:51:50.527Z" }, + { url = "https://files.pythonhosted.org/packages/73/5b/fbce55ce3d933afbc7ade04df826853e4a846aaa47d58d2fbb669b8f2d08/numba-0.65.0-cp314-cp314-win_amd64.whl", hash = "sha256:add297d3e1c08dd884f44100152612fa41e66a51d15fdf91307f9dde31d06830", size = 2752012, upload-time = "2026-04-01T03:51:52.691Z" }, + { url = "https://files.pythonhosted.org/packages/1e/ab/af705f4257d9388fb2fd6d7416573e98b6ca9c786e8b58f02720978557bd/numba-0.65.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:194a243ba53a9157c8538cbb3166ec015d785a8c5d584d06cdd88bee902233c7", size = 2683961, upload-time = "2026-04-01T03:51:54.281Z" }, + { url = "https://files.pythonhosted.org/packages/ff/e5/8267b0adb0c01b52b553df5062fbbb42c30ed5362d08b85cc913a36f838f/numba-0.65.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c7fa502960f7a2f3f5cb025bc7bff888a3551277b92431bfdc5ba2f11a375749", size = 3816373, upload-time = "2026-04-01T03:51:56.18Z" }, + { url = "https://files.pythonhosted.org/packages/b0/f5/b8397ca360971669a93706b9274592b6864e4367a37d498fbbcb62aa2d48/numba-0.65.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5046c63f783ca3eb6195f826a50797465e7c4ce811daa17c9bea47e310c9b964", size = 3532782, upload-time = "2026-04-01T03:51:58.387Z" }, + { url = "https://files.pythonhosted.org/packages/f5/21/1e73fa16bf0393ebb74c5bb208d712152ffdfc84600a8e93a3180317856e/numba-0.65.0-cp314-cp314t-win_amd64.whl", hash = "sha256:46fd679ae4f68c7a5d5721efbd29ecee0b0f3013211591891d79b51bfdf73113", size = 2757611, upload-time = "2026-04-01T03:52:00.083Z" }, +] + +[[package]] +name = "numpy" +version = "1.26.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", + "python_full_version < '3.11'", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/94/ace0fdea5241a27d13543ee117cbc65868e82213fb31a8eb7fe9ff23f313/numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0", size = 20631468, upload-time = "2024-02-05T23:48:01.194Z" }, + { url = "https://files.pythonhosted.org/packages/20/f7/b24208eba89f9d1b58c1668bc6c8c4fd472b20c45573cb767f59d49fb0f6/numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a", size = 13966411, upload-time = "2024-02-05T23:48:29.038Z" }, + { url = "https://files.pythonhosted.org/packages/fc/a5/4beee6488160798683eed5bdb7eead455892c3b4e1f78d79d8d3f3b084ac/numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4", size = 14219016, upload-time = "2024-02-05T23:48:54.098Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d7/ecf66c1cd12dc28b4040b15ab4d17b773b87fa9d29ca16125de01adb36cd/numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f", size = 18240889, upload-time = "2024-02-05T23:49:25.361Z" }, + { url = "https://files.pythonhosted.org/packages/24/03/6f229fe3187546435c4f6f89f6d26c129d4f5bed40552899fcf1f0bf9e50/numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a", size = 13876746, upload-time = "2024-02-05T23:49:51.983Z" }, + { url = "https://files.pythonhosted.org/packages/39/fe/39ada9b094f01f5a35486577c848fe274e374bbf8d8f472e1423a0bbd26d/numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2", size = 18078620, upload-time = "2024-02-05T23:50:22.515Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ef/6ad11d51197aad206a9ad2286dc1aac6a378059e06e8cf22cd08ed4f20dc/numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07", size = 5972659, upload-time = "2024-02-05T23:50:35.834Z" }, + { url = "https://files.pythonhosted.org/packages/19/77/538f202862b9183f54108557bfda67e17603fc560c384559e769321c9d92/numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5", size = 15808905, upload-time = "2024-02-05T23:51:03.701Z" }, + { url = "https://files.pythonhosted.org/packages/11/57/baae43d14fe163fa0e4c47f307b6b2511ab8d7d30177c491960504252053/numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71", size = 20630554, upload-time = "2024-02-05T23:51:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/1a/2e/151484f49fd03944c4a3ad9c418ed193cfd02724e138ac8a9505d056c582/numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef", size = 13997127, upload-time = "2024-02-05T23:52:15.314Z" }, + { url = "https://files.pythonhosted.org/packages/79/ae/7e5b85136806f9dadf4878bf73cf223fe5c2636818ba3ab1c585d0403164/numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e", size = 14222994, upload-time = "2024-02-05T23:52:47.569Z" }, + { url = "https://files.pythonhosted.org/packages/3a/d0/edc009c27b406c4f9cbc79274d6e46d634d139075492ad055e3d68445925/numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5", size = 18252005, upload-time = "2024-02-05T23:53:15.637Z" }, + { url = "https://files.pythonhosted.org/packages/09/bf/2b1aaf8f525f2923ff6cfcf134ae5e750e279ac65ebf386c75a0cf6da06a/numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a", size = 13885297, upload-time = "2024-02-05T23:53:42.16Z" }, + { url = "https://files.pythonhosted.org/packages/df/a0/4e0f14d847cfc2a633a1c8621d00724f3206cfeddeb66d35698c4e2cf3d2/numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a", size = 18093567, upload-time = "2024-02-05T23:54:11.696Z" }, + { url = "https://files.pythonhosted.org/packages/d2/b7/a734c733286e10a7f1a8ad1ae8c90f2d33bf604a96548e0a4a3a6739b468/numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20", size = 5968812, upload-time = "2024-02-05T23:54:26.453Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6b/5610004206cf7f8e7ad91c5a85a8c71b2f2f8051a0c0c4d5916b76d6cbb2/numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2", size = 15811913, upload-time = "2024-02-05T23:54:53.933Z" }, + { url = "https://files.pythonhosted.org/packages/95/12/8f2020a8e8b8383ac0177dc9570aad031a3beb12e38847f7129bacd96228/numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", size = 20335901, upload-time = "2024-02-05T23:55:32.801Z" }, + { url = "https://files.pythonhosted.org/packages/75/5b/ca6c8bd14007e5ca171c7c03102d17b4f4e0ceb53957e8c44343a9546dcc/numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", size = 13685868, upload-time = "2024-02-05T23:55:56.28Z" }, + { url = "https://files.pythonhosted.org/packages/79/f8/97f10e6755e2a7d027ca783f63044d5b1bc1ae7acb12afe6a9b4286eac17/numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", size = 13925109, upload-time = "2024-02-05T23:56:20.368Z" }, + { url = "https://files.pythonhosted.org/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", size = 17950613, upload-time = "2024-02-05T23:56:56.054Z" }, + { url = "https://files.pythonhosted.org/packages/4c/0c/9c603826b6465e82591e05ca230dfc13376da512b25ccd0894709b054ed0/numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", size = 13572172, upload-time = "2024-02-05T23:57:21.56Z" }, + { url = "https://files.pythonhosted.org/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", size = 17786643, upload-time = "2024-02-05T23:57:56.585Z" }, + { url = "https://files.pythonhosted.org/packages/28/4a/46d9e65106879492374999e76eb85f87b15328e06bd1550668f79f7b18c6/numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", size = 5677803, upload-time = "2024-02-05T23:58:08.963Z" }, + { url = "https://files.pythonhosted.org/packages/16/2e/86f24451c2d530c88daf997cb8d6ac622c1d40d19f5a031ed68a4b73a374/numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", size = 15517754, upload-time = "2024-02-05T23:58:36.364Z" }, +] + +[[package]] +name = "numpy" +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", +] +sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3e/ed6db5be21ce87955c0cbd3009f2803f59fa08df21b5df06862e2d8e2bdd/numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb", size = 21165245, upload-time = "2025-05-17T21:27:58.555Z" }, + { url = "https://files.pythonhosted.org/packages/22/c2/4b9221495b2a132cc9d2eb862e21d42a009f5a60e45fc44b00118c174bff/numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90", size = 14360048, upload-time = "2025-05-17T21:28:21.406Z" }, + { url = "https://files.pythonhosted.org/packages/fd/77/dc2fcfc66943c6410e2bf598062f5959372735ffda175b39906d54f02349/numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163", size = 5340542, upload-time = "2025-05-17T21:28:30.931Z" }, + { url = "https://files.pythonhosted.org/packages/7a/4f/1cb5fdc353a5f5cc7feb692db9b8ec2c3d6405453f982435efc52561df58/numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf", size = 6878301, upload-time = "2025-05-17T21:28:41.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/17/96a3acd228cec142fcb8723bd3cc39c2a474f7dcf0a5d16731980bcafa95/numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83", size = 14297320, upload-time = "2025-05-17T21:29:02.78Z" }, + { url = "https://files.pythonhosted.org/packages/b4/63/3de6a34ad7ad6646ac7d2f55ebc6ad439dbbf9c4370017c50cf403fb19b5/numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915", size = 16801050, upload-time = "2025-05-17T21:29:27.675Z" }, + { url = "https://files.pythonhosted.org/packages/07/b6/89d837eddef52b3d0cec5c6ba0456c1bf1b9ef6a6672fc2b7873c3ec4e2e/numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680", size = 15807034, upload-time = "2025-05-17T21:29:51.102Z" }, + { url = "https://files.pythonhosted.org/packages/01/c8/dc6ae86e3c61cfec1f178e5c9f7858584049b6093f843bca541f94120920/numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289", size = 18614185, upload-time = "2025-05-17T21:30:18.703Z" }, + { url = "https://files.pythonhosted.org/packages/5b/c5/0064b1b7e7c89137b471ccec1fd2282fceaae0ab3a9550f2568782d80357/numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d", size = 6527149, upload-time = "2025-05-17T21:30:29.788Z" }, + { url = "https://files.pythonhosted.org/packages/a3/dd/4b822569d6b96c39d1215dbae0582fd99954dcbcf0c1a13c61783feaca3f/numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3", size = 12904620, upload-time = "2025-05-17T21:30:48.994Z" }, + { url = "https://files.pythonhosted.org/packages/da/a8/4f83e2aa666a9fbf56d6118faaaf5f1974d456b1823fda0a176eff722839/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae", size = 21176963, upload-time = "2025-05-17T21:31:19.36Z" }, + { url = "https://files.pythonhosted.org/packages/b3/2b/64e1affc7972decb74c9e29e5649fac940514910960ba25cd9af4488b66c/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a", size = 14406743, upload-time = "2025-05-17T21:31:41.087Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/0121e375000b5e50ffdd8b25bf78d8e1a5aa4cca3f185d41265198c7b834/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42", size = 5352616, upload-time = "2025-05-17T21:31:50.072Z" }, + { url = "https://files.pythonhosted.org/packages/31/0d/b48c405c91693635fbe2dcd7bc84a33a602add5f63286e024d3b6741411c/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491", size = 6889579, upload-time = "2025-05-17T21:32:01.712Z" }, + { url = "https://files.pythonhosted.org/packages/52/b8/7f0554d49b565d0171eab6e99001846882000883998e7b7d9f0d98b1f934/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a", size = 14312005, upload-time = "2025-05-17T21:32:23.332Z" }, + { url = "https://files.pythonhosted.org/packages/b3/dd/2238b898e51bd6d389b7389ffb20d7f4c10066d80351187ec8e303a5a475/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf", size = 16821570, upload-time = "2025-05-17T21:32:47.991Z" }, + { url = "https://files.pythonhosted.org/packages/83/6c/44d0325722cf644f191042bf47eedad61c1e6df2432ed65cbe28509d404e/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1", size = 15818548, upload-time = "2025-05-17T21:33:11.728Z" }, + { url = "https://files.pythonhosted.org/packages/ae/9d/81e8216030ce66be25279098789b665d49ff19eef08bfa8cb96d4957f422/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab", size = 18620521, upload-time = "2025-05-17T21:33:39.139Z" }, + { url = "https://files.pythonhosted.org/packages/6a/fd/e19617b9530b031db51b0926eed5345ce8ddc669bb3bc0044b23e275ebe8/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47", size = 6525866, upload-time = "2025-05-17T21:33:50.273Z" }, + { url = "https://files.pythonhosted.org/packages/31/0a/f354fb7176b81747d870f7991dc763e157a934c717b67b58456bc63da3df/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303", size = 12907455, upload-time = "2025-05-17T21:34:09.135Z" }, + { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348, upload-time = "2025-05-17T21:34:39.648Z" }, + { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362, upload-time = "2025-05-17T21:35:01.241Z" }, + { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103, upload-time = "2025-05-17T21:35:10.622Z" }, + { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382, upload-time = "2025-05-17T21:35:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462, upload-time = "2025-05-17T21:35:42.174Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618, upload-time = "2025-05-17T21:36:06.711Z" }, + { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511, upload-time = "2025-05-17T21:36:29.965Z" }, + { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783, upload-time = "2025-05-17T21:36:56.883Z" }, + { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506, upload-time = "2025-05-17T21:37:07.368Z" }, + { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190, upload-time = "2025-05-17T21:37:26.213Z" }, + { url = "https://files.pythonhosted.org/packages/f9/5c/6657823f4f594f72b5471f1db1ab12e26e890bb2e41897522d134d2a3e81/numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84", size = 20867828, upload-time = "2025-05-17T21:37:56.699Z" }, + { url = "https://files.pythonhosted.org/packages/dc/9e/14520dc3dadf3c803473bd07e9b2bd1b69bc583cb2497b47000fed2fa92f/numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b", size = 14143006, upload-time = "2025-05-17T21:38:18.291Z" }, + { url = "https://files.pythonhosted.org/packages/4f/06/7e96c57d90bebdce9918412087fc22ca9851cceaf5567a45c1f404480e9e/numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d", size = 5076765, upload-time = "2025-05-17T21:38:27.319Z" }, + { url = "https://files.pythonhosted.org/packages/73/ed/63d920c23b4289fdac96ddbdd6132e9427790977d5457cd132f18e76eae0/numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566", size = 6617736, upload-time = "2025-05-17T21:38:38.141Z" }, + { url = "https://files.pythonhosted.org/packages/85/c5/e19c8f99d83fd377ec8c7e0cf627a8049746da54afc24ef0a0cb73d5dfb5/numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f", size = 14010719, upload-time = "2025-05-17T21:38:58.433Z" }, + { url = "https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f", size = 16526072, upload-time = "2025-05-17T21:39:22.638Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6c/04b5f47f4f32f7c2b0e7260442a8cbcf8168b0e1a41ff1495da42f42a14f/numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868", size = 15503213, upload-time = "2025-05-17T21:39:45.865Z" }, + { url = "https://files.pythonhosted.org/packages/17/0a/5cd92e352c1307640d5b6fec1b2ffb06cd0dabe7d7b8227f97933d378422/numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d", size = 18316632, upload-time = "2025-05-17T21:40:13.331Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3b/5cba2b1d88760ef86596ad0f3d484b1cbff7c115ae2429678465057c5155/numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd", size = 6244532, upload-time = "2025-05-17T21:43:46.099Z" }, + { url = "https://files.pythonhosted.org/packages/cb/3b/d58c12eafcb298d4e6d0d40216866ab15f59e55d148a5658bb3132311fcf/numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c", size = 12610885, upload-time = "2025-05-17T21:44:05.145Z" }, + { url = "https://files.pythonhosted.org/packages/6b/9e/4bf918b818e516322db999ac25d00c75788ddfd2d2ade4fa66f1f38097e1/numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6", size = 20963467, upload-time = "2025-05-17T21:40:44Z" }, + { url = "https://files.pythonhosted.org/packages/61/66/d2de6b291507517ff2e438e13ff7b1e2cdbdb7cb40b3ed475377aece69f9/numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda", size = 14225144, upload-time = "2025-05-17T21:41:05.695Z" }, + { url = "https://files.pythonhosted.org/packages/e4/25/480387655407ead912e28ba3a820bc69af9adf13bcbe40b299d454ec011f/numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40", size = 5200217, upload-time = "2025-05-17T21:41:15.903Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4a/6e313b5108f53dcbf3aca0c0f3e9c92f4c10ce57a0a721851f9785872895/numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8", size = 6712014, upload-time = "2025-05-17T21:41:27.321Z" }, + { url = "https://files.pythonhosted.org/packages/b7/30/172c2d5c4be71fdf476e9de553443cf8e25feddbe185e0bd88b096915bcc/numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f", size = 14077935, upload-time = "2025-05-17T21:41:49.738Z" }, + { url = "https://files.pythonhosted.org/packages/12/fb/9e743f8d4e4d3c710902cf87af3512082ae3d43b945d5d16563f26ec251d/numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa", size = 16600122, upload-time = "2025-05-17T21:42:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/12/75/ee20da0e58d3a66f204f38916757e01e33a9737d0b22373b3eb5a27358f9/numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571", size = 15586143, upload-time = "2025-05-17T21:42:37.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/95/bef5b37f29fc5e739947e9ce5179ad402875633308504a52d188302319c8/numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1", size = 18385260, upload-time = "2025-05-17T21:43:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/09/04/f2f83279d287407cf36a7a8053a5abe7be3622a4363337338f2585e4afda/numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff", size = 6377225, upload-time = "2025-05-17T21:43:16.254Z" }, + { url = "https://files.pythonhosted.org/packages/67/0e/35082d13c09c02c011cf21570543d202ad929d961c02a147493cb0c2bdf5/numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06", size = 12771374, upload-time = "2025-05-17T21:43:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/9e/3b/d94a75f4dbf1ef5d321523ecac21ef23a3cd2ac8b78ae2aac40873590229/numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d", size = 21040391, upload-time = "2025-05-17T21:44:35.948Z" }, + { url = "https://files.pythonhosted.org/packages/17/f4/09b2fa1b58f0fb4f7c7963a1649c64c4d315752240377ed74d9cd878f7b5/numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db", size = 6786754, upload-time = "2025-05-17T21:44:47.446Z" }, + { url = "https://files.pythonhosted.org/packages/af/30/feba75f143bdc868a1cc3f44ccfa6c4b9ec522b36458e738cd00f67b573f/numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543", size = 16643476, upload-time = "2025-05-17T21:45:11.871Z" }, + { url = "https://files.pythonhosted.org/packages/37/48/ac2a9584402fb6c0cd5b5d1a91dcf176b15760130dd386bbafdbfe3640bf/numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00", size = 12812666, upload-time = "2025-05-17T21:45:31.426Z" }, +] + +[[package]] +name = "numpy" +version = "2.3.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.13' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", +] +sdist = { url = "https://files.pythonhosted.org/packages/76/65/21b3bc86aac7b8f2862db1e808f1ea22b028e30a225a34a5ede9bf8678f2/numpy-2.3.5.tar.gz", hash = "sha256:784db1dcdab56bf0517743e746dfb0f885fc68d948aba86eeec2cba234bdf1c0", size = 20584950, upload-time = "2025-11-16T22:52:42.067Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/77/84dd1d2e34d7e2792a236ba180b5e8fcc1e3e414e761ce0253f63d7f572e/numpy-2.3.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:de5672f4a7b200c15a4127042170a694d4df43c992948f5e1af57f0174beed10", size = 17034641, upload-time = "2025-11-16T22:49:19.336Z" }, + { url = "https://files.pythonhosted.org/packages/2a/ea/25e26fa5837106cde46ae7d0b667e20f69cbbc0efd64cba8221411ab26ae/numpy-2.3.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:acfd89508504a19ed06ef963ad544ec6664518c863436306153e13e94605c218", size = 12528324, upload-time = "2025-11-16T22:49:22.582Z" }, + { url = "https://files.pythonhosted.org/packages/4d/1a/e85f0eea4cf03d6a0228f5c0256b53f2df4bc794706e7df019fc622e47f1/numpy-2.3.5-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:ffe22d2b05504f786c867c8395de703937f934272eb67586817b46188b4ded6d", size = 5356872, upload-time = "2025-11-16T22:49:25.408Z" }, + { url = "https://files.pythonhosted.org/packages/5c/bb/35ef04afd567f4c989c2060cde39211e4ac5357155c1833bcd1166055c61/numpy-2.3.5-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:872a5cf366aec6bb1147336480fef14c9164b154aeb6542327de4970282cd2f5", size = 6893148, upload-time = "2025-11-16T22:49:27.549Z" }, + { url = "https://files.pythonhosted.org/packages/f2/2b/05bbeb06e2dff5eab512dfc678b1cc5ee94d8ac5956a0885c64b6b26252b/numpy-2.3.5-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3095bdb8dd297e5920b010e96134ed91d852d81d490e787beca7e35ae1d89cf7", size = 14557282, upload-time = "2025-11-16T22:49:30.964Z" }, + { url = "https://files.pythonhosted.org/packages/65/fb/2b23769462b34398d9326081fad5655198fcf18966fcb1f1e49db44fbf31/numpy-2.3.5-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8cba086a43d54ca804ce711b2a940b16e452807acebe7852ff327f1ecd49b0d4", size = 16897903, upload-time = "2025-11-16T22:49:34.191Z" }, + { url = "https://files.pythonhosted.org/packages/ac/14/085f4cf05fc3f1e8aa95e85404e984ffca9b2275a5dc2b1aae18a67538b8/numpy-2.3.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6cf9b429b21df6b99f4dee7a1218b8b7ffbbe7df8764dc0bd60ce8a0708fed1e", size = 16341672, upload-time = "2025-11-16T22:49:37.2Z" }, + { url = "https://files.pythonhosted.org/packages/6f/3b/1f73994904142b2aa290449b3bb99772477b5fd94d787093e4f24f5af763/numpy-2.3.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:396084a36abdb603546b119d96528c2f6263921c50df3c8fd7cb28873a237748", size = 18838896, upload-time = "2025-11-16T22:49:39.727Z" }, + { url = "https://files.pythonhosted.org/packages/cd/b9/cf6649b2124f288309ffc353070792caf42ad69047dcc60da85ee85fea58/numpy-2.3.5-cp311-cp311-win32.whl", hash = "sha256:b0c7088a73aef3d687c4deef8452a3ac7c1be4e29ed8bf3b366c8111128ac60c", size = 6563608, upload-time = "2025-11-16T22:49:42.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/44/9fe81ae1dcc29c531843852e2874080dc441338574ccc4306b39e2ff6e59/numpy-2.3.5-cp311-cp311-win_amd64.whl", hash = "sha256:a414504bef8945eae5f2d7cb7be2d4af77c5d1cb5e20b296c2c25b61dff2900c", size = 13078442, upload-time = "2025-11-16T22:49:43.99Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a7/f99a41553d2da82a20a2f22e93c94f928e4490bb447c9ff3c4ff230581d3/numpy-2.3.5-cp311-cp311-win_arm64.whl", hash = "sha256:0cd00b7b36e35398fa2d16af7b907b65304ef8bb4817a550e06e5012929830fa", size = 10458555, upload-time = "2025-11-16T22:49:47.092Z" }, + { url = "https://files.pythonhosted.org/packages/44/37/e669fe6cbb2b96c62f6bbedc6a81c0f3b7362f6a59230b23caa673a85721/numpy-2.3.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:74ae7b798248fe62021dbf3c914245ad45d1a6b0cb4a29ecb4b31d0bfbc4cc3e", size = 16733873, upload-time = "2025-11-16T22:49:49.84Z" }, + { url = "https://files.pythonhosted.org/packages/c5/65/df0db6c097892c9380851ab9e44b52d4f7ba576b833996e0080181c0c439/numpy-2.3.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee3888d9ff7c14604052b2ca5535a30216aa0a58e948cdd3eeb8d3415f638769", size = 12259838, upload-time = "2025-11-16T22:49:52.863Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e1/1ee06e70eb2136797abe847d386e7c0e830b67ad1d43f364dd04fa50d338/numpy-2.3.5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:612a95a17655e213502f60cfb9bf9408efdc9eb1d5f50535cc6eb365d11b42b5", size = 5088378, upload-time = "2025-11-16T22:49:55.055Z" }, + { url = "https://files.pythonhosted.org/packages/6d/9c/1ca85fb86708724275103b81ec4cf1ac1d08f465368acfc8da7ab545bdae/numpy-2.3.5-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:3101e5177d114a593d79dd79658650fe28b5a0d8abeb8ce6f437c0e6df5be1a4", size = 6628559, upload-time = "2025-11-16T22:49:57.371Z" }, + { url = "https://files.pythonhosted.org/packages/74/78/fcd41e5a0ce4f3f7b003da85825acddae6d7ecb60cf25194741b036ca7d6/numpy-2.3.5-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b973c57ff8e184109db042c842423ff4f60446239bd585a5131cc47f06f789d", size = 14250702, upload-time = "2025-11-16T22:49:59.632Z" }, + { url = "https://files.pythonhosted.org/packages/b6/23/2a1b231b8ff672b4c450dac27164a8b2ca7d9b7144f9c02d2396518352eb/numpy-2.3.5-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d8163f43acde9a73c2a33605353a4f1bc4798745a8b1d73183b28e5b435ae28", size = 16606086, upload-time = "2025-11-16T22:50:02.127Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c5/5ad26fbfbe2012e190cc7d5003e4d874b88bb18861d0829edc140a713021/numpy-2.3.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:51c1e14eb1e154ebd80e860722f9e6ed6ec89714ad2db2d3aa33c31d7c12179b", size = 16025985, upload-time = "2025-11-16T22:50:04.536Z" }, + { url = "https://files.pythonhosted.org/packages/d2/fa/dd48e225c46c819288148d9d060b047fd2a6fb1eb37eae25112ee4cb4453/numpy-2.3.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b46b4ec24f7293f23adcd2d146960559aaf8020213de8ad1909dba6c013bf89c", size = 18542976, upload-time = "2025-11-16T22:50:07.557Z" }, + { url = "https://files.pythonhosted.org/packages/05/79/ccbd23a75862d95af03d28b5c6901a1b7da4803181513d52f3b86ed9446e/numpy-2.3.5-cp312-cp312-win32.whl", hash = "sha256:3997b5b3c9a771e157f9aae01dd579ee35ad7109be18db0e85dbdbe1de06e952", size = 6285274, upload-time = "2025-11-16T22:50:10.746Z" }, + { url = "https://files.pythonhosted.org/packages/2d/57/8aeaf160312f7f489dea47ab61e430b5cb051f59a98ae68b7133ce8fa06a/numpy-2.3.5-cp312-cp312-win_amd64.whl", hash = "sha256:86945f2ee6d10cdfd67bcb4069c1662dd711f7e2a4343db5cecec06b87cf31aa", size = 12782922, upload-time = "2025-11-16T22:50:12.811Z" }, + { url = "https://files.pythonhosted.org/packages/78/a6/aae5cc2ca78c45e64b9ef22f089141d661516856cf7c8a54ba434576900d/numpy-2.3.5-cp312-cp312-win_arm64.whl", hash = "sha256:f28620fe26bee16243be2b7b874da327312240a7cdc38b769a697578d2100013", size = 10194667, upload-time = "2025-11-16T22:50:16.16Z" }, + { url = "https://files.pythonhosted.org/packages/db/69/9cde09f36da4b5a505341180a3f2e6fadc352fd4d2b7096ce9778db83f1a/numpy-2.3.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d0f23b44f57077c1ede8c5f26b30f706498b4862d3ff0a7298b8411dd2f043ff", size = 16728251, upload-time = "2025-11-16T22:50:19.013Z" }, + { url = "https://files.pythonhosted.org/packages/79/fb/f505c95ceddd7027347b067689db71ca80bd5ecc926f913f1a23e65cf09b/numpy-2.3.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:aa5bc7c5d59d831d9773d1170acac7893ce3a5e130540605770ade83280e7188", size = 12254652, upload-time = "2025-11-16T22:50:21.487Z" }, + { url = "https://files.pythonhosted.org/packages/78/da/8c7738060ca9c31b30e9301ee0cf6c5ffdbf889d9593285a1cead337f9a5/numpy-2.3.5-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:ccc933afd4d20aad3c00bcef049cb40049f7f196e0397f1109dba6fed63267b0", size = 5083172, upload-time = "2025-11-16T22:50:24.562Z" }, + { url = "https://files.pythonhosted.org/packages/a4/b4/ee5bb2537fb9430fd2ef30a616c3672b991a4129bb1c7dcc42aa0abbe5d7/numpy-2.3.5-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:afaffc4393205524af9dfa400fa250143a6c3bc646c08c9f5e25a9f4b4d6a903", size = 6622990, upload-time = "2025-11-16T22:50:26.47Z" }, + { url = "https://files.pythonhosted.org/packages/95/03/dc0723a013c7d7c19de5ef29e932c3081df1c14ba582b8b86b5de9db7f0f/numpy-2.3.5-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c75442b2209b8470d6d5d8b1c25714270686f14c749028d2199c54e29f20b4d", size = 14248902, upload-time = "2025-11-16T22:50:28.861Z" }, + { url = "https://files.pythonhosted.org/packages/f5/10/ca162f45a102738958dcec8023062dad0cbc17d1ab99d68c4e4a6c45fb2b/numpy-2.3.5-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11e06aa0af8c0f05104d56450d6093ee639e15f24ecf62d417329d06e522e017", size = 16597430, upload-time = "2025-11-16T22:50:31.56Z" }, + { url = "https://files.pythonhosted.org/packages/2a/51/c1e29be863588db58175175f057286900b4b3327a1351e706d5e0f8dd679/numpy-2.3.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ed89927b86296067b4f81f108a2271d8926467a8868e554eaf370fc27fa3ccaf", size = 16024551, upload-time = "2025-11-16T22:50:34.242Z" }, + { url = "https://files.pythonhosted.org/packages/83/68/8236589d4dbb87253d28259d04d9b814ec0ecce7cb1c7fed29729f4c3a78/numpy-2.3.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51c55fe3451421f3a6ef9a9c1439e82101c57a2c9eab9feb196a62b1a10b58ce", size = 18533275, upload-time = "2025-11-16T22:50:37.651Z" }, + { url = "https://files.pythonhosted.org/packages/40/56/2932d75b6f13465239e3b7b7e511be27f1b8161ca2510854f0b6e521c395/numpy-2.3.5-cp313-cp313-win32.whl", hash = "sha256:1978155dd49972084bd6ef388d66ab70f0c323ddee6f693d539376498720fb7e", size = 6277637, upload-time = "2025-11-16T22:50:40.11Z" }, + { url = "https://files.pythonhosted.org/packages/0c/88/e2eaa6cffb115b85ed7c7c87775cb8bcf0816816bc98ca8dbfa2ee33fe6e/numpy-2.3.5-cp313-cp313-win_amd64.whl", hash = "sha256:00dc4e846108a382c5869e77c6ed514394bdeb3403461d25a829711041217d5b", size = 12779090, upload-time = "2025-11-16T22:50:42.503Z" }, + { url = "https://files.pythonhosted.org/packages/8f/88/3f41e13a44ebd4034ee17baa384acac29ba6a4fcc2aca95f6f08ca0447d1/numpy-2.3.5-cp313-cp313-win_arm64.whl", hash = "sha256:0472f11f6ec23a74a906a00b48a4dcf3849209696dff7c189714511268d103ae", size = 10194710, upload-time = "2025-11-16T22:50:44.971Z" }, + { url = "https://files.pythonhosted.org/packages/13/cb/71744144e13389d577f867f745b7df2d8489463654a918eea2eeb166dfc9/numpy-2.3.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:414802f3b97f3c1eef41e530aaba3b3c1620649871d8cb38c6eaff034c2e16bd", size = 16827292, upload-time = "2025-11-16T22:50:47.715Z" }, + { url = "https://files.pythonhosted.org/packages/71/80/ba9dc6f2a4398e7f42b708a7fdc841bb638d353be255655498edbf9a15a8/numpy-2.3.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5ee6609ac3604fa7780e30a03e5e241a7956f8e2fcfe547d51e3afa5247ac47f", size = 12378897, upload-time = "2025-11-16T22:50:51.327Z" }, + { url = "https://files.pythonhosted.org/packages/2e/6d/db2151b9f64264bcceccd51741aa39b50150de9b602d98ecfe7e0c4bff39/numpy-2.3.5-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:86d835afea1eaa143012a2d7a3f45a3adce2d7adc8b4961f0b362214d800846a", size = 5207391, upload-time = "2025-11-16T22:50:54.542Z" }, + { url = "https://files.pythonhosted.org/packages/80/ae/429bacace5ccad48a14c4ae5332f6aa8ab9f69524193511d60ccdfdc65fa/numpy-2.3.5-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:30bc11310e8153ca664b14c5f1b73e94bd0503681fcf136a163de856f3a50139", size = 6721275, upload-time = "2025-11-16T22:50:56.794Z" }, + { url = "https://files.pythonhosted.org/packages/74/5b/1919abf32d8722646a38cd527bc3771eb229a32724ee6ba340ead9b92249/numpy-2.3.5-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1062fde1dcf469571705945b0f221b73928f34a20c904ffb45db101907c3454e", size = 14306855, upload-time = "2025-11-16T22:50:59.208Z" }, + { url = "https://files.pythonhosted.org/packages/a5/87/6831980559434973bebc30cd9c1f21e541a0f2b0c280d43d3afd909b66d0/numpy-2.3.5-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce581db493ea1a96c0556360ede6607496e8bf9b3a8efa66e06477267bc831e9", size = 16657359, upload-time = "2025-11-16T22:51:01.991Z" }, + { url = "https://files.pythonhosted.org/packages/dd/91/c797f544491ee99fd00495f12ebb7802c440c1915811d72ac5b4479a3356/numpy-2.3.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:cc8920d2ec5fa99875b670bb86ddeb21e295cb07aa331810d9e486e0b969d946", size = 16093374, upload-time = "2025-11-16T22:51:05.291Z" }, + { url = "https://files.pythonhosted.org/packages/74/a6/54da03253afcbe7a72785ec4da9c69fb7a17710141ff9ac5fcb2e32dbe64/numpy-2.3.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:9ee2197ef8c4f0dfe405d835f3b6a14f5fee7782b5de51ba06fb65fc9b36e9f1", size = 18594587, upload-time = "2025-11-16T22:51:08.585Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/aff53abbdd41b0ecca94285f325aff42357c6b5abc482a3fcb4994290b18/numpy-2.3.5-cp313-cp313t-win32.whl", hash = "sha256:70b37199913c1bd300ff6e2693316c6f869c7ee16378faf10e4f5e3275b299c3", size = 6405940, upload-time = "2025-11-16T22:51:11.541Z" }, + { url = "https://files.pythonhosted.org/packages/d5/81/50613fec9d4de5480de18d4f8ef59ad7e344d497edbef3cfd80f24f98461/numpy-2.3.5-cp313-cp313t-win_amd64.whl", hash = "sha256:b501b5fa195cc9e24fe102f21ec0a44dffc231d2af79950b451e0d99cea02234", size = 12920341, upload-time = "2025-11-16T22:51:14.312Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ab/08fd63b9a74303947f34f0bd7c5903b9c5532c2d287bead5bdf4c556c486/numpy-2.3.5-cp313-cp313t-win_arm64.whl", hash = "sha256:a80afd79f45f3c4a7d341f13acbe058d1ca8ac017c165d3fa0d3de6bc1a079d7", size = 10262507, upload-time = "2025-11-16T22:51:16.846Z" }, + { url = "https://files.pythonhosted.org/packages/ba/97/1a914559c19e32d6b2e233cf9a6a114e67c856d35b1d6babca571a3e880f/numpy-2.3.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:bf06bc2af43fa8d32d30fae16ad965663e966b1a3202ed407b84c989c3221e82", size = 16735706, upload-time = "2025-11-16T22:51:19.558Z" }, + { url = "https://files.pythonhosted.org/packages/57/d4/51233b1c1b13ecd796311216ae417796b88b0616cfd8a33ae4536330748a/numpy-2.3.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:052e8c42e0c49d2575621c158934920524f6c5da05a1d3b9bab5d8e259e045f0", size = 12264507, upload-time = "2025-11-16T22:51:22.492Z" }, + { url = "https://files.pythonhosted.org/packages/45/98/2fe46c5c2675b8306d0b4a3ec3494273e93e1226a490f766e84298576956/numpy-2.3.5-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:1ed1ec893cff7040a02c8aa1c8611b94d395590d553f6b53629a4461dc7f7b63", size = 5093049, upload-time = "2025-11-16T22:51:25.171Z" }, + { url = "https://files.pythonhosted.org/packages/ce/0e/0698378989bb0ac5f1660c81c78ab1fe5476c1a521ca9ee9d0710ce54099/numpy-2.3.5-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:2dcd0808a421a482a080f89859a18beb0b3d1e905b81e617a188bd80422d62e9", size = 6626603, upload-time = "2025-11-16T22:51:27Z" }, + { url = "https://files.pythonhosted.org/packages/5e/a6/9ca0eecc489640615642a6cbc0ca9e10df70df38c4d43f5a928ff18d8827/numpy-2.3.5-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:727fd05b57df37dc0bcf1a27767a3d9a78cbbc92822445f32cc3436ba797337b", size = 14262696, upload-time = "2025-11-16T22:51:29.402Z" }, + { url = "https://files.pythonhosted.org/packages/c8/f6/07ec185b90ec9d7217a00eeeed7383b73d7e709dae2a9a021b051542a708/numpy-2.3.5-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fffe29a1ef00883599d1dc2c51aa2e5d80afe49523c261a74933df395c15c520", size = 16597350, upload-time = "2025-11-16T22:51:32.167Z" }, + { url = "https://files.pythonhosted.org/packages/75/37/164071d1dde6a1a84c9b8e5b414fa127981bad47adf3a6b7e23917e52190/numpy-2.3.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8f7f0e05112916223d3f438f293abf0727e1181b5983f413dfa2fefc4098245c", size = 16040190, upload-time = "2025-11-16T22:51:35.403Z" }, + { url = "https://files.pythonhosted.org/packages/08/3c/f18b82a406b04859eb026d204e4e1773eb41c5be58410f41ffa511d114ae/numpy-2.3.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2e2eb32ddb9ccb817d620ac1d8dae7c3f641c1e5f55f531a33e8ab97960a75b8", size = 18536749, upload-time = "2025-11-16T22:51:39.698Z" }, + { url = "https://files.pythonhosted.org/packages/40/79/f82f572bf44cf0023a2fe8588768e23e1592585020d638999f15158609e1/numpy-2.3.5-cp314-cp314-win32.whl", hash = "sha256:66f85ce62c70b843bab1fb14a05d5737741e74e28c7b8b5a064de10142fad248", size = 6335432, upload-time = "2025-11-16T22:51:42.476Z" }, + { url = "https://files.pythonhosted.org/packages/a3/2e/235b4d96619931192c91660805e5e49242389742a7a82c27665021db690c/numpy-2.3.5-cp314-cp314-win_amd64.whl", hash = "sha256:e6a0bc88393d65807d751a614207b7129a310ca4fe76a74e5c7da5fa5671417e", size = 12919388, upload-time = "2025-11-16T22:51:45.275Z" }, + { url = "https://files.pythonhosted.org/packages/07/2b/29fd75ce45d22a39c61aad74f3d718e7ab67ccf839ca8b60866054eb15f8/numpy-2.3.5-cp314-cp314-win_arm64.whl", hash = "sha256:aeffcab3d4b43712bb7a60b65f6044d444e75e563ff6180af8f98dd4b905dfd2", size = 10476651, upload-time = "2025-11-16T22:51:47.749Z" }, + { url = "https://files.pythonhosted.org/packages/17/e1/f6a721234ebd4d87084cfa68d081bcba2f5cfe1974f7de4e0e8b9b2a2ba1/numpy-2.3.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:17531366a2e3a9e30762c000f2c43a9aaa05728712e25c11ce1dbe700c53ad41", size = 16834503, upload-time = "2025-11-16T22:51:50.443Z" }, + { url = "https://files.pythonhosted.org/packages/5c/1c/baf7ffdc3af9c356e1c135e57ab7cf8d247931b9554f55c467efe2c69eff/numpy-2.3.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d21644de1b609825ede2f48be98dfde4656aefc713654eeee280e37cadc4e0ad", size = 12381612, upload-time = "2025-11-16T22:51:53.609Z" }, + { url = "https://files.pythonhosted.org/packages/74/91/f7f0295151407ddc9ba34e699013c32c3c91944f9b35fcf9281163dc1468/numpy-2.3.5-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:c804e3a5aba5460c73955c955bdbd5c08c354954e9270a2c1565f62e866bdc39", size = 5210042, upload-time = "2025-11-16T22:51:56.213Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3b/78aebf345104ec50dd50a4d06ddeb46a9ff5261c33bcc58b1c4f12f85ec2/numpy-2.3.5-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:cc0a57f895b96ec78969c34f682c602bf8da1a0270b09bc65673df2e7638ec20", size = 6724502, upload-time = "2025-11-16T22:51:58.584Z" }, + { url = "https://files.pythonhosted.org/packages/02/c6/7c34b528740512e57ef1b7c8337ab0b4f0bddf34c723b8996c675bc2bc91/numpy-2.3.5-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:900218e456384ea676e24ea6a0417f030a3b07306d29d7ad843957b40a9d8d52", size = 14308962, upload-time = "2025-11-16T22:52:01.698Z" }, + { url = "https://files.pythonhosted.org/packages/80/35/09d433c5262bc32d725bafc619e095b6a6651caf94027a03da624146f655/numpy-2.3.5-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09a1bea522b25109bf8e6f3027bd810f7c1085c64a0c7ce050c1676ad0ba010b", size = 16655054, upload-time = "2025-11-16T22:52:04.267Z" }, + { url = "https://files.pythonhosted.org/packages/7a/ab/6a7b259703c09a88804fa2430b43d6457b692378f6b74b356155283566ac/numpy-2.3.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:04822c00b5fd0323c8166d66c701dc31b7fbd252c100acd708c48f763968d6a3", size = 16091613, upload-time = "2025-11-16T22:52:08.651Z" }, + { url = "https://files.pythonhosted.org/packages/c2/88/330da2071e8771e60d1038166ff9d73f29da37b01ec3eb43cb1427464e10/numpy-2.3.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d6889ec4ec662a1a37eb4b4fb26b6100841804dac55bd9df579e326cdc146227", size = 18591147, upload-time = "2025-11-16T22:52:11.453Z" }, + { url = "https://files.pythonhosted.org/packages/51/41/851c4b4082402d9ea860c3626db5d5df47164a712cb23b54be028b184c1c/numpy-2.3.5-cp314-cp314t-win32.whl", hash = "sha256:93eebbcf1aafdf7e2ddd44c2923e2672e1010bddc014138b229e49725b4d6be5", size = 6479806, upload-time = "2025-11-16T22:52:14.641Z" }, + { url = "https://files.pythonhosted.org/packages/90/30/d48bde1dfd93332fa557cff1972fbc039e055a52021fbef4c2c4b1eefd17/numpy-2.3.5-cp314-cp314t-win_amd64.whl", hash = "sha256:c8a9958e88b65c3b27e22ca2a076311636850b612d6bbfb76e8d156aacde2aaf", size = 13105760, upload-time = "2025-11-16T22:52:17.975Z" }, + { url = "https://files.pythonhosted.org/packages/2d/fd/4b5eb0b3e888d86aee4d198c23acec7d214baaf17ea93c1adec94c9518b9/numpy-2.3.5-cp314-cp314t-win_arm64.whl", hash = "sha256:6203fdf9f3dc5bdaed7319ad8698e685c7a3be10819f41d32a0723e611733b42", size = 10545459, upload-time = "2025-11-16T22:52:20.55Z" }, + { url = "https://files.pythonhosted.org/packages/c6/65/f9dea8e109371ade9c782b4e4756a82edf9d3366bca495d84d79859a0b79/numpy-2.3.5-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:f0963b55cdd70fad460fa4c1341f12f976bb26cb66021a5580329bd498988310", size = 16910689, upload-time = "2025-11-16T22:52:23.247Z" }, + { url = "https://files.pythonhosted.org/packages/00/4f/edb00032a8fb92ec0a679d3830368355da91a69cab6f3e9c21b64d0bb986/numpy-2.3.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:f4255143f5160d0de972d28c8f9665d882b5f61309d8362fdd3e103cf7bf010c", size = 12457053, upload-time = "2025-11-16T22:52:26.367Z" }, + { url = "https://files.pythonhosted.org/packages/16/a4/e8a53b5abd500a63836a29ebe145fc1ab1f2eefe1cfe59276020373ae0aa/numpy-2.3.5-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:a4b9159734b326535f4dd01d947f919c6eefd2d9827466a696c44ced82dfbc18", size = 5285635, upload-time = "2025-11-16T22:52:29.266Z" }, + { url = "https://files.pythonhosted.org/packages/a3/2f/37eeb9014d9c8b3e9c55bc599c68263ca44fdbc12a93e45a21d1d56df737/numpy-2.3.5-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:2feae0d2c91d46e59fcd62784a3a83b3fb677fead592ce51b5a6fbb4f95965ff", size = 6801770, upload-time = "2025-11-16T22:52:31.421Z" }, + { url = "https://files.pythonhosted.org/packages/7d/e4/68d2f474df2cb671b2b6c2986a02e520671295647dad82484cde80ca427b/numpy-2.3.5-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffac52f28a7849ad7576293c0cb7b9f08304e8f7d738a8cb8a90ec4c55a998eb", size = 14391768, upload-time = "2025-11-16T22:52:33.593Z" }, + { url = "https://files.pythonhosted.org/packages/b8/50/94ccd8a2b141cb50651fddd4f6a48874acb3c91c8f0842b08a6afc4b0b21/numpy-2.3.5-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63c0e9e7eea69588479ebf4a8a270d5ac22763cc5854e9a7eae952a3908103f7", size = 16729263, upload-time = "2025-11-16T22:52:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/2d/ee/346fa473e666fe14c52fcdd19ec2424157290a032d4c41f98127bfb31ac7/numpy-2.3.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:f16417ec91f12f814b10bafe79ef77e70113a2f5f7018640e7425ff979253425", size = 12967213, upload-time = "2025-11-16T22:52:39.38Z" }, +] + +[[package]] +name = "nvidia-cublas" +version = "13.1.0.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/a5/fce49e2ae977e0ccc084e5adafceb4f0ac0c8333cb6863501618a7277f67/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c86fc7f7ae36d7528288c5d88098edcb7b02c633d262e7ddbb86b0ad91be5df2", size = 542851226, upload-time = "2025-10-09T08:59:04.818Z" }, + { url = "https://files.pythonhosted.org/packages/e7/44/423ac00af4dd95a5aeb27207e2c0d9b7118702149bf4704c3ddb55bb7429/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:ee8722c1f0145ab246bccb9e452153b5e0515fd094c3678df50b2a0888b8b171", size = 423133236, upload-time = "2025-10-09T08:59:32.536Z" }, + { url = "https://files.pythonhosted.org/packages/10/f5/f50bc3f5c2bb57ab8f5b4d78bc1146b57810d42cb8fcb28cbe2e14050376/nvidia_cublas-13.1.0.3-py3-none-win_amd64.whl", hash = "sha256:2a3b94a37def342471c59fad7856caee4926809a72dd5270155d6a31b5b277be", size = 404355960, upload-time = "2025-10-09T09:07:00.987Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, + { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, + { url = "https://files.pythonhosted.org/packages/ad/df/b74b10025c1205695c5676373f2edd3e87a7202cc62ead0dfbc373b0f6ea/nvidia_cuda_cupti-13.0.85-py3-none-win_amd64.whl", hash = "sha256:683f58d301548deeefcb8f6fac1b8d907691b9d8b18eccab417f51e362102f00", size = 7736776, upload-time = "2025-09-04T08:38:08.38Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, + { url = "https://files.pythonhosted.org/packages/4a/af/345fedb9f4c76c84ab4fa445b36bd4048a4d9db60e6bc76b4f913ff4b852/nvidia_cuda_nvrtc-13.0.88-py3-none-win_amd64.whl", hash = "sha256:6bcd4e7f8e205cbe644f5a98f2f799bef9556fefc89dd786e79a16312ce49872", size = 76807835, upload-time = "2025-09-04T08:39:15.274Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime" +version = "13.0.96" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, + { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, + { url = "https://files.pythonhosted.org/packages/b7/94/6b867483bec07da24ffa32736c79fabb94ef3a7af4d787a9d4a974868576/nvidia_cuda_runtime-13.0.96-py3-none-win_amd64.whl", hash = "sha256:f79298c8a098cec150a597c8eba58ecdab96e3bdc4b9bc4f9983635031740492", size = 2927037, upload-time = "2025-10-09T09:04:23.782Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu13" +version = "9.19.0.56" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas", marker = "sys_platform == 'linux' or extra == 'extra-16-transformer-lens-lit' or extra != 'extra-16-transformer-lens-vllm'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" }, + { url = "https://files.pythonhosted.org/packages/a3/22/0b4b932655d17a6da1b92fa92ab12844b053bb2ac2475e179ba6f043da1e/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:d20e1734305e9d68889a96e3f35094d733ff1f83932ebe462753973e53a572bf", size = 366066321, upload-time = "2026-02-03T20:44:52.837Z" }, + { url = "https://files.pythonhosted.org/packages/91/a2/f020386683ee9ab2c9a9f7f79290d9b0d07f7241de54dc746af2abd188d2/nvidia_cudnn_cu13-9.19.0.56-py3-none-win_amd64.whl", hash = "sha256:40d8c375005bcb01495f8edf375230b203a411a0c05fb6dc92a3781edcb23eac", size = 350547366, upload-time = "2026-02-03T20:50:49.563Z" }, +] + +[[package]] +name = "nvidia-cudnn-frontend" +version = "1.18.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/be/f5a1e633c524c13c0182213ab27dab42dca29a3c785be5ff74d2d185aed1/nvidia_cudnn_frontend-1.18.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:baa6fbc8e7c55f1c78c0374ed9a890e1cf81acaca0c92d6135d18a8e3c985244", size = 2023500, upload-time = "2026-01-27T23:31:34.747Z" }, + { url = "https://files.pythonhosted.org/packages/82/a7/765a17c6a9496196c34f269d17dfb902b6c618c0261c0962511e95302e81/nvidia_cudnn_frontend-1.18.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e4bcca42259e358002c8867e3624a558f66cd5dff2cc6c3aafd860ef2f41730", size = 2154278, upload-time = "2026-01-27T23:06:55.784Z" }, + { url = "https://files.pythonhosted.org/packages/19/a1/7caae2243540bc60e47eae95f0fd913c9baa05cf94df0471914f70d45158/nvidia_cudnn_frontend-1.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:06252021ef1e5a7256f1e70429a426b01792636c05cc547fe8e64c6885a9652e", size = 1590158, upload-time = "2026-01-27T23:08:26.703Z" }, + { url = "https://files.pythonhosted.org/packages/e2/9a/83d3d080118de4a7810fa019349edec634b8b37b9cafaacd05719de62dd6/nvidia_cudnn_frontend-1.18.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6d4d0b88d617b233a503c84980b54d840b60b2734497d1a7a071ec5293daec2", size = 2023709, upload-time = "2026-01-27T23:32:10.912Z" }, + { url = "https://files.pythonhosted.org/packages/13/c7/c3624b3ed77b102618f26295e816b27f1c3ebb1143730237a9f51d403c3f/nvidia_cudnn_frontend-1.18.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:382ea063b92cbfd5b442cb75ff8422932d78276aecf139e46713ed1ad3d07af4", size = 2155568, upload-time = "2026-01-27T23:07:13.277Z" }, + { url = "https://files.pythonhosted.org/packages/52/dd/8613dfd029d076b86a8a87efe3f4bb4ab73cec15fa8fc27e665098f4d167/nvidia_cudnn_frontend-1.18.0-cp311-cp311-win_amd64.whl", hash = "sha256:baa509effc4d299d3f04e549d4188f88bca8a8b527f483cbd2f66bc18f13a8b1", size = 1591244, upload-time = "2026-01-27T23:08:44.691Z" }, + { url = "https://files.pythonhosted.org/packages/e3/b4/604e230378680ee117849a4e1045baca092f93161a829291a84d5acce70c/nvidia_cudnn_frontend-1.18.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:310b417f2848a83d1437203fcaeea320a74fb7f28af20bf42bf5afc9c01f1c12", size = 2027408, upload-time = "2026-01-27T23:32:46.576Z" }, + { url = "https://files.pythonhosted.org/packages/c6/52/08f98262e77b1cbcc834cc1a5db494d0661ea1dbdea58c2e2d51a57fdaca/nvidia_cudnn_frontend-1.18.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c023539ca6de99234cf5102c3ec0d6af817f5396fc93028a22ba5b834a35b8a", size = 2159245, upload-time = "2026-01-27T23:07:32.664Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1f/751a5a8cfdc95fb4dc556192d37369ae488c30c473fe9a3ec720b23d07ea/nvidia_cudnn_frontend-1.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:e13f7dd46cdb4762dde87f181f06d1c5e15e9478bbdd547bfa74d9b11f415aae", size = 1591041, upload-time = "2026-01-27T23:09:04.118Z" }, + { url = "https://files.pythonhosted.org/packages/e8/bd/db791a26ebb6a6e1268f518e18c82d8ad18546f7008f4b0d5bde15f927de/nvidia_cudnn_frontend-1.18.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a6e2b7bd43705ffa4af3b187374fdd5e7d09fc228a4d65fc8b4b0a537a8e605", size = 2027249, upload-time = "2026-01-27T23:33:22.46Z" }, + { url = "https://files.pythonhosted.org/packages/19/74/3038cf496d5de7cfdff730f5202e438c17d9123de507059340e02ddff9d7/nvidia_cudnn_frontend-1.18.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0544206b02cae9da4f044ca3fe7416b99e0c8a8052285dd3e5a8fc445d34f9c", size = 2160001, upload-time = "2026-01-27T23:07:50.248Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5e/148cc6609dba326e620e4d949246020dfba05ca07d0387442e62b71d19b6/nvidia_cudnn_frontend-1.18.0-cp313-cp313-win_amd64.whl", hash = "sha256:7eefa5f10cc003df5f3593f82f1ee6c001fc3412bdc78430c751914dfceefd7f", size = 1591270, upload-time = "2026-01-27T23:09:21.435Z" }, + { url = "https://files.pythonhosted.org/packages/a3/0a/515209dd2afc6027bf1112bf415f575bfe9628d18877abe7424cb597dd7b/nvidia_cudnn_frontend-1.18.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b489da1b30f1d7da822b37b89cc4f68afd80e020eb57e4ab24921f8b57f6e946", size = 2028689, upload-time = "2026-02-11T21:32:04.235Z" }, + { url = "https://files.pythonhosted.org/packages/ab/57/52d18e1f50979eeabfafb408ec73068afc5a1e1ccd21636240317cd456d4/nvidia_cudnn_frontend-1.18.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:37688c81a34ac590aff9de4c34d2968bab949411af707baa327616ebd4b34ae1", size = 2160182, upload-time = "2026-02-11T21:25:18.437Z" }, + { url = "https://files.pythonhosted.org/packages/67/53/df2810b56d259ef96fa6beaa1381bd14c29fbe82836b409516e864c5e177/nvidia_cudnn_frontend-1.18.0-cp314-cp314-win_amd64.whl", hash = "sha256:5053b473fa74168b5fbf35934cd6187f88aa03b8447b9f2cd417332d5e5c9569", size = 1592759, upload-time = "2026-02-11T21:32:33.87Z" }, +] + +[[package]] +name = "nvidia-cufft" +version = "12.0.0.61" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or extra == 'extra-16-transformer-lens-lit' or extra != 'extra-16-transformer-lens-vllm'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, + { url = "https://files.pythonhosted.org/packages/85/b2/f8af21a2ed1beed337a6a02c5a28aeb85441f4d578ec3d529543c775ea4b/nvidia_cufft-12.0.0.61-py3-none-win_amd64.whl", hash = "sha256:2abce5b39d2f5ae12730fb7e5db6696533e36c26e2d3e8fd1750bdd2853364eb", size = 213342123, upload-time = "2025-09-04T08:40:51.145Z" }, +] + +[[package]] +name = "nvidia-cufile" +version = "1.15.1.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, +] + +[[package]] +name = "nvidia-curand" +version = "10.4.0.35" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, + { url = "https://files.pythonhosted.org/packages/99/27/72103153b1ffc00e09fdc40ac970235343dcd1ea8bd762e84d2d73219ffa/nvidia_curand-10.4.0.35-py3-none-win_amd64.whl", hash = "sha256:65b1710aa6961d326b411e314b374290904c5ddf41dc3f766ebc3f1d7d4ca69f", size = 55242481, upload-time = "2025-08-04T10:30:41.831Z" }, +] + +[[package]] +name = "nvidia-cusolver" +version = "12.0.4.66" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas", marker = "sys_platform == 'linux' or extra == 'extra-16-transformer-lens-lit' or extra != 'extra-16-transformer-lens-vllm'" }, + { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or extra == 'extra-16-transformer-lens-lit' or extra != 'extra-16-transformer-lens-vllm'" }, + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or extra == 'extra-16-transformer-lens-lit' or extra != 'extra-16-transformer-lens-vllm'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, + { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, + { url = "https://files.pythonhosted.org/packages/99/ef/332a0101260ca78a1daef046bf0b06199e8ed4dac1d2aa698289c358169c/nvidia_cusolver-12.0.4.66-py3-none-win_amd64.whl", hash = "sha256:16515bd33a8e76bb54d024cfa068fa68d30e80fc34b9e1090813ea9362e0cb65", size = 193551444, upload-time = "2025-09-04T08:41:46.813Z" }, +] + +[[package]] +name = "nvidia-cusparse" +version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "joblib" }, - { name = "regex" }, - { name = "tqdm" }, + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or extra == 'extra-16-transformer-lens-lit' or extra != 'extra-16-transformer-lens-vllm'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/74/a1/b3b4adf15585a5bc4c357adde150c01ebeeb642173ded4d871e89468767c/nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0", size = 2946864, upload-time = "2026-03-24T06:13:40.641Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f", size = 1552087, upload-time = "2026-03-24T06:13:38.47Z" }, + { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, + { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, + { url = "https://files.pythonhosted.org/packages/02/b0/b043d6f3480f102f885cf87fc3ffd3edcb5e23b855025a50e2ef4d059185/nvidia_cusparse-12.6.3.3-py3-none-win_amd64.whl", hash = "sha256:cbcf42feb737bd7ec15b4c0a63e62351886bd3f975027b8815d7f720a2b5ea79", size = 143783033, upload-time = "2025-09-04T08:42:12.391Z" }, ] [[package]] -name = "notebook" -version = "6.5.7" +name = "nvidia-cusparselt-cu13" +version = "0.8.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argon2-cffi" }, - { name = "ipykernel" }, - { name = "ipython-genutils" }, - { name = "jinja2" }, - { name = "jupyter-client" }, - { name = "jupyter-core" }, - { name = "nbclassic" }, - { name = "nbconvert" }, - { name = "nbformat" }, - { name = "nest-asyncio" }, - { name = "prometheus-client" }, - { name = "pyzmq" }, - { name = "send2trash" }, - { name = "terminado" }, - { name = "tornado" }, - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/05/bc/b025bac523640c13b0c1150466475eac3d79acbf187ef6c1967596c41c43/notebook-6.5.7.tar.gz", hash = "sha256:04eb9011dfac634fbd4442adaf0a8c27cd26beef831fe1d19faf930c327768e4", size = 5786526, upload-time = "2024-05-01T17:42:26.956Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/af/9c/0620631da9d7013e95a8f985043cad229a0d8fb537a7e3f8ff8467565a8c/notebook-6.5.7-py3-none-any.whl", hash = "sha256:a6afa9a4ff4d149a0771ff8b8c881a7a73b3835f9add0606696d6e9d98ac1cd0", size = 529829, upload-time = "2024-05-01T17:42:22.403Z" }, + { url = "https://files.pythonhosted.org/packages/46/10/8dcd1175260706a2fc92a16a52e306b71d4c1ea0b0cc4a9484183399818a/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:400c6ed1cf6780fc6efedd64ec9f1345871767e6a1a0a552a1ea0578117ea77c", size = 220791277, upload-time = "2025-08-13T19:22:40.982Z" }, + { url = "https://files.pythonhosted.org/packages/fd/53/43b0d71f4e702fa9733f8b4571fdca50a8813f1e450b656c239beff12315/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25e30a8a7323935d4ad0340b95a0b69926eee755767e8e0b1cf8dd85b197d3fd", size = 169884119, upload-time = "2025-08-13T19:23:41.967Z" }, + { url = "https://files.pythonhosted.org/packages/57/de/8f0578928b9b1246d7b1324db0528e6b9f9fb54496a49f40bf71f09f1a27/nvidia_cusparselt_cu13-0.8.0-py3-none-win_amd64.whl", hash = "sha256:e80212ed7b1afc97102fbb2b5c82487aa73f6a0edfa6d26c5a152593e520bb8f", size = 156459710, upload-time = "2025-08-13T19:24:18.043Z" }, ] [[package]] -name = "notebook-shim" -version = "0.2.4" +name = "nvidia-cutlass-dsl" +version = "4.5.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jupyter-server" }, + { name = "nvidia-cutlass-dsl-libs-base", marker = "sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/54/d2/92fa3243712b9a3e8bafaf60aac366da1cada3639ca767ff4b5b3654ec28/notebook_shim-0.2.4.tar.gz", hash = "sha256:b4b2cfa1b65d98307ca24361f5b30fe785b53c3fd07b7a47e89acb5e6ac638cb", size = 13167, upload-time = "2024-02-14T23:35:18.353Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/33/bd5b9137445ea4b680023eb0469b2bb969d61303dedb2aac6560ff3d14a1/notebook_shim-0.2.4-py3-none-any.whl", hash = "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef", size = 13307, upload-time = "2024-02-14T23:35:16.286Z" }, + { url = "https://files.pythonhosted.org/packages/d5/3b/b8ff6a6103d387ce7272e31ad15b74e46fdc93aaeceab1d7ca457f813200/nvidia_cutlass_dsl-4.5.3-py3-none-any.whl", hash = "sha256:c14ca121c2a4ce53ca4d5ee5d939e4181b97054b55ea7bb84aa9f99b98960060", size = 10178, upload-time = "2026-07-08T01:56:16.346Z" }, ] [[package]] -name = "numba" -version = "0.65.0" +name = "nvidia-cutlass-dsl-libs-base" +version = "4.5.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "llvmlite" }, - { name = "numpy" }, + { name = "cuda-python", marker = "sys_platform == 'linux'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/49/61/7299643b9c18d669e04be7c5bcb64d985070d07553274817b45b049e7bfe/numba-0.65.0.tar.gz", hash = "sha256:edad0d9f6682e93624c00125a471ae4df186175d71fd604c983c377cdc03e68b", size = 2764131, upload-time = "2026-04-01T03:52:01.946Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/23/9b/e8453d93d5cb3f53cc956f135024be09d52f4f99643acaf8fdca090a8f3c/numba-0.65.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:dff9fd5fbc9a35c517359c5823ea705d9b65f01fb46e42e35a2eabe5a52c2e96", size = 2680537, upload-time = "2026-04-01T03:51:17.325Z" }, - { url = "https://files.pythonhosted.org/packages/07/95/d6a2f0625e1092624228301eea11cdaff21ddcaf917ef3d631846a38b2f4/numba-0.65.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4c894c94afa5ffd627c7e3b693df10cb0d905bd5eb06de3dfc31775140cf4f89", size = 3739444, upload-time = "2026-04-01T03:51:19.629Z" }, - { url = "https://files.pythonhosted.org/packages/49/ed/fe518c97af035e4ec670c2edc3f0ff7a518cbed2f0b5053124d7c979bd8a/numba-0.65.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7325b1aab88f0339057288ee32f39dc660e14f93872a6fda14fa6eb9f95b047", size = 3446390, upload-time = "2026-04-01T03:51:21.55Z" }, - { url = "https://files.pythonhosted.org/packages/d0/06/5010939854249c290c6217e3fb7404914f4ed953f9923e340c3e166bcaf0/numba-0.65.0-cp310-cp310-win_amd64.whl", hash = "sha256:71e72e9ca2f619df4768f9c3962bfec60191a5a26fe2b6a8c6a07532b6146169", size = 2747200, upload-time = "2026-04-01T03:51:23.674Z" }, - { url = "https://files.pythonhosted.org/packages/ba/ce/d67c499703eb5479ce02420e8ccd65c5753d87d2e16d563f152d71405346/numba-0.65.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:28e547d0b18024f19cbaf9de02fc5c145790213d9be8a2c95b43f93ec162b9e4", size = 2680228, upload-time = "2026-04-01T03:51:25.401Z" }, - { url = "https://files.pythonhosted.org/packages/c1/a7/11e2b24251d57cf41fc9ad83f378d890d61a890e3f8eb6338b39833f67a4/numba-0.65.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:032b0b8e879512cd424d79eed6d772a1399c6387ded184c2cf3cc22c08d750a6", size = 3744674, upload-time = "2026-04-01T03:51:27.311Z" }, - { url = "https://files.pythonhosted.org/packages/fe/0b/7c63eb742859a6243f42288441f65ac9dac96ea59f409e43b713aafbe867/numba-0.65.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af143d823624033a128b5950c0aaf9ffc2386dfe954eb757119cf0432335534c", size = 3450620, upload-time = "2026-04-01T03:51:29.092Z" }, - { url = "https://files.pythonhosted.org/packages/53/ff/1371cbbe955be340a46093a10b61462437e0fadc7a63290473a0e584cb03/numba-0.65.0-cp311-cp311-win_amd64.whl", hash = "sha256:15d159578e59a39df246b83480f78d7794b0fca40153b5684d3849a99c48a0fb", size = 2747081, upload-time = "2026-04-01T03:51:30.785Z" }, - { url = "https://files.pythonhosted.org/packages/6c/2f/8bd31a1ea43c01ac215283d83aa5f8d5acbe7a36c85b82f1757bfe9ccb31/numba-0.65.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:b27ee4847e1bfb17e9604d100417ee7c1d10f15a6711c6213404b3da13a0b2aa", size = 2680705, upload-time = "2026-04-01T03:51:32.597Z" }, - { url = "https://files.pythonhosted.org/packages/73/36/88406bd58600cc696417b8e5dd6a056478da808f3eaf48d18e2421e0c2d9/numba-0.65.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a52d92ffd297c10364bce60cd1fcb88f99284ab5df085f2c6bcd1cb33b529a6f", size = 3801411, upload-time = "2026-04-01T03:51:34.321Z" }, - { url = "https://files.pythonhosted.org/packages/0c/61/ce753a1d7646dd477e16d15e89473703faebb8995d2f71d7ad69a540b565/numba-0.65.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da8e371e328c06d0010c3d8b44b21858652831b85bcfba78cb22c042e22dbd8e", size = 3501622, upload-time = "2026-04-01T03:51:36.348Z" }, - { url = "https://files.pythonhosted.org/packages/7d/86/db87a5393f1b1fabef53ac3ba4e6b938bb27e40a04ad7cc512098fcae032/numba-0.65.0-cp312-cp312-win_amd64.whl", hash = "sha256:59bb9f2bb9f1238dfd8e927ba50645c18ae769fef4f3d58ea0ea22a2683b91f5", size = 2749979, upload-time = "2026-04-01T03:51:37.88Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f8/eee0f1ff456218db036bfc9023995ec1f85a9dc8f2422f1594f6a87829e0/numba-0.65.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:c6334094563a456a695c812e6846288376ca02327cf246cdcc83e1bb27862367", size = 2680679, upload-time = "2026-04-01T03:51:39.491Z" }, - { url = "https://files.pythonhosted.org/packages/1b/8f/3d116e4b8e92f6abace431afa4b2b944f4d65bdee83af886f5c4b263df95/numba-0.65.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b8a9008411615c69d083d1dcf477f75a5aa727b30beb16e139799e2be945cdfd", size = 3809537, upload-time = "2026-04-01T03:51:41.42Z" }, - { url = "https://files.pythonhosted.org/packages/b5/2c/6a3ca4128e253cb67affe06deb47688f51ce968f5111e2a06d010e6f1fa6/numba-0.65.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af96c0cba53664efcb361528b8c75e011a6556c859c7e08424c2715201c6cf7a", size = 3508615, upload-time = "2026-04-01T03:51:43.444Z" }, - { url = "https://files.pythonhosted.org/packages/96/0e/267f9a36fb282c104a971d7eecb685b411c47dce2a740fe69cf5fc2945d9/numba-0.65.0-cp313-cp313-win_amd64.whl", hash = "sha256:6254e73b9c929dc736a1fbd3d6f5680789709a5067cae1fa7198707385129c04", size = 2749938, upload-time = "2026-04-01T03:51:45.218Z" }, - { url = "https://files.pythonhosted.org/packages/56/a4/90edb01e9176053578e343d7a7276bc28356741ee67059aed8ed2c1a4e59/numba-0.65.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:ee336b398a6fca51b1f626034de99f50cb1bd87d537a166275158a3cee744b82", size = 2680878, upload-time = "2026-04-01T03:51:46.91Z" }, - { url = "https://files.pythonhosted.org/packages/24/8d/e12d6ff4b9119db3cbf7b2db1ce257576441bd3c76388c786dea74f20b02/numba-0.65.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:05c0a9fdf75d85f57dee47b719e8d6415707b80aae45d75f63f9dc1b935c29f7", size = 3778456, upload-time = "2026-04-01T03:51:48.552Z" }, - { url = "https://files.pythonhosted.org/packages/17/89/abcd83e76f6a773276fe76244140671bcc5bf820f6e2ae1a15362ae4c8c9/numba-0.65.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:583680e0e8faf124d362df23b4b593f3221a8996341a63d1b664c122401bec2f", size = 3478464, upload-time = "2026-04-01T03:51:50.527Z" }, - { url = "https://files.pythonhosted.org/packages/73/5b/fbce55ce3d933afbc7ade04df826853e4a846aaa47d58d2fbb669b8f2d08/numba-0.65.0-cp314-cp314-win_amd64.whl", hash = "sha256:add297d3e1c08dd884f44100152612fa41e66a51d15fdf91307f9dde31d06830", size = 2752012, upload-time = "2026-04-01T03:51:52.691Z" }, - { url = "https://files.pythonhosted.org/packages/1e/ab/af705f4257d9388fb2fd6d7416573e98b6ca9c786e8b58f02720978557bd/numba-0.65.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:194a243ba53a9157c8538cbb3166ec015d785a8c5d584d06cdd88bee902233c7", size = 2683961, upload-time = "2026-04-01T03:51:54.281Z" }, - { url = "https://files.pythonhosted.org/packages/ff/e5/8267b0adb0c01b52b553df5062fbbb42c30ed5362d08b85cc913a36f838f/numba-0.65.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c7fa502960f7a2f3f5cb025bc7bff888a3551277b92431bfdc5ba2f11a375749", size = 3816373, upload-time = "2026-04-01T03:51:56.18Z" }, - { url = "https://files.pythonhosted.org/packages/b0/f5/b8397ca360971669a93706b9274592b6864e4367a37d498fbbcb62aa2d48/numba-0.65.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5046c63f783ca3eb6195f826a50797465e7c4ce811daa17c9bea47e310c9b964", size = 3532782, upload-time = "2026-04-01T03:51:58.387Z" }, - { url = "https://files.pythonhosted.org/packages/f5/21/1e73fa16bf0393ebb74c5bb208d712152ffdfc84600a8e93a3180317856e/numba-0.65.0-cp314-cp314t-win_amd64.whl", hash = "sha256:46fd679ae4f68c7a5d5721efbd29ecee0b0f3013211591891d79b51bfdf73113", size = 2757611, upload-time = "2026-04-01T03:52:00.083Z" }, + { url = "https://files.pythonhosted.org/packages/a0/b7/337838987380d610e2e09ab1b609929f46a0bd1cdd77a0512b1eb92e924c/nvidia_cutlass_dsl_libs_base-4.5.3-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:f788a83c4c6f62bca9ba01fced55b8899663cdd61eac0fea09e8aa8e1a5b4294", size = 75641336, upload-time = "2026-07-08T01:56:31.582Z" }, + { url = "https://files.pythonhosted.org/packages/74/fd/f1fbdc5847c2f415d3527cf7ab8d2a74165827c8bfe6b94f72cc892e2448/nvidia_cutlass_dsl_libs_base-4.5.3-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:feab418814e6973c57ca64b021e9147002d4fb13c9c745d390d232435991d8de", size = 74510856, upload-time = "2026-07-08T01:56:54.226Z" }, + { url = "https://files.pythonhosted.org/packages/61/c1/ff7cbdda6c1015ebcb5f8929ee2f565ccdbbcbc356defbee80730a104377/nvidia_cutlass_dsl_libs_base-4.5.3-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:224ceed6db71b632b318c19bea52f1d0efdafc321dbe4159a7508f93abe8c694", size = 75644212, upload-time = "2026-07-08T01:57:17.799Z" }, + { url = "https://files.pythonhosted.org/packages/a9/b3/c2e95f6c8c92d6d2ce3a30c0659b83fe9e59d757ed06c0796dd1e4657eac/nvidia_cutlass_dsl_libs_base-4.5.3-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:b36ba3ae4d5d9b0b1dc55f47d1939d2d4262d8eaad188aa675e366d4c58011cc", size = 74511395, upload-time = "2026-07-08T01:57:37.925Z" }, + { url = "https://files.pythonhosted.org/packages/c4/52/c16acdeb83983f49d1b3701add0d943ec682e45566877a91d35187285e0a/nvidia_cutlass_dsl_libs_base-4.5.3-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ef820069ee6fa79832a30efd3ba99f6bf289cf48a942f78119361710f3c0e0d1", size = 75644326, upload-time = "2026-07-08T01:58:01.27Z" }, + { url = "https://files.pythonhosted.org/packages/a6/18/170e4aca062632a7ad6d59604e29e3a7d0ed29af04fd33ce1ca0ecccd412/nvidia_cutlass_dsl_libs_base-4.5.3-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b5b5229c4ac9508b5f5554de62f35cfa162dadbd806a3dad4affb34775eb2a89", size = 74512695, upload-time = "2026-07-08T01:58:20.734Z" }, + { url = "https://files.pythonhosted.org/packages/89/39/510bc881b40d976b46d90814c005d7e2282b227c61806c2cb3ad8119d94e/nvidia_cutlass_dsl_libs_base-4.5.3-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:83070630bd0bab18a2fd4ef3c57632e11628c0e930fbf2aa070a3509a32a1079", size = 75643572, upload-time = "2026-07-08T01:58:41.114Z" }, + { url = "https://files.pythonhosted.org/packages/bd/8f/4c31eb7c71a00c70abb663f68e1fef2a8e10843f43346d91e913f9f7f32e/nvidia_cutlass_dsl_libs_base-4.5.3-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:d0e524fc5fe8901479571af6d4b8fcf14277cc2c61b791b287a7762e6c9f7a90", size = 74510664, upload-time = "2026-07-08T01:59:13.022Z" }, + { url = "https://files.pythonhosted.org/packages/39/d3/69392d89e378b4e08eb5bd6a7f6ae53f2d2ca4305fb0316ab5bb6abef21a/nvidia_cutlass_dsl_libs_base-4.5.3-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:5d29ff10d723f4525da8777ba668b107a0879b895fe2a11f1d60b555ec0a1c7c", size = 75641458, upload-time = "2026-07-08T01:59:42.113Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1c/495735145702db1ccf0e6d1806bb61b0b1f5d72094e836bb97618010e9c3/nvidia_cutlass_dsl_libs_base-4.5.3-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:acf61f778e1c26326407bb5aa2e92a236e0abe9a41abf2ae71222d90593f4b1d", size = 74510703, upload-time = "2026-07-08T02:00:07.785Z" }, + { url = "https://files.pythonhosted.org/packages/06/dd/64b2dfa3ac4140e551900710e107a2eca0c31817f63042e0956e10e54dea/nvidia_cutlass_dsl_libs_base-4.5.3-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:1bea2f2e5312a97a850be4bfe73ab0689861fa6f81888ad2abe47172f96afd1d", size = 75643870, upload-time = "2026-07-08T02:00:31.742Z" }, + { url = "https://files.pythonhosted.org/packages/85/3b/6a63abc658497740731ea387f570b6a42a42e12347fc4fe96737feba35f2/nvidia_cutlass_dsl_libs_base-4.5.3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:5516d76454f04f381111187b180bfa8651b4829e169b8283eeac8e794285d529", size = 74514616, upload-time = "2026-07-08T02:00:57.508Z" }, ] [[package]] -name = "numpy" -version = "1.26.4" +name = "nvidia-ml-py" +version = "13.610.43" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/b5/a8fbc356f768fa5c9cfd646668fd7d34bf55bdd1c6e20754642a64d930d4/nvidia_ml_py-13.610.43.tar.gz", hash = "sha256:65437eb73d68d0c62c931ca4d45038472faff03bd0b8729abba4b899f70d60f2", size = 52109, upload-time = "2026-06-01T18:54:08.829Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/94/ace0fdea5241a27d13543ee117cbc65868e82213fb31a8eb7fe9ff23f313/numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0", size = 20631468, upload-time = "2024-02-05T23:48:01.194Z" }, - { url = "https://files.pythonhosted.org/packages/20/f7/b24208eba89f9d1b58c1668bc6c8c4fd472b20c45573cb767f59d49fb0f6/numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a", size = 13966411, upload-time = "2024-02-05T23:48:29.038Z" }, - { url = "https://files.pythonhosted.org/packages/fc/a5/4beee6488160798683eed5bdb7eead455892c3b4e1f78d79d8d3f3b084ac/numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4", size = 14219016, upload-time = "2024-02-05T23:48:54.098Z" }, - { url = "https://files.pythonhosted.org/packages/4b/d7/ecf66c1cd12dc28b4040b15ab4d17b773b87fa9d29ca16125de01adb36cd/numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f", size = 18240889, upload-time = "2024-02-05T23:49:25.361Z" }, - { url = "https://files.pythonhosted.org/packages/24/03/6f229fe3187546435c4f6f89f6d26c129d4f5bed40552899fcf1f0bf9e50/numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a", size = 13876746, upload-time = "2024-02-05T23:49:51.983Z" }, - { url = "https://files.pythonhosted.org/packages/39/fe/39ada9b094f01f5a35486577c848fe274e374bbf8d8f472e1423a0bbd26d/numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2", size = 18078620, upload-time = "2024-02-05T23:50:22.515Z" }, - { url = "https://files.pythonhosted.org/packages/d5/ef/6ad11d51197aad206a9ad2286dc1aac6a378059e06e8cf22cd08ed4f20dc/numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07", size = 5972659, upload-time = "2024-02-05T23:50:35.834Z" }, - { url = "https://files.pythonhosted.org/packages/19/77/538f202862b9183f54108557bfda67e17603fc560c384559e769321c9d92/numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5", size = 15808905, upload-time = "2024-02-05T23:51:03.701Z" }, - { url = "https://files.pythonhosted.org/packages/11/57/baae43d14fe163fa0e4c47f307b6b2511ab8d7d30177c491960504252053/numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71", size = 20630554, upload-time = "2024-02-05T23:51:50.149Z" }, - { url = "https://files.pythonhosted.org/packages/1a/2e/151484f49fd03944c4a3ad9c418ed193cfd02724e138ac8a9505d056c582/numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef", size = 13997127, upload-time = "2024-02-05T23:52:15.314Z" }, - { url = "https://files.pythonhosted.org/packages/79/ae/7e5b85136806f9dadf4878bf73cf223fe5c2636818ba3ab1c585d0403164/numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e", size = 14222994, upload-time = "2024-02-05T23:52:47.569Z" }, - { url = "https://files.pythonhosted.org/packages/3a/d0/edc009c27b406c4f9cbc79274d6e46d634d139075492ad055e3d68445925/numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5", size = 18252005, upload-time = "2024-02-05T23:53:15.637Z" }, - { url = "https://files.pythonhosted.org/packages/09/bf/2b1aaf8f525f2923ff6cfcf134ae5e750e279ac65ebf386c75a0cf6da06a/numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a", size = 13885297, upload-time = "2024-02-05T23:53:42.16Z" }, - { url = "https://files.pythonhosted.org/packages/df/a0/4e0f14d847cfc2a633a1c8621d00724f3206cfeddeb66d35698c4e2cf3d2/numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a", size = 18093567, upload-time = "2024-02-05T23:54:11.696Z" }, - { url = "https://files.pythonhosted.org/packages/d2/b7/a734c733286e10a7f1a8ad1ae8c90f2d33bf604a96548e0a4a3a6739b468/numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20", size = 5968812, upload-time = "2024-02-05T23:54:26.453Z" }, - { url = "https://files.pythonhosted.org/packages/3f/6b/5610004206cf7f8e7ad91c5a85a8c71b2f2f8051a0c0c4d5916b76d6cbb2/numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2", size = 15811913, upload-time = "2024-02-05T23:54:53.933Z" }, - { url = "https://files.pythonhosted.org/packages/95/12/8f2020a8e8b8383ac0177dc9570aad031a3beb12e38847f7129bacd96228/numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", size = 20335901, upload-time = "2024-02-05T23:55:32.801Z" }, - { url = "https://files.pythonhosted.org/packages/75/5b/ca6c8bd14007e5ca171c7c03102d17b4f4e0ceb53957e8c44343a9546dcc/numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", size = 13685868, upload-time = "2024-02-05T23:55:56.28Z" }, - { url = "https://files.pythonhosted.org/packages/79/f8/97f10e6755e2a7d027ca783f63044d5b1bc1ae7acb12afe6a9b4286eac17/numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", size = 13925109, upload-time = "2024-02-05T23:56:20.368Z" }, - { url = "https://files.pythonhosted.org/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", size = 17950613, upload-time = "2024-02-05T23:56:56.054Z" }, - { url = "https://files.pythonhosted.org/packages/4c/0c/9c603826b6465e82591e05ca230dfc13376da512b25ccd0894709b054ed0/numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", size = 13572172, upload-time = "2024-02-05T23:57:21.56Z" }, - { url = "https://files.pythonhosted.org/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", size = 17786643, upload-time = "2024-02-05T23:57:56.585Z" }, - { url = "https://files.pythonhosted.org/packages/28/4a/46d9e65106879492374999e76eb85f87b15328e06bd1550668f79f7b18c6/numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", size = 5677803, upload-time = "2024-02-05T23:58:08.963Z" }, - { url = "https://files.pythonhosted.org/packages/16/2e/86f24451c2d530c88daf997cb8d6ac622c1d40d19f5a031ed68a4b73a374/numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", size = 15517754, upload-time = "2024-02-05T23:58:36.364Z" }, + { url = "https://files.pythonhosted.org/packages/23/45/caa600acfab94560807a20a64b5830d2cd3c3202b7f1328644d70b7d6bd8/nvidia_ml_py-13.610.43-py3-none-any.whl", hash = "sha256:f13c72698edef492f985cc225f14faafe68ae065a2e407f45bdf6f4b9b43fde8", size = 53163, upload-time = "2026-06-01T18:54:07.704Z" }, +] + +[[package]] +name = "nvidia-nccl-cu13" +version = "2.28.9" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/55/1920646a2e43ffd4fc958536b276197ed740e9e0c54105b4bb3521591fc7/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:01c873ba1626b54caa12272ed228dc5b2781545e0ae8ba3f432a8ef1c6d78643", size = 196561677, upload-time = "2025-11-18T05:49:03.45Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b4/878fefaad5b2bcc6fcf8d474a25e3e3774bc5133e4b58adff4d0bca238bc/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:e4553a30f34195f3fa1da02a6da3d6337d28f2003943aa0a3d247bbc25fefc42", size = 196493177, upload-time = "2025-11-18T05:49:17.677Z" }, ] [[package]] -name = "nvidia-cublas-cu12" -version = "12.8.4.1" +name = "nvidia-nvjitlink" +version = "13.0.88" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921, upload-time = "2025-03-07T01:44:31.254Z" }, + { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, + { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, + { url = "https://files.pythonhosted.org/packages/e4/01/07530b0e37546231052e30234540289c42eaffa486f1a34a87fed340157b/nvidia_nvjitlink-13.0.88-py3-none-win_amd64.whl", hash = "sha256:634e96e3da9ef845ae744097a1f289238ecf946ce0b82e93cdce14b9782e682f", size = 36035115, upload-time = "2025-09-04T08:43:03.001Z" }, ] [[package]] -name = "nvidia-cuda-cupti-cu12" -version = "12.8.90" +name = "nvidia-nvshmem-cu13" +version = "3.4.5" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621, upload-time = "2025-03-07T01:40:21.213Z" }, + { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, + { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, ] [[package]] -name = "nvidia-cuda-nvrtc-cu12" -version = "12.8.93" +name = "nvidia-nvtx" +version = "13.0.85" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029, upload-time = "2025-03-07T01:42:13.562Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, + { url = "https://files.pythonhosted.org/packages/d2/50/0e2220f8620a177de994211186ffc5bfa9f2ce1e1282797f8f90096f9f88/nvidia_nvtx-13.0.85-py3-none-win_amd64.whl", hash = "sha256:d66ea44254dd3c6eacc300047af6e1288d2269dd072b417e0adffbf479e18519", size = 137066, upload-time = "2025-09-04T08:39:25.649Z" }, ] [[package]] -name = "nvidia-cuda-runtime-cu12" -version = "12.8.90" +name = "openai" +version = "2.45.0" source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform == 'linux'" }, + { name = "distro", marker = "sys_platform == 'linux'" }, + { name = "httpx", marker = "sys_platform == 'linux'" }, + { name = "jiter", marker = "sys_platform == 'linux'" }, + { name = "pydantic", marker = "sys_platform == 'linux'" }, + { name = "sniffio", marker = "sys_platform == 'linux'" }, + { name = "tqdm", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/60/d4219875289b11d2c2f7da93c36283da224a2e55865ed865ab64e0ce9217/openai-2.45.0.tar.gz", hash = "sha256:10d34ca9c5643bce775852fddbfc172505cb1d4de1ccd101696c3ecff358765d", size = 1109653, upload-time = "2026-07-09T18:02:44.091Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765, upload-time = "2025-03-07T01:40:01.615Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b0/2291689e3ec4723fbf5bbf3b54afcd7b160f9ddc98ca7aedfd0132af5677/openai-2.45.0-py3-none-any.whl", hash = "sha256:5df105f5f8c9b711fcb9d06d2d3888cebc82506db216484c14a4e53cdf651777", size = 1629470, upload-time = "2026-07-09T18:02:42.21Z" }, ] [[package]] -name = "nvidia-cudnn-cu12" -version = "9.10.2.21" +name = "openai-harmony" +version = "0.0.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" }, + { name = "pydantic", marker = "sys_platform == 'linux'" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/3e/92/2d038d096f29179c7c9571b431f9e739f87a487121901725e23fe338dd9d/openai_harmony-0.0.8.tar.gz", hash = "sha256:6e43f98e6c242fa2de6f8ea12eab24af63fa2ed3e89c06341fb9d92632c5cbdf", size = 284777, upload-time = "2025-11-05T19:07:06.727Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, + { url = "https://files.pythonhosted.org/packages/45/c6/2502f416d46be3ec08bb66d696cccffb57781a499e3ff2e4d7c174af4e8f/openai_harmony-0.0.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:029ec25ca74abe48fdb58eb9fdd2a8c1618581fc33ce8e5653f8a1ffbfbd9326", size = 2627806, upload-time = "2025-11-05T19:06:57.063Z" }, + { url = "https://files.pythonhosted.org/packages/d3/d2/ce6953ca87db9cae3e775024184da7d1c5cb88cead19a2d75b42f00a959c/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4f709815924ec325b9a890e6ab2bbb0ceec8e319a4e257328eb752cf36b2efc", size = 2948463, upload-time = "2025-11-05T19:06:48.17Z" }, + { url = "https://files.pythonhosted.org/packages/fa/4c/b553c9651662d6ce102ca7f3629d268b23df1abe5841e24bed81e8a8e949/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5cfcfd963b50a41fc656c84d3440ca6eecdccd6c552158ce790b8f2e33dfb5a9", size = 2704083, upload-time = "2025-11-05T19:06:50.205Z" }, + { url = "https://files.pythonhosted.org/packages/9b/af/4eec8f9ab9c27bcdb444460c72cf43011d176fc44c79d6e113094ca1e152/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a3a16972aa1cee38ea958470cd04ac9a2d5ac38fdcf77ab686611246220c158", size = 2959765, upload-time = "2025-11-05T19:06:53.62Z" }, + { url = "https://files.pythonhosted.org/packages/11/3c/33f3374e4624e0e776f6b13b73c45a7ead7f9c4529f8369ed5bfcaa30cac/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4d5cfa168e74d08f8ba6d58a7e49bc7daef4d58951ec69b66b0d56f4927a68d", size = 3427031, upload-time = "2025-11-05T19:06:51.829Z" }, + { url = "https://files.pythonhosted.org/packages/25/3f/1a192b93bb47c6b44cd98ba8cc1d3d2a9308f1bb700c3017e6352da11bda/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c007d277218a50db8839e599ed78e0fffe5130f614c3f6d93ae257f282071a29", size = 2953260, upload-time = "2025-11-05T19:06:55.406Z" }, + { url = "https://files.pythonhosted.org/packages/5b/f8/93b582cad3531797c3db7c2db5400fd841538ccddfd9f5e3df61be99a630/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8565d4f5a0638da1bffde29832ed63c9e695c558611053add3b2dc0b56c92dbc", size = 3127044, upload-time = "2025-11-05T19:06:59.553Z" }, + { url = "https://files.pythonhosted.org/packages/1d/10/4327dbf87f75ae813405fd9a9b4a5cde63d506ffed0a096a440a4cabd89c/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:cbaa3bda75ef0d8836e1f8cc84af62f971b1d756d740efc95c38c3e04c0bfde2", size = 2932931, upload-time = "2025-11-05T19:07:01.437Z" }, + { url = "https://files.pythonhosted.org/packages/8a/c8/1774eec4f6f360ef57618fb8f52e3d3af245b2491bd0297513aa09eec04b/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:772922a9bd24e133950fad71eb1550836f415a88e8c77870e12d0c3bd688ddc2", size = 2996140, upload-time = "2025-11-05T19:07:03.438Z" }, + { url = "https://files.pythonhosted.org/packages/60/c3/3d1e01e2dba517a91760e4a03e4f20ffc75039a6fe584d0e6f9b5c78fd15/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:007b0476a1f331f8130783f901f1da6f5a7057af1a4891f1b6a31dec364189b5", size = 3205080, upload-time = "2025-11-05T19:07:05.078Z" }, + { url = "https://files.pythonhosted.org/packages/14/63/119de431572d7c70a7bf1037034a9be6ed0a7502a7498ba7302bca5b3242/openai_harmony-0.0.8-cp38-abi3-win32.whl", hash = "sha256:a9b5f893326b28d9e935ade14b4f655f5a840942473bc89b201c25f7a15af9cf", size = 2082457, upload-time = "2025-11-05T19:07:09.631Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/c83cf5a206c263ee70448a5ae4264682555f4d0b5bed0d2cc6ca1108103d/openai_harmony-0.0.8-cp38-abi3-win_amd64.whl", hash = "sha256:39d44f0d8f466bd56698e7ead708bead3141e27b9b87e3ab7d5a6d0e4a869ee5", size = 2438369, upload-time = "2025-11-05T19:07:08.1Z" }, ] [[package]] -name = "nvidia-cufft-cu12" -version = "11.3.3.83" +name = "opencv-python-headless" +version = "5.0.0.93" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/1d/99/76b7c80252aa83c1af16393454aafd125a0287101afe8deb0a6821af0e30/opencv_python_headless-5.0.0.93.tar.gz", hash = "sha256:b82f9831daab90b725c7c1ee1b36cb5732c367096ac76d119e64e14eb70d5f3c", size = 81817738, upload-time = "2026-07-02T07:01:06.039Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, + { url = "https://files.pythonhosted.org/packages/53/7c/8c8097891c509d98cd128493835c95631c80be6a8f37ed9d25716c2e16f1/opencv_python_headless-5.0.0.93-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:030ca5e0837a2963ab36ef896baa9767eb8d2b83353fb28af5a521e40dd8756f", size = 48322581, upload-time = "2026-07-02T05:50:34.207Z" }, + { url = "https://files.pythonhosted.org/packages/90/8c/eab2ad388c3cbab2a350c10c2ef19ce6bd099240afc31789032c996bab52/opencv_python_headless-5.0.0.93-cp37-abi3-macosx_14_0_x86_64.whl", hash = "sha256:1e55af3abfb462eeeabe5c775f12bdb36216d8a93a3583d69e6bd6e1d6ba7d00", size = 34782894, upload-time = "2026-07-02T05:51:39.856Z" }, + { url = "https://files.pythonhosted.org/packages/ec/78/afca939f40ffe2b2380bfa86f812b2f7d4acc5a27b27dc41b49cad7ce7b4/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10818d91510e05c04568ae12b5cd120779c70c01bf897b001a6221fe430df80f", size = 36521085, upload-time = "2026-07-02T06:55:24.429Z" }, + { url = "https://files.pythonhosted.org/packages/2b/97/8170e9819764c47e436c130d3ff6cfb73b58f923eae9d3a03d8982b04aec/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:09a872a157c1376ab922a69bbf22f9a95bcc7b658a9d8b436a60212b02b2eeb4", size = 56563598, upload-time = "2026-07-02T06:55:47.355Z" }, + { url = "https://files.pythonhosted.org/packages/3a/98/1a28a7101e31801042b3098871a74b76c61581d328ef40774ff4edb53a56/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:840bd717c21e5c11cadadc022a823315ea417f961213d06b4df010e019eb16f4", size = 39648433, upload-time = "2026-07-02T06:56:04.255Z" }, + { url = "https://files.pythonhosted.org/packages/9b/21/f6ef335f6e65724aa78b8d792b48d40a48c381715f1e62f5a5049e09d07e/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ed709fdf9aa0bd1f2ed8549e71d19449b03a675bb581eb292285f6861953be37", size = 61204038, upload-time = "2026-07-02T06:56:41.823Z" }, + { url = "https://files.pythonhosted.org/packages/d0/8f/b8756467ea991449a293797f6b3fa80fcfdd29598a0a60d1cd5715b96e61/opencv_python_headless-5.0.0.93-cp37-abi3-win32.whl", hash = "sha256:c6bcd96b185975ea240d22cfdb15a1f6d080cc95264cfbe2621f21bb144d89b9", size = 35411237, upload-time = "2026-07-02T05:50:12.901Z" }, + { url = "https://files.pythonhosted.org/packages/b8/88/763b967f7efd7226b82c9fae16d560cba049b1f0c036647e65c610fd636e/opencv_python_headless-5.0.0.93-cp37-abi3-win_amd64.whl", hash = "sha256:829717b6a95554f273e49e357cee3b3a2a26b6f4842fbc1bed2b45bdd8f87e0e", size = 43825962, upload-time = "2026-07-02T05:50:09.627Z" }, ] [[package]] -name = "nvidia-cufile-cu12" -version = "1.13.1.3" +name = "opentelemetry-api" +version = "1.43.0" source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/cc/e4c9584181f86494df0f6bdec1a4f3280c50db44704dc2a407e994fc87bb/opentelemetry_api-1.43.0.tar.gz", hash = "sha256:107d0d03857ea8fc7c5fcbbbd83f800c281f0d560553d61c1d675fccfd1761c1", size = 73476, upload-time = "2026-06-24T15:19:55.323Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834, upload-time = "2025-03-07T01:45:50.723Z" }, + { url = "https://files.pythonhosted.org/packages/17/83/6dba32b85f31868400440dc7ad2ca1eab94cbbf3a7b0459ed39f8311a9e2/opentelemetry_api-1.43.0-py3-none-any.whl", hash = "sha256:20acf45e9b21851926835292e4045d290acade1edd2ff3de86d2f069687ba1fd", size = 61912, upload-time = "2026-06-24T15:19:35.434Z" }, ] [[package]] -name = "nvidia-curand-cu12" -version = "10.3.9.90" +name = "opentelemetry-exporter-otlp" +version = "1.43.0" source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-exporter-otlp-proto-grpc", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/47/b77366bcbe719373a8cac2b4e8ad01f8ff3c9f2c223374d77ece280aae6f/opentelemetry_exporter_otlp-1.43.0.tar.gz", hash = "sha256:65aded6c50ee7dd2b9948c9d0e59ddb4ed4eea6e8532fba95cbe6a4a64a566ba", size = 6086, upload-time = "2026-06-24T15:19:58.003Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976, upload-time = "2025-03-07T01:46:23.323Z" }, + { url = "https://files.pythonhosted.org/packages/d9/26/d28cc854d2eb779f91351216c51ed0d54886f89f2dd56db9d493ba9fd429/opentelemetry_exporter_otlp-1.43.0-py3-none-any.whl", hash = "sha256:70f3fe740a64596d4157588a2ee7e4fd37d2acc0c0f522a2882b8c29316cd0f0", size = 6726, upload-time = "2026-06-24T15:19:38.437Z" }, ] [[package]] -name = "nvidia-cusolver-cu12" -version = "11.7.3.90" +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.43.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparse-cu12", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-proto", marker = "sys_platform == 'linux'" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/55/c1/e8098490ab15abf116dcaf9fa89ededcb35547c7d08d4b5a62f573dc1e63/opentelemetry_exporter_otlp_proto_common-1.43.0.tar.gz", hash = "sha256:c4e32ba6d6b13bdb2b8f6764c4fd28d00192826561aa04f6d14eedfce7ac076f", size = 20197, upload-time = "2026-06-24T15:20:00.247Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, + { url = "https://files.pythonhosted.org/packages/d0/b2/41ebc74ae1d5859901f1b69305de58724bf043381103d6ef413521cbc35a/opentelemetry_exporter_otlp_proto_common-1.43.0-py3-none-any.whl", hash = "sha256:123c3f9cc87218562490c63b36f497bf3a722faf174a515d1443f31ababa6264", size = 17048, upload-time = "2026-06-24T15:19:41.264Z" }, ] [[package]] -name = "nvidia-cusparse-cu12" -version = "12.5.8.93" +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.43.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, + { name = "googleapis-common-protos", marker = "sys_platform == 'linux'" }, + { name = "grpcio", version = "1.80.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.14' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version >= '3.14' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "grpcio", version = "1.82.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version < '3.14' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "opentelemetry-api", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-exporter-otlp-proto-common", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-proto", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-sdk", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/e2/1d/6336453716ca0a240d4417d19e6d5b77a5e7163e5670ec4f7ec4d3ede7bf/opentelemetry_exporter_otlp_proto_grpc-1.43.0.tar.gz", hash = "sha256:1b3e0627daa9bc21884d4a13946807c255eb558bfe5bdd543dffb6f4c9faee0d", size = 27213, upload-time = "2026-06-24T15:20:00.907Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, + { url = "https://files.pythonhosted.org/packages/6d/74/2700b5d5c946bf2dba87073fce3dfc198c46bc92ea3d5693f54bc51c90b1/opentelemetry_exporter_otlp_proto_grpc-1.43.0-py3-none-any.whl", hash = "sha256:6a10d1feacffffda19acacbf277b736094b1e2f4dbb98c90ccb2c6e1962e2ec6", size = 19626, upload-time = "2026-06-24T15:19:42.233Z" }, ] [[package]] -name = "nvidia-cusparselt-cu12" -version = "0.7.1" +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.43.0" source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-api", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-exporter-otlp-proto-common", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-proto", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-sdk", marker = "sys_platform == 'linux'" }, + { name = "requests", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/92/0b9f56412483a8891d4843890294796c9df8ab42417bd9bad8035d840cb3/opentelemetry_exporter_otlp_proto_http-1.43.0.tar.gz", hash = "sha256:fa8a42bb7d00ee5391f4c0b04d8e6a46c03caa437903296ab73a81dc11ba118f", size = 25406, upload-time = "2026-06-24T15:20:01.515Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691, upload-time = "2025-02-26T00:15:44.104Z" }, + { url = "https://files.pythonhosted.org/packages/b3/20/b685ed7af2e17c29ffc8af56f1fa8bc2033258fc30fb0d2b722f49d13ba0/opentelemetry_exporter_otlp_proto_http-1.43.0-py3-none-any.whl", hash = "sha256:647f603aa8efdbdb4dbff842e0729d0406a6fff26b295a72d3d60e7d963b2610", size = 21795, upload-time = "2026-06-24T15:19:43.164Z" }, ] [[package]] -name = "nvidia-nccl-cu12" -version = "2.27.5" +name = "opentelemetry-proto" +version = "1.43.0" source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/b9/d357faefb40bda1d4799913e6af611171ff22a2dedcb93576bc92242d056/opentelemetry_proto-1.43.0.tar.gz", hash = "sha256:224778df17e1f3fafeaaa21d874236ca5f6ffc2f86e0899298ec7351aac27924", size = 46481, upload-time = "2026-06-24T15:20:07.625Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ad730cf15cb5d25fe849c6e6ca9eb5b76db16a80f13f425ac68d8e2e55624457", size = 322348229, upload-time = "2025-06-26T04:11:28.385Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a7/3e5308cf548b8f72529c7db1afdb3a404211982376a12927fd7759f77bf3/opentelemetry_proto-1.43.0-py3-none-any.whl", hash = "sha256:c58f1f7ef84bc7dc2834016c0c37fe0081dde7ca9f6339be1970fbf9cdaaa90d", size = 72489, upload-time = "2026-06-24T15:19:51.164Z" }, ] [[package]] -name = "nvidia-nvjitlink-cu12" -version = "12.8.93" +name = "opentelemetry-sdk" +version = "1.43.0" source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-semantic-conventions", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3e/eb/5041074274ac0956b03637cc039d434569112468e875eddfcc9a0674ce06/opentelemetry_sdk-1.43.0.tar.gz", hash = "sha256:d8187c81c162df9913e4003dd6485f7390d9a24fc17026ec7387b8b8218b08e9", size = 254744, upload-time = "2026-06-24T15:20:08.467Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836, upload-time = "2025-03-07T01:49:55.661Z" }, + { url = "https://files.pythonhosted.org/packages/49/e3/b17be23af124201c9f52eececd4cc8ddfed1597d37b4ee771895d325805c/opentelemetry_sdk-1.43.0-py3-none-any.whl", hash = "sha256:d1323a547c1ce69d6a069a17a44b7da82bb8b332051ecb074041f87642c86823", size = 178852, upload-time = "2026-06-24T15:19:52.169Z" }, ] [[package]] -name = "nvidia-nvshmem-cu12" -version = "3.4.5" +name = "opentelemetry-semantic-conventions" +version = "0.64b0" source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/30/5f26df29509eccd86b99b481ac9ffa39da49ba9577cc69071c552ae30447/opentelemetry_semantic_conventions-0.64b0.tar.gz", hash = "sha256:72f76fb2d1582d9d033dd1fcd84532e961e6ff3d90d24ba6fabc72975a83864c", size = 148340, upload-time = "2026-06-24T15:20:09.267Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/09/6ea3ea725f82e1e76684f0708bbedd871fc96da89945adeba65c3835a64c/nvidia_nvshmem_cu12-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:042f2500f24c021db8a06c5eec2539027d57460e1c1a762055a6554f72c369bd", size = 139103095, upload-time = "2025-09-06T00:32:31.266Z" }, + { url = "https://files.pythonhosted.org/packages/f2/ca/23ba87a221b574a7c5a99d48849d80bfe8b047624681357e2b002e566187/opentelemetry_semantic_conventions-0.64b0-py3-none-any.whl", hash = "sha256:ea77e85e354b8f604ddbe5f3d9135216f982fa4d77e5859ac30f6d8a50505aa6", size = 203713, upload-time = "2026-06-24T15:19:53.339Z" }, ] [[package]] -name = "nvidia-nvtx-cu12" -version = "12.8.90" +name = "opentelemetry-semantic-conventions-ai" +version = "0.5.1" source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-sdk", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-semantic-conventions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/24/02/10aeacc37a38a3a8fa16ff67bec1ae3bf882539f6f9efb0f70acf802ca2d/opentelemetry_semantic_conventions_ai-0.5.1.tar.gz", hash = "sha256:153906200d8c1d2f8e09bd78dbef526916023de85ac3dab35912bfafb69ff04c", size = 26533, upload-time = "2026-03-26T14:20:38.73Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" }, + { url = "https://files.pythonhosted.org/packages/55/22/41fb05f1dc5fda2c468e05a41814c20859016c85117b66c8a257cae814f6/opentelemetry_semantic_conventions_ai-0.5.1-py3-none-any.whl", hash = "sha256:25aeb22bd261543b4898a73824026d96770e5351209c7d07a0b1314762b1f6e4", size = 11250, upload-time = "2026-03-26T14:20:37.108Z" }, ] [[package]] @@ -4242,7 +5991,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, { name = "ninja" }, - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "safetensors" }, { name = "torch" }, ] @@ -4332,6 +6083,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/d1/facb5b5051fabb0ef9d26c6544d87ef19a939a9a001198655d0d891062dd/orjson-3.11.8-cp314-cp314-win_arm64.whl", hash = "sha256:6ccdea2c213cf9f3d9490cbd5d427693c870753df41e6cb375bd79bcbafc8817", size = 127330, upload-time = "2026-03-31T16:16:25.496Z" }, ] +[[package]] +name = "outlines-core" +version = "0.2.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/04/4a0812eb27c086cfd2e66e7ec9150f33e105912a9b7f8b335e3479f03a06/outlines_core-0.2.14.tar.gz", hash = "sha256:64808deed1591ca3029ff64346ceb974cd5d780c916ea82504951fe83523039e", size = 191539, upload-time = "2026-01-09T15:59:10.016Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d0/e7719044b3ed57fde6f700211be67682eec4a2735fcf8d4199400ac8f7a3/outlines_core-0.2.14-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:056f656ea6e4807338963377afb50b9d936593ba3545a819f1aba56fd6e14920", size = 2050100, upload-time = "2026-01-09T15:58:07.323Z" }, + { url = "https://files.pythonhosted.org/packages/fb/ff/f41c933bc5b69b7080e763af375d7632d419859b6cd4492c5a12d5c89c80/outlines_core-0.2.14-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:c76f28feb6ea71b1ff4b0ba5901dc383273a32b156213dc1bc753fc634645a1e", size = 2200812, upload-time = "2026-01-09T15:58:08.876Z" }, + { url = "https://files.pythonhosted.org/packages/ab/3c/05b3e5b9fa4f7f40da459022b2292c6bca48492df920eebd3572844e7fe6/outlines_core-0.2.14-cp310-cp310-macosx_15_0_arm64.whl", hash = "sha256:e604925d6525f669253160568397df6d6c8124b2e01f1fde553e3b9f28ce9e21", size = 2050299, upload-time = "2026-01-09T15:58:10.176Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ee/abcee1d35c7a7fde984ce4299de2db72add3c8a88a0f6e9e7f8c4836151d/outlines_core-0.2.14-cp310-cp310-macosx_15_0_x86_64.whl", hash = "sha256:f0e5037153b5b3abfb617f6dfdc3ff28b6fab50f0de5936ea6995f5675d23e0b", size = 2197822, upload-time = "2026-01-09T15:58:11.253Z" }, + { url = "https://files.pythonhosted.org/packages/19/cc/6f5d1b92e79b30c2ac187b23ab66b0365e06007a9405b35709d02e94615b/outlines_core-0.2.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e4b5b7c8e50489bea444b095692ddb5d8fb92ea6b949c4a6a3381eca9b691a7", size = 2339071, upload-time = "2026-01-09T15:58:12.488Z" }, + { url = "https://files.pythonhosted.org/packages/66/f7/252c0d4ecc5020d698b23257b9860fbab020c1120c15417ed02a9fd82d19/outlines_core-0.2.14-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:b582b5d2f773cff966f37d7a5680d97506792647c93fb2e522283e8a14726e9d", size = 2236310, upload-time = "2026-01-09T15:58:13.942Z" }, + { url = "https://files.pythonhosted.org/packages/b6/60/5599fef9e4184a99684c9cc285081a1b5f4a741b2d6c676466a0ca365d39/outlines_core-0.2.14-cp310-cp310-win32.whl", hash = "sha256:f753edd430ac27e6dcde5a614665888db72b78c666aa160c478afd1eb986fb8b", size = 1841873, upload-time = "2026-01-09T15:58:15.282Z" }, + { url = "https://files.pythonhosted.org/packages/8b/5f/da4daf605ab9c26e854a741c3567eb717e8622d4d17bcd751da5238b039f/outlines_core-0.2.14-cp310-cp310-win_amd64.whl", hash = "sha256:060a0174a6262bfd378763f210e374e52011776849a3a767df9863fb6839c142", size = 2136404, upload-time = "2026-01-09T15:58:16.939Z" }, + { url = "https://files.pythonhosted.org/packages/05/12/a67f0be9546776f71c5df373f38ce6db965abc9845fbcd291b393a20712e/outlines_core-0.2.14-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:7770b5e0497e6f4548a8923299d4438d7dd61dc17c2f58acfd5df4d3101bb991", size = 2050098, upload-time = "2026-01-09T15:58:18.399Z" }, + { url = "https://files.pythonhosted.org/packages/34/31/f2e19cc32ea97c1bac4882dbfa693671175a330ad5a735af5b97c2258056/outlines_core-0.2.14-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:a2795dc2047821b229457f941a303639e0c14e4c3c5718797540a27b529a062e", size = 2200792, upload-time = "2026-01-09T15:58:19.775Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ea/19e859d4cfcbeceace30ad490f5369c87eab81767238593e20c17f55a390/outlines_core-0.2.14-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:4daa22d677dc6a74c44f9266ec9e3151332dcea4250dd019ea0c75b98ae32938", size = 2050363, upload-time = "2026-01-09T15:58:20.981Z" }, + { url = "https://files.pythonhosted.org/packages/c9/db/188aecb87008ddd293b8d315f26017750a1d7f9e95b8e2756d4a3af08196/outlines_core-0.2.14-cp311-cp311-macosx_15_0_x86_64.whl", hash = "sha256:813b28813b22025c3d079b3b8a20cf5a28c6d5ba29ec21c5b1093442aa5d4e91", size = 2197869, upload-time = "2026-01-09T15:58:22.11Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/e0be45d4c8ad7d301cdc9917d22ff39211da1e830f92fb07b29c9221b5c4/outlines_core-0.2.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:615566bf8257d2bba8ac192cdfc29d1c4357f57b53672fbd622e821215e4f1bd", size = 2338968, upload-time = "2026-01-09T15:58:23.317Z" }, + { url = "https://files.pythonhosted.org/packages/f2/67/9dab90313460eb250f926e7985d62cebfc33c7580197be8a496de6e9f7c4/outlines_core-0.2.14-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:81d01cfae29de5671bc5013fd6b2008621157bec3d8be284da7da2dc0672745c", size = 2236169, upload-time = "2026-01-09T15:58:24.575Z" }, + { url = "https://files.pythonhosted.org/packages/ef/91/289996bae3457cf3917ff21e0082e4950cf27a101d0870e16fee94c917e0/outlines_core-0.2.14-cp311-cp311-win32.whl", hash = "sha256:8a5e5f34961fe4d04c389d00f92d624c6318ab3ff00467fbf7c93324458886d9", size = 1841978, upload-time = "2026-01-09T15:58:26.309Z" }, + { url = "https://files.pythonhosted.org/packages/be/65/2d59be2f8c0cca118a6235ab2286615e3c1b2fa9d6768c4ea4b86b556204/outlines_core-0.2.14-cp311-cp311-win_amd64.whl", hash = "sha256:babf97a54662330c55a79fdcab8994f96faa6dcb71b458d4b18c4fb538f5d461", size = 2136353, upload-time = "2026-01-09T15:58:27.443Z" }, + { url = "https://files.pythonhosted.org/packages/66/93/30b9188648a479b32be429a24166db47a7bfdb0f9a8aac4c6dcf569e0a52/outlines_core-0.2.14-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:95e6476d9702d2fcc4e85370dbbfb6933a46c816e9c90107f6ce36eb68b5d64a", size = 2049651, upload-time = "2026-01-09T15:58:28.549Z" }, + { url = "https://files.pythonhosted.org/packages/0d/06/f3557daa8e87d5b95f64de269a301d73ec3c2202ab897c3e1f1cb93eb1db/outlines_core-0.2.14-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:f04731a5e29a190e2cc9f692a1f3fb2414a645355ca7d01b83df43439c38bea8", size = 2201046, upload-time = "2026-01-09T15:58:29.958Z" }, + { url = "https://files.pythonhosted.org/packages/0c/67/d8acf778990964c951080d568284e858d466f27dfd6f2674781927faba1c/outlines_core-0.2.14-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:0e4c69f0a8565edb56464c4c9b6c291a10805f3a96dff84182980e90ae1a5e2f", size = 2049558, upload-time = "2026-01-09T15:58:31.003Z" }, + { url = "https://files.pythonhosted.org/packages/17/e1/0320b14b49b8379ced1ab195ecf5875dbd2267b90148847541f43bfde6c1/outlines_core-0.2.14-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:63f53cfd9614e754499ae86dd699f3abcecf42d6a4e58d80fd80347881d85960", size = 2197854, upload-time = "2026-01-09T15:58:32.39Z" }, + { url = "https://files.pythonhosted.org/packages/29/29/3a04944407207a5d214879ca5ca33c2bd3e65199a4e927051c1bdaaa4d50/outlines_core-0.2.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bb2060c240c4507f334965a8948dbeeb22007560d797f6debd92346c0b620cb", size = 2341426, upload-time = "2026-01-09T15:58:33.553Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a7/a77f746272504bac3f628047d56ea1731b61549a3e1d9bbfd226f2968246/outlines_core-0.2.14-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1de34681c7e0e7e1551fc9036e4fa3c57986336c905a10536591ceb6d869c258", size = 2236941, upload-time = "2026-01-09T15:58:35.118Z" }, + { url = "https://files.pythonhosted.org/packages/99/0d/9f599d938923ab8ceeff26fdf2f9ea53bea3c962085c4927a08338a32349/outlines_core-0.2.14-cp312-cp312-win32.whl", hash = "sha256:870e8e038853818cb202ccc8cde92251f300f96805bfcc3be1c883adda7b5297", size = 1842940, upload-time = "2026-01-09T15:58:36.544Z" }, + { url = "https://files.pythonhosted.org/packages/f8/df/0f145c52ebd156d80273e2f5278227ea57e0275b2aa863bed33f44f77923/outlines_core-0.2.14-cp312-cp312-win_amd64.whl", hash = "sha256:87b42440478764cce1353a87d8560ef82f3b39b9d753bfe93195ea3584f369e3", size = 2137266, upload-time = "2026-01-09T15:58:37.831Z" }, + { url = "https://files.pythonhosted.org/packages/13/9d/e6c81c975c123f0639d5f6909c987e510d43e07c2e1e6495b21639c4dec6/outlines_core-0.2.14-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8b3e8d668188282a1f7666732bb8a01958ab134db35bb792e7442a40e55ff1e7", size = 2049297, upload-time = "2026-01-09T15:58:39.184Z" }, + { url = "https://files.pythonhosted.org/packages/7a/d1/5ce55ef724aed0915edc877b6dd610d39b3169e4341154bb53daa022065a/outlines_core-0.2.14-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:66e695b375b180725fb534d9adf298531c152ec3d881e3b9e01c82b5dd269f52", size = 2200944, upload-time = "2026-01-09T15:58:40.257Z" }, + { url = "https://files.pythonhosted.org/packages/32/e3/60ad781251eedcf1496317ecd58eb2e4488717ba63b10494ab49dfd05e5d/outlines_core-0.2.14-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:6bd166d3b07acef2f60d4ede44592a26d3f7d8712876bfc8e22150045def5857", size = 2049607, upload-time = "2026-01-09T15:58:41.635Z" }, + { url = "https://files.pythonhosted.org/packages/bc/2d/662d6a76face5b4b3481f888900d00856c37aa2927341a023866457da212/outlines_core-0.2.14-cp313-cp313-macosx_15_0_x86_64.whl", hash = "sha256:9d45462d7548aa0e17176a691ae73447f3e6bed9658a0cd96fe72eadf7474475", size = 2197755, upload-time = "2026-01-09T15:58:42.861Z" }, + { url = "https://files.pythonhosted.org/packages/c1/9a/4b62903de006d991b58674ff033c1b6fb92be5767360376fc961f6771bdb/outlines_core-0.2.14-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6453e23f01d98ec48e3a4141d7112792ce77001dfb28d91d6fd89f47009f91ef", size = 2341051, upload-time = "2026-01-09T15:58:44.415Z" }, + { url = "https://files.pythonhosted.org/packages/50/36/1532f7d9ab16c676812d94528e89964aa0d15f12adcb285e6ed86f86f2fe/outlines_core-0.2.14-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:7deef6df74cb247f2a3a62f03438ba967456504b0555ec7029f8db834e054448", size = 2236778, upload-time = "2026-01-09T15:58:45.437Z" }, + { url = "https://files.pythonhosted.org/packages/a8/5a/dfd94f15f4c04e691e7fdf30cf8b9b22bf2cbc426b3ef270af3e200596d5/outlines_core-0.2.14-cp313-cp313-win32.whl", hash = "sha256:bb008c7ecc034bcfda0ddc10a4d1f2181a4b61ec1643ee56183dd6fa64139c9d", size = 1842727, upload-time = "2026-01-09T15:58:46.723Z" }, + { url = "https://files.pythonhosted.org/packages/34/35/e24ab5d2116812464380587435297d8ece2f0218c2ba8afc9f541e3a6911/outlines_core-0.2.14-cp313-cp313-win_amd64.whl", hash = "sha256:eb27e92204b296a063ac58f361153be4e78c8103a96e0b1c085b22d4fc3534cf", size = 2137108, upload-time = "2026-01-09T15:58:47.784Z" }, + { url = "https://files.pythonhosted.org/packages/1b/28/22fe8ee3bdf9cf13ab88a9d9b96729d9966c791c25227d0b7ca45c8d118f/outlines_core-0.2.14-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:69410e5b55bcbaad8c865d94bd01e7bff8a57996dcd2251b7d50dec70d7d9a63", size = 2050470, upload-time = "2026-01-09T15:58:49.217Z" }, + { url = "https://files.pythonhosted.org/packages/d4/3e/30ce0b13e4c4c82de606c8bbf60775ac6fca1978efa54cd553893795fd0b/outlines_core-0.2.14-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:adf96395759d7fdf6efeb8a67d3f36f520c1546bfd4df0752306db8c7cb7d6c5", size = 2202138, upload-time = "2026-01-09T15:58:50.281Z" }, + { url = "https://files.pythonhosted.org/packages/53/13/bd2ff9e90b28fa0dcc345c9196731ed9126e366733c8ccbc559149e34893/outlines_core-0.2.14-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:b02bb0fc21c5e23e2ff9b2d1459db2c1c3e813a7646c9d5db091c6931edb9c85", size = 2050325, upload-time = "2026-01-09T15:58:51.596Z" }, + { url = "https://files.pythonhosted.org/packages/1e/25/fc0ae7d04345d17267d4dd5c693ed9e86c7f44419cc04ad92348472781be/outlines_core-0.2.14-cp314-cp314-macosx_15_0_x86_64.whl", hash = "sha256:e75395b1cccecdf85d8d8265aba28841ddeb1e8da406f4b1e0135df5a6e9960f", size = 2199081, upload-time = "2026-01-09T15:58:53.17Z" }, + { url = "https://files.pythonhosted.org/packages/d5/63/dfa000239e46f17b47e6dc9bec3aab8a8136fe400312f1916320e02c8f38/outlines_core-0.2.14-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1776ae984574461f249fe590314a439992eb9b883f4091b8fa7fc56f29f3717", size = 2343210, upload-time = "2026-01-09T15:58:54.282Z" }, + { url = "https://files.pythonhosted.org/packages/36/4f/0e63da06c6054f154ef22b5ef3c6b9030cb22da9c03d2d2dd82836a1e795/outlines_core-0.2.14-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:7eba2b41dac03d6e6e8d5ea0aecbbc03dacb4c57de3b1fc944d0bafb022941f7", size = 2238206, upload-time = "2026-01-09T15:58:55.705Z" }, + { url = "https://files.pythonhosted.org/packages/74/4e/382271ab5ffe768055f11dddb50e82a0c46487f3766bf08a06cfcd35388b/outlines_core-0.2.14-cp314-cp314-win32.whl", hash = "sha256:0cd8ce3ce61df44fd9c5450d9744e2280586c2a6e6e3dfefa0dab1944764b424", size = 1845364, upload-time = "2026-01-09T15:58:56.795Z" }, + { url = "https://files.pythonhosted.org/packages/0d/11/13adf2d02c681b599c1eb550b0dbd763d1b818a106a13bd693019bdb5637/outlines_core-0.2.14-cp314-cp314-win_amd64.whl", hash = "sha256:3e67fc23b1a3ac9562488fb50f409c171538b76f64aa5f7e25d9b0bf14770204", size = 2139979, upload-time = "2026-01-09T15:58:57.984Z" }, +] + [[package]] name = "overrides" version = "7.7.0" @@ -4355,7 +6154,9 @@ name = "pandas" version = "2.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "python-dateutil" }, { name = "pytz" }, { name = "tzdata" }, @@ -4432,6 +6233,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/ac/dac4a63f978e4dcb3c6d3a78c4d8e0192a113d288502a1216950c41b1027/parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18", size = 103650, upload-time = "2024-04-05T09:43:53.299Z" }, ] +[[package]] +name = "partial-json-parser" +version = "0.2.1.1.post7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/6d/eed37d7ebc1e0bcd27b831c0cf1fe94881934316187c4b30d23f29ea0bd4/partial_json_parser-0.2.1.1.post7.tar.gz", hash = "sha256:86590e1ba6bcb6739a2dfc17d2323f028cb5884f4c6ce23db376999132c9a922", size = 10296, upload-time = "2025-11-17T07:27:41.202Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/32/658973117bf0fd82a24abbfb94fe73a5e86216e49342985e10acce54775a/partial_json_parser-0.2.1.1.post7-py3-none-any.whl", hash = "sha256:145119e5eabcf80cbb13844a6b50a85c68bf99d376f8ed771e2a3c3b03e653ae", size = 10877, upload-time = "2025-11-17T07:27:40.457Z" }, +] + [[package]] name = "pathlib-abc" version = "0.5.2" @@ -4464,7 +6274,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "python_full_version < '3.13' or sys_platform != 'win32' or extra == 'extra-16-transformer-lens-lit' or extra != 'extra-16-transformer-lens-vllm'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -4605,7 +6415,7 @@ name = "plumbum" version = "1.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pywin32", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'win32'" }, + { name = "pywin32", marker = "(platform_python_implementation != 'PyPy' and sys_platform == 'win32') or (platform_python_implementation == 'PyPy' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'win32' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f0/5d/49ba324ad4ae5b1a4caefafbce7a1648540129344481f2ed4ef6bb68d451/plumbum-1.9.0.tar.gz", hash = "sha256:e640062b72642c3873bd5bdc3effed75ba4d3c70ef6b6a7b907357a84d909219", size = 319083, upload-time = "2024-10-05T05:59:27.059Z" } wheels = [ @@ -4638,7 +6448,7 @@ name = "portalocker" version = "3.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "pywin32", marker = "sys_platform == 'win32' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5e/77/65b857a69ed876e1951e88aaba60f5ce6120c33703f7cb61a3c894b8c1b6/portalocker-3.2.0.tar.gz", hash = "sha256:1f3002956a54a8c3730586c5c77bf18fae4149e07eaf1c29fc3faf4d5a3f89ac", size = 95644, upload-time = "2025-06-14T13:20:40.03Z" } wheels = [ @@ -4666,6 +6476,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/ae/ec06af4fe3ee72d16973474f122541746196aaa16cea6f66d18b963c6177/prometheus_client-0.22.1-py3-none-any.whl", hash = "sha256:cca895342e308174341b2cbf99a56bef291fbc0ef7b9e5412a0f26d653ba7094", size = 58694, upload-time = "2025-06-02T14:29:00.068Z" }, ] +[[package]] +name = "prometheus-fastapi-instrumentator" +version = "8.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "prometheus-client", marker = "sys_platform == 'linux'" }, + { name = "starlette", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1b/e9/2065686d1dfa62296fdc158b6e8fd25b0cb3dca09b0632cabeb5ae81fe4d/prometheus_fastapi_instrumentator-8.0.2.tar.gz", hash = "sha256:3c252e748151768a7aefd66824a04a870144f71de48a67aed211749a9ca2a548", size = 21342, upload-time = "2026-06-23T09:39:31.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/c7/fa2b3f469a2e6001b829e0d6bc8680755349aa1329a87bf48731e9d5d30a/prometheus_fastapi_instrumentator-8.0.2-py3-none-any.whl", hash = "sha256:746002ec1e2c58b93f61444e1d104de959a9463a6a3f1c8909ac3757e16c3866", size = 20549, upload-time = "2026-06-23T09:39:32.616Z" }, +] + [[package]] name = "prompt-toolkit" version = "3.0.51" @@ -4781,16 +6604,17 @@ wheels = [ [[package]] name = "protobuf" -version = "6.31.1" +version = "7.35.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/f3/b9655a711b32c19720253f6f06326faf90580834e2e83f840472d752bc8b/protobuf-6.31.1.tar.gz", hash = "sha256:d8cac4c982f0b957a4dc73a80e2ea24fab08e679c0de9deb835f4a12d69aca9a", size = 441797, upload-time = "2025-05-28T19:25:54.947Z" } +sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/6f/6ab8e4bf962fd5570d3deaa2d5c38f0a363f57b4501047b5ebeb83ab1125/protobuf-6.31.1-cp310-abi3-win32.whl", hash = "sha256:7fa17d5a29c2e04b7d90e5e32388b8bfd0e7107cd8e616feef7ed3fa6bdab5c9", size = 423603, upload-time = "2025-05-28T19:25:41.198Z" }, - { url = "https://files.pythonhosted.org/packages/44/3a/b15c4347dd4bf3a1b0ee882f384623e2063bb5cf9fa9d57990a4f7df2fb6/protobuf-6.31.1-cp310-abi3-win_amd64.whl", hash = "sha256:426f59d2964864a1a366254fa703b8632dcec0790d8862d30034d8245e1cd447", size = 435283, upload-time = "2025-05-28T19:25:44.275Z" }, - { url = "https://files.pythonhosted.org/packages/6a/c9/b9689a2a250264a84e66c46d8862ba788ee7a641cdca39bccf64f59284b7/protobuf-6.31.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:6f1227473dc43d44ed644425268eb7c2e488ae245d51c6866d19fe158e207402", size = 425604, upload-time = "2025-05-28T19:25:45.702Z" }, - { url = "https://files.pythonhosted.org/packages/76/a1/7a5a94032c83375e4fe7e7f56e3976ea6ac90c5e85fac8576409e25c39c3/protobuf-6.31.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:a40fc12b84c154884d7d4c4ebd675d5b3b5283e155f324049ae396b95ddebc39", size = 322115, upload-time = "2025-05-28T19:25:47.128Z" }, - { url = "https://files.pythonhosted.org/packages/fa/b1/b59d405d64d31999244643d88c45c8241c58f17cc887e73bcb90602327f8/protobuf-6.31.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:4ee898bf66f7a8b0bd21bce523814e6fbd8c6add948045ce958b73af7e8878c6", size = 321070, upload-time = "2025-05-28T19:25:50.036Z" }, - { url = "https://files.pythonhosted.org/packages/f7/af/ab3c51ab7507a7325e98ffe691d9495ee3d3aa5f589afad65ec920d39821/protobuf-6.31.1-py3-none-any.whl", hash = "sha256:720a6c7e6b77288b85063569baae8536671b39f15cc22037ec7045658d80489e", size = 168724, upload-time = "2025-05-28T19:25:53.926Z" }, + { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" }, + { url = "https://files.pythonhosted.org/packages/37/4b/dfb89eb0e652a1ff073c39a59fb5e3a83cfe9b57a2c83fa6d78270101767/protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799", size = 328847, upload-time = "2026-06-11T21:55:34.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/58/dc12f2cd484951524af6e3382c785869b9b3fb5e52ee95ae23add53ee8f9/protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4", size = 344030, upload-time = "2026-06-11T21:55:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4", size = 327130, upload-time = "2026-06-11T21:55:35.921Z" }, + { url = "https://files.pythonhosted.org/packages/d8/bc/6d6c7ba8709c85f8f2c390b2b118d6fb08a783676a572271851bf45a7d22/protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30", size = 428945, upload-time = "2026-06-11T21:55:37.034Z" }, + { url = "https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87", size = 439996, upload-time = "2026-06-11T21:55:38.123Z" }, + { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, ] [[package]] @@ -4826,6 +6650,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, +] + [[package]] name = "pyarrow" version = "21.0.0" @@ -4890,6 +6723,179 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, ] +[[package]] +name = "pybase64" +version = "1.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/b8/4ed5c7ad5ec15b08d35cc79ace6145d5c1ae426e46435f4987379439dfea/pybase64-1.4.3.tar.gz", hash = "sha256:c2ed274c9e0ba9c8f9c4083cfe265e66dd679126cd9c2027965d807352f3f053", size = 137272, upload-time = "2025-12-06T13:27:04.013Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/47/16d7af6fae7803f4c691856bc0d8d433ccf30e106432e2ef7707ee19a38a/pybase64-1.4.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f63aa7f29139b8a05ce5f97cdb7fad63d29071e5bdc8a638a343311fe996112a", size = 38241, upload-time = "2025-12-06T13:22:27.396Z" }, + { url = "https://files.pythonhosted.org/packages/4d/3e/268beb8d2240ab55396af4d1b45d2494935982212549b92a5f5b57079bd3/pybase64-1.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f5943ec1ae87a8b4fe310905bb57205ea4330c75e2c628433a7d9dd52295b588", size = 31672, upload-time = "2025-12-06T13:22:28.854Z" }, + { url = "https://files.pythonhosted.org/packages/80/14/4365fa33222edcc46b6db4973f9e22bda82adfb6ab2a01afff591f1e41c8/pybase64-1.4.3-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5f2b8aef86f35cd5894c13681faf433a1fffc5b2e76544dcb5416a514a1a8347", size = 65978, upload-time = "2025-12-06T13:22:30.191Z" }, + { url = "https://files.pythonhosted.org/packages/1c/22/e89739d8bc9b96c68ead44b4eec42fe555683d9997e4ba65216d384920fc/pybase64-1.4.3-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6ec7e53dd09b0a8116ccf5c3265c7c7fce13c980747525be76902aef36a514a", size = 68903, upload-time = "2025-12-06T13:22:31.29Z" }, + { url = "https://files.pythonhosted.org/packages/77/e1/7e59a19f8999cdefe9eb0d56bfd701dd38263b0f6fb4a4d29fce165a1b36/pybase64-1.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7528604cd69c538e1dbaafded46e9e4915a2adcd6f2a60fcef6390d87ca922ea", size = 57516, upload-time = "2025-12-06T13:22:32.395Z" }, + { url = "https://files.pythonhosted.org/packages/42/ad/f47dc7e6fe32022b176868b88b671a32dab389718c8ca905cab79280aaaf/pybase64-1.4.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:4ec645f32b50593879031e09158f8681a1db9f5df0f72af86b3969a1c5d1fa2b", size = 54533, upload-time = "2025-12-06T13:22:33.457Z" }, + { url = "https://files.pythonhosted.org/packages/7c/9a/7ab312b5a324833953b00e47b23eb4f83d45bd5c5c854b4b4e51b2a0cf5b/pybase64-1.4.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:634a000c5b3485ccc18bb9b244e0124f74b6fbc7f43eade815170237a7b34c64", size = 57187, upload-time = "2025-12-06T13:22:34.566Z" }, + { url = "https://files.pythonhosted.org/packages/2c/84/80acab1fcbaaae103e6b862ef5019192c8f2cd8758433595a202179a0d1d/pybase64-1.4.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:309ea32ad07639a485580af1be0ad447a434deb1924e76adced63ac2319cfe15", size = 57730, upload-time = "2025-12-06T13:22:35.581Z" }, + { url = "https://files.pythonhosted.org/packages/1f/24/84256d472400ea3163d7d69c44bb7e2e1027f0f1d4d20c47629a7dc4578e/pybase64-1.4.3-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:d10d517566b748d3f25f6ac7162af779360c1c6426ad5f962927ee205990d27c", size = 53036, upload-time = "2025-12-06T13:22:36.621Z" }, + { url = "https://files.pythonhosted.org/packages/a3/0f/33aecbed312ee0431798a73fa25e00dedbffdd91389ee23121fed397c550/pybase64-1.4.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a74cc0f4d835400857cc5c6d27ec854f7949491e07a04e6d66e2137812831f4c", size = 56321, upload-time = "2025-12-06T13:22:37.7Z" }, + { url = "https://files.pythonhosted.org/packages/dc/1c/a341b050746658cbec8cab3c733aeb3ef52ce8f11e60d0d47adbdf729ebf/pybase64-1.4.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b591d774ac09d5eb73c156a03277cb271438fbd8042bae4109ff3a827cd218c", size = 50114, upload-time = "2025-12-06T13:22:38.752Z" }, + { url = "https://files.pythonhosted.org/packages/ba/d3/f7e6680ae6dc4ddff39112ad66e0fa6b2ec346e73881bafc08498c560bc0/pybase64-1.4.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5eb588d35a04302ef6157d17db62354a787ac6f8b1585dd0b90c33d63a97a550", size = 66570, upload-time = "2025-12-06T13:22:40.221Z" }, + { url = "https://files.pythonhosted.org/packages/4c/71/774748eecc7fe23869b7e5df028e3c4c2efa16b506b83ea3fa035ea95dc2/pybase64-1.4.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:df8b122d5be2c96962231cc4831d9c2e1eae6736fb12850cec4356d8b06fe6f8", size = 55700, upload-time = "2025-12-06T13:22:41.289Z" }, + { url = "https://files.pythonhosted.org/packages/b3/91/dd15075bb2fe0086193e1cd4bad80a43652c38d8a572f9218d46ba721802/pybase64-1.4.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31b7a85c661fc591bbcce82fb8adaebe2941e6a83b08444b0957b77380452a4b", size = 52491, upload-time = "2025-12-06T13:22:42.628Z" }, + { url = "https://files.pythonhosted.org/packages/7b/27/f357d63ea3774c937fc47160e040419ed528827aa3d4306d5ec9826259c0/pybase64-1.4.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e6d7beaae65979fef250e25e66cf81c68a8f81910bcda1a2f43297ab486a7e4e", size = 53957, upload-time = "2025-12-06T13:22:44.615Z" }, + { url = "https://files.pythonhosted.org/packages/b3/c3/243693771701a54e67ff5ccbf4c038344f429613f5643169a7befc51f007/pybase64-1.4.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4a6276bc3a3962d172a2b5aba544d89881c4037ea954517b86b00892c703d007", size = 68422, upload-time = "2025-12-06T13:22:45.641Z" }, + { url = "https://files.pythonhosted.org/packages/75/95/f987081bf6bc1d1eda3012dae1b06ad427732ef9933a632cb8b58f9917f8/pybase64-1.4.3-cp310-cp310-win32.whl", hash = "sha256:4bdd07ef017515204ee6eaab17e1ad05f83c0ccb5af8ae24a0fe6d9cb5bb0b7a", size = 33622, upload-time = "2025-12-06T13:22:47.348Z" }, + { url = "https://files.pythonhosted.org/packages/79/28/c169a769fe90128f16d394aad87b2096dd4bf2f035ae0927108a46b617df/pybase64-1.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:5db0b6bbda15110db2740c61970a8fda3bf9c93c3166a3f57f87c7865ed1125c", size = 35799, upload-time = "2025-12-06T13:22:48.731Z" }, + { url = "https://files.pythonhosted.org/packages/ab/f2/bdbe6af0bd4f3fe5bc70e77ead7f7d523bb9d3ca3ad50ac42b9adbb9ca14/pybase64-1.4.3-cp310-cp310-win_arm64.whl", hash = "sha256:f96367dfc82598569aa02b1103ebd419298293e59e1151abda2b41728703284b", size = 31158, upload-time = "2025-12-06T13:22:50.021Z" }, + { url = "https://files.pythonhosted.org/packages/2b/63/21e981e9d3f1f123e0b0ee2130112b1956cad9752309f574862c7ae77c08/pybase64-1.4.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:70b0d4a4d54e216ce42c2655315378b8903933ecfa32fced453989a92b4317b2", size = 38237, upload-time = "2025-12-06T13:22:52.159Z" }, + { url = "https://files.pythonhosted.org/packages/92/fb/3f448e139516404d2a3963915cc10dc9dde7d3a67de4edba2f827adfef17/pybase64-1.4.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8127f110cdee7a70e576c5c9c1d4e17e92e76c191869085efbc50419f4ae3c72", size = 31673, upload-time = "2025-12-06T13:22:53.241Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/bb06a5b9885e7d853ac1e801c4d8abfdb4c8506deee33e53d55aa6690e67/pybase64-1.4.3-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f9ef0388878bc15a084bd9bf73ec1b2b4ee513d11009b1506375e10a7aae5032", size = 68331, upload-time = "2025-12-06T13:22:54.197Z" }, + { url = "https://files.pythonhosted.org/packages/64/15/8d60b9ec5e658185fc2ee3333e01a6e30d717cf677b24f47cbb3a859d13c/pybase64-1.4.3-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:95a57cccf106352a72ed8bc8198f6820b16cc7d55aa3867a16dea7011ae7c218", size = 71370, upload-time = "2025-12-06T13:22:55.517Z" }, + { url = "https://files.pythonhosted.org/packages/ac/29/a3e5c1667cc8c38d025a4636855de0fc117fc62e2afeb033a3c6f12c6a22/pybase64-1.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cd1c47dfceb9c7bd3de210fb4e65904053ed2d7c9dce6d107f041ff6fbd7e21", size = 59834, upload-time = "2025-12-06T13:22:56.682Z" }, + { url = "https://files.pythonhosted.org/packages/a9/00/8ffcf9810bd23f3984698be161cf7edba656fd639b818039a7be1d6405d4/pybase64-1.4.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:9fe9922698f3e2f72874b26890d53a051c431d942701bb3a37aae94da0b12107", size = 56652, upload-time = "2025-12-06T13:22:57.724Z" }, + { url = "https://files.pythonhosted.org/packages/81/62/379e347797cdea4ab686375945bc77ad8d039c688c0d4d0cfb09d247beb9/pybase64-1.4.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:af5f4bd29c86b59bb4375e0491d16ec8a67548fa99c54763aaedaf0b4b5a6632", size = 59382, upload-time = "2025-12-06T13:22:58.758Z" }, + { url = "https://files.pythonhosted.org/packages/c6/f2/9338ffe2f487086f26a2c8ca175acb3baa86fce0a756ff5670a0822bb877/pybase64-1.4.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c302f6ca7465262908131411226e02100f488f531bb5e64cb901aa3f439bccd9", size = 59990, upload-time = "2025-12-06T13:23:01.007Z" }, + { url = "https://files.pythonhosted.org/packages/f9/a4/85a6142b65b4df8625b337727aa81dc199642de3d09677804141df6ee312/pybase64-1.4.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:2f3f439fa4d7fde164ebbbb41968db7d66b064450ab6017c6c95cef0afa2b349", size = 54923, upload-time = "2025-12-06T13:23:02.369Z" }, + { url = "https://files.pythonhosted.org/packages/ac/00/e40215d25624012bf5b7416ca37f168cb75f6dd15acdb91ea1f2ea4dc4e7/pybase64-1.4.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7a23c6866551043f8b681a5e1e0d59469148b2920a3b4fc42b1275f25ea4217a", size = 58664, upload-time = "2025-12-06T13:23:03.378Z" }, + { url = "https://files.pythonhosted.org/packages/b0/73/d7e19a63e795c13837f2356268d95dc79d1180e756f57ced742a1e52fdeb/pybase64-1.4.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:56e6526f8565642abc5f84338cc131ce298a8ccab696b19bdf76fa6d7dc592ef", size = 52338, upload-time = "2025-12-06T13:23:04.458Z" }, + { url = "https://files.pythonhosted.org/packages/f2/32/3c746d7a310b69bdd9df77ffc85c41b80bce00a774717596f869b0d4a20e/pybase64-1.4.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6a792a8b9d866ffa413c9687d9b611553203753987a3a582d68cbc51cf23da45", size = 68993, upload-time = "2025-12-06T13:23:05.526Z" }, + { url = "https://files.pythonhosted.org/packages/5d/b3/63cec68f9d6f6e4c0b438d14e5f1ef536a5fe63ce14b70733ac5e31d7ab8/pybase64-1.4.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:62ad29a5026bb22cfcd1ca484ec34b0a5ced56ddba38ceecd9359b2818c9c4f9", size = 58055, upload-time = "2025-12-06T13:23:06.931Z" }, + { url = "https://files.pythonhosted.org/packages/d5/cb/7acf7c3c06f9692093c07f109668725dc37fb9a3df0fa912b50add645195/pybase64-1.4.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:11b9d1d2d32ec358c02214363b8fc3651f6be7dd84d880ecd597a6206a80e121", size = 54430, upload-time = "2025-12-06T13:23:07.936Z" }, + { url = "https://files.pythonhosted.org/packages/33/39/4eb33ff35d173bfff4002e184ce8907f5d0a42d958d61cd9058ef3570179/pybase64-1.4.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0aebaa7f238caa0a0d373616016e2040c6c879ebce3ba7ab3c59029920f13640", size = 56272, upload-time = "2025-12-06T13:23:09.253Z" }, + { url = "https://files.pythonhosted.org/packages/19/97/a76d65c375a254e65b730c6f56bf528feca91305da32eceab8bcc08591e6/pybase64-1.4.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e504682b20c63c2b0c000e5f98a80ea867f8d97642e042a5a39818e44ba4d599", size = 70904, upload-time = "2025-12-06T13:23:10.336Z" }, + { url = "https://files.pythonhosted.org/packages/5e/2c/8338b6d3da3c265002839e92af0a80d6db88385c313c73f103dfb800c857/pybase64-1.4.3-cp311-cp311-win32.whl", hash = "sha256:e9a8b81984e3c6fb1db9e1614341b0a2d98c0033d693d90c726677db1ffa3a4c", size = 33639, upload-time = "2025-12-06T13:23:11.9Z" }, + { url = "https://files.pythonhosted.org/packages/39/dc/32efdf2f5927e5449cc341c266a1bbc5fecd5319a8807d9c5405f76e6d02/pybase64-1.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:a90a8fa16a901fabf20de824d7acce07586e6127dc2333f1de05f73b1f848319", size = 35797, upload-time = "2025-12-06T13:23:13.174Z" }, + { url = "https://files.pythonhosted.org/packages/da/59/eda4f9cb0cbce5a45f0cd06131e710674f8123a4d570772c5b9694f88559/pybase64-1.4.3-cp311-cp311-win_arm64.whl", hash = "sha256:61d87de5bc94d143622e94390ec3e11b9c1d4644fe9be3a81068ab0f91056f59", size = 31160, upload-time = "2025-12-06T13:23:15.696Z" }, + { url = "https://files.pythonhosted.org/packages/86/a7/efcaa564f091a2af7f18a83c1c4875b1437db56ba39540451dc85d56f653/pybase64-1.4.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:18d85e5ab8b986bb32d8446aca6258ed80d1bafe3603c437690b352c648f5967", size = 38167, upload-time = "2025-12-06T13:23:16.821Z" }, + { url = "https://files.pythonhosted.org/packages/db/c7/c7ad35adff2d272bf2930132db2b3eea8c44bb1b1f64eb9b2b8e57cde7b4/pybase64-1.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3f5791a3491d116d0deaf4d83268f48792998519698f8751efb191eac84320e9", size = 31673, upload-time = "2025-12-06T13:23:17.835Z" }, + { url = "https://files.pythonhosted.org/packages/43/1b/9a8cab0042b464e9a876d5c65fe5127445a2436da36fda64899b119b1a1b/pybase64-1.4.3-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f0b3f200c3e06316f6bebabd458b4e4bcd4c2ca26af7c0c766614d91968dee27", size = 68210, upload-time = "2025-12-06T13:23:18.813Z" }, + { url = "https://files.pythonhosted.org/packages/62/f7/965b79ff391ad208b50e412b5d3205ccce372a2d27b7218ae86d5295b105/pybase64-1.4.3-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bb632edfd132b3eaf90c39c89aa314beec4e946e210099b57d40311f704e11d4", size = 71599, upload-time = "2025-12-06T13:23:20.195Z" }, + { url = "https://files.pythonhosted.org/packages/03/4b/a3b5175130b3810bbb8ccfa1edaadbd3afddb9992d877c8a1e2f274b476e/pybase64-1.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:356ef1d74648ce997f5a777cf8f1aefecc1c0b4fe6201e0ef3ec8a08170e1b54", size = 59922, upload-time = "2025-12-06T13:23:21.487Z" }, + { url = "https://files.pythonhosted.org/packages/da/5d/c38d1572027fc601b62d7a407721688b04b4d065d60ca489912d6893e6cf/pybase64-1.4.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:c48361f90db32bacaa5518419d4eb9066ba558013aaf0c7781620279ecddaeb9", size = 56712, upload-time = "2025-12-06T13:23:22.77Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d4/4e04472fef485caa8f561d904d4d69210a8f8fc1608ea15ebd9012b92655/pybase64-1.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:702bcaa16ae02139d881aeaef5b1c8ffb4a3fae062fe601d1e3835e10310a517", size = 59300, upload-time = "2025-12-06T13:23:24.543Z" }, + { url = "https://files.pythonhosted.org/packages/86/e7/16e29721b86734b881d09b7e23dfd7c8408ad01a4f4c7525f3b1088e25ec/pybase64-1.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:53d0ffe1847b16b647c6413d34d1de08942b7724273dd57e67dcbdb10c574045", size = 60278, upload-time = "2025-12-06T13:23:25.608Z" }, + { url = "https://files.pythonhosted.org/packages/b1/02/18515f211d7c046be32070709a8efeeef8a0203de4fd7521e6b56404731b/pybase64-1.4.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9a1792e8b830a92736dae58f0c386062eb038dfe8004fb03ba33b6083d89cd43", size = 54817, upload-time = "2025-12-06T13:23:26.633Z" }, + { url = "https://files.pythonhosted.org/packages/e7/be/14e29d8e1a481dbff151324c96dd7b5d2688194bb65dc8a00ca0e1ad1e86/pybase64-1.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d468b1b1ac5ad84875a46eaa458663c3721e8be5f155ade356406848d3701f6", size = 58611, upload-time = "2025-12-06T13:23:27.684Z" }, + { url = "https://files.pythonhosted.org/packages/b4/8a/a2588dfe24e1bbd742a554553778ab0d65fdf3d1c9a06d10b77047d142aa/pybase64-1.4.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e97b7bdbd62e71898cd542a6a9e320d9da754ff3ebd02cb802d69087ee94d468", size = 52404, upload-time = "2025-12-06T13:23:28.714Z" }, + { url = "https://files.pythonhosted.org/packages/27/fc/afcda7445bebe0cbc38cafdd7813234cdd4fc5573ff067f1abf317bb0cec/pybase64-1.4.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b33aeaa780caaa08ffda87fc584d5eab61e3d3bbb5d86ead02161dc0c20d04bc", size = 68817, upload-time = "2025-12-06T13:23:30.079Z" }, + { url = "https://files.pythonhosted.org/packages/d3/3a/87c3201e555ed71f73e961a787241a2438c2bbb2ca8809c29ddf938a3157/pybase64-1.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c0efcf78f11cf866bed49caa7b97552bc4855a892f9cc2372abcd3ed0056f0d", size = 57854, upload-time = "2025-12-06T13:23:31.17Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7d/931c2539b31a7b375e7d595b88401eeb5bd6c5ce1059c9123f9b608aaa14/pybase64-1.4.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66e3791f2ed725a46593f8bd2761ff37d01e2cdad065b1dceb89066f476e50c6", size = 54333, upload-time = "2025-12-06T13:23:32.422Z" }, + { url = "https://files.pythonhosted.org/packages/de/5e/537601e02cc01f27e9d75f440f1a6095b8df44fc28b1eef2cd739aea8cec/pybase64-1.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:72bb0b6bddadab26e1b069bb78e83092711a111a80a0d6b9edcb08199ad7299b", size = 56492, upload-time = "2025-12-06T13:23:33.515Z" }, + { url = "https://files.pythonhosted.org/packages/96/97/2a2e57acf8f5c9258d22aba52e71f8050e167b29ed2ee1113677c1b600c1/pybase64-1.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5b3365dbcbcdb0a294f0f50af0c0a16b27a232eddeeb0bceeefd844ef30d2a23", size = 70974, upload-time = "2025-12-06T13:23:36.27Z" }, + { url = "https://files.pythonhosted.org/packages/75/2e/a9e28941c6dab6f06e6d3f6783d3373044be9b0f9a9d3492c3d8d2260ac0/pybase64-1.4.3-cp312-cp312-win32.whl", hash = "sha256:7bca1ed3a5df53305c629ca94276966272eda33c0d71f862d2d3d043f1e1b91a", size = 33686, upload-time = "2025-12-06T13:23:37.848Z" }, + { url = "https://files.pythonhosted.org/packages/83/e3/507ab649d8c3512c258819c51d25c45d6e29d9ca33992593059e7b646a33/pybase64-1.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:9f2da8f56d9b891b18b4daf463a0640eae45a80af548ce435be86aa6eff3603b", size = 35833, upload-time = "2025-12-06T13:23:38.877Z" }, + { url = "https://files.pythonhosted.org/packages/bc/8a/6eba66cd549a2fc74bb4425fd61b839ba0ab3022d3c401b8a8dc2cc00c7a/pybase64-1.4.3-cp312-cp312-win_arm64.whl", hash = "sha256:0631d8a2d035de03aa9bded029b9513e1fee8ed80b7ddef6b8e9389ffc445da0", size = 31185, upload-time = "2025-12-06T13:23:39.908Z" }, + { url = "https://files.pythonhosted.org/packages/3a/50/b7170cb2c631944388fe2519507fe3835a4054a6a12a43f43781dae82be1/pybase64-1.4.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:ea4b785b0607d11950b66ce7c328f452614aefc9c6d3c9c28bae795dc7f072e1", size = 33901, upload-time = "2025-12-06T13:23:40.951Z" }, + { url = "https://files.pythonhosted.org/packages/48/8b/69f50578e49c25e0a26e3ee72c39884ff56363344b79fc3967f5af420ed6/pybase64-1.4.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:6a10b6330188c3026a8b9c10e6b9b3f2e445779cf16a4c453d51a072241c65a2", size = 40807, upload-time = "2025-12-06T13:23:42.006Z" }, + { url = "https://files.pythonhosted.org/packages/5c/8d/20b68f11adfc4c22230e034b65c71392e3e338b413bf713c8945bd2ccfb3/pybase64-1.4.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:27fdff227a0c0e182e0ba37a99109645188978b920dfb20d8b9c17eeee370d0d", size = 30932, upload-time = "2025-12-06T13:23:43.348Z" }, + { url = "https://files.pythonhosted.org/packages/f7/79/b1b550ac6bff51a4880bf6e089008b2e1ca16f2c98db5e039a08ac3ad157/pybase64-1.4.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2a8204f1fdfec5aa4184249b51296c0de95445869920c88123978304aad42df1", size = 31394, upload-time = "2025-12-06T13:23:44.317Z" }, + { url = "https://files.pythonhosted.org/packages/82/70/b5d7c5932bf64ee1ec5da859fbac981930b6a55d432a603986c7f509c838/pybase64-1.4.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:874fc2a3777de6baf6aa921a7aa73b3be98295794bea31bd80568a963be30767", size = 38078, upload-time = "2025-12-06T13:23:45.348Z" }, + { url = "https://files.pythonhosted.org/packages/56/fe/e66fe373bce717c6858427670736d54297938dad61c5907517ab4106bd90/pybase64-1.4.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2dc64a94a9d936b8e3449c66afabbaa521d3cc1a563d6bbaaa6ffa4535222e4b", size = 38158, upload-time = "2025-12-06T13:23:46.872Z" }, + { url = "https://files.pythonhosted.org/packages/80/a9/b806ed1dcc7aed2ea3dd4952286319e6f3a8b48615c8118f453948e01999/pybase64-1.4.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e48f86de1c145116ccf369a6e11720ce696c2ec02d285f440dfb57ceaa0a6cb4", size = 31672, upload-time = "2025-12-06T13:23:47.88Z" }, + { url = "https://files.pythonhosted.org/packages/1c/c9/24b3b905cf75e23a9a4deaf203b35ffcb9f473ac0e6d8257f91a05dfce62/pybase64-1.4.3-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:1d45c8fe8fe82b65c36b227bb4a2cf623d9ada16bed602ce2d3e18c35285b72a", size = 68244, upload-time = "2025-12-06T13:23:49.026Z" }, + { url = "https://files.pythonhosted.org/packages/f8/cd/d15b0c3e25e5859fab0416dc5b96d34d6bd2603c1c96a07bb2202b68ab92/pybase64-1.4.3-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ad70c26ba091d8f5167e9d4e1e86a0483a5414805cdb598a813db635bd3be8b8", size = 71620, upload-time = "2025-12-06T13:23:50.081Z" }, + { url = "https://files.pythonhosted.org/packages/0d/31/4ca953cc3dcde2b3711d6bfd70a6f4ad2ca95a483c9698076ba605f1520f/pybase64-1.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e98310b7c43145221e7194ac9fa7fffc84763c87bfc5e2f59f9f92363475bdc1", size = 59930, upload-time = "2025-12-06T13:23:51.68Z" }, + { url = "https://files.pythonhosted.org/packages/60/55/e7f7bdcd0fd66e61dda08db158ffda5c89a306bbdaaf5a062fbe4e48f4a1/pybase64-1.4.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:398685a76034e91485a28aeebcb49e64cd663212fd697b2497ac6dfc1df5e671", size = 56425, upload-time = "2025-12-06T13:23:52.732Z" }, + { url = "https://files.pythonhosted.org/packages/cb/65/b592c7f921e51ca1aca3af5b0d201a98666d0a36b930ebb67e7c2ed27395/pybase64-1.4.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7e46400a6461187ccb52ed75b0045d937529e801a53a9cd770b350509f9e4d50", size = 59327, upload-time = "2025-12-06T13:23:53.856Z" }, + { url = "https://files.pythonhosted.org/packages/23/95/1613d2fb82dbb1548595ad4179f04e9a8451bfa18635efce18b631eabe3f/pybase64-1.4.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1b62b9f2f291d94f5e0b76ab499790b7dcc78a009d4ceea0b0428770267484b6", size = 60294, upload-time = "2025-12-06T13:23:54.937Z" }, + { url = "https://files.pythonhosted.org/packages/9d/73/40431f37f7d1b3eab4673e7946ff1e8f5d6bd425ec257e834dae8a6fc7b0/pybase64-1.4.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:f30ceb5fa4327809dede614be586efcbc55404406d71e1f902a6fdcf322b93b2", size = 54858, upload-time = "2025-12-06T13:23:56.031Z" }, + { url = "https://files.pythonhosted.org/packages/a7/84/f6368bcaf9f743732e002a9858646fd7a54f428490d427dd6847c5cfe89e/pybase64-1.4.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0d5f18ed53dfa1d4cf8b39ee542fdda8e66d365940e11f1710989b3cf4a2ed66", size = 58629, upload-time = "2025-12-06T13:23:57.12Z" }, + { url = "https://files.pythonhosted.org/packages/43/75/359532f9adb49c6b546cafc65c46ed75e2ccc220d514ba81c686fbd83965/pybase64-1.4.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:119d31aa4b58b85a8ebd12b63c07681a138c08dfc2fe5383459d42238665d3eb", size = 52448, upload-time = "2025-12-06T13:23:58.298Z" }, + { url = "https://files.pythonhosted.org/packages/92/6c/ade2ba244c3f33ed920a7ed572ad772eb0b5f14480b72d629d0c9e739a40/pybase64-1.4.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:3cf0218b0e2f7988cf7d738a73b6a1d14f3be6ce249d7c0f606e768366df2cce", size = 68841, upload-time = "2025-12-06T13:23:59.886Z" }, + { url = "https://files.pythonhosted.org/packages/a0/51/b345139cd236be382f2d4d4453c21ee6299e14d2f759b668e23080f8663f/pybase64-1.4.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:12f4ee5e988bc5c0c1106b0d8fc37fb0508f12dab76bac1b098cb500d148da9d", size = 57910, upload-time = "2025-12-06T13:24:00.994Z" }, + { url = "https://files.pythonhosted.org/packages/1a/b8/9f84bdc4f1c4f0052489396403c04be2f9266a66b70c776001eaf0d78c1f/pybase64-1.4.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:937826bc7b6b95b594a45180e81dd4d99bd4dd4814a443170e399163f7ff3fb6", size = 54335, upload-time = "2025-12-06T13:24:02.046Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c7/be63b617d284de46578a366da77ede39c8f8e815ed0d82c7c2acca560fab/pybase64-1.4.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:88995d1460971ef80b13e3e007afbe4b27c62db0508bc7250a2ab0a0b4b91362", size = 56486, upload-time = "2025-12-06T13:24:03.141Z" }, + { url = "https://files.pythonhosted.org/packages/5e/96/f252c8f9abd6ded3ef1ccd3cdbb8393a33798007f761b23df8de1a2480e6/pybase64-1.4.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:72326fe163385ed3e1e806dd579d47fde5d8a59e51297a60fc4e6cbc1b4fc4ed", size = 70978, upload-time = "2025-12-06T13:24:04.221Z" }, + { url = "https://files.pythonhosted.org/packages/af/51/0f5714af7aeef96e30f968e4371d75ad60558aaed3579d7c6c8f1c43c18a/pybase64-1.4.3-cp313-cp313-win32.whl", hash = "sha256:b1623730c7892cf5ed0d6355e375416be6ef8d53ab9b284f50890443175c0ac3", size = 33684, upload-time = "2025-12-06T13:24:05.29Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ad/0cea830a654eb08563fb8214150ef57546ece1cc421c09035f0e6b0b5ea9/pybase64-1.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:8369887590f1646a5182ca2fb29252509da7ae31d4923dbb55d3e09da8cc4749", size = 35832, upload-time = "2025-12-06T13:24:06.35Z" }, + { url = "https://files.pythonhosted.org/packages/b4/0d/eec2a8214989c751bc7b4cad1860eb2c6abf466e76b77508c0f488c96a37/pybase64-1.4.3-cp313-cp313-win_arm64.whl", hash = "sha256:860b86bca71e5f0237e2ab8b2d9c4c56681f3513b1bf3e2117290c1963488390", size = 31175, upload-time = "2025-12-06T13:24:07.419Z" }, + { url = "https://files.pythonhosted.org/packages/db/c9/e23463c1a2913686803ef76b1a5ae7e6fac868249a66e48253d17ad7232c/pybase64-1.4.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:eb51db4a9c93215135dccd1895dca078e8785c357fabd983c9f9a769f08989a9", size = 38497, upload-time = "2025-12-06T13:24:08.873Z" }, + { url = "https://files.pythonhosted.org/packages/71/83/343f446b4b7a7579bf6937d2d013d82f1a63057cf05558e391ab6039d7db/pybase64-1.4.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a03ef3f529d85fd46b89971dfb00c634d53598d20ad8908fb7482955c710329d", size = 32076, upload-time = "2025-12-06T13:24:09.975Z" }, + { url = "https://files.pythonhosted.org/packages/46/fc/cb64964c3b29b432f54d1bce5e7691d693e33bbf780555151969ffd95178/pybase64-1.4.3-cp313-cp313t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:2e745f2ce760c6cf04d8a72198ef892015ddb89f6ceba489e383518ecbdb13ab", size = 72317, upload-time = "2025-12-06T13:24:11.129Z" }, + { url = "https://files.pythonhosted.org/packages/0a/b7/fab2240da6f4e1ad46f71fa56ec577613cf5df9dce2d5b4cfaa4edd0e365/pybase64-1.4.3-cp313-cp313t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6fac217cd9de8581a854b0ac734c50fd1fa4b8d912396c1fc2fce7c230efe3a7", size = 75534, upload-time = "2025-12-06T13:24:12.433Z" }, + { url = "https://files.pythonhosted.org/packages/91/3b/3e2f2b6e68e3d83ddb9fa799f3548fb7449765daec9bbd005a9fbe296d7f/pybase64-1.4.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:da1ee8fa04b283873de2d6e8fa5653e827f55b86bdf1a929c5367aaeb8d26f8a", size = 65399, upload-time = "2025-12-06T13:24:13.928Z" }, + { url = "https://files.pythonhosted.org/packages/6b/08/476ac5914c3b32e0274a2524fc74f01cbf4f4af4513d054e41574eb018f6/pybase64-1.4.3-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:b0bf8e884ee822ca7b1448eeb97fa131628fe0ff42f60cae9962789bd562727f", size = 60487, upload-time = "2025-12-06T13:24:15.177Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b8/618a92915330cc9cba7880299b546a1d9dab1a21fd6c0292ee44a4fe608c/pybase64-1.4.3-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1bf749300382a6fd1f4f255b183146ef58f8e9cb2f44a077b3a9200dfb473a77", size = 63959, upload-time = "2025-12-06T13:24:16.854Z" }, + { url = "https://files.pythonhosted.org/packages/a5/52/af9d8d051652c3051862c442ec3861259c5cdb3fc69774bc701470bd2a59/pybase64-1.4.3-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:153a0e42329b92337664cfc356f2065248e6c9a1bd651bbcd6dcaf15145d3f06", size = 64874, upload-time = "2025-12-06T13:24:18.328Z" }, + { url = "https://files.pythonhosted.org/packages/e4/51/5381a7adf1f381bd184d33203692d3c57cf8ae9f250f380c3fecbdbe554b/pybase64-1.4.3-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:86ee56ac7f2184ca10217ed1c655c1a060273e233e692e9086da29d1ae1768db", size = 58572, upload-time = "2025-12-06T13:24:19.417Z" }, + { url = "https://files.pythonhosted.org/packages/e0/f0/578ee4ffce5818017de4fdf544e066c225bc435e73eb4793cde28a689d0b/pybase64-1.4.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:0e71a4db76726bf830b47477e7d830a75c01b2e9b01842e787a0836b0ba741e3", size = 63636, upload-time = "2025-12-06T13:24:20.497Z" }, + { url = "https://files.pythonhosted.org/packages/b9/ad/8ae94814bf20159ea06310b742433e53d5820aa564c9fdf65bf2d79f8799/pybase64-1.4.3-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:2ba7799ec88540acd9861b10551d24656ca3c2888ecf4dba2ee0a71544a8923f", size = 56193, upload-time = "2025-12-06T13:24:21.559Z" }, + { url = "https://files.pythonhosted.org/packages/d1/31/6438cfcc3d3f0fa84d229fa125c243d5094e72628e525dfefadf3bcc6761/pybase64-1.4.3-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:2860299e4c74315f5951f0cf3e72ba0f201c3356c8a68f95a3ab4e620baf44e9", size = 72655, upload-time = "2025-12-06T13:24:22.673Z" }, + { url = "https://files.pythonhosted.org/packages/a3/0d/2bbc9e9c3fc12ba8a6e261482f03a544aca524f92eae0b4908c0a10ba481/pybase64-1.4.3-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:bb06015db9151f0c66c10aae8e3603adab6b6cd7d1f7335a858161d92fc29618", size = 62471, upload-time = "2025-12-06T13:24:23.8Z" }, + { url = "https://files.pythonhosted.org/packages/2c/0b/34d491e7f49c1dbdb322ea8da6adecda7c7cd70b6644557c6e4ca5c6f7c7/pybase64-1.4.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:242512a070817272865d37c8909059f43003b81da31f616bb0c391ceadffe067", size = 58119, upload-time = "2025-12-06T13:24:24.994Z" }, + { url = "https://files.pythonhosted.org/packages/ce/17/c21d0cde2a6c766923ae388fc1f78291e1564b0d38c814b5ea8a0e5e081c/pybase64-1.4.3-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:5d8277554a12d3e3eed6180ebda62786bf9fc8d7bb1ee00244258f4a87ca8d20", size = 60791, upload-time = "2025-12-06T13:24:26.046Z" }, + { url = "https://files.pythonhosted.org/packages/92/b2/eaa67038916a48de12b16f4c384bcc1b84b7ec731b23613cb05f27673294/pybase64-1.4.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f40b7ddd698fc1e13a4b64fbe405e4e0e1279e8197e37050e24154655f5f7c4e", size = 74701, upload-time = "2025-12-06T13:24:27.466Z" }, + { url = "https://files.pythonhosted.org/packages/42/10/abb7757c330bb869ebb95dab0c57edf5961ffbd6c095c8209cbbf75d117d/pybase64-1.4.3-cp313-cp313t-win32.whl", hash = "sha256:46d75c9387f354c5172582a9eaae153b53a53afeb9c19fcf764ea7038be3bd8b", size = 33965, upload-time = "2025-12-06T13:24:28.548Z" }, + { url = "https://files.pythonhosted.org/packages/63/a0/2d4e5a59188e9e6aed0903d580541aaea72dcbbab7bf50fb8b83b490b6c3/pybase64-1.4.3-cp313-cp313t-win_amd64.whl", hash = "sha256:d7344625591d281bec54e85cbfdab9e970f6219cac1570f2aa140b8c942ccb81", size = 36207, upload-time = "2025-12-06T13:24:29.646Z" }, + { url = "https://files.pythonhosted.org/packages/1f/05/95b902e8f567b4d4b41df768ccc438af618f8d111e54deaf57d2df46bd76/pybase64-1.4.3-cp313-cp313t-win_arm64.whl", hash = "sha256:28a3c60c55138e0028313f2eccd321fec3c4a0be75e57a8d3eb883730b1b0880", size = 31505, upload-time = "2025-12-06T13:24:30.687Z" }, + { url = "https://files.pythonhosted.org/packages/e4/80/4bd3dff423e5a91f667ca41982dc0b79495b90ec0c0f5d59aca513e50f8c/pybase64-1.4.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:015bb586a1ea1467f69d57427abe587469392215f59db14f1f5c39b52fdafaf5", size = 33835, upload-time = "2025-12-06T13:24:31.767Z" }, + { url = "https://files.pythonhosted.org/packages/45/60/a94d94cc1e3057f602e0b483c9ebdaef40911d84a232647a2fe593ab77bb/pybase64-1.4.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:d101e3a516f837c3dcc0e5a0b7db09582ebf99ed670865223123fb2e5839c6c0", size = 40673, upload-time = "2025-12-06T13:24:32.82Z" }, + { url = "https://files.pythonhosted.org/packages/e3/71/cf62b261d431857e8e054537a5c3c24caafa331de30daede7b2c6c558501/pybase64-1.4.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8f183ac925a48046abe047360fe3a1b28327afb35309892132fe1915d62fb282", size = 30939, upload-time = "2025-12-06T13:24:34.001Z" }, + { url = "https://files.pythonhosted.org/packages/24/3e/d12f92a3c1f7c6ab5d53c155bff9f1084ba997a37a39a4f781ccba9455f3/pybase64-1.4.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:30bf3558e24dcce4da5248dcf6d73792adfcf4f504246967e9db155be4c439ad", size = 31401, upload-time = "2025-12-06T13:24:35.11Z" }, + { url = "https://files.pythonhosted.org/packages/9b/3d/9c27440031fea0d05146f8b70a460feb95d8b4e3d9ca8f45c972efb4c3d3/pybase64-1.4.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a674b419de318d2ce54387dd62646731efa32b4b590907800f0bd40675c1771d", size = 38075, upload-time = "2025-12-06T13:24:36.53Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d4/6c0e0cf0efd53c254173fbcd84a3d8fcbf5e0f66622473da425becec32a5/pybase64-1.4.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:720104fd7303d07bac302be0ff8f7f9f126f2f45c1edb4f48fdb0ff267e69fe1", size = 38257, upload-time = "2025-12-06T13:24:38.049Z" }, + { url = "https://files.pythonhosted.org/packages/50/eb/27cb0b610d5cd70f5ad0d66c14ad21c04b8db930f7139818e8fbdc14df4d/pybase64-1.4.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:83f1067f73fa5afbc3efc0565cecc6ed53260eccddef2ebe43a8ce2b99ea0e0a", size = 31685, upload-time = "2025-12-06T13:24:40.327Z" }, + { url = "https://files.pythonhosted.org/packages/db/26/b136a4b65e5c94ff06217f7726478df3f31ab1c777c2c02cf698e748183f/pybase64-1.4.3-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:b51204d349a4b208287a8aa5b5422be3baa88abf6cc8ff97ccbda34919bbc857", size = 68460, upload-time = "2025-12-06T13:24:41.735Z" }, + { url = "https://files.pythonhosted.org/packages/68/6d/84ce50e7ee1ae79984d689e05a9937b2460d4efa1e5b202b46762fb9036c/pybase64-1.4.3-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:30f2fd53efecbdde4bdca73a872a68dcb0d1bf8a4560c70a3e7746df973e1ef3", size = 71688, upload-time = "2025-12-06T13:24:42.908Z" }, + { url = "https://files.pythonhosted.org/packages/e3/57/6743e420416c3ff1b004041c85eb0ebd9c50e9cf05624664bfa1dc8b5625/pybase64-1.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0932b0c5cfa617091fd74f17d24549ce5de3628791998c94ba57be808078eeaf", size = 60040, upload-time = "2025-12-06T13:24:44.37Z" }, + { url = "https://files.pythonhosted.org/packages/3b/68/733324e28068a89119af2921ce548e1c607cc5c17d354690fc51c302e326/pybase64-1.4.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:acb61f5ab72bec808eb0d4ce8b87ec9f38d7d750cb89b1371c35eb8052a29f11", size = 56478, upload-time = "2025-12-06T13:24:45.815Z" }, + { url = "https://files.pythonhosted.org/packages/b5/9e/f3f4aa8cfe3357a3cdb0535b78eb032b671519d3ecc08c58c4c6b72b5a91/pybase64-1.4.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:2bc2d5bc15168f5c04c53bdfe5a1e543b2155f456ed1e16d7edce9ce73842021", size = 59463, upload-time = "2025-12-06T13:24:46.938Z" }, + { url = "https://files.pythonhosted.org/packages/aa/d1/53286038e1f0df1cf58abcf4a4a91b0f74ab44539c2547b6c31001ddd054/pybase64-1.4.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:8a7bc3cd23880bdca59758bcdd6f4ef0674f2393782763910a7466fab35ccb98", size = 60360, upload-time = "2025-12-06T13:24:48.039Z" }, + { url = "https://files.pythonhosted.org/packages/00/9a/5cc6ce95db2383d27ff4d790b8f8b46704d360d701ab77c4f655bcfaa6a7/pybase64-1.4.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:ad15acf618880d99792d71e3905b0e2508e6e331b76a1b34212fa0f11e01ad28", size = 54999, upload-time = "2025-12-06T13:24:49.547Z" }, + { url = "https://files.pythonhosted.org/packages/64/e7/c3c1d09c3d7ae79e3aa1358c6d912d6b85f29281e47aa94fc0122a415a2f/pybase64-1.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:448158d417139cb4851200e5fee62677ae51f56a865d50cda9e0d61bda91b116", size = 58736, upload-time = "2025-12-06T13:24:50.641Z" }, + { url = "https://files.pythonhosted.org/packages/db/d5/0baa08e3d8119b15b588c39f0d39fd10472f0372e3c54ca44649cbefa256/pybase64-1.4.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:9058c49b5a2f3e691b9db21d37eb349e62540f9f5fc4beabf8cbe3c732bead86", size = 52298, upload-time = "2025-12-06T13:24:51.791Z" }, + { url = "https://files.pythonhosted.org/packages/00/87/fc6f11474a1de7e27cd2acbb8d0d7508bda3efa73dfe91c63f968728b2a3/pybase64-1.4.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ce561724f6522907a66303aca27dce252d363fcd85884972d348f4403ba3011a", size = 69049, upload-time = "2025-12-06T13:24:53.253Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/7fb5566f669ac18b40aa5fc1c438e24df52b843c1bdc5da47d46d4c1c630/pybase64-1.4.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:63316560a94ac449fe86cb8b9e0a13714c659417e92e26a5cbf085cd0a0c838d", size = 57952, upload-time = "2025-12-06T13:24:54.342Z" }, + { url = "https://files.pythonhosted.org/packages/de/cc/ceb949232dbbd3ec4ee0190d1df4361296beceee9840390a63df8bc31784/pybase64-1.4.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7ecd796f2ac0be7b73e7e4e232b8c16422014de3295d43e71d2b19fd4a4f5368", size = 54484, upload-time = "2025-12-06T13:24:55.774Z" }, + { url = "https://files.pythonhosted.org/packages/a7/69/659f3c8e6a5d7b753b9c42a4bd9c42892a0f10044e9c7351a4148d413a33/pybase64-1.4.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d01e102a12fb2e1ed3dc11611c2818448626637857ec3994a9cf4809dfd23477", size = 56542, upload-time = "2025-12-06T13:24:57Z" }, + { url = "https://files.pythonhosted.org/packages/85/2c/29c9e6c9c82b72025f9676f9e82eb1fd2339ad038cbcbf8b9e2ac02798fc/pybase64-1.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ebff797a93c2345f22183f454fd8607a34d75eca5a3a4a969c1c75b304cee39d", size = 71045, upload-time = "2025-12-06T13:24:58.179Z" }, + { url = "https://files.pythonhosted.org/packages/b9/84/5a3dce8d7a0040a5c0c14f0fe1311cd8db872913fa04438071b26b0dac04/pybase64-1.4.3-cp314-cp314-win32.whl", hash = "sha256:28b2a1bb0828c0595dc1ea3336305cd97ff85b01c00d81cfce4f92a95fb88f56", size = 34200, upload-time = "2025-12-06T13:24:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/57/bc/ce7427c12384adee115b347b287f8f3cf65860b824d74fe2c43e37e81c1f/pybase64-1.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:33338d3888700ff68c3dedfcd49f99bfc3b887570206130926791e26b316b029", size = 36323, upload-time = "2025-12-06T13:25:01.708Z" }, + { url = "https://files.pythonhosted.org/packages/9a/1b/2b8ffbe9a96eef7e3f6a5a7be75995eebfb6faaedc85b6da6b233e50c778/pybase64-1.4.3-cp314-cp314-win_arm64.whl", hash = "sha256:62725669feb5acb186458da2f9353e88ae28ef66bb9c4c8d1568b12a790dfa94", size = 31584, upload-time = "2025-12-06T13:25:02.801Z" }, + { url = "https://files.pythonhosted.org/packages/ac/d8/6824c2e6fb45b8fa4e7d92e3c6805432d5edc7b855e3e8e1eedaaf6efb7c/pybase64-1.4.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:153fe29be038948d9372c3e77ae7d1cab44e4ba7d9aaf6f064dbeea36e45b092", size = 38601, upload-time = "2025-12-06T13:25:04.222Z" }, + { url = "https://files.pythonhosted.org/packages/ea/e5/10d2b3a4ad3a4850be2704a2f70cd9c0cf55725c8885679872d3bc846c67/pybase64-1.4.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f7fe3decaa7c4a9e162327ec7bd81ce183d2b16f23c6d53b606649c6e0203e9e", size = 32078, upload-time = "2025-12-06T13:25:05.362Z" }, + { url = "https://files.pythonhosted.org/packages/43/04/8b15c34d3c2282f1c1b0850f1113a249401b618a382646a895170bc9b5e7/pybase64-1.4.3-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:a5ae04ea114c86eb1da1f6e18d75f19e3b5ae39cb1d8d3cd87c29751a6a22780", size = 72474, upload-time = "2025-12-06T13:25:06.434Z" }, + { url = "https://files.pythonhosted.org/packages/42/00/f34b4d11278f8fdc68bc38f694a91492aa318f7c6f1bd7396197ac0f8b12/pybase64-1.4.3-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1755b3dce3a2a5c7d17ff6d4115e8bee4a1d5aeae74469db02e47c8f477147da", size = 75706, upload-time = "2025-12-06T13:25:07.636Z" }, + { url = "https://files.pythonhosted.org/packages/bb/5d/71747d4ad7fe16df4c4c852bdbdeb1f2cf35677b48d7c34d3011a7a6ad3a/pybase64-1.4.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fb852f900e27ffc4ec1896817535a0fa19610ef8875a096b59f21d0aa42ff172", size = 65589, upload-time = "2025-12-06T13:25:08.809Z" }, + { url = "https://files.pythonhosted.org/packages/49/b1/d1e82bd58805bb5a3a662864800bab83a83a36ba56e7e3b1706c708002a5/pybase64-1.4.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:9cf21ea8c70c61eddab3421fbfce061fac4f2fb21f7031383005a1efdb13d0b9", size = 60670, upload-time = "2025-12-06T13:25:10.04Z" }, + { url = "https://files.pythonhosted.org/packages/15/67/16c609b7a13d1d9fc87eca12ba2dce5e67f949eeaab61a41bddff843cbb0/pybase64-1.4.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:afff11b331fdc27692fc75e85ae083340a35105cea1a3c4552139e2f0e0d174f", size = 64194, upload-time = "2025-12-06T13:25:11.48Z" }, + { url = "https://files.pythonhosted.org/packages/3c/11/37bc724e42960f0106c2d33dc957dcec8f760c91a908cc6c0df7718bc1a8/pybase64-1.4.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9a5143df542c1ce5c1f423874b948c4d689b3f05ec571f8792286197a39ba02", size = 64984, upload-time = "2025-12-06T13:25:12.645Z" }, + { url = "https://files.pythonhosted.org/packages/6e/66/b2b962a6a480dd5dae3029becf03ea1a650d326e39bf1c44ea3db78bb010/pybase64-1.4.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:d62e9861019ad63624b4a7914dff155af1cc5d6d79df3be14edcaedb5fdad6f9", size = 58750, upload-time = "2025-12-06T13:25:13.848Z" }, + { url = "https://files.pythonhosted.org/packages/2b/15/9b6d711035e29b18b2e1c03d47f41396d803d06ef15b6c97f45b75f73f04/pybase64-1.4.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:84cfd4d92668ef5766cc42a9c9474b88960ac2b860767e6e7be255c6fddbd34a", size = 63816, upload-time = "2025-12-06T13:25:15.356Z" }, + { url = "https://files.pythonhosted.org/packages/b4/21/e2901381ed0df62e2308380f30d9c4d87d6b74e33a84faed3478d33a7197/pybase64-1.4.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:60fc025437f9a7c2cc45e0c19ed68ed08ba672be2c5575fd9d98bdd8f01dd61f", size = 56348, upload-time = "2025-12-06T13:25:16.559Z" }, + { url = "https://files.pythonhosted.org/packages/c4/16/3d788388a178a0407aa814b976fe61bfa4af6760d9aac566e59da6e4a8b4/pybase64-1.4.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:edc8446196f04b71d3af76c0bd1fe0a45066ac5bffecca88adb9626ee28c266f", size = 72842, upload-time = "2025-12-06T13:25:18.055Z" }, + { url = "https://files.pythonhosted.org/packages/a6/63/c15b1f8bd47ea48a5a2d52a4ec61f037062932ea6434ab916107b58e861e/pybase64-1.4.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e99f6fa6509c037794da57f906ade271f52276c956d00f748e5b118462021d48", size = 62651, upload-time = "2025-12-06T13:25:19.191Z" }, + { url = "https://files.pythonhosted.org/packages/bd/b8/f544a2e37c778d59208966d4ef19742a0be37c12fc8149ff34483c176616/pybase64-1.4.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d94020ef09f624d841aa9a3a6029df8cf65d60d7a6d5c8687579fa68bd679b65", size = 58295, upload-time = "2025-12-06T13:25:20.822Z" }, + { url = "https://files.pythonhosted.org/packages/03/99/1fae8a3b7ac181e36f6e7864a62d42d5b1f4fa7edf408c6711e28fba6b4d/pybase64-1.4.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f64ce70d89942a23602dee910dec9b48e5edf94351e1b378186b74fcc00d7f66", size = 60960, upload-time = "2025-12-06T13:25:22.099Z" }, + { url = "https://files.pythonhosted.org/packages/9d/9e/cd4c727742345ad8384569a4466f1a1428f4e5cc94d9c2ab2f53d30be3fe/pybase64-1.4.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8ea99f56e45c469818b9781903be86ba4153769f007ba0655fa3b46dc332803d", size = 74863, upload-time = "2025-12-06T13:25:23.442Z" }, + { url = "https://files.pythonhosted.org/packages/28/86/a236ecfc5b494e1e922da149689f690abc84248c7c1358f5605b8c9fdd60/pybase64-1.4.3-cp314-cp314t-win32.whl", hash = "sha256:343b1901103cc72362fd1f842524e3bb24978e31aea7ff11e033af7f373f66ab", size = 34513, upload-time = "2025-12-06T13:25:24.592Z" }, + { url = "https://files.pythonhosted.org/packages/56/ce/ca8675f8d1352e245eb012bfc75429ee9cf1f21c3256b98d9a329d44bf0f/pybase64-1.4.3-cp314-cp314t-win_amd64.whl", hash = "sha256:57aff6f7f9dea6705afac9d706432049642de5b01080d3718acc23af87c5af76", size = 36702, upload-time = "2025-12-06T13:25:25.72Z" }, + { url = "https://files.pythonhosted.org/packages/3b/30/4a675864877397179b09b720ee5fcb1cf772cf7bebc831989aff0a5f79c1/pybase64-1.4.3-cp314-cp314t-win_arm64.whl", hash = "sha256:e906aa08d4331e799400829e0f5e4177e76a3281e8a4bc82ba114c6b30e405c9", size = 31904, upload-time = "2025-12-06T13:25:26.826Z" }, + { url = "https://files.pythonhosted.org/packages/b2/7c/545fd4935a0e1ddd7147f557bf8157c73eecec9cffd523382fa7af2557de/pybase64-1.4.3-graalpy311-graalpy242_311_native-macosx_10_9_x86_64.whl", hash = "sha256:d27c1dfdb0c59a5e758e7a98bd78eaca5983c22f4a811a36f4f980d245df4611", size = 38393, upload-time = "2025-12-06T13:26:19.535Z" }, + { url = "https://files.pythonhosted.org/packages/c3/ca/ae7a96be9ddc96030d4e9dffc43635d4e136b12058b387fd47eb8301b60f/pybase64-1.4.3-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0f1a0c51d6f159511e3431b73c25db31095ee36c394e26a4349e067c62f434e5", size = 32109, upload-time = "2025-12-06T13:26:20.72Z" }, + { url = "https://files.pythonhosted.org/packages/bf/44/d4b7adc7bf4fd5b52d8d099121760c450a52c390223806b873f0b6a2d551/pybase64-1.4.3-graalpy311-graalpy242_311_native-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a492518f3078a4e3faaef310697d21df9c6bc71908cebc8c2f6fbfa16d7d6b1f", size = 43227, upload-time = "2025-12-06T13:26:21.845Z" }, + { url = "https://files.pythonhosted.org/packages/08/86/2ba2d8734ef7939debeb52cf9952e457ba7aa226cae5c0e6dd631f9b851f/pybase64-1.4.3-graalpy311-graalpy242_311_native-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cae1a0f47784fd16df90d8acc32011c8d5fcdd9ab392c9ec49543e5f6a9c43a4", size = 35804, upload-time = "2025-12-06T13:26:23.149Z" }, + { url = "https://files.pythonhosted.org/packages/4f/5b/19c725dc3aaa6281f2ce3ea4c1628d154a40dd99657d1381995f8096768b/pybase64-1.4.3-graalpy311-graalpy242_311_native-win_amd64.whl", hash = "sha256:03cea70676ffbd39a1ab7930a2d24c625b416cacc9d401599b1d29415a43ab6a", size = 35880, upload-time = "2025-12-06T13:26:24.663Z" }, + { url = "https://files.pythonhosted.org/packages/17/45/92322aec1b6979e789b5710f73c59f2172bc37c8ce835305434796824b7b/pybase64-1.4.3-graalpy312-graalpy250_312_native-macosx_10_13_x86_64.whl", hash = "sha256:2baaa092f3475f3a9c87ac5198023918ea8b6c125f4c930752ab2cbe3cd1d520", size = 38746, upload-time = "2025-12-06T13:26:25.869Z" }, + { url = "https://files.pythonhosted.org/packages/11/94/f1a07402870388fdfc2ecec0c718111189732f7d0f2d7fe1386e19e8fad0/pybase64-1.4.3-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:cde13c0764b1af07a631729f26df019070dad759981d6975527b7e8ecb465b6c", size = 32573, upload-time = "2025-12-06T13:26:27.792Z" }, + { url = "https://files.pythonhosted.org/packages/fa/8f/43c3bb11ca9bacf81cb0b7a71500bb65b2eda6d5fe07433c09b543de97f3/pybase64-1.4.3-graalpy312-graalpy250_312_native-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5c29a582b0ea3936d02bd6fe9bf674ab6059e6e45ab71c78404ab2c913224414", size = 43461, upload-time = "2025-12-06T13:26:28.906Z" }, + { url = "https://files.pythonhosted.org/packages/2d/4c/2a5258329200be57497d3972b5308558c6de42e3749c6cc2aa1cbe34b25a/pybase64-1.4.3-graalpy312-graalpy250_312_native-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6b664758c804fa919b4f1257aa8cf68e95db76fc331de5f70bfc3a34655afe1", size = 36058, upload-time = "2025-12-06T13:26:30.092Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6d/41faa414cde66ec023b0ca8402a8f11cb61731c3dc27c082909cbbd1f929/pybase64-1.4.3-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:f7537fa22ae56a0bf51e4b0ffc075926ad91c618e1416330939f7ef366b58e3b", size = 36231, upload-time = "2025-12-06T13:26:31.656Z" }, + { url = "https://files.pythonhosted.org/packages/2a/cf/6e712491bd665ea8633efb0b484121893ea838d8e830e06f39f2aae37e58/pybase64-1.4.3-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:94cf50c36bb2f8618982ee5a978c4beed9db97d35944fa96e8586dd953c7994a", size = 38007, upload-time = "2025-12-06T13:26:32.804Z" }, + { url = "https://files.pythonhosted.org/packages/38/c0/9272cae1c49176337dcdbd97511e2843faae1aaf5a5fb48569093c6cd4ce/pybase64-1.4.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:01bc3ff5ca1341685c6d2d945b035f442f7b9c3b068a5c6ee8408a41fda5754e", size = 31538, upload-time = "2025-12-06T13:26:34.001Z" }, + { url = "https://files.pythonhosted.org/packages/20/f2/17546f97befe429c73f622bbd869ceebb518c40fdb0dec4c4f98312e80a5/pybase64-1.4.3-pp310-pypy310_pp73-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:03d0aa3761a99034960496280c02aa063f856a3cc9b33771bc4eab0e4e72b5c2", size = 40682, upload-time = "2025-12-06T13:26:35.168Z" }, + { url = "https://files.pythonhosted.org/packages/92/a0/464b36d5dfb61f3da17858afaeaa876a9342d58e9f17803ce7f28b5de9e8/pybase64-1.4.3-pp310-pypy310_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7ca5b1ce768520acd6440280cdab35235b27ad2faacfcec064bc9c3377066ef1", size = 41306, upload-time = "2025-12-06T13:26:36.351Z" }, + { url = "https://files.pythonhosted.org/packages/07/c9/a748dfc0969a8d960ecf1e82c8a2a16046ffec22f8e7ece582aa3b1c6cf9/pybase64-1.4.3-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3caa1e2ddad1c50553ffaaa1c86b74b3f9fbd505bea9970326ab88fc68c4c184", size = 35452, upload-time = "2025-12-06T13:26:37.772Z" }, + { url = "https://files.pythonhosted.org/packages/95/b7/4d37bd3577d1aa6c732dc099087fe027c48873e223de3784b095e5653f8b/pybase64-1.4.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:bd47076f736b27a8b0f9b30d93b6bb4f5af01b0dc8971f883ed3b75934f39a99", size = 36125, upload-time = "2025-12-06T13:26:39.78Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/160dded493c00d3376d4ad0f38a2119c5345de4a6693419ad39c3565959b/pybase64-1.4.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:277de6e03cc9090fb359365c686a2a3036d23aee6cd20d45d22b8c89d1247f17", size = 37939, upload-time = "2025-12-06T13:26:41.014Z" }, + { url = "https://files.pythonhosted.org/packages/b7/b8/a0f10be8d648d6f8f26e560d6e6955efa7df0ff1e009155717454d76f601/pybase64-1.4.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ab1dd8b1ed2d1d750260ed58ab40defaa5ba83f76a30e18b9ebd5646f6247ae5", size = 31466, upload-time = "2025-12-06T13:26:42.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/22/832a2f9e76cdf39b52e01e40d8feeb6a04cf105494f2c3e3126d0149717f/pybase64-1.4.3-pp311-pypy311_pp73-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:bd4d2293de9fd212e294c136cec85892460b17d24e8c18a6ba18750928037750", size = 40681, upload-time = "2025-12-06T13:26:43.782Z" }, + { url = "https://files.pythonhosted.org/packages/12/d7/6610f34a8972415fab3bb4704c174a1cc477bffbc3c36e526428d0f3957d/pybase64-1.4.3-pp311-pypy311_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2af6d0d3a691911cc4c9a625f3ddcd3af720738c21be3d5c72de05629139d393", size = 41294, upload-time = "2025-12-06T13:26:44.936Z" }, + { url = "https://files.pythonhosted.org/packages/64/25/ed24400948a6c974ab1374a233cb7e8af0a5373cea0dd8a944627d17c34a/pybase64-1.4.3-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5cfc8c49a28322d82242088378f8542ce97459866ba73150b062a7073e82629d", size = 35447, upload-time = "2025-12-06T13:26:46.098Z" }, + { url = "https://files.pythonhosted.org/packages/ee/2b/e18ee7c5ee508a82897f021c1981533eca2940b5f072fc6ed0906c03a7a7/pybase64-1.4.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:debf737e09b8bf832ba86f5ecc3d3dbd0e3021d6cd86ba4abe962d6a5a77adb3", size = 36134, upload-time = "2025-12-06T13:26:47.35Z" }, +] + [[package]] name = "pycln" version = "2.5.0" @@ -4906,6 +6912,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/20/ea99d5f0b3b4bc863ae892e2ff9a1a342e5ca709cb9b9927eeb705dd5984/pycln-2.5.0-py3-none-any.whl", hash = "sha256:6aec7a5b8df47e23399842b1f8470da4164956e26391f9b86c5edced5344da92", size = 38788, upload-time = "2025-01-06T19:21:33.636Z" }, ] +[[package]] +name = "pycountry" +version = "26.2.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/1d/061b9e7a48b85cfd69f33c33d2ef784a531c359399ad764243399673c8f5/pycountry-26.2.16.tar.gz", hash = "sha256:5b6027d453fcd6060112b951dd010f01f168b51b4bf8a1f1fc8c95c8d94a0801", size = 7711342, upload-time = "2026-02-17T03:42:52.367Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/42/7703bd45b62fecd44cd7d3495423097e2f7d28bc2e99e7c1af68892ab157/pycountry-26.2.16-py3-none-any.whl", hash = "sha256:115c4baf7cceaa30f59a4694d79483c9167dbce7a9de4d3d571c5f3ea77c305a", size = 8044600, upload-time = "2026-02-17T03:42:49.777Z" }, +] + [[package]] name = "pycparser" version = "2.22" @@ -4930,6 +6945,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, ] +[package.optional-dependencies] +email = [ + { name = "email-validator", marker = "sys_platform == 'linux'" }, +] + [[package]] name = "pydantic-core" version = "2.46.4" @@ -5046,6 +7066,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, ] +[[package]] +name = "pydantic-extra-types" +version = "2.11.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/71/dba38ee2651f84f7842206adbd2233d8bbdb59fb85e9fa14232486a8c471/pydantic_extra_types-2.11.1.tar.gz", hash = "sha256:46792d2307383859e923d8fcefa82108b1a141f8a9c0198982b3832ab5ef1049", size = 172002, upload-time = "2026-03-16T08:08:03.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl", hash = "sha256:1722ea2bddae5628ace25f2aa685b69978ef533123e5638cfbddb999e0100ec1", size = 79526, upload-time = "2026-03-16T08:08:02.533Z" }, +] + +[package.optional-dependencies] +pycountry = [ + { name = "pycountry", marker = "sys_platform == 'linux'" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic", marker = "sys_platform == 'linux'" }, + { name = "python-dotenv", marker = "sys_platform == 'linux'" }, + { name = "typing-inspection", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/b5/8f48e906c3e0205276e8bd8cb7512217a87b2685304d64be27cad5b3019f/pydantic_settings-2.14.2.tar.gz", hash = "sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f", size = 237700, upload-time = "2026-06-19T13:44:56.324Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" }, +] + [[package]] name = "pydub" version = "0.25.1" @@ -5064,6 +7116,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] +[[package]] +name = "pyjwt" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11' and sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography", marker = "sys_platform == 'linux'" }, +] + [[package]] name = "pyparsing" version = "3.3.2" @@ -5096,13 +7165,13 @@ name = "pytest" version = "8.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, { name = "pygments" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714, upload-time = "2025-06-18T05:48:06.109Z" } wheels = [ @@ -5318,7 +7387,7 @@ name = "pyzmq" version = "27.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "implementation_name == 'pypy'" }, + { name = "cffi", marker = "implementation_name == 'pypy' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f1/06/50a4e9648b3e8b992bef8eb632e457307553a89d294103213cfd47b3da69/pyzmq-27.0.0.tar.gz", hash = "sha256:b1f08eeb9ce1510e6939b6e5dcd46a17765e2333daae78ecf4606808442e52cf", size = 280478, upload-time = "2025-06-13T14:09:07.087Z" } wheels = [ @@ -5373,6 +7442,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/8a/4a3764a68abc02e2fbb0668d225b6fda5cd39586dd099cee8b2ed6ab0452/pyzmq-27.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:9df43a2459cd3a3563404c1456b2c4c69564daa7dbaf15724c09821a3329ce46", size = 544726, upload-time = "2025-06-13T14:08:49.903Z" }, ] +[[package]] +name = "quack-kernels" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apache-tvm-ffi", marker = "sys_platform == 'linux'" }, + { name = "einops", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cutlass-dsl", marker = "sys_platform == 'linux'" }, + { name = "torch", marker = "sys_platform == 'linux'" }, + { name = "torch-c-dlpack-ext", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/94/ee76e3a3dc74d986b7b24c5928f1d14b01bd5152375688c2ede369f6d19b/quack_kernels-0.5.0.tar.gz", hash = "sha256:c7c7338b67243397b6ca166e648bba161076e99f3858b532e1c877dcc6eaa03d", size = 366426, upload-time = "2026-05-29T05:00:25.985Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/2b/a8f171d5e172880885571bf89e93204aaf231a0e92c4c84714eaf18c271a/quack_kernels-0.5.0-py3-none-any.whl", hash = "sha256:08821ebfb8e638cc20308d5c59410c6dbb3b637ccc7b07bd57c7a9261a06af74", size = 327709, upload-time = "2026-05-29T05:00:24.679Z" }, +] + [[package]] name = "rapidfuzz" version = "3.14.5" @@ -5470,7 +7555,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, { name = "rpds-py" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2f/db/98b5c277be99dd18bfd91dd04e1b759cad18d1a338188c936e92f921c7e2/referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa", size = 74744, upload-time = "2025-01-25T08:48:16.138Z" } wheels = [ @@ -5659,6 +7744,141 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e3/30/3c4d035596d3cf444529e0b2953ad0466f6049528a879d27534700580395/rich-14.1.0-py3-none-any.whl", hash = "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f", size = 243368, upload-time = "2025-07-25T07:32:56.73Z" }, ] +[[package]] +name = "rich-toolkit" +version = "0.20.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click", marker = "sys_platform == 'linux'" }, + { name = "rich", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/3a/a258c2fbc6c6bdf428611388f5698ba5d57ffdf0755e1cab474d9cc47813/rich_toolkit-0.20.3.tar.gz", hash = "sha256:223dd2cfba325ed55e94933b9e53f3aca13e9fdf76622bd564c18109a2273c1b", size = 205355, upload-time = "2026-07-13T14:38:06.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/ce/639d0d0ce3d25c5edbd1afecd308bb35dc04883a45ac9f0855c8aee4e919/rich_toolkit-0.20.3-py3-none-any.whl", hash = "sha256:419aa87516d5f3849cca553c6dcf707c02a36d508fcf996946606725d34a3002", size = 36195, upload-time = "2026-07-13T14:38:05.687Z" }, +] + +[[package]] +name = "rignore" +version = "0.7.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/f5/8bed2310abe4ae04b67a38374a4d311dd85220f5d8da56f47ae9361be0b0/rignore-0.7.6.tar.gz", hash = "sha256:00d3546cd793c30cb17921ce674d2c8f3a4b00501cb0e3dd0e82217dbeba2671", size = 57140, upload-time = "2025-11-05T21:41:21.968Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/7a/b970cd0138b0ece72eb28f086e933f9ed75b795716ad3de5ab22994b3b54/rignore-0.7.6-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:f3c74a7e5ee77aea669c95fdb3933f2a6c7549893700082e759128a29cf67e45", size = 884999, upload-time = "2025-11-05T20:42:38.373Z" }, + { url = "https://files.pythonhosted.org/packages/ca/05/23faca29616d8966ada63fb0e13c214107811fa9a0aba2275e4c7ca63bd5/rignore-0.7.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b7202404958f5fe3474bac91f65350f0b1dde1a5e05089f2946549b7e91e79ec", size = 824824, upload-time = "2025-11-05T20:42:22.1Z" }, + { url = "https://files.pythonhosted.org/packages/fa/2e/05a1e61f04cf2548524224f0b5f21ca19ea58f7273a863bac10846b8ff69/rignore-0.7.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bde7c5835fa3905bfb7e329a4f1d7eccb676de63da7a3f934ddd5c06df20597", size = 899121, upload-time = "2025-11-05T20:40:48.94Z" }, + { url = "https://files.pythonhosted.org/packages/ff/35/71518847e10bdbf359badad8800e4681757a01f4777b3c5e03dbde8a42d8/rignore-0.7.6-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:626c3d4ba03af266694d25101bc1d8d16eda49c5feb86cedfec31c614fceca7d", size = 873813, upload-time = "2025-11-05T20:41:04.71Z" }, + { url = "https://files.pythonhosted.org/packages/f6/c8/32ae405d3e7fd4d9f9b7838f2fcca0a5005bb87fa514b83f83fd81c0df22/rignore-0.7.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0a43841e651e7a05a4274b9026cc408d1912e64016ede8cd4c145dae5d0635be", size = 1168019, upload-time = "2025-11-05T20:41:20.723Z" }, + { url = "https://files.pythonhosted.org/packages/25/98/013c955982bc5b4719bf9a5bea58be317eea28aa12bfd004025e3cd7c000/rignore-0.7.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7978c498dbf7f74d30cdb8859fe612167d8247f0acd377ae85180e34490725da", size = 942822, upload-time = "2025-11-05T20:41:36.99Z" }, + { url = "https://files.pythonhosted.org/packages/90/fb/9a3f3156c6ed30bcd597e63690353edac1fcffe9d382ad517722b56ac195/rignore-0.7.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d22f72ab695c07d2d96d2a645208daff17084441b5d58c07378c9dd6f9c4c87", size = 959820, upload-time = "2025-11-05T20:42:06.364Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b2/93bf609633021e9658acaff24cfb055d8cdaf7f5855d10ebb35307900dda/rignore-0.7.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d5bd8e1a91ed1a789b2cbe39eeea9204a6719d4f2cf443a9544b521a285a295f", size = 985050, upload-time = "2025-11-05T20:41:51.124Z" }, + { url = "https://files.pythonhosted.org/packages/69/bc/ec2d040469bdfd7b743df10f2201c5d285009a4263d506edbf7a06a090bb/rignore-0.7.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:bc1fc03efad5789365018e94ac4079f851a999bc154d1551c45179f7fcf45322", size = 1079164, upload-time = "2025-11-05T21:40:10.368Z" }, + { url = "https://files.pythonhosted.org/packages/df/26/4b635f4ea5baf4baa8ba8eee06163f6af6e76dfbe72deb57da34bb24b19d/rignore-0.7.6-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:ce2617fe28c51367fd8abfd4eeea9e61664af63c17d4ea00353d8ef56dfb95fa", size = 1139028, upload-time = "2025-11-05T21:40:27.977Z" }, + { url = "https://files.pythonhosted.org/packages/6a/54/a3147ebd1e477b06eb24e2c2c56d951ae5faa9045b7b36d7892fec5080d9/rignore-0.7.6-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:7c4ad2cee85068408e7819a38243043214e2c3047e9bd4c506f8de01c302709e", size = 1119024, upload-time = "2025-11-05T21:40:45.148Z" }, + { url = "https://files.pythonhosted.org/packages/fb/f4/27475db769a57cff18fe7e7267b36e6cdb5b1281caa185ba544171106cba/rignore-0.7.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:02cd240bfd59ecc3907766f4839cbba20530a2e470abca09eaa82225e4d946fb", size = 1128531, upload-time = "2025-11-05T21:41:02.734Z" }, + { url = "https://files.pythonhosted.org/packages/97/32/6e782d3b352e4349fa0e90bf75b13cb7f11d8908b36d9e2b262224b65d9a/rignore-0.7.6-cp310-cp310-win32.whl", hash = "sha256:fe2bd8fa1ff555259df54c376abc73855cb02628a474a40d51b358c3a1ddc55b", size = 646817, upload-time = "2025-11-05T21:41:47.51Z" }, + { url = "https://files.pythonhosted.org/packages/c0/8a/53185c69abb3bb362e8a46b8089999f820bf15655629ff8395107633c8ab/rignore-0.7.6-cp310-cp310-win_amd64.whl", hash = "sha256:d80afd6071c78baf3765ec698841071b19e41c326f994cfa69b5a1df676f5d39", size = 727001, upload-time = "2025-11-05T21:41:32.778Z" }, + { url = "https://files.pythonhosted.org/packages/25/41/b6e2be3069ef3b7f24e35d2911bd6deb83d20ed5642ad81d5a6d1c015473/rignore-0.7.6-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:40be8226e12d6653abbebaffaea2885f80374c1c8f76fe5ca9e0cadd120a272c", size = 885285, upload-time = "2025-11-05T20:42:39.763Z" }, + { url = "https://files.pythonhosted.org/packages/52/66/ba7f561b6062402022887706a7f2b2c2e2e2a28f1e3839202b0a2f77e36d/rignore-0.7.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:182f4e5e4064d947c756819446a7d4cdede8e756b8c81cf9e509683fe38778d7", size = 823882, upload-time = "2025-11-05T20:42:23.488Z" }, + { url = "https://files.pythonhosted.org/packages/f5/81/4087453df35a90b07370647b19017029324950c1b9137d54bf1f33843f17/rignore-0.7.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16b63047648a916a87be1e51bb5c009063f1b8b6f5afe4f04f875525507e63dc", size = 899362, upload-time = "2025-11-05T20:40:51.111Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c9/390a8fdfabb76d71416be773bd9f162977bd483084f68daf19da1dec88a6/rignore-0.7.6-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ba5524f5178deca4d7695e936604ebc742acb8958f9395776e1fcb8133f8257a", size = 873633, upload-time = "2025-11-05T20:41:06.193Z" }, + { url = "https://files.pythonhosted.org/packages/df/c9/79404fcb0faa76edfbc9df0901f8ef18568d1104919ebbbad6d608c888d1/rignore-0.7.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:62020dbb89a1dd4b84ab3d60547b3b2eb2723641d5fb198463643f71eaaed57d", size = 1167633, upload-time = "2025-11-05T20:41:22.491Z" }, + { url = "https://files.pythonhosted.org/packages/6e/8d/b3466d32d445d158a0aceb80919085baaae495b1f540fb942f91d93b5e5b/rignore-0.7.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b34acd532769d5a6f153a52a98dcb81615c949ab11697ce26b2eb776af2e174d", size = 941434, upload-time = "2025-11-05T20:41:38.151Z" }, + { url = "https://files.pythonhosted.org/packages/e8/40/9cd949761a7af5bc27022a939c91ff622d29c7a0b66d0c13a863097dde2d/rignore-0.7.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c5e53b752f9de44dff7b3be3c98455ce3bf88e69d6dc0cf4f213346c5e3416c", size = 959461, upload-time = "2025-11-05T20:42:08.476Z" }, + { url = "https://files.pythonhosted.org/packages/b5/87/1e1a145731f73bdb7835e11f80da06f79a00d68b370d9a847de979575e6d/rignore-0.7.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:25b3536d13a5d6409ce85f23936f044576eeebf7b6db1d078051b288410fc049", size = 985323, upload-time = "2025-11-05T20:41:52.735Z" }, + { url = "https://files.pythonhosted.org/packages/6c/31/1ecff992fc3f59c4fcdcb6c07d5f6c1e6dfb55ccda19c083aca9d86fa1c6/rignore-0.7.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6e01cad2b0b92f6b1993f29fc01f23f2d78caf4bf93b11096d28e9d578eb08ce", size = 1079173, upload-time = "2025-11-05T21:40:12.007Z" }, + { url = "https://files.pythonhosted.org/packages/17/18/162eedadb4c2282fa4c521700dbf93c9b14b8842e8354f7d72b445b8d593/rignore-0.7.6-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5991e46ab9b4868334c9e372ab0892b0150f3f586ff2b1e314272caeb38aaedb", size = 1139012, upload-time = "2025-11-05T21:40:29.399Z" }, + { url = "https://files.pythonhosted.org/packages/78/96/a9ca398a8af74bb143ad66c2a31303c894111977e28b0d0eab03867f1b43/rignore-0.7.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6c8ae562e5d1246cba5eaeb92a47b2a279e7637102828dde41dcbe291f529a3e", size = 1118827, upload-time = "2025-11-05T21:40:46.6Z" }, + { url = "https://files.pythonhosted.org/packages/9f/22/1c1a65047df864def9a047dbb40bc0b580b8289a4280e62779cd61ae21f2/rignore-0.7.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aaf938530dcc0b47c4cfa52807aa2e5bfd5ca6d57a621125fe293098692f6345", size = 1128182, upload-time = "2025-11-05T21:41:04.239Z" }, + { url = "https://files.pythonhosted.org/packages/bd/f4/1526eb01fdc2235aca1fd9d0189bee4021d009a8dcb0161540238c24166e/rignore-0.7.6-cp311-cp311-win32.whl", hash = "sha256:166ebce373105dd485ec213a6a2695986346e60c94ff3d84eb532a237b24a4d5", size = 646547, upload-time = "2025-11-05T21:41:49.439Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c8/dda0983e1845706beb5826459781549a840fe5a7eb934abc523e8cd17814/rignore-0.7.6-cp311-cp311-win_amd64.whl", hash = "sha256:44f35ee844b1a8cea50d056e6a595190ce9d42d3cccf9f19d280ae5f3058973a", size = 727139, upload-time = "2025-11-05T21:41:34.367Z" }, + { url = "https://files.pythonhosted.org/packages/e3/47/eb1206b7bf65970d41190b879e1723fc6bbdb2d45e53565f28991a8d9d96/rignore-0.7.6-cp311-cp311-win_arm64.whl", hash = "sha256:14b58f3da4fa3d5c3fa865cab49821675371f5e979281c683e131ae29159a581", size = 657598, upload-time = "2025-11-05T21:41:23.758Z" }, + { url = "https://files.pythonhosted.org/packages/0b/0e/012556ef3047a2628842b44e753bb15f4dc46806780ff090f1e8fe4bf1eb/rignore-0.7.6-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:03e82348cb7234f8d9b2834f854400ddbbd04c0f8f35495119e66adbd37827a8", size = 883488, upload-time = "2025-11-05T20:42:41.359Z" }, + { url = "https://files.pythonhosted.org/packages/93/b0/d4f1f3fe9eb3f8e382d45ce5b0547ea01c4b7e0b4b4eb87bcd66a1d2b888/rignore-0.7.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b9e624f6be6116ea682e76c5feb71ea91255c67c86cb75befe774365b2931961", size = 820411, upload-time = "2025-11-05T20:42:24.782Z" }, + { url = "https://files.pythonhosted.org/packages/4a/c8/dea564b36dedac8de21c18e1851789545bc52a0c22ece9843444d5608a6a/rignore-0.7.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bda49950d405aa8d0ebe26af807c4e662dd281d926530f03f29690a2e07d649a", size = 897821, upload-time = "2025-11-05T20:40:52.613Z" }, + { url = "https://files.pythonhosted.org/packages/b3/2b/ee96db17ac1835e024c5d0742eefb7e46de60020385ac883dd3d1cde2c1f/rignore-0.7.6-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5fd5ab3840b8c16851d327ed06e9b8be6459702a53e5ab1fc4073b684b3789e", size = 873963, upload-time = "2025-11-05T20:41:07.49Z" }, + { url = "https://files.pythonhosted.org/packages/a5/8c/ad5a57bbb9d14d5c7e5960f712a8a0b902472ea3f4a2138cbf70d1777b75/rignore-0.7.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ced2a248352636a5c77504cb755dc02c2eef9a820a44d3f33061ce1bb8a7f2d2", size = 1169216, upload-time = "2025-11-05T20:41:23.73Z" }, + { url = "https://files.pythonhosted.org/packages/80/e6/5b00bc2a6bc1701e6878fca798cf5d9125eb3113193e33078b6fc0d99123/rignore-0.7.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a04a3b73b75ddc12c9c9b21efcdaab33ca3832941d6f1d67bffd860941cd448a", size = 942942, upload-time = "2025-11-05T20:41:39.393Z" }, + { url = "https://files.pythonhosted.org/packages/85/e5/7f99bd0cc9818a91d0e8b9acc65b792e35750e3bdccd15a7ee75e64efca4/rignore-0.7.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d24321efac92140b7ec910ac7c53ab0f0c86a41133d2bb4b0e6a7c94967f44dd", size = 959787, upload-time = "2025-11-05T20:42:09.765Z" }, + { url = "https://files.pythonhosted.org/packages/55/54/2ffea79a7c1eabcede1926347ebc2a81bc6b81f447d05b52af9af14948b9/rignore-0.7.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:73c7aa109d41e593785c55fdaa89ad80b10330affa9f9d3e3a51fa695f739b20", size = 984245, upload-time = "2025-11-05T20:41:54.062Z" }, + { url = "https://files.pythonhosted.org/packages/41/f7/e80f55dfe0f35787fa482aa18689b9c8251e045076c35477deb0007b3277/rignore-0.7.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1734dc49d1e9501b07852ef44421f84d9f378da9fbeda729e77db71f49cac28b", size = 1078647, upload-time = "2025-11-05T21:40:13.463Z" }, + { url = "https://files.pythonhosted.org/packages/d4/cf/2c64f0b6725149f7c6e7e5a909d14354889b4beaadddaa5fff023ec71084/rignore-0.7.6-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5719ea14ea2b652c0c0894be5dfde954e1853a80dea27dd2fbaa749618d837f5", size = 1139186, upload-time = "2025-11-05T21:40:31.27Z" }, + { url = "https://files.pythonhosted.org/packages/75/95/a86c84909ccc24af0d094b50d54697951e576c252a4d9f21b47b52af9598/rignore-0.7.6-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8e23424fc7ce35726854f639cb7968151a792c0c3d9d082f7f67e0c362cfecca", size = 1117604, upload-time = "2025-11-05T21:40:48.07Z" }, + { url = "https://files.pythonhosted.org/packages/7f/5e/13b249613fd5d18d58662490ab910a9f0be758981d1797789913adb4e918/rignore-0.7.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3efdcf1dd84d45f3e2bd2f93303d9be103888f56dfa7c3349b5bf4f0657ec696", size = 1127725, upload-time = "2025-11-05T21:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/c7/28/fa5dcd1e2e16982c359128664e3785f202d3eca9b22dd0b2f91c4b3d242f/rignore-0.7.6-cp312-cp312-win32.whl", hash = "sha256:ccca9d1a8b5234c76b71546fc3c134533b013f40495f394a65614a81f7387046", size = 646145, upload-time = "2025-11-05T21:41:51.096Z" }, + { url = "https://files.pythonhosted.org/packages/26/87/69387fb5dd81a0f771936381431780b8cf66fcd2cfe9495e1aaf41548931/rignore-0.7.6-cp312-cp312-win_amd64.whl", hash = "sha256:c96a285e4a8bfec0652e0bfcf42b1aabcdda1e7625f5006d188e3b1c87fdb543", size = 726090, upload-time = "2025-11-05T21:41:36.485Z" }, + { url = "https://files.pythonhosted.org/packages/24/5f/e8418108dcda8087fb198a6f81caadbcda9fd115d61154bf0df4d6d3619b/rignore-0.7.6-cp312-cp312-win_arm64.whl", hash = "sha256:a64a750e7a8277a323f01ca50b7784a764845f6cce2fe38831cb93f0508d0051", size = 656317, upload-time = "2025-11-05T21:41:25.305Z" }, + { url = "https://files.pythonhosted.org/packages/b7/8a/a4078f6e14932ac7edb171149c481de29969d96ddee3ece5dc4c26f9e0c3/rignore-0.7.6-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:2bdab1d31ec9b4fb1331980ee49ea051c0d7f7bb6baa28b3125ef03cdc48fdaf", size = 883057, upload-time = "2025-11-05T20:42:42.741Z" }, + { url = "https://files.pythonhosted.org/packages/f9/8f/f8daacd177db4bf7c2223bab41e630c52711f8af9ed279be2058d2fe4982/rignore-0.7.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:90f0a00ce0c866c275bf888271f1dc0d2140f29b82fcf33cdbda1e1a6af01010", size = 820150, upload-time = "2025-11-05T20:42:26.545Z" }, + { url = "https://files.pythonhosted.org/packages/36/31/b65b837e39c3f7064c426754714ac633b66b8c2290978af9d7f513e14aa9/rignore-0.7.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1ad295537041dc2ed4b540fb1a3906bd9ede6ccdad3fe79770cd89e04e3c73c", size = 897406, upload-time = "2025-11-05T20:40:53.854Z" }, + { url = "https://files.pythonhosted.org/packages/ca/58/1970ce006c427e202ac7c081435719a076c478f07b3a23f469227788dc23/rignore-0.7.6-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f782dbd3a65a5ac85adfff69e5c6b101285ef3f845c3a3cae56a54bebf9fe116", size = 874050, upload-time = "2025-11-05T20:41:08.922Z" }, + { url = "https://files.pythonhosted.org/packages/d4/00/eb45db9f90137329072a732273be0d383cb7d7f50ddc8e0bceea34c1dfdf/rignore-0.7.6-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65cece3b36e5b0826d946494734c0e6aaf5a0337e18ff55b071438efe13d559e", size = 1167835, upload-time = "2025-11-05T20:41:24.997Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f1/6f1d72ddca41a64eed569680587a1236633587cc9f78136477ae69e2c88a/rignore-0.7.6-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d7e4bb66c13cd7602dc8931822c02dfbbd5252015c750ac5d6152b186f0a8be0", size = 941945, upload-time = "2025-11-05T20:41:40.628Z" }, + { url = "https://files.pythonhosted.org/packages/48/6f/2f178af1c1a276a065f563ec1e11e7a9e23d4996fd0465516afce4b5c636/rignore-0.7.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:297e500c15766e196f68aaaa70e8b6db85fa23fdc075b880d8231fdfba738cd7", size = 959067, upload-time = "2025-11-05T20:42:11.09Z" }, + { url = "https://files.pythonhosted.org/packages/5b/db/423a81c4c1e173877c7f9b5767dcaf1ab50484a94f60a0b2ed78be3fa765/rignore-0.7.6-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a07084211a8d35e1a5b1d32b9661a5ed20669970b369df0cf77da3adea3405de", size = 984438, upload-time = "2025-11-05T20:41:55.443Z" }, + { url = "https://files.pythonhosted.org/packages/31/eb/c4f92cc3f2825d501d3c46a244a671eb737fc1bcf7b05a3ecd34abb3e0d7/rignore-0.7.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:181eb2a975a22256a1441a9d2f15eb1292839ea3f05606620bd9e1938302cf79", size = 1078365, upload-time = "2025-11-05T21:40:15.148Z" }, + { url = "https://files.pythonhosted.org/packages/26/09/99442f02794bd7441bfc8ed1c7319e890449b816a7493b2db0e30af39095/rignore-0.7.6-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:7bbcdc52b5bf9f054b34ce4af5269df5d863d9c2456243338bc193c28022bd7b", size = 1139066, upload-time = "2025-11-05T21:40:32.771Z" }, + { url = "https://files.pythonhosted.org/packages/2c/88/bcfc21e520bba975410e9419450f4b90a2ac8236b9a80fd8130e87d098af/rignore-0.7.6-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f2e027a6da21a7c8c0d87553c24ca5cc4364def18d146057862c23a96546238e", size = 1118036, upload-time = "2025-11-05T21:40:49.646Z" }, + { url = "https://files.pythonhosted.org/packages/e2/25/d37215e4562cda5c13312636393aea0bafe38d54d4e0517520a4cc0753ec/rignore-0.7.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee4a18b82cbbc648e4aac1510066682fe62beb5dc88e2c67c53a83954e541360", size = 1127550, upload-time = "2025-11-05T21:41:07.648Z" }, + { url = "https://files.pythonhosted.org/packages/dc/76/a264ab38bfa1620ec12a8ff1c07778da89e16d8c0f3450b0333020d3d6dc/rignore-0.7.6-cp313-cp313-win32.whl", hash = "sha256:a7d7148b6e5e95035d4390396895adc384d37ff4e06781a36fe573bba7c283e5", size = 646097, upload-time = "2025-11-05T21:41:53.201Z" }, + { url = "https://files.pythonhosted.org/packages/62/44/3c31b8983c29ea8832b6082ddb1d07b90379c2d993bd20fce4487b71b4f4/rignore-0.7.6-cp313-cp313-win_amd64.whl", hash = "sha256:b037c4b15a64dced08fc12310ee844ec2284c4c5c1ca77bc37d0a04f7bff386e", size = 726170, upload-time = "2025-11-05T21:41:38.131Z" }, + { url = "https://files.pythonhosted.org/packages/aa/41/e26a075cab83debe41a42661262f606166157df84e0e02e2d904d134c0d8/rignore-0.7.6-cp313-cp313-win_arm64.whl", hash = "sha256:e47443de9b12fe569889bdbe020abe0e0b667516ee2ab435443f6d0869bd2804", size = 656184, upload-time = "2025-11-05T21:41:27.396Z" }, + { url = "https://files.pythonhosted.org/packages/9a/b9/1f5bd82b87e5550cd843ceb3768b4a8ef274eb63f29333cf2f29644b3d75/rignore-0.7.6-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:8e41be9fa8f2f47239ded8920cc283699a052ac4c371f77f5ac017ebeed75732", size = 882632, upload-time = "2025-11-05T20:42:44.063Z" }, + { url = "https://files.pythonhosted.org/packages/e9/6b/07714a3efe4a8048864e8a5b7db311ba51b921e15268b17defaebf56d3db/rignore-0.7.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6dc1e171e52cefa6c20e60c05394a71165663b48bca6c7666dee4f778f2a7d90", size = 820760, upload-time = "2025-11-05T20:42:27.885Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0f/348c829ea2d8d596e856371b14b9092f8a5dfbb62674ec9b3f67e4939a9d/rignore-0.7.6-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ce2268837c3600f82ab8db58f5834009dc638ee17103582960da668963bebc5", size = 899044, upload-time = "2025-11-05T20:40:55.336Z" }, + { url = "https://files.pythonhosted.org/packages/f0/30/2e1841a19b4dd23878d73edd5d82e998a83d5ed9570a89675f140ca8b2ad/rignore-0.7.6-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:690a3e1b54bfe77e89c4bacb13f046e642f8baadafc61d68f5a726f324a76ab6", size = 874144, upload-time = "2025-11-05T20:41:10.195Z" }, + { url = "https://files.pythonhosted.org/packages/c2/bf/0ce9beb2e5f64c30e3580bef09f5829236889f01511a125f98b83169b993/rignore-0.7.6-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09d12ac7a0b6210c07bcd145007117ebd8abe99c8eeb383e9e4673910c2754b2", size = 1168062, upload-time = "2025-11-05T20:41:26.511Z" }, + { url = "https://files.pythonhosted.org/packages/b9/8b/571c178414eb4014969865317da8a02ce4cf5241a41676ef91a59aab24de/rignore-0.7.6-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a2b2b74a8c60203b08452479b90e5ce3dbe96a916214bc9eb2e5af0b6a9beb0", size = 942542, upload-time = "2025-11-05T20:41:41.838Z" }, + { url = "https://files.pythonhosted.org/packages/19/62/7a3cf601d5a45137a7e2b89d10c05b5b86499190c4b7ca5c3c47d79ee519/rignore-0.7.6-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fc5a531ef02131e44359419a366bfac57f773ea58f5278c2cdd915f7d10ea94", size = 958739, upload-time = "2025-11-05T20:42:12.463Z" }, + { url = "https://files.pythonhosted.org/packages/5f/1f/4261f6a0d7caf2058a5cde2f5045f565ab91aa7badc972b57d19ce58b14e/rignore-0.7.6-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b7a1f77d9c4cd7e76229e252614d963442686bfe12c787a49f4fe481df49e7a9", size = 984138, upload-time = "2025-11-05T20:41:56.775Z" }, + { url = "https://files.pythonhosted.org/packages/2b/bf/628dfe19c75e8ce1f45f7c248f5148b17dfa89a817f8e3552ab74c3ae812/rignore-0.7.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ead81f728682ba72b5b1c3d5846b011d3e0174da978de87c61645f2ed36659a7", size = 1079299, upload-time = "2025-11-05T21:40:16.639Z" }, + { url = "https://files.pythonhosted.org/packages/af/a5/be29c50f5c0c25c637ed32db8758fdf5b901a99e08b608971cda8afb293b/rignore-0.7.6-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:12ffd50f520c22ffdabed8cd8bfb567d9ac165b2b854d3e679f4bcaef11a9441", size = 1139618, upload-time = "2025-11-05T21:40:34.507Z" }, + { url = "https://files.pythonhosted.org/packages/2a/40/3c46cd7ce4fa05c20b525fd60f599165e820af66e66f2c371cd50644558f/rignore-0.7.6-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:e5a16890fbe3c894f8ca34b0fcacc2c200398d4d46ae654e03bc9b3dbf2a0a72", size = 1117626, upload-time = "2025-11-05T21:40:51.494Z" }, + { url = "https://files.pythonhosted.org/packages/8c/b9/aea926f263b8a29a23c75c2e0d8447965eb1879d3feb53cfcf84db67ed58/rignore-0.7.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3abab3bf99e8a77488ef6c7c9a799fac22224c28fe9f25cc21aa7cc2b72bfc0b", size = 1128144, upload-time = "2025-11-05T21:41:09.169Z" }, + { url = "https://files.pythonhosted.org/packages/a4/f6/0d6242f8d0df7f2ecbe91679fefc1f75e7cd2072cb4f497abaab3f0f8523/rignore-0.7.6-cp314-cp314-win32.whl", hash = "sha256:eeef421c1782953c4375aa32f06ecae470c1285c6381eee2a30d2e02a5633001", size = 646385, upload-time = "2025-11-05T21:41:55.105Z" }, + { url = "https://files.pythonhosted.org/packages/d5/38/c0dcd7b10064f084343d6af26fe9414e46e9619c5f3224b5272e8e5d9956/rignore-0.7.6-cp314-cp314-win_amd64.whl", hash = "sha256:6aeed503b3b3d5af939b21d72a82521701a4bd3b89cd761da1e7dc78621af304", size = 725738, upload-time = "2025-11-05T21:41:39.736Z" }, + { url = "https://files.pythonhosted.org/packages/d9/7a/290f868296c1ece914d565757ab363b04730a728b544beb567ceb3b2d96f/rignore-0.7.6-cp314-cp314-win_arm64.whl", hash = "sha256:104f215b60b3c984c386c3e747d6ab4376d5656478694e22c7bd2f788ddd8304", size = 656008, upload-time = "2025-11-05T21:41:29.028Z" }, + { url = "https://files.pythonhosted.org/packages/ca/d2/3c74e3cd81fe8ea08a8dcd2d755c09ac2e8ad8fe409508904557b58383d3/rignore-0.7.6-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:bb24a5b947656dd94cb9e41c4bc8b23cec0c435b58be0d74a874f63c259549e8", size = 882835, upload-time = "2025-11-05T20:42:45.443Z" }, + { url = "https://files.pythonhosted.org/packages/77/61/a772a34b6b63154877433ac2d048364815b24c2dd308f76b212c408101a2/rignore-0.7.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5b1e33c9501cefe24b70a1eafd9821acfd0ebf0b35c3a379430a14df089993e3", size = 820301, upload-time = "2025-11-05T20:42:29.226Z" }, + { url = "https://files.pythonhosted.org/packages/71/30/054880b09c0b1b61d17eeb15279d8bf729c0ba52b36c3ada52fb827cbb3c/rignore-0.7.6-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bec3994665a44454df86deb762061e05cd4b61e3772f5b07d1882a8a0d2748d5", size = 897611, upload-time = "2025-11-05T20:40:56.475Z" }, + { url = "https://files.pythonhosted.org/packages/1e/40/b2d1c169f833d69931bf232600eaa3c7998ba4f9a402e43a822dad2ea9f2/rignore-0.7.6-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:26cba2edfe3cff1dfa72bddf65d316ddebf182f011f2f61538705d6dbaf54986", size = 873875, upload-time = "2025-11-05T20:41:11.561Z" }, + { url = "https://files.pythonhosted.org/packages/55/59/ca5ae93d83a1a60e44b21d87deb48b177a8db1b85e82fc8a9abb24a8986d/rignore-0.7.6-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ffa86694fec604c613696cb91e43892aa22e1fec5f9870e48f111c603e5ec4e9", size = 1167245, upload-time = "2025-11-05T20:41:28.29Z" }, + { url = "https://files.pythonhosted.org/packages/a5/52/cf3dce392ba2af806cba265aad6bcd9c48bb2a6cb5eee448d3319f6e505b/rignore-0.7.6-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48efe2ed95aa8104145004afb15cdfa02bea5cdde8b0344afeb0434f0d989aa2", size = 941750, upload-time = "2025-11-05T20:41:43.111Z" }, + { url = "https://files.pythonhosted.org/packages/ec/be/3f344c6218d779395e785091d05396dfd8b625f6aafbe502746fcd880af2/rignore-0.7.6-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8dcae43eb44b7f2457fef7cc87f103f9a0013017a6f4e62182c565e924948f21", size = 958896, upload-time = "2025-11-05T20:42:13.784Z" }, + { url = "https://files.pythonhosted.org/packages/c9/34/d3fa71938aed7d00dcad87f0f9bcb02ad66c85d6ffc83ba31078ce53646a/rignore-0.7.6-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2cd649a7091c0dad2f11ef65630d30c698d505cbe8660dd395268e7c099cc99f", size = 983992, upload-time = "2025-11-05T20:41:58.022Z" }, + { url = "https://files.pythonhosted.org/packages/24/a4/52a697158e9920705bdbd0748d59fa63e0f3233fb92e9df9a71afbead6ca/rignore-0.7.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:42de84b0289d478d30ceb7ae59023f7b0527786a9a5b490830e080f0e4ea5aeb", size = 1078181, upload-time = "2025-11-05T21:40:18.151Z" }, + { url = "https://files.pythonhosted.org/packages/ac/65/aa76dbcdabf3787a6f0fd61b5cc8ed1e88580590556d6c0207960d2384bb/rignore-0.7.6-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:875a617e57b53b4acbc5a91de418233849711c02e29cc1f4f9febb2f928af013", size = 1139232, upload-time = "2025-11-05T21:40:35.966Z" }, + { url = "https://files.pythonhosted.org/packages/08/44/31b31a49b3233c6842acc1c0731aa1e7fb322a7170612acf30327f700b44/rignore-0.7.6-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8703998902771e96e49968105207719f22926e4431b108450f3f430b4e268b7c", size = 1117349, upload-time = "2025-11-05T21:40:53.013Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ae/1b199a2302c19c658cf74e5ee1427605234e8c91787cfba0015f2ace145b/rignore-0.7.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:602ef33f3e1b04c1e9a10a3c03f8bc3cef2d2383dcc250d309be42b49923cabc", size = 1127702, upload-time = "2025-11-05T21:41:10.881Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d3/18210222b37e87e36357f7b300b7d98c6dd62b133771e71ae27acba83a4f/rignore-0.7.6-cp314-cp314t-win32.whl", hash = "sha256:c1d8f117f7da0a4a96a8daef3da75bc090e3792d30b8b12cfadc240c631353f9", size = 647033, upload-time = "2025-11-05T21:42:00.095Z" }, + { url = "https://files.pythonhosted.org/packages/3e/87/033eebfbee3ec7d92b3bb1717d8f68c88e6fc7de54537040f3b3a405726f/rignore-0.7.6-cp314-cp314t-win_amd64.whl", hash = "sha256:ca36e59408bec81de75d307c568c2d0d410fb880b1769be43611472c61e85c96", size = 725647, upload-time = "2025-11-05T21:41:44.449Z" }, + { url = "https://files.pythonhosted.org/packages/79/62/b88e5879512c55b8ee979c666ee6902adc4ed05007226de266410ae27965/rignore-0.7.6-cp314-cp314t-win_arm64.whl", hash = "sha256:b83adabeb3e8cf662cabe1931b83e165b88c526fa6af6b3aa90429686e474896", size = 656035, upload-time = "2025-11-05T21:41:31.13Z" }, + { url = "https://files.pythonhosted.org/packages/85/12/62d690b4644c330d7ac0f739b7f078190ab4308faa909a60842d0e4af5b2/rignore-0.7.6-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c3d3a523af1cd4ed2c0cba8d277a32d329b0c96ef9901fb7ca45c8cfaccf31a5", size = 887462, upload-time = "2025-11-05T20:42:50.804Z" }, + { url = "https://files.pythonhosted.org/packages/05/bc/6528a0e97ed2bd7a7c329183367d1ffbc5b9762ae8348d88dae72cc9d1f5/rignore-0.7.6-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:990853566e65184a506e1e2af2d15045afad3ebaebb8859cb85b882081915110", size = 826918, upload-time = "2025-11-05T20:42:33.689Z" }, + { url = "https://files.pythonhosted.org/packages/3e/2c/7d7bad116e09a04e9e1688c6f891fa2d4fd33f11b69ac0bd92419ddebeae/rignore-0.7.6-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cab9ff2e436ce7240d7ee301c8ef806ed77c1fd6b8a8239ff65f9bbbcb5b8a3", size = 900922, upload-time = "2025-11-05T20:41:00.361Z" }, + { url = "https://files.pythonhosted.org/packages/09/ba/e5ea89fbde8e37a90ce456e31c5e9d85512cef5ae38e0f4d2426eb776a19/rignore-0.7.6-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d1a6671b2082c13bfd9a5cf4ce64670f832a6d41470556112c4ab0b6519b2fc4", size = 876987, upload-time = "2025-11-05T20:41:16.219Z" }, + { url = "https://files.pythonhosted.org/packages/d0/fb/93d14193f0ec0c3d35b763f0a000e9780f63b2031f3d3756442c2152622d/rignore-0.7.6-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2468729b4c5295c199d084ab88a40afcb7c8b974276805105239c07855bbacee", size = 1171110, upload-time = "2025-11-05T20:41:32.631Z" }, + { url = "https://files.pythonhosted.org/packages/9e/46/08436312ff96ffa29cfa4e1a987efc37e094531db46ba5e9fda9bb792afd/rignore-0.7.6-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:775710777fd71e5fdf54df69cdc249996a1d6f447a2b5bfb86dbf033fddd9cf9", size = 943339, upload-time = "2025-11-05T20:41:47.128Z" }, + { url = "https://files.pythonhosted.org/packages/34/28/3b3c51328f505cfaf7e53f408f78a1e955d561135d02f9cb0341ea99f69a/rignore-0.7.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4565407f4a77f72cf9d91469e75d15d375f755f0a01236bb8aaa176278cc7085", size = 961680, upload-time = "2025-11-05T20:42:18.061Z" }, + { url = "https://files.pythonhosted.org/packages/5c/9e/cbff75c8676d4f4a90bd58a1581249d255c7305141b0868f0abc0324836b/rignore-0.7.6-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dc44c33f8fb2d5c9da748de7a6e6653a78aa740655e7409895e94a247ffa97c8", size = 987045, upload-time = "2025-11-05T20:42:02.315Z" }, + { url = "https://files.pythonhosted.org/packages/8c/25/d802d1d369502a7ddb8816059e7c79d2d913e17df975b863418e0aca4d8a/rignore-0.7.6-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:8f32478f05540513c11923e8838afab9efef0131d66dca7f67f0e1bbd118af6a", size = 1080310, upload-time = "2025-11-05T21:40:23.184Z" }, + { url = "https://files.pythonhosted.org/packages/43/f0/250b785c2e473b1ab763eaf2be820934c2a5409a722e94b279dddac21c7d/rignore-0.7.6-pp310-pypy310_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:1b63a3dd76225ea35b01dd6596aa90b275b5d0f71d6dc28fce6dd295d98614aa", size = 1140998, upload-time = "2025-11-05T21:40:40.603Z" }, + { url = "https://files.pythonhosted.org/packages/f5/d6/bb42fd2a8bba6aea327962656e20621fd495523259db40cfb4c5f760f05c/rignore-0.7.6-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:fe6c41175c36554a4ef0994cd1b4dbd6d73156fca779066456b781707402048e", size = 1121178, upload-time = "2025-11-05T21:40:57.585Z" }, + { url = "https://files.pythonhosted.org/packages/97/f4/aeb548374129dce3dc191a4bb598c944d9ed663f467b9af830315d86059c/rignore-0.7.6-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:9a0c6792406ae36f4e7664dc772da909451d46432ff8485774526232d4885063", size = 1130190, upload-time = "2025-11-05T21:41:16.403Z" }, + { url = "https://files.pythonhosted.org/packages/82/78/a6250ff0c49a3cdb943910ada4116e708118e9b901c878cfae616c80a904/rignore-0.7.6-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a20b6fb61bcced9a83dfcca6599ad45182b06ba720cff7c8d891e5b78db5b65f", size = 886470, upload-time = "2025-11-05T20:42:52.314Z" }, + { url = "https://files.pythonhosted.org/packages/35/af/c69c0c51b8f9f7914d95c4ea91c29a2ac067572048cae95dd6d2efdbe05d/rignore-0.7.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:392dcabfecbe176c9ebbcb40d85a5e86a5989559c4f988c2741da7daf1b5be25", size = 825976, upload-time = "2025-11-05T20:42:35.118Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d2/1b264f56132264ea609d3213ab603d6a27016b19559a1a1ede1a66a03dcd/rignore-0.7.6-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22baa462abdc36fdd5a5e2dae423107723351b85ff093762f9261148b9d0a04a", size = 899739, upload-time = "2025-11-05T20:41:01.518Z" }, + { url = "https://files.pythonhosted.org/packages/55/e4/b3c5dfdd8d8a10741dfe7199ef45d19a0e42d0c13aa377c83bd6caf65d90/rignore-0.7.6-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53fb28882d2538cb2d231972146c4927a9d9455e62b209f85d634408c4103538", size = 874843, upload-time = "2025-11-05T20:41:17.687Z" }, + { url = "https://files.pythonhosted.org/packages/cc/10/d6f3750233881a2a154cefc9a6a0a9b19da526b19f7f08221b552c6f827d/rignore-0.7.6-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87409f7eeb1103d6b77f3472a3a0d9a5953e3ae804a55080bdcb0120ee43995b", size = 1170348, upload-time = "2025-11-05T20:41:34.21Z" }, + { url = "https://files.pythonhosted.org/packages/6e/10/ad98ca05c9771c15af734cee18114a3c280914b6e34fde9ffea2e61e88aa/rignore-0.7.6-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684014e42e4341ab3ea23a203551857fcc03a7f8ae96ca3aefb824663f55db32", size = 942315, upload-time = "2025-11-05T20:41:48.508Z" }, + { url = "https://files.pythonhosted.org/packages/de/00/ab5c0f872acb60d534e687e629c17e0896c62da9b389c66d3aa16b817aa8/rignore-0.7.6-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77356ebb01ba13f8a425c3d30fcad40e57719c0e37670d022d560884a30e4767", size = 961047, upload-time = "2025-11-05T20:42:19.403Z" }, + { url = "https://files.pythonhosted.org/packages/b8/86/3030fdc363a8f0d1cd155b4c453d6db9bab47a24fcc64d03f61d9d78fe6a/rignore-0.7.6-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6cbd8a48abbd3747a6c830393cd578782fab5d43f4deea48c5f5e344b8fed2b0", size = 986090, upload-time = "2025-11-05T20:42:03.581Z" }, + { url = "https://files.pythonhosted.org/packages/33/b8/133aa4002cee0ebbb39362f94e4898eec7fbd09cec9fcbce1cd65b355b7f/rignore-0.7.6-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2673225dcec7f90497e79438c35e34638d0d0391ccea3cbb79bfb9adc0dc5bd7", size = 1079656, upload-time = "2025-11-05T21:40:24.89Z" }, + { url = "https://files.pythonhosted.org/packages/67/56/36d5d34210e5e7dfcd134eed8335b19e80ae940ee758f493e4f2b344dd70/rignore-0.7.6-pp311-pypy311_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:c081f17290d8a2b96052b79207622aa635686ea39d502b976836384ede3d303c", size = 1139789, upload-time = "2025-11-05T21:40:42.119Z" }, + { url = "https://files.pythonhosted.org/packages/6b/5b/bb4f9420802bf73678033a4a55ab1bede36ce2e9b41fec5f966d83d932b3/rignore-0.7.6-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:57e8327aacc27f921968cb2a174f9e47b084ce9a7dd0122c8132d22358f6bd79", size = 1120308, upload-time = "2025-11-05T21:40:59.402Z" }, + { url = "https://files.pythonhosted.org/packages/ce/8b/a1299085b28a2f6135e30370b126e3c5055b61908622f2488ade67641479/rignore-0.7.6-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:d8955b57e42f2a5434670d5aa7b75eaf6e74602ccd8955dddf7045379cd762fb", size = 1129444, upload-time = "2025-11-05T21:41:17.906Z" }, +] + [[package]] name = "rouge-score" version = "0.1.2" @@ -5666,7 +7886,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "absl-py" }, { name = "nltk" }, - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "six" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e2/c5/9136736c37022a6ad27fea38f3111eb8f02fe75d067f9a985cc358653102/rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04", size = 17400, upload-time = "2022-07-22T22:46:22.909Z" } @@ -5830,7 +8052,9 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama" }, { name = "lxml" }, - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "portalocker" }, { name = "regex" }, { name = "tabulate" }, @@ -5879,9 +8103,9 @@ name = "saliency" version = "0.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, - { name = "scikit-image", version = "0.25.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scikit-image", version = "0.26.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } }, + { name = "scikit-image", version = "0.25.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "scikit-image", version = "0.26.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4f/63/0fb70dfa4b10c912e4613c0d0f99a3e46e615c8cc34657e14b0b511687e3/saliency-0.2.1.tar.gz", hash = "sha256:79a3f64393a3ce89620bf46629af120c36a061019eff51b32b173378c8b18c63", size = 54245, upload-time = "2024-03-20T19:51:30.647Z" } wheels = [ @@ -5893,14 +8117,13 @@ name = "scikit-image" version = "0.25.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", + "python_full_version < '3.11'", ] dependencies = [ { name = "imageio", marker = "python_full_version < '3.11'" }, { name = "lazy-loader", marker = "python_full_version < '3.11'" }, { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "packaging", marker = "python_full_version < '3.11'" }, { name = "pillow", marker = "python_full_version < '3.11'" }, { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -5936,20 +8159,16 @@ name = "scikit-image" version = "0.26.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.12.*' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", - "python_full_version >= '3.14' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and sys_platform == 'linux'", - "python_full_version >= '3.14' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and sys_platform != 'linux'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", ] dependencies = [ { name = "imageio", marker = "python_full_version >= '3.11'" }, { name = "lazy-loader", marker = "python_full_version >= '3.11'" }, { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "numpy", marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "packaging", marker = "python_full_version >= '3.11'" }, { name = "pillow", marker = "python_full_version >= '3.11'" }, { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -6012,14 +8231,18 @@ name = "scikit-learn" version = "1.7.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", + "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", ] dependencies = [ - { name = "joblib", marker = "python_full_version < '3.11'" }, - { name = "numpy", marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "threadpoolctl", marker = "python_full_version < '3.11'" }, + { name = "joblib", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "threadpoolctl", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/c2/a7855e41c9d285dfe86dc50b250978105dce513d6e459ea66a6aeb0e1e0c/scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda", size = 7193136, upload-time = "2025-09-09T08:21:29.075Z" } wheels = [ @@ -6060,20 +8283,34 @@ name = "scikit-learn" version = "1.8.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.12.*' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", - "python_full_version >= '3.14' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and sys_platform == 'linux'", - "python_full_version >= '3.14' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and sys_platform != 'linux'", -] -dependencies = [ - { name = "joblib", marker = "python_full_version >= '3.11'" }, - { name = "numpy", marker = "python_full_version >= '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "threadpoolctl", marker = "python_full_version >= '3.11'" }, + "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.13' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", +] +dependencies = [ + { name = "joblib", marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "threadpoolctl", marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" } wheels = [ @@ -6120,11 +8357,15 @@ name = "scipy" version = "1.15.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", + "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", ] dependencies = [ - { name = "numpy", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -6180,17 +8421,31 @@ name = "scipy" version = "1.17.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.12.*' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", - "python_full_version >= '3.14' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and sys_platform == 'linux'", - "python_full_version >= '3.14' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and sys_platform != 'linux'", -] -dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.11'" }, + "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.14' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.13.*' and extra == 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.12.*' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version == '3.11.*' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", + "python_full_version >= '3.13' and extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm'", +] +dependencies = [ + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -6360,6 +8615,90 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/dc/4d825d5eb6e924dfcc6a91c8185578a7b0a5c41fd2416a6f49c8226d6ef9/sentry_sdk-2.33.2-py2.py3-none-any.whl", hash = "sha256:8d57a3b4861b243aa9d558fda75509ad487db14f488cbdb6c78c614979d77632", size = 356692, upload-time = "2025-07-22T10:41:16.531Z" }, ] +[[package]] +name = "setproctitle" +version = "1.3.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/48/49393a96a2eef1ab418b17475fb92b8fcfad83d099e678751b05472e69de/setproctitle-1.3.7.tar.gz", hash = "sha256:bc2bc917691c1537d5b9bca1468437176809c7e11e5694ca79a9ca12345dcb9e", size = 27002, upload-time = "2025-09-05T12:51:25.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/48/fb401ec8c4953d519d05c87feca816ad668b8258448ff60579ac7a1c1386/setproctitle-1.3.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cf555b6299f10a6eb44e4f96d2f5a3884c70ce25dc5c8796aaa2f7b40e72cb1b", size = 18079, upload-time = "2025-09-05T12:49:07.732Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a3/c2b0333c2716fb3b4c9a973dd113366ac51b4f8d56b500f4f8f704b4817a/setproctitle-1.3.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:690b4776f9c15aaf1023bb07d7c5b797681a17af98a4a69e76a1d504e41108b7", size = 13099, upload-time = "2025-09-05T12:49:09.222Z" }, + { url = "https://files.pythonhosted.org/packages/0e/f8/17bda581c517678260e6541b600eeb67745f53596dc077174141ba2f6702/setproctitle-1.3.7-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:00afa6fc507967d8c9d592a887cdc6c1f5742ceac6a4354d111ca0214847732c", size = 31793, upload-time = "2025-09-05T12:49:10.297Z" }, + { url = "https://files.pythonhosted.org/packages/27/d1/76a33ae80d4e788ecab9eb9b53db03e81cfc95367ec7e3fbf4989962fedd/setproctitle-1.3.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9e02667f6b9fc1238ba753c0f4b0a37ae184ce8f3bbbc38e115d99646b3f4cd3", size = 32779, upload-time = "2025-09-05T12:49:12.157Z" }, + { url = "https://files.pythonhosted.org/packages/59/27/1a07c38121967061564f5e0884414a5ab11a783260450172d4fc68c15621/setproctitle-1.3.7-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:83fcd271567d133eb9532d3b067c8a75be175b2b3b271e2812921a05303a693f", size = 34578, upload-time = "2025-09-05T12:49:13.393Z" }, + { url = "https://files.pythonhosted.org/packages/d8/d4/725e6353935962d8bb12cbf7e7abba1d0d738c7f6935f90239d8e1ccf913/setproctitle-1.3.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13fe37951dda1a45c35d77d06e3da5d90e4f875c4918a7312b3b4556cfa7ff64", size = 32030, upload-time = "2025-09-05T12:49:15.362Z" }, + { url = "https://files.pythonhosted.org/packages/67/24/e4677ae8e1cb0d549ab558b12db10c175a889be0974c589c428fece5433e/setproctitle-1.3.7-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a05509cfb2059e5d2ddff701d38e474169e9ce2a298cf1b6fd5f3a213a553fe5", size = 33363, upload-time = "2025-09-05T12:49:16.829Z" }, + { url = "https://files.pythonhosted.org/packages/55/d4/69ce66e4373a48fdbb37489f3ded476bb393e27f514968c3a69a67343ae0/setproctitle-1.3.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6da835e76ae18574859224a75db6e15c4c2aaa66d300a57efeaa4c97ca4c7381", size = 31508, upload-time = "2025-09-05T12:49:18.032Z" }, + { url = "https://files.pythonhosted.org/packages/4b/5a/42c1ed0e9665d068146a68326529b5686a1881c8b9197c2664db4baf6aeb/setproctitle-1.3.7-cp310-cp310-win32.whl", hash = "sha256:9e803d1b1e20240a93bac0bc1025363f7f80cb7eab67dfe21efc0686cc59ad7c", size = 12558, upload-time = "2025-09-05T12:49:19.742Z" }, + { url = "https://files.pythonhosted.org/packages/dc/fe/dd206cc19a25561921456f6cb12b405635319299b6f366e0bebe872abc18/setproctitle-1.3.7-cp310-cp310-win_amd64.whl", hash = "sha256:a97200acc6b64ec4cada52c2ecaf1fba1ef9429ce9c542f8a7db5bcaa9dcbd95", size = 13245, upload-time = "2025-09-05T12:49:21.023Z" }, + { url = "https://files.pythonhosted.org/packages/04/cd/1b7ba5cad635510720ce19d7122154df96a2387d2a74217be552887c93e5/setproctitle-1.3.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a600eeb4145fb0ee6c287cb82a2884bd4ec5bbb076921e287039dcc7b7cc6dd0", size = 18085, upload-time = "2025-09-05T12:49:22.183Z" }, + { url = "https://files.pythonhosted.org/packages/8f/1a/b2da0a620490aae355f9d72072ac13e901a9fec809a6a24fc6493a8f3c35/setproctitle-1.3.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:97a090fed480471bb175689859532709e28c085087e344bca45cf318034f70c4", size = 13097, upload-time = "2025-09-05T12:49:23.322Z" }, + { url = "https://files.pythonhosted.org/packages/18/2e/bd03ff02432a181c1787f6fc2a678f53b7dacdd5ded69c318fe1619556e8/setproctitle-1.3.7-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1607b963e7b53e24ec8a2cb4e0ab3ae591d7c6bf0a160feef0551da63452b37f", size = 32191, upload-time = "2025-09-05T12:49:24.567Z" }, + { url = "https://files.pythonhosted.org/packages/28/78/1e62fc0937a8549f2220445ed2175daacee9b6764c7963b16148119b016d/setproctitle-1.3.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a20fb1a3974e2dab857870cf874b325b8705605cb7e7e8bcbb915bca896f52a9", size = 33203, upload-time = "2025-09-05T12:49:25.871Z" }, + { url = "https://files.pythonhosted.org/packages/a0/3c/65edc65db3fa3df400cf13b05e9d41a3c77517b4839ce873aa6b4043184f/setproctitle-1.3.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8d961bba676e07d77665204f36cffaa260f526e7b32d07ab3df6a2c1dfb44ba", size = 34963, upload-time = "2025-09-05T12:49:27.044Z" }, + { url = "https://files.pythonhosted.org/packages/a1/32/89157e3de997973e306e44152522385f428e16f92f3cf113461489e1e2ee/setproctitle-1.3.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:db0fd964fbd3a9f8999b502f65bd2e20883fdb5b1fae3a424e66db9a793ed307", size = 32398, upload-time = "2025-09-05T12:49:28.909Z" }, + { url = "https://files.pythonhosted.org/packages/4a/18/77a765a339ddf046844cb4513353d8e9dcd8183da9cdba6e078713e6b0b2/setproctitle-1.3.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:db116850fcf7cca19492030f8d3b4b6e231278e8fe097a043957d22ce1bdf3ee", size = 33657, upload-time = "2025-09-05T12:49:30.323Z" }, + { url = "https://files.pythonhosted.org/packages/6b/63/f0b6205c64d74d2a24a58644a38ec77bdbaa6afc13747e75973bf8904932/setproctitle-1.3.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:316664d8b24a5c91ee244460bdaf7a74a707adaa9e14fbe0dc0a53168bb9aba1", size = 31836, upload-time = "2025-09-05T12:49:32.309Z" }, + { url = "https://files.pythonhosted.org/packages/ba/51/e1277f9ba302f1a250bbd3eedbbee747a244b3cc682eb58fb9733968f6d8/setproctitle-1.3.7-cp311-cp311-win32.whl", hash = "sha256:b74774ca471c86c09b9d5037c8451fff06bb82cd320d26ae5a01c758088c0d5d", size = 12556, upload-time = "2025-09-05T12:49:33.529Z" }, + { url = "https://files.pythonhosted.org/packages/b6/7b/822a23f17e9003dfdee92cd72758441ca2a3680388da813a371b716fb07f/setproctitle-1.3.7-cp311-cp311-win_amd64.whl", hash = "sha256:acb9097213a8dd3410ed9f0dc147840e45ca9797785272928d4be3f0e69e3be4", size = 13243, upload-time = "2025-09-05T12:49:34.553Z" }, + { url = "https://files.pythonhosted.org/packages/fb/f0/2dc88e842077719d7384d86cc47403e5102810492b33680e7dadcee64cd8/setproctitle-1.3.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2dc99aec591ab6126e636b11035a70991bc1ab7a261da428491a40b84376654e", size = 18049, upload-time = "2025-09-05T12:49:36.241Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b4/50940504466689cda65680c9e9a1e518e5750c10490639fa687489ac7013/setproctitle-1.3.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cdd8aa571b7aa39840fdbea620e308a19691ff595c3a10231e9ee830339dd798", size = 13079, upload-time = "2025-09-05T12:49:38.088Z" }, + { url = "https://files.pythonhosted.org/packages/d0/99/71630546b9395b095f4082be41165d1078204d1696c2d9baade3de3202d0/setproctitle-1.3.7-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2906b6c7959cdb75f46159bf0acd8cc9906cf1361c9e1ded0d065fe8f9039629", size = 32932, upload-time = "2025-09-05T12:49:39.271Z" }, + { url = "https://files.pythonhosted.org/packages/50/22/cee06af4ffcfb0e8aba047bd44f5262e644199ae7527ae2c1f672b86495c/setproctitle-1.3.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6915964a6dda07920a1159321dcd6d94fc7fc526f815ca08a8063aeca3c204f1", size = 33736, upload-time = "2025-09-05T12:49:40.565Z" }, + { url = "https://files.pythonhosted.org/packages/5c/00/a5949a8bb06ef5e7df214fc393bb2fb6aedf0479b17214e57750dfdd0f24/setproctitle-1.3.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cff72899861c765bd4021d1ff1c68d60edc129711a2fdba77f9cb69ef726a8b6", size = 35605, upload-time = "2025-09-05T12:49:42.362Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3a/50caca532a9343828e3bf5778c7a84d6c737a249b1796d50dd680290594d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7cb05bd446687ff816a3aaaf831047fc4c364feff7ada94a66024f1367b448c", size = 33143, upload-time = "2025-09-05T12:49:43.515Z" }, + { url = "https://files.pythonhosted.org/packages/ca/14/b843a251296ce55e2e17c017d6b9f11ce0d3d070e9265de4ecad948b913d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3a57b9a00de8cae7e2a1f7b9f0c2ac7b69372159e16a7708aa2f38f9e5cc987a", size = 34434, upload-time = "2025-09-05T12:49:45.31Z" }, + { url = "https://files.pythonhosted.org/packages/c8/b7/06145c238c0a6d2c4bc881f8be230bb9f36d2bf51aff7bddcb796d5eed67/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d8828b356114f6b308b04afe398ed93803d7fca4a955dd3abe84430e28d33739", size = 32795, upload-time = "2025-09-05T12:49:46.419Z" }, + { url = "https://files.pythonhosted.org/packages/ef/dc/ef76a81fac9bf27b84ed23df19c1f67391a753eed6e3c2254ebcb5133f56/setproctitle-1.3.7-cp312-cp312-win32.whl", hash = "sha256:b0304f905efc845829ac2bc791ddebb976db2885f6171f4a3de678d7ee3f7c9f", size = 12552, upload-time = "2025-09-05T12:49:47.635Z" }, + { url = "https://files.pythonhosted.org/packages/e2/5b/a9fe517912cd6e28cf43a212b80cb679ff179a91b623138a99796d7d18a0/setproctitle-1.3.7-cp312-cp312-win_amd64.whl", hash = "sha256:9888ceb4faea3116cf02a920ff00bfbc8cc899743e4b4ac914b03625bdc3c300", size = 13247, upload-time = "2025-09-05T12:49:49.16Z" }, + { url = "https://files.pythonhosted.org/packages/5d/2f/fcedcade3b307a391b6e17c774c6261a7166aed641aee00ed2aad96c63ce/setproctitle-1.3.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c3736b2a423146b5e62230502e47e08e68282ff3b69bcfe08a322bee73407922", size = 18047, upload-time = "2025-09-05T12:49:50.271Z" }, + { url = "https://files.pythonhosted.org/packages/23/ae/afc141ca9631350d0a80b8f287aac79a76f26b6af28fd8bf92dae70dc2c5/setproctitle-1.3.7-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3384e682b158d569e85a51cfbde2afd1ab57ecf93ea6651fe198d0ba451196ee", size = 13073, upload-time = "2025-09-05T12:49:51.46Z" }, + { url = "https://files.pythonhosted.org/packages/87/ed/0a4f00315bc02510395b95eec3d4aa77c07192ee79f0baae77ea7b9603d8/setproctitle-1.3.7-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0564a936ea687cd24dffcea35903e2a20962aa6ac20e61dd3a207652401492dd", size = 33284, upload-time = "2025-09-05T12:49:52.741Z" }, + { url = "https://files.pythonhosted.org/packages/fc/e4/adf3c4c0a2173cb7920dc9df710bcc67e9bcdbf377e243b7a962dc31a51a/setproctitle-1.3.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a5d1cb3f81531f0eb40e13246b679a1bdb58762b170303463cb06ecc296f26d0", size = 34104, upload-time = "2025-09-05T12:49:54.416Z" }, + { url = "https://files.pythonhosted.org/packages/52/4f/6daf66394152756664257180439d37047aa9a1cfaa5e4f5ed35e93d1dc06/setproctitle-1.3.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a7d159e7345f343b44330cbba9194169b8590cb13dae940da47aa36a72aa9929", size = 35982, upload-time = "2025-09-05T12:49:56.295Z" }, + { url = "https://files.pythonhosted.org/packages/1b/62/f2c0595403cf915db031f346b0e3b2c0096050e90e0be658a64f44f4278a/setproctitle-1.3.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0b5074649797fd07c72ca1f6bff0406f4a42e1194faac03ecaab765ce605866f", size = 33150, upload-time = "2025-09-05T12:49:58.025Z" }, + { url = "https://files.pythonhosted.org/packages/a0/29/10dd41cde849fb2f9b626c846b7ea30c99c81a18a5037a45cc4ba33c19a7/setproctitle-1.3.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:61e96febced3f61b766115381d97a21a6265a0f29188a791f6df7ed777aef698", size = 34463, upload-time = "2025-09-05T12:49:59.424Z" }, + { url = "https://files.pythonhosted.org/packages/71/3c/cedd8eccfaf15fb73a2c20525b68c9477518917c9437737fa0fda91e378f/setproctitle-1.3.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:047138279f9463f06b858e579cc79580fbf7a04554d24e6bddf8fe5dddbe3d4c", size = 32848, upload-time = "2025-09-05T12:50:01.107Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3e/0a0e27d1c9926fecccfd1f91796c244416c70bf6bca448d988638faea81d/setproctitle-1.3.7-cp313-cp313-win32.whl", hash = "sha256:7f47accafac7fe6535ba8ba9efd59df9d84a6214565108d0ebb1199119c9cbbd", size = 12544, upload-time = "2025-09-05T12:50:15.81Z" }, + { url = "https://files.pythonhosted.org/packages/36/1b/6bf4cb7acbbd5c846ede1c3f4d6b4ee52744d402e43546826da065ff2ab7/setproctitle-1.3.7-cp313-cp313-win_amd64.whl", hash = "sha256:fe5ca35aeec6dc50cabab9bf2d12fbc9067eede7ff4fe92b8f5b99d92e21263f", size = 13235, upload-time = "2025-09-05T12:50:16.89Z" }, + { url = "https://files.pythonhosted.org/packages/e6/a4/d588d3497d4714750e3eaf269e9e8985449203d82b16b933c39bd3fc52a1/setproctitle-1.3.7-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:10e92915c4b3086b1586933a36faf4f92f903c5554f3c34102d18c7d3f5378e9", size = 18058, upload-time = "2025-09-05T12:50:02.501Z" }, + { url = "https://files.pythonhosted.org/packages/05/77/7637f7682322a7244e07c373881c7e982567e2cb1dd2f31bd31481e45500/setproctitle-1.3.7-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:de879e9c2eab637f34b1a14c4da1e030c12658cdc69ee1b3e5be81b380163ce5", size = 13072, upload-time = "2025-09-05T12:50:03.601Z" }, + { url = "https://files.pythonhosted.org/packages/52/09/f366eca0973cfbac1470068d1313fa3fe3de4a594683385204ec7f1c4101/setproctitle-1.3.7-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c18246d88e227a5b16248687514f95642505000442165f4b7db354d39d0e4c29", size = 34490, upload-time = "2025-09-05T12:50:04.948Z" }, + { url = "https://files.pythonhosted.org/packages/71/36/611fc2ed149fdea17c3677e1d0df30d8186eef9562acc248682b91312706/setproctitle-1.3.7-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7081f193dab22df2c36f9fc6d113f3793f83c27891af8fe30c64d89d9a37e152", size = 35267, upload-time = "2025-09-05T12:50:06.015Z" }, + { url = "https://files.pythonhosted.org/packages/88/a4/64e77d0671446bd5a5554387b69e1efd915274686844bea733714c828813/setproctitle-1.3.7-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9cc9b901ce129350637426a89cfd650066a4adc6899e47822e2478a74023ff7c", size = 37376, upload-time = "2025-09-05T12:50:07.484Z" }, + { url = "https://files.pythonhosted.org/packages/89/bc/ad9c664fe524fb4a4b2d3663661a5c63453ce851736171e454fa2cdec35c/setproctitle-1.3.7-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:80e177eff2d1ec172188d0d7fd9694f8e43d3aab76a6f5f929bee7bf7894e98b", size = 33963, upload-time = "2025-09-05T12:50:09.056Z" }, + { url = "https://files.pythonhosted.org/packages/ab/01/a36de7caf2d90c4c28678da1466b47495cbbad43badb4e982d8db8167ed4/setproctitle-1.3.7-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:23e520776c445478a67ee71b2a3c1ffdafbe1f9f677239e03d7e2cc635954e18", size = 35550, upload-time = "2025-09-05T12:50:10.791Z" }, + { url = "https://files.pythonhosted.org/packages/dd/68/17e8aea0ed5ebc17fbf03ed2562bfab277c280e3625850c38d92a7b5fcd9/setproctitle-1.3.7-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5fa1953126a3b9bd47049d58c51b9dac72e78ed120459bd3aceb1bacee72357c", size = 33727, upload-time = "2025-09-05T12:50:12.032Z" }, + { url = "https://files.pythonhosted.org/packages/b2/33/90a3bf43fe3a2242b4618aa799c672270250b5780667898f30663fd94993/setproctitle-1.3.7-cp313-cp313t-win32.whl", hash = "sha256:4a5e212bf438a4dbeece763f4962ad472c6008ff6702e230b4f16a037e2f6f29", size = 12549, upload-time = "2025-09-05T12:50:13.074Z" }, + { url = "https://files.pythonhosted.org/packages/0b/0e/50d1f07f3032e1f23d814ad6462bc0a138f369967c72494286b8a5228e40/setproctitle-1.3.7-cp313-cp313t-win_amd64.whl", hash = "sha256:cf2727b733e90b4f874bac53e3092aa0413fe1ea6d4f153f01207e6ce65034d9", size = 13243, upload-time = "2025-09-05T12:50:14.146Z" }, + { url = "https://files.pythonhosted.org/packages/89/c7/43ac3a98414f91d1b86a276bc2f799ad0b4b010e08497a95750d5bc42803/setproctitle-1.3.7-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:80c36c6a87ff72eabf621d0c79b66f3bdd0ecc79e873c1e9f0651ee8bf215c63", size = 18052, upload-time = "2025-09-05T12:50:17.928Z" }, + { url = "https://files.pythonhosted.org/packages/cd/2c/dc258600a25e1a1f04948073826bebc55e18dbd99dc65a576277a82146fa/setproctitle-1.3.7-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b53602371a52b91c80aaf578b5ada29d311d12b8a69c0c17fbc35b76a1fd4f2e", size = 13071, upload-time = "2025-09-05T12:50:19.061Z" }, + { url = "https://files.pythonhosted.org/packages/ab/26/8e3bb082992f19823d831f3d62a89409deb6092e72fc6940962983ffc94f/setproctitle-1.3.7-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fcb966a6c57cf07cc9448321a08f3be6b11b7635be502669bc1d8745115d7e7f", size = 33180, upload-time = "2025-09-05T12:50:20.395Z" }, + { url = "https://files.pythonhosted.org/packages/f1/af/ae692a20276d1159dd0cf77b0bcf92cbb954b965655eb4a69672099bb214/setproctitle-1.3.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46178672599b940368d769474fe13ecef1b587d58bb438ea72b9987f74c56ea5", size = 34043, upload-time = "2025-09-05T12:50:22.454Z" }, + { url = "https://files.pythonhosted.org/packages/34/b2/6a092076324dd4dac1a6d38482bedebbff5cf34ef29f58585ec76e47bc9d/setproctitle-1.3.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7f9e9e3ff135cbcc3edd2f4cf29b139f4aca040d931573102742db70ff428c17", size = 35892, upload-time = "2025-09-05T12:50:23.937Z" }, + { url = "https://files.pythonhosted.org/packages/1c/1a/8836b9f28cee32859ac36c3df85aa03e1ff4598d23ea17ca2e96b5845a8f/setproctitle-1.3.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:14c7eba8d90c93b0e79c01f0bd92a37b61983c27d6d7d5a3b5defd599113d60e", size = 32898, upload-time = "2025-09-05T12:50:25.617Z" }, + { url = "https://files.pythonhosted.org/packages/ef/22/8fabdc24baf42defb599714799d8445fe3ae987ec425a26ec8e80ea38f8e/setproctitle-1.3.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:9e64e98077fb30b6cf98073d6c439cd91deb8ebbf8fc62d9dbf52bd38b0c6ac0", size = 34308, upload-time = "2025-09-05T12:50:26.827Z" }, + { url = "https://files.pythonhosted.org/packages/15/1b/b9bee9de6c8cdcb3b3a6cb0b3e773afdb86bbbc1665a3bfa424a4294fda2/setproctitle-1.3.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b91387cc0f02a00ac95dcd93f066242d3cca10ff9e6153de7ee07069c6f0f7c8", size = 32536, upload-time = "2025-09-05T12:50:28.5Z" }, + { url = "https://files.pythonhosted.org/packages/37/0c/75e5f2685a5e3eda0b39a8b158d6d8895d6daf3ba86dec9e3ba021510272/setproctitle-1.3.7-cp314-cp314-win32.whl", hash = "sha256:52b054a61c99d1b72fba58b7f5486e04b20fefc6961cd76722b424c187f362ed", size = 12731, upload-time = "2025-09-05T12:50:43.955Z" }, + { url = "https://files.pythonhosted.org/packages/d2/ae/acddbce90d1361e1786e1fb421bc25baeb0c22ef244ee5d0176511769ec8/setproctitle-1.3.7-cp314-cp314-win_amd64.whl", hash = "sha256:5818e4080ac04da1851b3ec71e8a0f64e3748bf9849045180566d8b736702416", size = 13464, upload-time = "2025-09-05T12:50:45.057Z" }, + { url = "https://files.pythonhosted.org/packages/01/6d/20886c8ff2e6d85e3cabadab6aab9bb90acaf1a5cfcb04d633f8d61b2626/setproctitle-1.3.7-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:6fc87caf9e323ac426910306c3e5d3205cd9f8dcac06d233fcafe9337f0928a3", size = 18062, upload-time = "2025-09-05T12:50:29.78Z" }, + { url = "https://files.pythonhosted.org/packages/9a/60/26dfc5f198715f1343b95c2f7a1c16ae9ffa45bd89ffd45a60ed258d24ea/setproctitle-1.3.7-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6134c63853d87a4897ba7d5cc0e16abfa687f6c66fc09f262bb70d67718f2309", size = 13075, upload-time = "2025-09-05T12:50:31.604Z" }, + { url = "https://files.pythonhosted.org/packages/21/9c/980b01f50d51345dd513047e3ba9e96468134b9181319093e61db1c47188/setproctitle-1.3.7-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1403d2abfd32790b6369916e2313dffbe87d6b11dca5bbd898981bcde48e7a2b", size = 34744, upload-time = "2025-09-05T12:50:32.777Z" }, + { url = "https://files.pythonhosted.org/packages/86/b4/82cd0c86e6d1c4538e1a7eb908c7517721513b801dff4ba3f98ef816a240/setproctitle-1.3.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e7c5bfe4228ea22373e3025965d1a4116097e555ee3436044f5c954a5e63ac45", size = 35589, upload-time = "2025-09-05T12:50:34.13Z" }, + { url = "https://files.pythonhosted.org/packages/8a/4f/9f6b2a7417fd45673037554021c888b31247f7594ff4bd2239918c5cd6d0/setproctitle-1.3.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:585edf25e54e21a94ccb0fe81ad32b9196b69ebc4fc25f81da81fb8a50cca9e4", size = 37698, upload-time = "2025-09-05T12:50:35.524Z" }, + { url = "https://files.pythonhosted.org/packages/20/92/927b7d4744aac214d149c892cb5fa6dc6f49cfa040cb2b0a844acd63dcaf/setproctitle-1.3.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:96c38cdeef9036eb2724c2210e8d0b93224e709af68c435d46a4733a3675fee1", size = 34201, upload-time = "2025-09-05T12:50:36.697Z" }, + { url = "https://files.pythonhosted.org/packages/0a/0c/fd4901db5ba4b9d9013e62f61d9c18d52290497f956745cd3e91b0d80f90/setproctitle-1.3.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:45e3ef48350abb49cf937d0a8ba15e42cee1e5ae13ca41a77c66d1abc27a5070", size = 35801, upload-time = "2025-09-05T12:50:38.314Z" }, + { url = "https://files.pythonhosted.org/packages/e7/e3/54b496ac724e60e61cc3447f02690105901ca6d90da0377dffe49ff99fc7/setproctitle-1.3.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1fae595d032b30dab4d659bece20debd202229fce12b55abab978b7f30783d73", size = 33958, upload-time = "2025-09-05T12:50:39.841Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a8/c84bb045ebf8c6fdc7f7532319e86f8380d14bbd3084e6348df56bdfe6fd/setproctitle-1.3.7-cp314-cp314t-win32.whl", hash = "sha256:02432f26f5d1329ab22279ff863c83589894977063f59e6c4b4845804a08f8c2", size = 12745, upload-time = "2025-09-05T12:50:41.377Z" }, + { url = "https://files.pythonhosted.org/packages/08/b6/3a5a4f9952972791a9114ac01dfc123f0df79903577a3e0a7a404a695586/setproctitle-1.3.7-cp314-cp314t-win_amd64.whl", hash = "sha256:cbc388e3d86da1f766d8fc2e12682e446064c01cea9f88a88647cfe7c011de6a", size = 13469, upload-time = "2025-09-05T12:50:42.67Z" }, + { url = "https://files.pythonhosted.org/packages/34/8a/aff5506ce89bc3168cb492b18ba45573158d528184e8a9759a05a09088a9/setproctitle-1.3.7-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:eb440c5644a448e6203935ed60466ec8d0df7278cd22dc6cf782d07911bcbea6", size = 12654, upload-time = "2025-09-05T12:51:17.141Z" }, + { url = "https://files.pythonhosted.org/packages/41/89/5b6f2faedd6ced3d3c085a5efbd91380fb1f61f4c12bc42acad37932f4e9/setproctitle-1.3.7-pp310-pypy310_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:502b902a0e4c69031b87870ff4986c290ebbb12d6038a70639f09c331b18efb2", size = 14284, upload-time = "2025-09-05T12:51:18.393Z" }, + { url = "https://files.pythonhosted.org/packages/0a/c0/4312fed3ca393a29589603fd48f17937b4ed0638b923bac75a728382e730/setproctitle-1.3.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f6f268caeabb37ccd824d749e7ce0ec6337c4ed954adba33ec0d90cc46b0ab78", size = 13282, upload-time = "2025-09-05T12:51:19.703Z" }, + { url = "https://files.pythonhosted.org/packages/c3/5b/5e1c117ac84e3cefcf8d7a7f6b2461795a87e20869da065a5c087149060b/setproctitle-1.3.7-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b1cac6a4b0252b8811d60b6d8d0f157c0fdfed379ac89c25a914e6346cf355a1", size = 12587, upload-time = "2025-09-05T12:51:21.195Z" }, + { url = "https://files.pythonhosted.org/packages/73/02/b9eadc226195dcfa90eed37afe56b5dd6fa2f0e5220ab8b7867b8862b926/setproctitle-1.3.7-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f1704c9e041f2b1dc38f5be4552e141e1432fba3dd52c72eeffd5bc2db04dc65", size = 14286, upload-time = "2025-09-05T12:51:22.61Z" }, + { url = "https://files.pythonhosted.org/packages/28/26/1be1d2a53c2a91ec48fa2ff4a409b395f836798adf194d99de9c059419ea/setproctitle-1.3.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:b08b61976ffa548bd5349ce54404bf6b2d51bd74d4f1b241ed1b0f25bce09c3a", size = 13282, upload-time = "2025-09-05T12:51:24.094Z" }, +] + [[package]] name = "setuptools" version = "80.9.0" @@ -6376,13 +8715,13 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cloudpickle" }, { name = "numba" }, - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } }, { name = "packaging" }, { name = "pandas" }, - { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "slicer" }, { name = "tqdm" }, ] @@ -6487,7 +8826,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "alabaster" }, { name = "babel" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "docutils" }, { name = "imagesize" }, { name = "jinja2" }, @@ -6501,7 +8840,7 @@ dependencies = [ { name = "sphinxcontrib-jsmath" }, { name = "sphinxcontrib-qthelp" }, { name = "sphinxcontrib-serializinghtml" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5b/be/50e50cb4f2eff47df05673d361095cafd95521d2a22521b920c67a372dcb/sphinx-7.4.7.tar.gz", hash = "sha256:242f92a7ea7e6c5b406fdc2615413890ba9f699114a9c09192d7dfead2ee9cfe", size = 8067911, upload-time = "2024-07-20T14:46:56.059Z" } wheels = [ @@ -6607,6 +8946,19 @@ version = "2.1.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/12/9a/7620d1e9dcb02839ed6d4b14064e609cdd7a8ae1e47289aa0456796dd9ca/sqlitedict-2.1.0.tar.gz", hash = "sha256:03d9cfb96d602996f1d4c2db2856f1224b96a9c431bdd16e78032a72940f9e8c", size = 21846, upload-time = "2022-12-03T13:39:13.102Z" } +[[package]] +name = "sse-starlette" +version = "3.4.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform == 'linux'" }, + { name = "starlette", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d2/1b/bc9e3e7a72dcdad7dc7888758f5d00f56f8909ed5cfdff822bd72bb4c520/sse_starlette-3.4.5.tar.gz", hash = "sha256:83072538bc211a2f68b7b0422226c4af3e9b62e106e07034664b832ca019842a", size = 35249, upload-time = "2026-06-20T17:36:58.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/75/c88d3f5dafd59c791da1ce27650d30bf5b70cbf1cbf01cd00e5f9e360915/sse_starlette-3.4.5-py3-none-any.whl", hash = "sha256:e71bad53323f65573c3864a6c3bd0c1eb6e5f092b2e48082b0c35927d19ca296", size = 16518, upload-time = "2026-06-20T17:36:56.729Z" }, +] + [[package]] name = "stack-data" version = "0.6.3" @@ -6627,13 +8979,22 @@ version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/81/69/17425771797c36cded50b7fe44e850315d039f28b15901ab44839e70b593/starlette-1.0.0.tar.gz", hash = "sha256:6a4beaf1f81bb472fd19ea9b918b50dc3a77a6f2e190a12954b25e6ed5eea149", size = 2655289, upload-time = "2026-03-22T18:29:46.779Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0b/c9/584bc9651441b4ba60cc4d557d8a547b5aff901af35bda3a4ee30c819b82/starlette-1.0.0-py3-none-any.whl", hash = "sha256:d3ec55e0bb321692d275455ddfd3df75fff145d009685eb40dc91fc66b03d38b", size = 72651, upload-time = "2026-03-22T18:29:45.111Z" }, ] +[[package]] +name = "supervisor" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/b5/37e7a3706de436a8a2d75334711dad1afb4ddffab09f25e31d89e467542f/supervisor-4.3.0.tar.gz", hash = "sha256:4a2bf149adf42997e1bb44b70c43b613275ec9852c3edacca86a9166b27e945e", size = 468912, upload-time = "2025-08-23T18:25:02.418Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/65/5e726c372da8a5e35022a94388b12252710aad0c2351699c3d76ae8dba78/supervisor-4.3.0-py2.py3-none-any.whl", hash = "sha256:0bcb763fddafba410f35cbde226aa7f8514b9fb82eb05a0c85f6588d1c13f8db", size = 320736, upload-time = "2025-08-23T18:25:00.767Z" }, +] + [[package]] name = "sympy" version = "1.14.0" @@ -6700,8 +9061,8 @@ name = "terminado" version = "0.18.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess", marker = "os_name != 'nt'" }, - { name = "pywinpty", marker = "os_name == 'nt' and sys_platform != 'linux'" }, + { name = "ptyprocess", marker = "os_name != 'nt' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "pywinpty", marker = "(os_name == 'nt' and sys_platform != 'darwin' and sys_platform != 'linux') or (os_name == 'nt' and sys_platform == 'darwin' and extra == 'extra-16-transformer-lens-lit') or (os_name == 'nt' and sys_platform == 'darwin' and extra != 'extra-16-transformer-lens-vllm') or (os_name == 'nt' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-lit') or (os_name == 'nt' and sys_platform == 'linux' and extra != 'extra-16-transformer-lens-vllm') or (os_name != 'nt' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "tornado" }, ] sdist = { url = "https://files.pythonhosted.org/packages/8a/11/965c6fd8e5cc254f1fe142d547387da17a8ebfd75a3455f637c663fb38a0/terminado-0.18.1.tar.gz", hash = "sha256:de09f2c4b85de4765f7714688fff57d3e75bad1f909b589fde880460c753fd2e", size = 32701, upload-time = "2024-03-12T14:34:39.026Z" } @@ -6739,11 +9100,10 @@ name = "tifffile" version = "2025.5.10" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", + "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/44/d0/18fed0fc0916578a4463f775b0fbd9c5fed2392152d039df2fb533bfdd5d/tifffile-2025.5.10.tar.gz", hash = "sha256:018335d34283aa3fd8c263bae5c3c2b661ebc45548fde31504016fcae7bf1103", size = 365290, upload-time = "2025-05-10T19:22:34.386Z" } wheels = [ @@ -6755,17 +9115,13 @@ name = "tifffile" version = "2026.3.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.12.*' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", - "python_full_version >= '3.14' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and sys_platform == 'linux'", - "python_full_version >= '3.14' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and sys_platform != 'linux'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", ] dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c5/cb/2f6d79c7576e22c116352a801f4c3c8ace5957e9aced862012430b62e14f/tifffile-2026.3.3.tar.gz", hash = "sha256:d9a1266bed6f2ee1dd0abde2018a38b4f8b2935cb843df381d70ac4eac5458b7", size = 388745, upload-time = "2026-03-03T19:14:38.134Z" } wheels = [ @@ -6833,6 +9189,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/5b/f2aa703a4fc5d2dff73460a7d46cc2f3f44aa0f3dd8eeb20d2a0ecf68862/tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07", size = 918110, upload-time = "2026-05-15T04:51:17.237Z" }, ] +[[package]] +name = "tilelang" +version = "0.1.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apache-tvm-ffi", marker = "sys_platform == 'linux'" }, + { name = "cloudpickle", marker = "sys_platform == 'linux'" }, + { name = "ml-dtypes", marker = "sys_platform == 'linux'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "psutil", marker = "sys_platform == 'linux'" }, + { name = "torch", marker = "sys_platform == 'linux'" }, + { name = "torch-c-dlpack-ext", marker = "python_full_version < '3.14' and sys_platform == 'linux'" }, + { name = "tqdm", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "z3-solver", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/56/70/5051f65821baa30a3d61fc48f8ba10c776490315e8c90f82559b92089756/tilelang-0.1.9.tar.gz", hash = "sha256:287f727c913bb648fcf6c1968809ba3390e55eeed257a5c6bb9a80bc05966af4", size = 93395292, upload-time = "2026-04-22T09:19:11.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/db/4dd76da8c8585c605639a21bc098d504e317fe324a72f01ce3c7370250b4/tilelang-0.1.9-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:00ed594fdeb229c5505b9ffa895c3c5daeb28641c78f783fa1f724cf1e08cecd", size = 36599020, upload-time = "2026-04-22T09:14:39.366Z" }, + { url = "https://files.pythonhosted.org/packages/f7/8a/1cbeee79d62abaa02441c2d00621554e41aa62dbf3b94a4feb3867184b01/tilelang-0.1.9-cp38-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bbccfe9035aed775ffafb6dc25a5994504b24e2c5d95d0f39643edfafa7bf12", size = 45419374, upload-time = "2026-04-22T09:15:56.014Z" }, + { url = "https://files.pythonhosted.org/packages/c6/a7/f4bfb86f87e107703146e703204cec2c0eae2492b633e0052b0ace3febb6/tilelang-0.1.9-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:77ab0ee2f40f66ea015b6b21426d482751e28cbc635ef9d1198cbd6502454a7c", size = 42110365, upload-time = "2026-04-22T09:17:18.292Z" }, +] + [[package]] name = "timm" version = "1.0.27" @@ -6936,115 +9316,162 @@ wheels = [ [[package]] name = "torch" -version = "2.10.0" +version = "2.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-bindings", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "cuda-bindings", marker = "sys_platform == 'linux' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "filelock" }, { name = "fsspec" }, { name = "jinja2" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cufile-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "setuptools", marker = "python_full_version >= '3.12'" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "setuptools" }, { name = "sympy" }, - { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "triton", marker = "sys_platform == 'linux' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "typing-extensions" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/30/bfebdd8ec77db9a79775121789992d6b3b75ee5494971294d7b4b7c999bc/torch-2.10.0-2-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:2b980edd8d7c0a68c4e951ee1856334a43193f98730d97408fbd148c1a933313", size = 79411457, upload-time = "2026-02-10T21:44:59.189Z" }, - { url = "https://files.pythonhosted.org/packages/0f/8b/4b61d6e13f7108f36910df9ab4b58fd389cc2520d54d81b88660804aad99/torch-2.10.0-2-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:418997cb02d0a0f1497cf6a09f63166f9f5df9f3e16c8a716ab76a72127c714f", size = 79423467, upload-time = "2026-02-10T21:44:48.711Z" }, - { url = "https://files.pythonhosted.org/packages/d3/54/a2ba279afcca44bbd320d4e73675b282fcee3d81400ea1b53934efca6462/torch-2.10.0-2-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:13ec4add8c3faaed8d13e0574f5cd4a323c11655546f91fbe6afa77b57423574", size = 79498202, upload-time = "2026-02-10T21:44:52.603Z" }, - { url = "https://files.pythonhosted.org/packages/ec/23/2c9fe0c9c27f7f6cb865abcea8a4568f29f00acaeadfc6a37f6801f84cb4/torch-2.10.0-2-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:e521c9f030a3774ed770a9c011751fb47c4d12029a3d6522116e48431f2ff89e", size = 79498254, upload-time = "2026-02-10T21:44:44.095Z" }, - { url = "https://files.pythonhosted.org/packages/16/ee/efbd56687be60ef9af0c9c0ebe106964c07400eade5b0af8902a1d8cd58c/torch-2.10.0-3-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a1ff626b884f8c4e897c4c33782bdacdff842a165fee79817b1dd549fdda1321", size = 915510070, upload-time = "2026-03-11T14:16:39.386Z" }, - { url = "https://files.pythonhosted.org/packages/36/ab/7b562f1808d3f65414cd80a4f7d4bb00979d9355616c034c171249e1a303/torch-2.10.0-3-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:ac5bdcbb074384c66fa160c15b1ead77839e3fe7ed117d667249afce0acabfac", size = 915518691, upload-time = "2026-03-11T14:15:43.147Z" }, - { url = "https://files.pythonhosted.org/packages/b3/7a/abada41517ce0011775f0f4eacc79659bc9bc6c361e6bfe6f7052a6b9363/torch-2.10.0-3-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:98c01b8bb5e3240426dcde1446eed6f40c778091c8544767ef1168fc663a05a6", size = 915622781, upload-time = "2026-03-11T14:17:11.354Z" }, - { url = "https://files.pythonhosted.org/packages/ab/c6/4dfe238342ffdcec5aef1c96c457548762d33c40b45a1ab7033bb26d2ff2/torch-2.10.0-3-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:80b1b5bfe38eb0e9f5ff09f206dcac0a87aadd084230d4a36eea5ec5232c115b", size = 915627275, upload-time = "2026-03-11T14:16:11.325Z" }, - { url = "https://files.pythonhosted.org/packages/d8/f0/72bf18847f58f877a6a8acf60614b14935e2f156d942483af1ffc081aea0/torch-2.10.0-3-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:46b3574d93a2a8134b3f5475cfb98e2eb46771794c57015f6ad1fb795ec25e49", size = 915523474, upload-time = "2026-03-11T14:17:44.422Z" }, - { url = "https://files.pythonhosted.org/packages/f4/39/590742415c3030551944edc2ddc273ea1fdfe8ffb2780992e824f1ebee98/torch-2.10.0-3-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:b1d5e2aba4eb7f8e87fbe04f86442887f9167a35f092afe4c237dfcaaef6e328", size = 915632474, upload-time = "2026-03-11T14:15:13.666Z" }, - { url = "https://files.pythonhosted.org/packages/b6/8e/34949484f764dde5b222b7fe3fede43e4a6f0da9d7f8c370bb617d629ee2/torch-2.10.0-3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:0228d20b06701c05a8f978357f657817a4a63984b0c90745def81c18aedfa591", size = 915523882, upload-time = "2026-03-11T14:14:46.311Z" }, - { url = "https://files.pythonhosted.org/packages/0c/1a/c61f36cfd446170ec27b3a4984f072fd06dab6b5d7ce27e11adb35d6c838/torch-2.10.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:5276fa790a666ee8becaffff8acb711922252521b28fbce5db7db5cf9cb2026d", size = 145992962, upload-time = "2026-01-21T16:24:14.04Z" }, - { url = "https://files.pythonhosted.org/packages/b5/60/6662535354191e2d1555296045b63e4279e5a9dbad49acf55a5d38655a39/torch-2.10.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:aaf663927bcd490ae971469a624c322202a2a1e68936eb952535ca4cd3b90444", size = 915599237, upload-time = "2026-01-21T16:23:25.497Z" }, - { url = "https://files.pythonhosted.org/packages/40/b8/66bbe96f0d79be2b5c697b2e0b187ed792a15c6c4b8904613454651db848/torch-2.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:a4be6a2a190b32ff5c8002a0977a25ea60e64f7ba46b1be37093c141d9c49aeb", size = 113720931, upload-time = "2026-01-21T16:24:23.743Z" }, - { url = "https://files.pythonhosted.org/packages/76/bb/d820f90e69cda6c8169b32a0c6a3ab7b17bf7990b8f2c680077c24a3c14c/torch-2.10.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:35e407430795c8d3edb07a1d711c41cc1f9eaddc8b2f1cc0a165a6767a8fb73d", size = 79411450, upload-time = "2026-01-21T16:25:30.692Z" }, - { url = "https://files.pythonhosted.org/packages/78/89/f5554b13ebd71e05c0b002f95148033e730d3f7067f67423026cc9c69410/torch-2.10.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:3282d9febd1e4e476630a099692b44fdc214ee9bf8ee5377732d9d9dfe5712e4", size = 145992610, upload-time = "2026-01-21T16:25:26.327Z" }, - { url = "https://files.pythonhosted.org/packages/ae/30/a3a2120621bf9c17779b169fc17e3dc29b230c29d0f8222f499f5e159aa8/torch-2.10.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a2f9edd8dbc99f62bc4dfb78af7bf89499bca3d753423ac1b4e06592e467b763", size = 915607863, upload-time = "2026-01-21T16:25:06.696Z" }, - { url = "https://files.pythonhosted.org/packages/6f/3d/c87b33c5f260a2a8ad68da7147e105f05868c281c63d65ed85aa4da98c66/torch-2.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:29b7009dba4b7a1c960260fc8ac85022c784250af43af9fb0ebafc9883782ebd", size = 113723116, upload-time = "2026-01-21T16:25:21.916Z" }, - { url = "https://files.pythonhosted.org/packages/61/d8/15b9d9d3a6b0c01b883787bd056acbe5cc321090d4b216d3ea89a8fcfdf3/torch-2.10.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:b7bd80f3477b830dd166c707c5b0b82a898e7b16f59a7d9d42778dd058272e8b", size = 79423461, upload-time = "2026-01-21T16:24:50.266Z" }, - { url = "https://files.pythonhosted.org/packages/cc/af/758e242e9102e9988969b5e621d41f36b8f258bb4a099109b7a4b4b50ea4/torch-2.10.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5fd4117d89ffd47e3dcc71e71a22efac24828ad781c7e46aaaf56bf7f2796acf", size = 145996088, upload-time = "2026-01-21T16:24:44.171Z" }, - { url = "https://files.pythonhosted.org/packages/23/8e/3c74db5e53bff7ed9e34c8123e6a8bfef718b2450c35eefab85bb4a7e270/torch-2.10.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:787124e7db3b379d4f1ed54dd12ae7c741c16a4d29b49c0226a89bea50923ffb", size = 915711952, upload-time = "2026-01-21T16:23:53.503Z" }, - { url = "https://files.pythonhosted.org/packages/6e/01/624c4324ca01f66ae4c7cd1b74eb16fb52596dce66dbe51eff95ef9e7a4c/torch-2.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:2c66c61f44c5f903046cc696d088e21062644cbe541c7f1c4eaae88b2ad23547", size = 113757972, upload-time = "2026-01-21T16:24:39.516Z" }, - { url = "https://files.pythonhosted.org/packages/c9/5c/dee910b87c4d5c0fcb41b50839ae04df87c1cfc663cf1b5fca7ea565eeaa/torch-2.10.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:6d3707a61863d1c4d6ebba7be4ca320f42b869ee657e9b2c21c736bf17000294", size = 79498198, upload-time = "2026-01-21T16:24:34.704Z" }, - { url = "https://files.pythonhosted.org/packages/c9/6f/f2e91e34e3fcba2e3fc8d8f74e7d6c22e74e480bbd1db7bc8900fdf3e95c/torch-2.10.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:5c4d217b14741e40776dd7074d9006fd28b8a97ef5654db959d8635b2fe5f29b", size = 146004247, upload-time = "2026-01-21T16:24:29.335Z" }, - { url = "https://files.pythonhosted.org/packages/98/fb/5160261aeb5e1ee12ee95fe599d0541f7c976c3701d607d8fc29e623229f/torch-2.10.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6b71486353fce0f9714ca0c9ef1c850a2ae766b409808acd58e9678a3edb7738", size = 915716445, upload-time = "2026-01-21T16:22:45.353Z" }, - { url = "https://files.pythonhosted.org/packages/6a/16/502fb1b41e6d868e8deb5b0e3ae926bbb36dab8ceb0d1b769b266ad7b0c3/torch-2.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:c2ee399c644dc92ef7bc0d4f7e74b5360c37cdbe7c5ba11318dda49ffac2bc57", size = 113757050, upload-time = "2026-01-21T16:24:19.204Z" }, - { url = "https://files.pythonhosted.org/packages/1a/0b/39929b148f4824bc3ad6f9f72a29d4ad865bcf7ebfc2fa67584773e083d2/torch-2.10.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:3202429f58309b9fa96a614885eace4b7995729f44beb54d3e4a47773649d382", size = 79851305, upload-time = "2026-01-21T16:24:09.209Z" }, - { url = "https://files.pythonhosted.org/packages/d8/14/21fbce63bc452381ba5f74a2c0a959fdf5ad5803ccc0c654e752e0dbe91a/torch-2.10.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:aae1b29cd68e50a9397f5ee897b9c24742e9e306f88a807a27d617f07adb3bd8", size = 146005472, upload-time = "2026-01-21T16:22:29.022Z" }, - { url = "https://files.pythonhosted.org/packages/54/fd/b207d1c525cb570ef47f3e9f836b154685011fce11a2f444ba8a4084d042/torch-2.10.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:6021db85958db2f07ec94e1bc77212721ba4920c12a18dc552d2ae36a3eb163f", size = 915612644, upload-time = "2026-01-21T16:21:47.019Z" }, - { url = "https://files.pythonhosted.org/packages/36/53/0197f868c75f1050b199fe58f9bf3bf3aecac9b4e85cc9c964383d745403/torch-2.10.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ff43db38af76fda183156153983c9a096fc4c78d0cd1e07b14a2314c7f01c2c8", size = 113997015, upload-time = "2026-01-21T16:23:00.767Z" }, - { url = "https://files.pythonhosted.org/packages/0e/13/e76b4d9c160e89fff48bf16b449ea324bda84745d2ab30294c37c2434c0d/torch-2.10.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:cdf2a523d699b70d613243211ecaac14fe9c5df8a0b0a9c02add60fb2a413e0f", size = 79498248, upload-time = "2026-01-21T16:23:09.315Z" }, - { url = "https://files.pythonhosted.org/packages/4f/93/716b5ac0155f1be70ed81bacc21269c3ece8dba0c249b9994094110bfc51/torch-2.10.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:bf0d9ff448b0218e0433aeb198805192346c4fd659c852370d5cc245f602a06a", size = 79464992, upload-time = "2026-01-21T16:23:05.162Z" }, - { url = "https://files.pythonhosted.org/packages/69/2b/51e663ff190c9d16d4a8271203b71bc73a16aa7619b9f271a69b9d4a936b/torch-2.10.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:233aed0659a2503b831d8a67e9da66a62c996204c0bba4f4c442ccc0c68a3f60", size = 146018567, upload-time = "2026-01-21T16:22:23.393Z" }, - { url = "https://files.pythonhosted.org/packages/5e/cd/4b95ef7f293b927c283db0b136c42be91c8ec6845c44de0238c8c23bdc80/torch-2.10.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:682497e16bdfa6efeec8cde66531bc8d1fbbbb4d8788ec6173c089ed3cc2bfe5", size = 915721646, upload-time = "2026-01-21T16:21:16.983Z" }, - { url = "https://files.pythonhosted.org/packages/56/97/078a007208f8056d88ae43198833469e61a0a355abc0b070edd2c085eb9a/torch-2.10.0-cp314-cp314-win_amd64.whl", hash = "sha256:6528f13d2a8593a1a412ea07a99812495bec07e9224c28b2a25c0a30c7da025c", size = 113752373, upload-time = "2026-01-21T16:22:13.471Z" }, - { url = "https://files.pythonhosted.org/packages/d8/94/71994e7d0d5238393df9732fdab607e37e2b56d26a746cb59fdb415f8966/torch-2.10.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f5ab4ba32383061be0fb74bda772d470140a12c1c3b58a0cfbf3dae94d164c28", size = 79850324, upload-time = "2026-01-21T16:22:09.494Z" }, - { url = "https://files.pythonhosted.org/packages/e2/65/1a05346b418ea8ccd10360eef4b3e0ce688fba544e76edec26913a8d0ee0/torch-2.10.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:716b01a176c2a5659c98f6b01bf868244abdd896526f1c692712ab36dbaf9b63", size = 146006482, upload-time = "2026-01-21T16:22:18.42Z" }, - { url = "https://files.pythonhosted.org/packages/1d/b9/5f6f9d9e859fc3235f60578fa64f52c9c6e9b4327f0fe0defb6de5c0de31/torch-2.10.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:d8f5912ba938233f86361e891789595ff35ca4b4e2ac8fe3670895e5976731d6", size = 915613050, upload-time = "2026-01-21T16:20:49.035Z" }, - { url = "https://files.pythonhosted.org/packages/66/4d/35352043ee0eaffdeff154fad67cd4a31dbed7ff8e3be1cc4549717d6d51/torch-2.10.0-cp314-cp314t-win_amd64.whl", hash = "sha256:71283a373f0ee2c89e0f0d5f446039bdabe8dbc3c9ccf35f0f784908b0acd185", size = 113995816, upload-time = "2026-01-21T16:22:05.312Z" }, + { url = "https://files.pythonhosted.org/packages/ac/f2/c1690994afe461aae2d0cac62251e6802a703dec0a6c549c02ecd0de92a9/torch-2.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2c0d7fcfbc0c4e8bb5ebc3907cbc0c6a0da1b8f82b1fc6e14e914fa0b9baf74e", size = 80526521, upload-time = "2026-03-23T18:12:06.86Z" }, + { url = "https://files.pythonhosted.org/packages/a4/f0/98ae802fa8c09d3149b0c8690741f3f5753c90e779bd28c9613257295945/torch-2.11.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:4cf8687f4aec3900f748d553483ef40e0ac38411c3c48d0a86a438f6d7a99b18", size = 419723025, upload-time = "2026-03-23T18:11:43.774Z" }, + { url = "https://files.pythonhosted.org/packages/f9/1e/18a9b10b4bd34f12d4e561c52b0ae7158707b8193c6cfc0aad2b48167090/torch-2.11.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:1b32ceda909818a03b112006709b02be1877240c31750a8d9c6b7bf5f2d8a6e5", size = 530589207, upload-time = "2026-03-23T18:11:23.756Z" }, + { url = "https://files.pythonhosted.org/packages/35/40/2d532e8c0e23705be9d1debce5bc37b68d59a39bda7584c26fe9668076fe/torch-2.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:b3c712ae6fb8e7a949051a953fc412fe0a6940337336c3b6f905e905dac5157f", size = 114518313, upload-time = "2026-03-23T18:11:58.281Z" }, + { url = "https://files.pythonhosted.org/packages/ae/0d/98b410492609e34a155fa8b121b55c7dca229f39636851c3a9ec20edea21/torch-2.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7b6a60d48062809f58595509c524b88e6ddec3ebe25833d6462eeab81e5f2ce4", size = 80529712, upload-time = "2026-03-23T18:12:02.608Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/acea680005f098f79fd70c1d9d5ccc0cb4296ec2af539a0450108232fc0c/torch-2.11.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d91aac77f24082809d2c5a93f52a5f085032740a1ebc9252a7b052ef5a4fddc6", size = 419718178, upload-time = "2026-03-23T18:10:46.675Z" }, + { url = "https://files.pythonhosted.org/packages/8c/8b/d7be22fbec9ffee6cff31a39f8750d4b3a65d349a286cf4aec74c2375662/torch-2.11.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7aa2f9bbc6d4595ba72138026b2074be1233186150e9292865e04b7a63b8c67a", size = 530604548, upload-time = "2026-03-23T18:10:03.569Z" }, + { url = "https://files.pythonhosted.org/packages/d1/bd/9912d30b68845256aabbb4a40aeefeef3c3b20db5211ccda653544ada4b6/torch-2.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:73e24aaf8f36ab90d95cd1761208b2eb70841c2a9ca1a3f9061b39fc5331b708", size = 114519675, upload-time = "2026-03-23T18:11:52.995Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8b/69e3008d78e5cee2b30183340cc425081b78afc5eff3d080daab0adda9aa/torch-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b5866312ee6e52ea625cd211dcb97d6a2cdc1131a5f15cc0d87eec948f6dd34", size = 80606338, upload-time = "2026-03-23T18:11:34.781Z" }, + { url = "https://files.pythonhosted.org/packages/13/16/42e5915ebe4868caa6bac83a8ed59db57f12e9a61b7d749d584776ed53d5/torch-2.11.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f99924682ef0aa6a4ab3b1b76f40dc6e273fca09f367d15a524266db100a723f", size = 419731115, upload-time = "2026-03-23T18:11:06.944Z" }, + { url = "https://files.pythonhosted.org/packages/1a/c9/82638ef24d7877510f83baf821f5619a61b45568ce21c0a87a91576510aa/torch-2.11.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0f68f4ac6d95d12e896c3b7a912b5871619542ec54d3649cf48cc1edd4dd2756", size = 530712279, upload-time = "2026-03-23T18:10:31.481Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ff/6756f1c7ee302f6d202120e0f4f05b432b839908f9071157302cedfc5232/torch-2.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:fbf39280699d1b869f55eac536deceaa1b60bd6788ba74f399cc67e60a5fab10", size = 114556047, upload-time = "2026-03-23T18:10:55.931Z" }, + { url = "https://files.pythonhosted.org/packages/87/89/5ea6722763acee56b045435fb84258db7375c48165ec8be7880ab2b281c5/torch-2.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1e6debd97ccd3205bbb37eb806a9d8219e1139d15419982c09e23ef7d4369d18", size = 80606801, upload-time = "2026-03-23T18:10:18.649Z" }, + { url = "https://files.pythonhosted.org/packages/32/d1/8ed2173589cbfe744ed54e5a73efc107c0085ba5777ee93a5f4c1ab90553/torch-2.11.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:63a68fa59de8f87acc7e85a5478bb2dddbb3392b7593ec3e78827c793c4b73fd", size = 419732382, upload-time = "2026-03-23T18:08:30.835Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e1/b73f7c575a4b8f87a5928f50a1e35416b5e27295d8be9397d5293e7e8d4c/torch-2.11.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:cc89b9b173d9adfab59fd227f0ab5e5516d9a52b658ae41d64e59d2e55a418db", size = 530711509, upload-time = "2026-03-23T18:08:47.213Z" }, + { url = "https://files.pythonhosted.org/packages/66/82/3e3fcdd388fbe54e29fd3f991f36846ff4ac90b0d0181e9c8f7236565f82/torch-2.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:4dda3b3f52d121063a731ddb835f010dc137b920d7fec2778e52f60d8e4bf0cd", size = 114555842, upload-time = "2026-03-23T18:09:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/db/38/8ac78069621b8c2b4979c2f96dc8409ef5e9c4189f6aac629189a78677ca/torch-2.11.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8b394322f49af4362d4f80e424bcaca7efcd049619af03a4cf4501520bdf0fb4", size = 80959574, upload-time = "2026-03-23T18:10:14.214Z" }, + { url = "https://files.pythonhosted.org/packages/6d/6c/56bfb37073e7136e6dd86bfc6af7339946dd684e0ecf2155ac0eee687ae1/torch-2.11.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:2658f34ce7e2dabf4ec73b45e2ca68aedad7a5be87ea756ad656eaf32bf1e1ea", size = 419732324, upload-time = "2026-03-23T18:09:36.604Z" }, + { url = "https://files.pythonhosted.org/packages/07/f4/1b666b6d61d3394cca306ea543ed03a64aad0a201b6cd159f1d41010aeb1/torch-2.11.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:98bb213c3084cfe176302949bdc360074b18a9da7ab59ef2edc9d9f742504778", size = 530596026, upload-time = "2026-03-23T18:09:20.842Z" }, + { url = "https://files.pythonhosted.org/packages/48/6b/30d1459fa7e4b67e9e3fe1685ca1d8bb4ce7c62ef436c3a615963c6c866c/torch-2.11.0-cp313-cp313t-win_amd64.whl", hash = "sha256:a97b94bbf62992949b4730c6cd2cc9aee7b335921ee8dc207d930f2ed09ae2db", size = 114793702, upload-time = "2026-03-23T18:09:47.304Z" }, + { url = "https://files.pythonhosted.org/packages/26/0d/8603382f61abd0db35841148ddc1ffd607bf3100b11c6e1dab6d2fc44e72/torch-2.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:01018087326984a33b64e04c8cb5c2795f9120e0d775ada1f6638840227b04d7", size = 80573442, upload-time = "2026-03-23T18:09:10.117Z" }, + { url = "https://files.pythonhosted.org/packages/c7/86/7cd7c66cb9cec6be330fff36db5bd0eef386d80c031b581ec81be1d4b26c/torch-2.11.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:2bb3cc54bd0dea126b0060bb1ec9de0f9c7f7342d93d436646516b0330cd5be7", size = 419749385, upload-time = "2026-03-23T18:07:33.77Z" }, + { url = "https://files.pythonhosted.org/packages/47/e8/b98ca2d39b2e0e4730c0ee52537e488e7008025bc77ca89552ff91021f7c/torch-2.11.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4dc8b3809469b6c30b411bb8c4cad3828efd26236153d9beb6a3ec500f211a60", size = 530716756, upload-time = "2026-03-23T18:07:50.02Z" }, + { url = "https://files.pythonhosted.org/packages/78/88/d4a4cda8362f8a30d1ed428564878c3cafb0d87971fbd3947d4c84552095/torch-2.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:2b4e811728bd0cc58fb2b0948fe939a1ee2bf1422f6025be2fca4c7bd9d79718", size = 114552300, upload-time = "2026-03-23T18:09:05.617Z" }, + { url = "https://files.pythonhosted.org/packages/bf/46/4419098ed6d801750f26567b478fc185c3432e11e2cad712bc6b4c2ab0d0/torch-2.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8245477871c3700d4370352ffec94b103cfcb737229445cf9946cddb7b2ca7cd", size = 80959460, upload-time = "2026-03-23T18:09:00.818Z" }, + { url = "https://files.pythonhosted.org/packages/fd/66/54a56a4a6ceaffb567231994a9745821d3af922a854ed33b0b3a278e0a99/torch-2.11.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:ab9a8482f475f9ba20e12db84b0e55e2f58784bdca43a854a6ccd3fd4b9f75e6", size = 419735835, upload-time = "2026-03-23T18:07:18.974Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e7/0b6665f533aa9e337662dc190425abc0af1fe3234088f4454c52393ded61/torch-2.11.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:563ed3d25542d7e7bbc5b235ccfacfeb97fb470c7fee257eae599adb8005c8a2", size = 530613405, upload-time = "2026-03-23T18:08:07.014Z" }, + { url = "https://files.pythonhosted.org/packages/cf/bf/c8d12a2c86dbfd7f40fb2f56fbf5a505ccf2d9ce131eb559dfc7c51e1a04/torch-2.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b2a43985ff5ef6ddd923bbcf99943e5f58059805787c5c9a2622bf05ca2965b0", size = 114792991, upload-time = "2026-03-23T18:08:19.216Z" }, +] + +[[package]] +name = "torch-c-dlpack-ext" +version = "0.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "torch", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/de/921b6491efce5c389a5ef9bbed3d2d6660005840dae488124173180859ab/torch_c_dlpack_ext-0.1.5.tar.gz", hash = "sha256:d06f0357d575d22a168cc77acb9020fc4bae30968ceb6718a055dcbe92bacabe", size = 12913, upload-time = "2026-01-12T11:25:08.484Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/49/67a66932ab2fcdda3c5a4dcf606e713d86883a4a9a99a3bb832815b52b8e/torch_c_dlpack_ext-0.1.5-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:e0f6c197d5293884898b9ebf13d07501de39cb94799b374ed43f91731087d557", size = 7056755, upload-time = "2026-01-12T11:24:31.817Z" }, + { url = "https://files.pythonhosted.org/packages/ae/28/d2d6bf90e01a1f4da3277c9a56d9ecac648b6d6adaa8e20c17f802deb7fb/torch_c_dlpack_ext-0.1.5-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba3d88f0f7d5e1d9c3d4a3179037fc8e261c3b77ac1fad23edc0d3a9214ef193", size = 432066, upload-time = "2026-01-12T11:24:33.619Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e9/a1f9584a3af4ac6ae5ad5cf86927d8c3a9b6bb50d54e54d19313411216a0/torch_c_dlpack_ext-0.1.5-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7468df84ec152d930fbc3acf460c44a60b3462b95af3d3a676d133629c7e176", size = 879488, upload-time = "2026-01-12T11:24:34.837Z" }, + { url = "https://files.pythonhosted.org/packages/6c/08/478cfcb5814e29f9b720111bdef315fc2fbc8b276e4b1183c8b9c9414a4f/torch_c_dlpack_ext-0.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:78dd4904bd26170a2dd7c0eab56367756ee0a15672ce9b84146169e68f0c6ddc", size = 1461437, upload-time = "2026-01-12T11:24:36.385Z" }, + { url = "https://files.pythonhosted.org/packages/65/66/c12a9bb3a5ddc0962c00467891bf1ffdda39a4d4780bf0fbbf54523ff34e/torch_c_dlpack_ext-0.1.5-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:56bd25a2af19280bf8a06aa62cff5510106f43235b9327d8561b3e9a659c4d84", size = 5076782, upload-time = "2026-01-12T11:24:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/20/e1/64e1e579d107064785549e70758e38a42376ab7e73d86897ed4beab10e74/torch_c_dlpack_ext-0.1.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fba674110e1fab0b176bb5a28223e157db65c90767d4ba74abdbee9f537b0e9d", size = 440949, upload-time = "2026-01-12T11:24:39.716Z" }, + { url = "https://files.pythonhosted.org/packages/64/5c/3e1382a620824f92920ab3fae132d8fb4e85898284c99e0c6a7764e452ce/torch_c_dlpack_ext-0.1.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3448c4f0d64104d0b2e58080a7efa72304a04960c18f338024b80b13cd3eca26", size = 897768, upload-time = "2026-01-12T11:24:41.209Z" }, + { url = "https://files.pythonhosted.org/packages/54/4f/76ea1006b9038b496d01e916c91efd17cb782abde2491a261cf203f57e30/torch_c_dlpack_ext-0.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:74676474e0afa9a4216c4755ea7cf05e8158be1d168f6bda669ba91097c263f2", size = 1479088, upload-time = "2026-01-12T11:24:42.436Z" }, + { url = "https://files.pythonhosted.org/packages/b1/67/10d236698525d7b7db4d74ec0a4b01f5b2db33968995fdd9ac6b4635e327/torch_c_dlpack_ext-0.1.5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:c0f2bd51fcd99c0e5b50314e1985f2728c4941bfa821f065e6c30951d1f995ca", size = 5291237, upload-time = "2026-01-12T11:24:44.011Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8d760997307a5c3be4384424667bf31aae0a42060838c532c7d846516175/torch_c_dlpack_ext-0.1.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3562ee411258676f9c38b8ad39306d1c8d027b6a86f6a87c920d2d009a9d1510", size = 443069, upload-time = "2026-01-12T11:24:45.451Z" }, + { url = "https://files.pythonhosted.org/packages/e2/79/a914539b4785f3e44f891aa012a886edb8bc10fe081c440981c57543ce21/torch_c_dlpack_ext-0.1.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e6f9da4bb9af70e27facc777458be62e10dbbbddda7672d16138db0553c5a524", size = 897846, upload-time = "2026-01-12T11:24:48.168Z" }, + { url = "https://files.pythonhosted.org/packages/3a/e6/7d7a97a3953208d6d6ce749180c34d1dab48464ded9a76cecabe9d021ce6/torch_c_dlpack_ext-0.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:670fbbab70123cc228bed41693a3720757af57a0ad22669063c9db25321e8f55", size = 1482855, upload-time = "2026-01-12T11:24:49.581Z" }, + { url = "https://files.pythonhosted.org/packages/ca/c6/65346a201d921b616731311fc9941f15137672b444cebdad702cb52ccee0/torch_c_dlpack_ext-0.1.5-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:74acea2ed395cadda63342845b9e9ee7cd4537846223dacfb4431b4610109265", size = 1993243, upload-time = "2026-01-12T11:24:51.079Z" }, + { url = "https://files.pythonhosted.org/packages/fd/ec/faf10be09a5812b1c5ec9922b53fb5def5fc4080b81a653b9347bb169ebb/torch_c_dlpack_ext-0.1.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49f1e99d13c64e22dac0a34a1560e9e5a398a49a9fa81df83053e04fde6ec5bd", size = 443798, upload-time = "2026-01-12T11:24:52.754Z" }, + { url = "https://files.pythonhosted.org/packages/2d/68/f434b48700f3e04f33882f54d8d3910327b935f55e14ec49da7d607bf470/torch_c_dlpack_ext-0.1.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:debe62e5ef93e631065d6b9f6e60d3d39bae6b89fa1b25d9523f40b3efbf8aba", size = 755004, upload-time = "2026-01-12T11:24:54.004Z" }, + { url = "https://files.pythonhosted.org/packages/03/a8/cc64e563f05ea99bd79bdb43f71f0f46452d3acd734da4843ede5fc73a35/torch_c_dlpack_ext-0.1.5-cp313-cp313-win_amd64.whl", hash = "sha256:30e3eab616dbc81dfdb7492aca557be551a9163ba9b585f97394a42b336b113a", size = 999126, upload-time = "2026-01-12T11:24:55.44Z" }, + { url = "https://files.pythonhosted.org/packages/96/5e/449324ca8e81573e650b6851fc31c1038f750d1de85d0b185d788e1c7a3a/torch_c_dlpack_ext-0.1.5-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:cac94a4905d391889e679a8da31e46dc325af5d55d13b7c70c0ce3d71d1ced6d", size = 1982154, upload-time = "2026-01-12T11:24:58.038Z" }, + { url = "https://files.pythonhosted.org/packages/20/62/11c05b99f69aa5152bca0313e0dfa6d125a020cf890dc888ef009aa7891c/torch_c_dlpack_ext-0.1.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a58fdf45fb0bda7bc459632cec891570f31c11636d5851c825cf308ec8b73c2", size = 163825, upload-time = "2026-01-12T11:24:59.474Z" }, + { url = "https://files.pythonhosted.org/packages/15/b5/be613cd8e71c9982bd07af530f86c5a7f30df7831d14cec5414857af7149/torch_c_dlpack_ext-0.1.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b985a324c68241cf83a9474b28015524b66775b12a91930dd4c0760aa628d01", size = 171740, upload-time = "2026-01-12T11:25:00.776Z" }, + { url = "https://files.pythonhosted.org/packages/5c/11/52e291f1659e2ec70a09f5ca4ad27e015eb4f0a1371ae68d23a9fbd1c704/torch_c_dlpack_ext-0.1.5-cp314-cp314-win_amd64.whl", hash = "sha256:d794e19fa3f330ab7a29987c07e031fc08e4953aec516d35701d0827863e356b", size = 277086, upload-time = "2026-01-12T11:25:01.901Z" }, +] + +[[package]] +name = "torchaudio" +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/d9/357eb5fe4e19a861e6fa1af4d9f535e8fa8692336e6cf436e8a21262e054/torchaudio-2.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6ebb59c694909eccb5d61b7cc199d297692012c43286e36d92983aa7bad7586d", size = 684145, upload-time = "2026-03-23T18:13:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/2a/79/90de77e73f395bba2fe477f8e82e4ae1d14d6452a706838765e850a5e80c/torchaudio-2.11.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:be7ad472acb16d16e98c005f0219b0db06a47dfe8f7b4d177062e1638f871e3b", size = 1626521, upload-time = "2026-03-23T18:13:40.98Z" }, + { url = "https://files.pythonhosted.org/packages/66/dc/5757ed7d8d11a6c14336bcb54e63980979f00005555fec80fb4aa4de5eff/torchaudio-2.11.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:5847fe2022b17c6580aeb39c8797a443411cc09edfd9183cd50ac1a3b8ccf97c", size = 1771929, upload-time = "2026-03-23T18:13:43.432Z" }, + { url = "https://files.pythonhosted.org/packages/cf/f4/8ce2417eac66296e45b7aaa69858403fb6a52b1323f8635ec37b4b0f1fa3/torchaudio-2.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:7e2da1df4f6fe885c46db350a0dc90a0dff4b54541dff8846faa904d255e2bfe", size = 328661, upload-time = "2026-03-23T18:13:45.77Z" }, + { url = "https://files.pythonhosted.org/packages/94/77/0eec7f175d88f312296bd5b11c23bd58da37c1021f53da3db4df449ce3ee/torchaudio-2.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:492dd64645e9d0bb843e94f1d9a4d1e31426262ffc594fafecc1697df9df5eb9", size = 684142, upload-time = "2026-03-23T18:13:36.805Z" }, + { url = "https://files.pythonhosted.org/packages/b3/f9/6f7ebe071b44592c85269762b55b63ab0a091b5f479f73544738f7564a1e/torchaudio-2.11.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:73dab4841f94d888bc7c2aed7b5547c643edc974306919fe1adfb65d57cccf4b", size = 1626527, upload-time = "2026-03-23T18:13:39.011Z" }, + { url = "https://files.pythonhosted.org/packages/ac/70/17408e0d154d0c894537a88dcbadc48e8ad3b6e1ef4a1dabda5d40245ee0/torchaudio-2.11.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:1a07ec72fd6f26a588c39b5f029e0130d16bb40bc4221635580bf8fb18fcbc80", size = 1771930, upload-time = "2026-03-23T18:13:37.963Z" }, + { url = "https://files.pythonhosted.org/packages/c9/75/b6d03fc75b409bdaec597274d1bdd4213db716ed16f6801386b31d59c551/torchaudio-2.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:bb59ba4452bbbe95d75ad3ef18df9824955625f36698ce9a5998a4a9f3c1ba1d", size = 328658, upload-time = "2026-03-23T18:13:44.545Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b1/77658817acacd01a72b714440c62f419efc4d90170e704e8e7a2c0918988/torchaudio-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a1cf1acc883bee9cb906a933572fed6a8a933f86ef34e9ea7d803f72317e8c1b", size = 684226, upload-time = "2026-03-23T18:13:40.023Z" }, + { url = "https://files.pythonhosted.org/packages/78/28/c7adc053039f286c2aca0038b766cbe3294e66fec6b29a820e95128f9ede/torchaudio-2.11.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:bc653defca1c16154398517a1adc98d0fb7f1dd08e58ced217558d213c2c6e29", size = 1626670, upload-time = "2026-03-23T18:13:42.162Z" }, + { url = "https://files.pythonhosted.org/packages/88/d8/d6d0f896e064aa67377484efef4911cdcc07bce2929474e1417cc0af18c2/torchaudio-2.11.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6503c0bdb29daf2e6281bb70ea2dfe2c3553b782b619eb5d73bdadd8a3f7cecf", size = 1771992, upload-time = "2026-03-23T18:13:33.188Z" }, + { url = "https://files.pythonhosted.org/packages/23/a8/941277ecc39f7a0a169d554302a1f1afd87c1d94a8aec828891916cea59a/torchaudio-2.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:478110f981e5d40a8d82221732c57a56c85a1d5895fb8fe646e86ee15eded3bd", size = 328663, upload-time = "2026-03-23T18:13:19.218Z" }, + { url = "https://files.pythonhosted.org/packages/fb/9e/f76fcd9877c8c78f258ee34e0fb8291fdb91e6218d582d9ca66b1e4bd4ae/torchaudio-2.11.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e3f9696a9ef1d49acc452159b052370c636406d072e9d8f10895fda87b591ea9", size = 679904, upload-time = "2026-03-23T18:13:28.329Z" }, + { url = "https://files.pythonhosted.org/packages/85/70/249c1498ebdad3e7752866635ec0855fc0dcf898beccda5a9d2b9df8e4d0/torchaudio-2.11.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:b034d7672f1c415434f48ef17807f2cce47f29e8795338c751d4e596c9fbe8b5", size = 1618523, upload-time = "2026-03-23T18:13:15.703Z" }, + { url = "https://files.pythonhosted.org/packages/4f/98/be13fe35d9aa5c26381c0e453c828a789d15c007f8f7d08c95341d19974d/torchaudio-2.11.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1c1101c1243ef0e4063ec63298977e2d3655c15cf88d9eb0a1bd4fe2db9f47ea", size = 1771992, upload-time = "2026-03-23T18:13:35.343Z" }, + { url = "https://files.pythonhosted.org/packages/e2/8b/2bbb3dca6ff28cba0de250874d5ef4fc2822c47a934b59b3974cff3219ef/torchaudio-2.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:986f4df5ed17b003dc52489468601720090e65f964f8bebccf90eb45bba75744", size = 328662, upload-time = "2026-03-23T18:13:18.308Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ce/52c652d30af7d6e96c8f1735d26131e94708e3f38d852b8fa97958804dd8/torchaudio-2.11.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:bda09ea630ae7207384fb0f28c35e4f8c0d82dd6eba020b6b335ad0caa9fed49", size = 680814, upload-time = "2026-03-23T18:13:17.08Z" }, + { url = "https://files.pythonhosted.org/packages/06/95/1ad1507482e7263e556709a3f5f87fecd375a0742cdaf238806c8e72eaad/torchaudio-2.11.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:9fe3083c62e035646483a14e180d33561bdc2eed436c9ab1259c137fb7120b4a", size = 1618546, upload-time = "2026-03-23T18:13:29.686Z" }, + { url = "https://files.pythonhosted.org/packages/98/4c/480328ba07487eb9890406720304d0d460dd7a6a64098614f5aa53b662ca/torchaudio-2.11.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:13cff988697ccbad539987599f9dc672f40c417bed67570b365e4e5002bbd096", size = 1771991, upload-time = "2026-03-23T18:13:30.843Z" }, + { url = "https://files.pythonhosted.org/packages/3e/98/5d4790e2d6548768999acd34999d5aeefce8bcc23a07afaa5f03e723f557/torchaudio-2.11.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ed404c4399ad7f172c86a47c1b25293d322d1d58e26b10b0456a86cf67d37d84", size = 328661, upload-time = "2026-03-23T18:13:34.359Z" }, + { url = "https://files.pythonhosted.org/packages/39/fe/ffa618b4f0d9732d7df7a2fa2bd48657d896599bc224e5af3c70d46c546b/torchaudio-2.11.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:cc09cd1f6015b8549e7fe255fb1be5346b57e7fee06541d3f3dbb012d8c4715f", size = 679901, upload-time = "2026-03-23T18:13:25.472Z" }, + { url = "https://files.pythonhosted.org/packages/5c/54/f414d7b92dd0b3094a2409c95a97bd6c49aa0620da722a0e55462f9bd9cb/torchaudio-2.11.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:79fb3cb99169fd41bd9719647261402a164da0d105a4d81f42a3260844ec5e79", size = 1618527, upload-time = "2026-03-23T18:13:26.68Z" }, + { url = "https://files.pythonhosted.org/packages/a8/a8/bf2e1f6ce24c990192400ae49b4acc1a0d0295b6c6a06bceecdc46ce08de/torchaudio-2.11.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:00e9f71ab9c656f0abdb40c515bd65d4658ab0ad380dee27a2efd7d51dabd3d6", size = 1771995, upload-time = "2026-03-23T18:13:23.373Z" }, + { url = "https://files.pythonhosted.org/packages/83/6f/b0efb44e0bfe8dd4d78d76ae3be280354e1fb5c8631c782785d74cd8a7b1/torchaudio-2.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:1424638adb8bb40087bc7b6eb103e8e4fe398210f09076f33b7b5e61501b5d66", size = 328662, upload-time = "2026-03-23T18:13:32.243Z" }, + { url = "https://files.pythonhosted.org/packages/60/84/1c792b0b700eac9a96772cfd9f96c097b17bca3234a2fde3c64b8063660d/torchaudio-2.11.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:da2725e250866da42a12934c9a6552f65a18b7187fd7a6221387f0e605fb3b96", size = 679926, upload-time = "2026-03-23T18:13:24.452Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a0/62a5842062f739239691f2e57523e0570dd06704ad987755f7644a3afa23/torchaudio-2.11.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:1be3767064364ae82705bdf2b15c1e8b41fea82c4cd04d47428a8684b634b6ed", size = 1618552, upload-time = "2026-03-23T18:13:21.09Z" }, + { url = "https://files.pythonhosted.org/packages/6d/89/c293d818f9f899db93bf291b42401c05ae29acfb2e53d5341c30ea703e62/torchaudio-2.11.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:67f6edac29ed004652c11db5c19d9debb5d835695930574f564efc8bdd061bba", size = 1771986, upload-time = "2026-03-23T18:13:22.153Z" }, + { url = "https://files.pythonhosted.org/packages/93/f7/ee5da8c03f1a3c7662c6c6a119f24a4b3e646da94be56dce3201e3a6ee9b/torchaudio-2.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:88fb5e29f670a33d9bac6aabb1d2734460cf6e461bde5cdc352826035851b16d", size = 328661, upload-time = "2026-03-23T18:13:20.1Z" }, ] [[package]] name = "torchvision" -version = "0.25.0" +version = "0.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "pillow" }, { name = "torch" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/50/ae/cbf727421eb73f1cf907fbe5788326a08f111b3f6b6ddca15426b53fec9a/torchvision-0.25.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a95c47abb817d4e90ea1a8e57bd0d728e3e6b533b3495ae77d84d883c4d11f56", size = 1874919, upload-time = "2026-01-21T16:27:47.617Z" }, - { url = "https://files.pythonhosted.org/packages/64/68/dc7a224f606d53ea09f9a85196a3921ec3a801b0b1d17e84c73392f0c029/torchvision-0.25.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:acc339aba4a858192998c2b91f635827e40d9c469d9cf1455bafdda6e4c28ea4", size = 2343220, upload-time = "2026-01-21T16:27:44.26Z" }, - { url = "https://files.pythonhosted.org/packages/f9/fa/8cce5ca7ffd4da95193232493703d20aa06303f37b119fd23a65df4f239a/torchvision-0.25.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0d9a3f925a081dd2ebb0b791249b687c2ef2c2717d027946654607494b9b64b6", size = 8068106, upload-time = "2026-01-21T16:27:37.805Z" }, - { url = "https://files.pythonhosted.org/packages/8b/b9/a53bcf8f78f2cd89215e9ded70041765d50ef13bf301f9884ec6041a9421/torchvision-0.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:b57430fbe9e9b697418a395041bb615124d9c007710a2712fda6e35fb310f264", size = 3697295, upload-time = "2026-01-21T16:27:36.574Z" }, - { url = "https://files.pythonhosted.org/packages/3e/be/c704bceaf11c4f6b19d64337a34a877fcdfe3bd68160a8c9ae9bea4a35a3/torchvision-0.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db74a551946b75d19f9996c419a799ffdf6a223ecf17c656f90da011f1d75b20", size = 1874923, upload-time = "2026-01-21T16:27:46.574Z" }, - { url = "https://files.pythonhosted.org/packages/ae/e9/f143cd71232430de1f547ceab840f68c55e127d72558b1061a71d0b193cd/torchvision-0.25.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:f49964f96644dbac2506dffe1a0a7ec0f2bf8cf7a588c3319fed26e6329ffdf3", size = 2344808, upload-time = "2026-01-21T16:27:43.191Z" }, - { url = "https://files.pythonhosted.org/packages/43/ae/ad5d6165797de234c9658752acb4fce65b78a6a18d82efdf8367c940d8da/torchvision-0.25.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:153c0d2cbc34b7cf2da19d73450f24ba36d2b75ec9211b9962b5022fb9e4ecee", size = 8070752, upload-time = "2026-01-21T16:27:33.748Z" }, - { url = "https://files.pythonhosted.org/packages/23/19/55b28aecdc7f38df57b8eb55eb0b14a62b470ed8efeb22cdc74224df1d6a/torchvision-0.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:ea580ffd6094cc01914ad32f8c8118174f18974629af905cea08cb6d5d48c7b7", size = 4038722, upload-time = "2026-01-21T16:27:41.355Z" }, - { url = "https://files.pythonhosted.org/packages/56/3a/6ea0d73f49a9bef38a1b3a92e8dd455cea58470985d25635beab93841748/torchvision-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2abe430c90b1d5e552680037d68da4eb80a5852ebb1c811b2b89d299b10573b", size = 1874920, upload-time = "2026-01-21T16:27:45.348Z" }, - { url = "https://files.pythonhosted.org/packages/51/f8/c0e1ef27c66e15406fece94930e7d6feee4cb6374bbc02d945a630d6426e/torchvision-0.25.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b75deafa2dfea3e2c2a525559b04783515e3463f6e830cb71de0fb7ea36fe233", size = 2344556, upload-time = "2026-01-21T16:27:40.125Z" }, - { url = "https://files.pythonhosted.org/packages/68/2f/f24b039169db474e8688f649377de082a965fbf85daf4e46c44412f1d15a/torchvision-0.25.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f25aa9e380865b11ea6e9d99d84df86b9cc959f1a007cd966fc6f1ab2ed0e248", size = 8072351, upload-time = "2026-01-21T16:27:21.074Z" }, - { url = "https://files.pythonhosted.org/packages/ad/16/8f650c2e288977cf0f8f85184b90ee56ed170a4919347fc74ee99286ed6f/torchvision-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:f9c55ae8d673ab493325d1267cbd285bb94d56f99626c00ac4644de32a59ede3", size = 4303059, upload-time = "2026-01-21T16:27:11.08Z" }, - { url = "https://files.pythonhosted.org/packages/f5/5b/1562a04a6a5a4cf8cf40016a0cdeda91ede75d6962cff7f809a85ae966a5/torchvision-0.25.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:24e11199e4d84ba9c5ee7825ebdf1cd37ce8deec225117f10243cae984ced3ec", size = 1874918, upload-time = "2026-01-21T16:27:39.02Z" }, - { url = "https://files.pythonhosted.org/packages/36/b1/3d6c42f62c272ce34fcce609bb8939bdf873dab5f1b798fd4e880255f129/torchvision-0.25.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:5f271136d2d2c0b7a24c5671795c6e4fd8da4e0ea98aeb1041f62bc04c4370ef", size = 2309106, upload-time = "2026-01-21T16:27:30.624Z" }, - { url = "https://files.pythonhosted.org/packages/c7/60/59bb9c8b67cce356daeed4cb96a717caa4f69c9822f72e223a0eae7a9bd9/torchvision-0.25.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:855c0dc6d37f462482da7531c6788518baedca1e0847f3df42a911713acdfe52", size = 8071522, upload-time = "2026-01-21T16:27:29.392Z" }, - { url = "https://files.pythonhosted.org/packages/32/a5/9a9b1de0720f884ea50dbf9acb22cbe5312e51d7b8c4ac6ba9b51efd9bba/torchvision-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:cef0196be31be421f6f462d1e9da1101be7332d91984caa6f8022e6c78a5877f", size = 4321911, upload-time = "2026-01-21T16:27:35.195Z" }, - { url = "https://files.pythonhosted.org/packages/52/99/dca81ed21ebaeff2b67cc9f815a20fdaa418b69f5f9ea4c6ed71721470db/torchvision-0.25.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a8f8061284395ce31bcd460f2169013382ccf411148ceb2ee38e718e9860f5a7", size = 1896209, upload-time = "2026-01-21T16:27:32.159Z" }, - { url = "https://files.pythonhosted.org/packages/28/cc/2103149761fdb4eaed58a53e8437b2d716d48f05174fab1d9fcf1e2a2244/torchvision-0.25.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:146d02c9876858420adf41f3189fe90e3d6a409cbfa65454c09f25fb33bf7266", size = 2310735, upload-time = "2026-01-21T16:27:22.327Z" }, - { url = "https://files.pythonhosted.org/packages/76/ad/f4c985ad52ddd3b22711c588501be1b330adaeaf6850317f66751711b78c/torchvision-0.25.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:c4d395cb2c4a2712f6eb93a34476cdf7aae74bb6ea2ea1917f858e96344b00aa", size = 8089557, upload-time = "2026-01-21T16:27:27.666Z" }, - { url = "https://files.pythonhosted.org/packages/63/cc/0ea68b5802e5e3c31f44b307e74947bad5a38cc655231d845534ed50ddb8/torchvision-0.25.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5e6b449e9fa7d642142c0e27c41e5a43b508d57ed8e79b7c0a0c28652da8678c", size = 4344260, upload-time = "2026-01-21T16:27:17.018Z" }, - { url = "https://files.pythonhosted.org/packages/9e/1f/fa839532660e2602b7e704d65010787c5bb296258b44fa8b9c1cd6175e7d/torchvision-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:620a236288d594dcec7634c754484542dc0a5c1b0e0b83a34bda5e91e9b7c3a1", size = 1896193, upload-time = "2026-01-21T16:27:24.785Z" }, - { url = "https://files.pythonhosted.org/packages/80/ed/d51889da7ceaf5ff7a0574fb28f9b6b223df19667265395891f81b364ab3/torchvision-0.25.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b5e7f50002a8145a98c5694a018e738c50e2972608310c7e88e1bd4c058f6ce", size = 2309331, upload-time = "2026-01-21T16:27:19.97Z" }, - { url = "https://files.pythonhosted.org/packages/90/a5/f93fcffaddd8f12f9e812256830ec9c9ca65abbf1bc369379f9c364d1ff4/torchvision-0.25.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:632db02300e83793812eee4f61ae6a2686dab10b4cfd628b620dc47747aa9d03", size = 8088713, upload-time = "2026-01-21T16:27:15.281Z" }, - { url = "https://files.pythonhosted.org/packages/1f/eb/d0096eed5690d962853213f2ee00d91478dfcb586b62dbbb449fb8abc3a6/torchvision-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:d1abd5ed030c708f5dbf4812ad5f6fbe9384b63c40d6bd79f8df41a4a759a917", size = 4325058, upload-time = "2026-01-21T16:27:26.165Z" }, - { url = "https://files.pythonhosted.org/packages/97/36/96374a4c7ab50dea9787ce987815614ccfe988a42e10ac1a2e3e5b60319a/torchvision-0.25.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ad9a8a5877782944d99186e4502a614770fe906626d76e9cd32446a0ac3075f2", size = 1896207, upload-time = "2026-01-21T16:27:23.383Z" }, - { url = "https://files.pythonhosted.org/packages/b5/e2/7abb10a867db79b226b41da419b63b69c0bd5b82438c4a4ed50e084c552f/torchvision-0.25.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:40a122c3cf4d14b651f095e0f672b688dde78632783fc5cd3d4d5e4f6a828563", size = 2310741, upload-time = "2026-01-21T16:27:18.712Z" }, - { url = "https://files.pythonhosted.org/packages/08/e6/0927784e6ffc340b6676befde1c60260bd51641c9c574b9298d791a9cda4/torchvision-0.25.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:846890161b825b38aa85fc37fb3ba5eea74e7091ff28bab378287111483b6443", size = 8089772, upload-time = "2026-01-21T16:27:14.048Z" }, - { url = "https://files.pythonhosted.org/packages/b6/37/e7ca4ec820d434c0f23f824eb29f0676a0c3e7a118f1514f5b949c3356da/torchvision-0.25.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f07f01d27375ad89d72aa2b3f2180f07da95dd9d2e4c758e015c0acb2da72977", size = 4425879, upload-time = "2026-01-21T16:27:12.579Z" }, + { url = "https://files.pythonhosted.org/packages/74/b4/cdfee31e0402ea035135462cb0ab496e974d56fab6b4e7a1f0cbccb8cd28/torchvision-0.26.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a06d4772a8e13e772906ed736cc53ec6639e5e60554f8e5fa6ca165aabebc464", size = 1863503, upload-time = "2026-03-23T18:13:01.384Z" }, + { url = "https://files.pythonhosted.org/packages/e4/74/11fee109841e80ad14e5ca2d80bff6b10eb11b7838ff06f35bfeaa9f7251/torchvision-0.26.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:2adfbe438473236191ff077a4a9a0c767436879c89628aa97137e959b0c11a94", size = 7766423, upload-time = "2026-03-23T18:12:56.049Z" }, + { url = "https://files.pythonhosted.org/packages/5e/00/24d8c7845c3f270153fb81395a5135b2778e2538e81d14c6aea5106c689c/torchvision-0.26.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b6f9ad1ecc0eab52647298b379ee9426845f8903703e6127973f8f3d049a798b", size = 7518249, upload-time = "2026-03-23T18:12:51.743Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ed/e53cd7c0da7ae002e5e929c1796ebbe7ec0c700c29f7a0a6696497fb3d8b/torchvision-0.26.0-cp310-cp310-win_amd64.whl", hash = "sha256:f13f12b3791a266de2d599cb8162925261622a037d87fc03132848343cf68f75", size = 3669784, upload-time = "2026-03-23T18:12:49.949Z" }, + { url = "https://files.pythonhosted.org/packages/b4/bd/d552a2521bade3295b2c6e7a4a0d1022261cab7ca7011f4e2a330dbb3caa/torchvision-0.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:55bd6ad4ae77be01ba67a410b05b51f53b0d0ee45f146eb6a0dfb9007e70ab3c", size = 1863499, upload-time = "2026-03-23T18:12:58.696Z" }, + { url = "https://files.pythonhosted.org/packages/33/bf/21b899792b08cae7a298551c68398a79e333697479ed311b3b067aab4bdc/torchvision-0.26.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:1c55dc8affbcc0eb2060fbabbe996ae9e5839b24bb6419777f17848945a411b1", size = 7767527, upload-time = "2026-03-23T18:12:44.348Z" }, + { url = "https://files.pythonhosted.org/packages/9a/45/57bbf9e216850d065e66dd31a50f57424b607f1d878ab8956e56a1f4e36b/torchvision-0.26.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:fd10b5f994c210f4f6d6761cf686f82d748554adf486cb0979770c3252868c8f", size = 7519925, upload-time = "2026-03-23T18:12:53.283Z" }, + { url = "https://files.pythonhosted.org/packages/10/58/ed8f7754299f3e91d6414b6dc09f62b3fa7c6e5d63dfe48d69ab81498a37/torchvision-0.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:de6424b12887ad884f39a0ee446994ae3cd3b6a00a9cafe1bead85a031132af0", size = 3983834, upload-time = "2026-03-23T18:13:00.224Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e7/56b47cc3b132aea90ccce22bcb8975dec688b002150012acc842846039d0/torchvision-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c409e1c3fdebec7a3834465086dbda8bf7680eff79abf7fd2f10c6b59520a7a4", size = 1863502, upload-time = "2026-03-23T18:12:57.326Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ec/5c31c92c08b65662fe9604a4067ae8232582805949f11ddc042cebe818ed/torchvision-0.26.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:406557718e62fdf10f5706e88d8a5ec000f872da913bf629aab9297622585547", size = 7767944, upload-time = "2026-03-23T18:12:42.805Z" }, + { url = "https://files.pythonhosted.org/packages/f5/d8/cb6ccda1a1f35a6597645818641701207b3e8e13553e75fce5d86bac74b2/torchvision-0.26.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d61a5abb6b42a0c0c311996c2ac4b83a94418a97182c83b055a2a4ae985e05aa", size = 7522205, upload-time = "2026-03-23T18:12:54.654Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a9/c272623a0f735c35f0f6cd6dc74784d4f970e800cf063bb76687895a2ab9/torchvision-0.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:7993c01648e7c61d191b018e84d38fe0825c8fcb2720cd0f37caf7ba14404aa1", size = 4255155, upload-time = "2026-03-23T18:12:32.652Z" }, + { url = "https://files.pythonhosted.org/packages/da/80/0762f77f53605d10c9477be39bb47722cc8e383bbbc2531471ce0e396c07/torchvision-0.26.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5d63dd43162691258b1b3529b9041bac7d54caa37eae0925f997108268cbf7c4", size = 1860809, upload-time = "2026-03-23T18:12:47.629Z" }, + { url = "https://files.pythonhosted.org/packages/e6/81/0b3e58d1478c660a5af4268713486b2df7203f35abd9195fea87348a5178/torchvision-0.26.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a39c7a26538c41fda453f9a9692b5ff9b35a5437db1d94f3027f6f509c160eac", size = 7727494, upload-time = "2026-03-23T18:12:46.062Z" }, + { url = "https://files.pythonhosted.org/packages/b6/dc/d9ab5d29115aa05e12e30f1397a3eeae1d88a511241dc3bce48dc4342675/torchvision-0.26.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:b7e6213620bbf97742e5f79832f9e9d769e6cf0f744c5b53dad80b76db633691", size = 7521747, upload-time = "2026-03-23T18:12:36.815Z" }, + { url = "https://files.pythonhosted.org/packages/a9/1b/f1bc86a918c5f6feab1eeff11982e2060f4704332e96185463d27855bdf5/torchvision-0.26.0-cp313-cp313-win_amd64.whl", hash = "sha256:4280c35ec8cba1fcc8294fb87e136924708726864c379e4c54494797d86bc474", size = 4319880, upload-time = "2026-03-23T18:12:38.168Z" }, + { url = "https://files.pythonhosted.org/packages/66/28/b4ad0a723ed95b003454caffcc41894b34bd8379df340848cae2c33871de/torchvision-0.26.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:358fc4726d0c08615b6d83b3149854f11efb2a564ed1acb6fce882e151412d23", size = 1951973, upload-time = "2026-03-23T18:12:48.781Z" }, + { url = "https://files.pythonhosted.org/packages/71/e2/7a89096e6cf2f3336353b5338ba925e0addf9d8601920340e6bdf47e8eb3/torchvision-0.26.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:3daf9cc149cf3cdcbd4df9c59dae69ffca86c6823250442c3bbfd63fc2e26c61", size = 7728679, upload-time = "2026-03-23T18:12:26.196Z" }, + { url = "https://files.pythonhosted.org/packages/69/1d/4e1eebc17d18ce080a11dcf3df3f8f717f0efdfa00983f06e8ba79259f61/torchvision-0.26.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:82c3965eca27e86a316e31e4c3e5a16d353e0bcbe0ef8efa2e66502c54493c4b", size = 7609138, upload-time = "2026-03-23T18:12:35.327Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a4/f1155e943ae5b32400d7000adc81c79bb0392b16ceb33bcf13e02e48cced/torchvision-0.26.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ebc043cc5a4f0bf22e7680806dbba37ffb19e70f6953bbb44ed1a90aeb5c9bea", size = 4248202, upload-time = "2026-03-23T18:12:41.423Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c8/9bffa9c7f7bdf95b2a0a2dc535c290b9f1cc580c3fb3033ab1246ffffdeb/torchvision-0.26.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:eb61804eb9dbe88c5a2a6c4da8dec1d80d2d0a6f18c999c524e32266cb1ebcd3", size = 1860813, upload-time = "2026-03-23T18:12:39.636Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ac/48f28ffd227991f2e14f4392dde7e8dc14352bb9428c1ef4a4bbf5f7ed85/torchvision-0.26.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:9a904f2131cbfadab4df828088a9f66291ad33f49ff853872aed1f86848ef776", size = 7727777, upload-time = "2026-03-23T18:12:22.549Z" }, + { url = "https://files.pythonhosted.org/packages/a4/21/a2266f7f1b0e58e624ff15fd6f01041f59182c49551ece0db9a183071329/torchvision-0.26.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:0f3e572efe62ad645017ea847e0b5e4f2f638d4e39f05bc011d1eb9ac68d4806", size = 7522174, upload-time = "2026-03-23T18:12:29.565Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ba/1666f90bc0bdd77aaa11dcc42bb9f621a9c3668819c32430452e3d404730/torchvision-0.26.0-cp314-cp314-win_amd64.whl", hash = "sha256:114bec0c0e98aa4ba446f63e2fe7a2cbca37b39ac933987ee4804f65de121800", size = 4348469, upload-time = "2026-03-23T18:12:24.44Z" }, + { url = "https://files.pythonhosted.org/packages/45/8f/1f0402ac55c2ae15651ff831957d083fe70b2d12282e72612a30ba601512/torchvision-0.26.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:b7d3e295624a28b3b1769228ce1345d94cf4d390dd31136766f76f2d20f718da", size = 1860826, upload-time = "2026-03-23T18:12:34.1Z" }, + { url = "https://files.pythonhosted.org/packages/d2/6a/18a582fe3c5ee26f49b5c9fb21ad8016b4d1c06d10178894a58653946fda/torchvision-0.26.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:7058c5878262937e876f20c25867b33724586aa4499e2853b2d52b99a5e51953", size = 7729089, upload-time = "2026-03-23T18:12:31.394Z" }, + { url = "https://files.pythonhosted.org/packages/c5/9b/f7e119b59499edc00c55c03adc9ec3bd96144d9b81c46852c431f9c64a9a/torchvision-0.26.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:8008474855623c6ba52876589dc52df0aa66e518c25eca841445348e5f79844c", size = 7522704, upload-time = "2026-03-23T18:12:20.301Z" }, + { url = "https://files.pythonhosted.org/packages/d0/6a/09f3844c10643f6c0de5d95abc863420cfaf194c88c7dffd0ac523e2015f/torchvision-0.26.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e9d0e022c19a78552fb055d0414d47fecb4a649309b9968573daea160ba6869c", size = 4454275, upload-time = "2026-03-23T18:12:27.487Z" }, ] [[package]] @@ -7071,7 +9498,7 @@ name = "tqdm" version = "4.67.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" } wheels = [ @@ -7100,7 +9527,9 @@ dependencies = [ { name = "fancy-einsum" }, { name = "huggingface-hub" }, { name = "jaxtyping" }, - { name = "numpy", marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and python_full_version < '3.13' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "packaging" }, { name = "pandas" }, { name = "protobuf" }, @@ -7126,6 +9555,9 @@ inspect = [ lit = [ { name = "lit-nlp" }, ] +vllm = [ + { name = "vllm", marker = "sys_platform == 'linux'" }, +] [package.dev-dependencies] demo = [ @@ -7161,8 +9593,8 @@ docs = [ { name = "tabulate" }, ] jupyter = [ - { name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "ipython", version = "9.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "ipython", version = "9.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "ipywidgets" }, { name = "jupyter" }, { name = "jupyterlab" }, @@ -7204,9 +9636,10 @@ requires-dist = [ { name = "transformers-stream-generator", specifier = ">=0.0.5,<0.1" }, { name = "typeguard", specifier = ">=4.2,<5" }, { name = "typing-extensions" }, + { name = "vllm", marker = "sys_platform == 'linux' and extra == 'vllm'", specifier = ">=0.20.2,<0.21" }, { name = "wandb", specifier = ">=0.13.5" }, ] -provides-extras = ["evals", "inspect", "lit"] +provides-extras = ["evals", "inspect", "lit", "vllm"] [package.metadata.requires-dev] demo = [ @@ -7263,7 +9696,9 @@ version = "5.8.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, - { name = "numpy" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-16-transformer-lens-lit'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra != 'extra-16-transformer-lens-lit') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "packaging" }, { name = "pyyaml" }, { name = "regex" }, @@ -7291,12 +9726,19 @@ name = "triton" version = "3.6.0" source = { registry = "https://pypi.org/simple" } wheels = [ + { url = "https://files.pythonhosted.org/packages/44/ba/b1b04f4b291a3205d95ebd24465de0e5bf010a2df27a4e58a9b5f039d8f2/triton-3.6.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c723cfb12f6842a0ae94ac307dba7e7a44741d720a40cf0e270ed4a4e3be781", size = 175972180, upload-time = "2026-01-20T16:15:53.664Z" }, { url = "https://files.pythonhosted.org/packages/8c/f7/f1c9d3424ab199ac53c2da567b859bcddbb9c9e7154805119f8bd95ec36f/triton-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a6550fae429e0667e397e5de64b332d1e5695b73650ee75a6146e2e902770bea", size = 188105201, upload-time = "2026-01-20T16:00:29.272Z" }, + { url = "https://files.pythonhosted.org/packages/0f/2c/96f92f3c60387e14cc45aed49487f3486f89ea27106c1b1376913c62abe4/triton-3.6.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49df5ef37379c0c2b5c0012286f80174fcf0e073e5ade1ca9a86c36814553651", size = 176081190, upload-time = "2026-01-20T16:16:00.523Z" }, { url = "https://files.pythonhosted.org/packages/e0/12/b05ba554d2c623bffa59922b94b0775673de251f468a9609bc9e45de95e9/triton-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8e323d608e3a9bfcc2d9efcc90ceefb764a82b99dea12a86d643c72539ad5d3", size = 188214640, upload-time = "2026-01-20T16:00:35.869Z" }, + { url = "https://files.pythonhosted.org/packages/17/5d/08201db32823bdf77a0e2b9039540080b2e5c23a20706ddba942924ebcd6/triton-3.6.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:374f52c11a711fd062b4bfbb201fd9ac0a5febd28a96fb41b4a0f51dde3157f4", size = 176128243, upload-time = "2026-01-20T16:16:07.857Z" }, { url = "https://files.pythonhosted.org/packages/ab/a8/cdf8b3e4c98132f965f88c2313a4b493266832ad47fb52f23d14d4f86bb5/triton-3.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74caf5e34b66d9f3a429af689c1c7128daba1d8208df60e81106b115c00d6fca", size = 188266850, upload-time = "2026-01-20T16:00:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/3c/12/34d71b350e89a204c2c7777a9bba0dcf2f19a5bfdd70b57c4dbc5ffd7154/triton-3.6.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:448e02fe6dc898e9e5aa89cf0ee5c371e99df5aa5e8ad976a80b93334f3494fd", size = 176133521, upload-time = "2026-01-20T16:16:13.321Z" }, { url = "https://files.pythonhosted.org/packages/f9/0b/37d991d8c130ce81a8728ae3c25b6e60935838e9be1b58791f5997b24a54/triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10c7f76c6e72d2ef08df639e3d0d30729112f47a56b0c81672edc05ee5116ac9", size = 188289450, upload-time = "2026-01-20T16:00:49.136Z" }, + { url = "https://files.pythonhosted.org/packages/ce/4e/41b0c8033b503fd3cfcd12392cdd256945026a91ff02452bef40ec34bee7/triton-3.6.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1722e172d34e32abc3eb7711d0025bb69d7959ebea84e3b7f7a341cd7ed694d6", size = 176276087, upload-time = "2026-01-20T16:16:18.989Z" }, { url = "https://files.pythonhosted.org/packages/35/f8/9c66bfc55361ec6d0e4040a0337fb5924ceb23de4648b8a81ae9d33b2b38/triton-3.6.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d002e07d7180fd65e622134fbd980c9a3d4211fb85224b56a0a0efbd422ab72f", size = 188400296, upload-time = "2026-01-20T16:00:56.042Z" }, + { url = "https://files.pythonhosted.org/packages/49/55/5ecf0dcaa0f2fbbd4420f7ef227ee3cb172e91e5fede9d0ecaddc43363b4/triton-3.6.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef5523241e7d1abca00f1d240949eebdd7c673b005edbbce0aca95b8191f1d43", size = 176138577, upload-time = "2026-01-20T16:16:25.426Z" }, { url = "https://files.pythonhosted.org/packages/df/3d/9e7eee57b37c80cec63322c0231bb6da3cfe535a91d7a4d64896fcb89357/triton-3.6.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a17a5d5985f0ac494ed8a8e54568f092f7057ef60e1b0fa09d3fd1512064e803", size = 188273063, upload-time = "2026-01-20T16:01:07.278Z" }, + { url = "https://files.pythonhosted.org/packages/48/db/56ee649cab5eaff4757541325aca81f52d02d4a7cd3506776cad2451e060/triton-3.6.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b3a97e8ed304dfa9bd23bb41ca04cdf6b2e617d5e782a8653d616037a5d537d", size = 176274804, upload-time = "2026-01-20T16:16:31.528Z" }, { url = "https://files.pythonhosted.org/packages/f6/56/6113c23ff46c00aae423333eb58b3e60bdfe9179d542781955a5e1514cb3/triton-3.6.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46bd1c1af4b6704e554cad2eeb3b0a6513a980d470ccfa63189737340c7746a7", size = 188397994, upload-time = "2026-01-20T16:01:14.236Z" }, ] @@ -7333,17 +9775,17 @@ datetime = [ [[package]] name = "typer" -version = "0.25.1" +version = "0.26.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, - { name = "click" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, { name = "rich" }, { name = "shellingham" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e4/51/9aed62104cea109b820bbd6c14245af756112017d309da813ef107d42e7e/typer-0.25.1.tar.gz", hash = "sha256:9616eb8853a09ffeabab1698952f33c6f29ffdbceb4eaeecf571880e8d7664cc", size = 122276, upload-time = "2026-04-30T19:32:16.964Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/f7/68adc395201b20b872d68e975386832e8005ffeacedd43a1d837a32815be/typer-0.26.8.tar.gz", hash = "sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e", size = 202097, upload-time = "2026-06-26T09:22:45.705Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/f9/2b3ff4e56e5fa7debfaf9eb135d0da96f3e9a1d5b27222223c7296336e5f/typer-0.25.1-py3-none-any.whl", hash = "sha256:75caa44ed46a03fb2dab8808753ffacdbfea88495e74c85a28c5eefcf5f39c89", size = 58409, upload-time = "2026-04-30T19:32:18.271Z" }, + { url = "https://files.pythonhosted.org/packages/80/87/b9fd69c92c6102a066e1b86a35243f53e70bd4c709f2a26d9f4fee4f4dc0/typer-0.26.8-py3-none-any.whl", hash = "sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c", size = 122564, upload-time = "2026-06-26T09:22:44.72Z" }, ] [[package]] @@ -7432,16 +9874,151 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "h11" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5e/da/6eee1ff8b6cbeed47eeb5229749168e81eb4b7b999a1a15a7176e51410c9/uvicorn-0.44.0.tar.gz", hash = "sha256:6c942071b68f07e178264b9152f1f16dfac5da85880c4ce06366a96d70d4f31e", size = 86947, upload-time = "2026-04-06T09:23:22.826Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/23/a5bbd9600dd607411fa644c06ff4951bec3a4d82c4b852374024359c19c0/uvicorn-0.44.0-py3-none-any.whl", hash = "sha256:ce937c99a2cc70279556967274414c087888e8cec9f9c94644dfca11bd3ced89", size = 69425, upload-time = "2026-04-06T09:23:21.524Z" }, ] +[package.optional-dependencies] +standard = [ + { name = "httptools", marker = "sys_platform == 'linux'" }, + { name = "python-dotenv", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'linux'" }, + { name = "watchfiles", marker = "sys_platform == 'linux'" }, + { name = "websockets", marker = "sys_platform == 'linux'" }, +] + +[[package]] +name = "uvloop" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/14/ecceb239b65adaaf7fde510aa8bd534075695d1e5f8dadfa32b5723d9cfb/uvloop-0.22.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ef6f0d4cc8a9fa1f6a910230cd53545d9a14479311e87e3cb225495952eb672c", size = 1343335, upload-time = "2025-10-16T22:16:11.43Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ae/6f6f9af7f590b319c94532b9567409ba11f4fa71af1148cab1bf48a07048/uvloop-0.22.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7cd375a12b71d33d46af85a3343b35d98e8116134ba404bd657b3b1d15988792", size = 742903, upload-time = "2025-10-16T22:16:12.979Z" }, + { url = "https://files.pythonhosted.org/packages/09/bd/3667151ad0702282a1f4d5d29288fce8a13c8b6858bf0978c219cd52b231/uvloop-0.22.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac33ed96229b7790eb729702751c0e93ac5bc3bcf52ae9eccbff30da09194b86", size = 3648499, upload-time = "2025-10-16T22:16:14.451Z" }, + { url = "https://files.pythonhosted.org/packages/b3/f6/21657bb3beb5f8c57ce8be3b83f653dd7933c2fd00545ed1b092d464799a/uvloop-0.22.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:481c990a7abe2c6f4fc3d98781cc9426ebd7f03a9aaa7eb03d3bfc68ac2a46bd", size = 3700133, upload-time = "2025-10-16T22:16:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/09/e0/604f61d004ded805f24974c87ddd8374ef675644f476f01f1df90e4cdf72/uvloop-0.22.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a592b043a47ad17911add5fbd087c76716d7c9ccc1d64ec9249ceafd735f03c2", size = 3512681, upload-time = "2025-10-16T22:16:18.07Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ce/8491fd370b0230deb5eac69c7aae35b3be527e25a911c0acdffb922dc1cd/uvloop-0.22.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1489cf791aa7b6e8c8be1c5a080bae3a672791fcb4e9e12249b05862a2ca9cec", size = 3615261, upload-time = "2025-10-16T22:16:19.596Z" }, + { url = "https://files.pythonhosted.org/packages/c7/d5/69900f7883235562f1f50d8184bb7dd84a2fb61e9ec63f3782546fdbd057/uvloop-0.22.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c60ebcd36f7b240b30788554b6f0782454826a0ed765d8430652621b5de674b9", size = 1352420, upload-time = "2025-10-16T22:16:21.187Z" }, + { url = "https://files.pythonhosted.org/packages/a8/73/c4e271b3bce59724e291465cc936c37758886a4868787da0278b3b56b905/uvloop-0.22.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b7f102bf3cb1995cfeaee9321105e8f5da76fdb104cdad8986f85461a1b7b77", size = 748677, upload-time = "2025-10-16T22:16:22.558Z" }, + { url = "https://files.pythonhosted.org/packages/86/94/9fb7fad2f824d25f8ecac0d70b94d0d48107ad5ece03769a9c543444f78a/uvloop-0.22.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:53c85520781d84a4b8b230e24a5af5b0778efdb39142b424990ff1ef7c48ba21", size = 3753819, upload-time = "2025-10-16T22:16:23.903Z" }, + { url = "https://files.pythonhosted.org/packages/74/4f/256aca690709e9b008b7108bc85fba619a2bc37c6d80743d18abad16ee09/uvloop-0.22.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56a2d1fae65fd82197cb8c53c367310b3eabe1bbb9fb5a04d28e3e3520e4f702", size = 3804529, upload-time = "2025-10-16T22:16:25.246Z" }, + { url = "https://files.pythonhosted.org/packages/7f/74/03c05ae4737e871923d21a76fe28b6aad57f5c03b6e6bfcfa5ad616013e4/uvloop-0.22.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:40631b049d5972c6755b06d0bfe8233b1bd9a8a6392d9d1c45c10b6f9e9b2733", size = 3621267, upload-time = "2025-10-16T22:16:26.819Z" }, + { url = "https://files.pythonhosted.org/packages/75/be/f8e590fe61d18b4a92070905497aec4c0e64ae1761498cad09023f3f4b3e/uvloop-0.22.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:535cc37b3a04f6cd2c1ef65fa1d370c9a35b6695df735fcff5427323f2cd5473", size = 3723105, upload-time = "2025-10-16T22:16:28.252Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42", size = 1359936, upload-time = "2025-10-16T22:16:29.436Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c6/e5d433f88fd54d81ef4be58b2b7b0cea13c442454a1db703a1eea0db1a59/uvloop-0.22.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6", size = 752769, upload-time = "2025-10-16T22:16:30.493Z" }, + { url = "https://files.pythonhosted.org/packages/24/68/a6ac446820273e71aa762fa21cdcc09861edd3536ff47c5cd3b7afb10eeb/uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370", size = 4317413, upload-time = "2025-10-16T22:16:31.644Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4", size = 4426307, upload-time = "2025-10-16T22:16:32.917Z" }, + { url = "https://files.pythonhosted.org/packages/90/60/97362554ac21e20e81bcef1150cb2a7e4ffdaf8ea1e5b2e8bf7a053caa18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2", size = 4131970, upload-time = "2025-10-16T22:16:34.015Z" }, + { url = "https://files.pythonhosted.org/packages/99/39/6b3f7d234ba3964c428a6e40006340f53ba37993f46ed6e111c6e9141d18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0", size = 4296343, upload-time = "2025-10-16T22:16:35.149Z" }, + { url = "https://files.pythonhosted.org/packages/89/8c/182a2a593195bfd39842ea68ebc084e20c850806117213f5a299dfc513d9/uvloop-0.22.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:561577354eb94200d75aca23fbde86ee11be36b00e52a4eaf8f50fb0c86b7705", size = 1358611, upload-time = "2025-10-16T22:16:36.833Z" }, + { url = "https://files.pythonhosted.org/packages/d2/14/e301ee96a6dc95224b6f1162cd3312f6d1217be3907b79173b06785f2fe7/uvloop-0.22.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cdf5192ab3e674ca26da2eada35b288d2fa49fdd0f357a19f0e7c4e7d5077c8", size = 751811, upload-time = "2025-10-16T22:16:38.275Z" }, + { url = "https://files.pythonhosted.org/packages/b7/02/654426ce265ac19e2980bfd9ea6590ca96a56f10c76e63801a2df01c0486/uvloop-0.22.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2ea3d6190a2968f4a14a23019d3b16870dd2190cd69c8180f7c632d21de68d", size = 4288562, upload-time = "2025-10-16T22:16:39.375Z" }, + { url = "https://files.pythonhosted.org/packages/15/c0/0be24758891ef825f2065cd5db8741aaddabe3e248ee6acc5e8a80f04005/uvloop-0.22.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0530a5fbad9c9e4ee3f2b33b148c6a64d47bbad8000ea63704fa8260f4cf728e", size = 4366890, upload-time = "2025-10-16T22:16:40.547Z" }, + { url = "https://files.pythonhosted.org/packages/d2/53/8369e5219a5855869bcee5f4d317f6da0e2c669aecf0ef7d371e3d084449/uvloop-0.22.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc5ef13bbc10b5335792360623cc378d52d7e62c2de64660616478c32cd0598e", size = 4119472, upload-time = "2025-10-16T22:16:41.694Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" }, + { url = "https://files.pythonhosted.org/packages/90/cd/b62bdeaa429758aee8de8b00ac0dd26593a9de93d302bff3d21439e9791d/uvloop-0.22.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3879b88423ec7e97cd4eba2a443aa26ed4e59b45e6b76aabf13fe2f27023a142", size = 1362067, upload-time = "2025-10-16T22:16:44.503Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f8/a132124dfda0777e489ca86732e85e69afcd1ff7686647000050ba670689/uvloop-0.22.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4baa86acedf1d62115c1dc6ad1e17134476688f08c6efd8a2ab076e815665c74", size = 752423, upload-time = "2025-10-16T22:16:45.968Z" }, + { url = "https://files.pythonhosted.org/packages/a3/94/94af78c156f88da4b3a733773ad5ba0b164393e357cc4bd0ab2e2677a7d6/uvloop-0.22.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35", size = 4272437, upload-time = "2025-10-16T22:16:47.451Z" }, + { url = "https://files.pythonhosted.org/packages/b5/35/60249e9fd07b32c665192cec7af29e06c7cd96fa1d08b84f012a56a0b38e/uvloop-0.22.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1955d5a1dd43198244d47664a5858082a3239766a839b2102a269aaff7a4e25", size = 4292101, upload-time = "2025-10-16T22:16:49.318Z" }, + { url = "https://files.pythonhosted.org/packages/02/62/67d382dfcb25d0a98ce73c11ed1a6fba5037a1a1d533dcbb7cab033a2636/uvloop-0.22.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b31dc2fccbd42adc73bc4e7cdbae4fc5086cf378979e53ca5d0301838c5682c6", size = 4114158, upload-time = "2025-10-16T22:16:50.517Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/f1171b4a882a5d13c8b7576f348acfe6074d72eaf52cccef752f748d4a9f/uvloop-0.22.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93f617675b2d03af4e72a5333ef89450dfaa5321303ede6e67ba9c9d26878079", size = 4177360, upload-time = "2025-10-16T22:16:52.646Z" }, + { url = "https://files.pythonhosted.org/packages/79/7b/b01414f31546caf0919da80ad57cbfe24c56b151d12af68cee1b04922ca8/uvloop-0.22.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:37554f70528f60cad66945b885eb01f1bb514f132d92b6eeed1c90fd54ed6289", size = 1454790, upload-time = "2025-10-16T22:16:54.355Z" }, + { url = "https://files.pythonhosted.org/packages/d4/31/0bb232318dd838cad3fa8fb0c68c8b40e1145b32025581975e18b11fab40/uvloop-0.22.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:b76324e2dc033a0b2f435f33eb88ff9913c156ef78e153fb210e03c13da746b3", size = 796783, upload-time = "2025-10-16T22:16:55.906Z" }, + { url = "https://files.pythonhosted.org/packages/42/38/c9b09f3271a7a723a5de69f8e237ab8e7803183131bc57c890db0b6bb872/uvloop-0.22.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:badb4d8e58ee08dad957002027830d5c3b06aea446a6a3744483c2b3b745345c", size = 4647548, upload-time = "2025-10-16T22:16:57.008Z" }, + { url = "https://files.pythonhosted.org/packages/c1/37/945b4ca0ac27e3dc4952642d4c900edd030b3da6c9634875af6e13ae80e5/uvloop-0.22.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b91328c72635f6f9e0282e4a57da7470c7350ab1c9f48546c0f2866205349d21", size = 4467065, upload-time = "2025-10-16T22:16:58.206Z" }, + { url = "https://files.pythonhosted.org/packages/97/cc/48d232f33d60e2e2e0b42f4e73455b146b76ebe216487e862700457fbf3c/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:daf620c2995d193449393d6c62131b3fbd40a63bf7b307a1527856ace637fe88", size = 4328384, upload-time = "2025-10-16T22:16:59.36Z" }, + { url = "https://files.pythonhosted.org/packages/e4/16/c1fd27e9549f3c4baf1dc9c20c456cd2f822dbf8de9f463824b0c0357e06/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e", size = 4296730, upload-time = "2025-10-16T22:17:00.744Z" }, +] + +[[package]] +name = "vllm" +version = "0.20.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp", marker = "sys_platform == 'linux'" }, + { name = "anthropic", marker = "sys_platform == 'linux'" }, + { name = "apache-tvm-ffi", marker = "sys_platform == 'linux'" }, + { name = "blake3", marker = "sys_platform == 'linux'" }, + { name = "cachetools", marker = "sys_platform == 'linux'" }, + { name = "cbor2", marker = "sys_platform == 'linux'" }, + { name = "cloudpickle", marker = "sys_platform == 'linux'" }, + { name = "compressed-tensors", marker = "sys_platform == 'linux'" }, + { name = "depyf", marker = "sys_platform == 'linux'" }, + { name = "diskcache", marker = "sys_platform == 'linux'" }, + { name = "einops", marker = "sys_platform == 'linux'" }, + { name = "fastapi", extra = ["standard"], marker = "(sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "fastsafetensors", marker = "sys_platform == 'linux'" }, + { name = "filelock", marker = "sys_platform == 'linux'" }, + { name = "flashinfer-cubin", marker = "sys_platform == 'linux'" }, + { name = "flashinfer-python", marker = "sys_platform == 'linux'" }, + { name = "gguf", marker = "sys_platform == 'linux'" }, + { name = "ijson", marker = "sys_platform == 'linux'" }, + { name = "lark", marker = "sys_platform == 'linux'" }, + { name = "llguidance", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'arm64' and sys_platform == 'linux') or (platform_machine == 'ppc64le' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "lm-format-enforcer", marker = "sys_platform == 'linux'" }, + { name = "mcp", marker = "sys_platform == 'linux'" }, + { name = "mistral-common", extra = ["image"], marker = "(sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "model-hosting-container-standards", marker = "sys_platform == 'linux'" }, + { name = "msgspec", marker = "sys_platform == 'linux'" }, + { name = "ninja", marker = "sys_platform == 'linux'" }, + { name = "numba", marker = "sys_platform == 'linux'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "nvidia-cudnn-frontend", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cutlass-dsl", marker = "sys_platform == 'linux'" }, + { name = "openai", marker = "sys_platform == 'linux'" }, + { name = "openai-harmony", marker = "sys_platform == 'linux'" }, + { name = "opencv-python-headless", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-api", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-exporter-otlp", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-sdk", marker = "sys_platform == 'linux'" }, + { name = "opentelemetry-semantic-conventions-ai", marker = "sys_platform == 'linux'" }, + { name = "outlines-core", marker = "sys_platform == 'linux'" }, + { name = "partial-json-parser", marker = "sys_platform == 'linux'" }, + { name = "pillow", marker = "sys_platform == 'linux'" }, + { name = "prometheus-client", marker = "sys_platform == 'linux'" }, + { name = "prometheus-fastapi-instrumentator", marker = "sys_platform == 'linux'" }, + { name = "protobuf", marker = "sys_platform == 'linux'" }, + { name = "psutil", marker = "sys_platform == 'linux'" }, + { name = "py-cpuinfo", marker = "sys_platform == 'linux'" }, + { name = "pybase64", marker = "sys_platform == 'linux'" }, + { name = "pydantic", marker = "sys_platform == 'linux'" }, + { name = "python-json-logger", marker = "sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'linux'" }, + { name = "pyzmq", marker = "sys_platform == 'linux'" }, + { name = "quack-kernels", marker = "sys_platform == 'linux'" }, + { name = "regex", marker = "sys_platform == 'linux'" }, + { name = "requests", marker = "sys_platform == 'linux'" }, + { name = "sentencepiece", marker = "sys_platform == 'linux'" }, + { name = "setproctitle", marker = "sys_platform == 'linux'" }, + { name = "setuptools", marker = "python_full_version >= '3.12' and sys_platform == 'linux'" }, + { name = "six", marker = "python_full_version >= '3.12' and sys_platform == 'linux'" }, + { name = "tiktoken", marker = "sys_platform == 'linux'" }, + { name = "tilelang", marker = "sys_platform == 'linux'" }, + { name = "tokenizers", marker = "sys_platform == 'linux'" }, + { name = "torch", marker = "sys_platform == 'linux'" }, + { name = "torchaudio", marker = "sys_platform == 'linux'" }, + { name = "torchvision", marker = "sys_platform == 'linux'" }, + { name = "tqdm", marker = "sys_platform == 'linux'" }, + { name = "transformers", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, + { name = "watchfiles", marker = "sys_platform == 'linux'" }, + { name = "xgrammar", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'arm64' and sys_platform == 'linux') or (platform_machine == 'ppc64le' and sys_platform == 'linux') or (platform_machine == 's390x' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/23/4a5dc23600be07407835e404d06a2fcd587c8dcaebeab314b5c3593509ce/vllm-0.20.2.tar.gz", hash = "sha256:58809377798c5335c6e2fe30092abda54d9200b5b8a717b3735a63f5daa0e383", size = 33526607, upload-time = "2026-05-10T07:39:27.942Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/d9/7bef6cf9d7508b4313237602e96e10054c7491f12f48403813c9c2d6f6f1/vllm-0.20.2-cp38-abi3-manylinux_2_35_aarch64.whl", hash = "sha256:76ccf4c0554556c06f6b0fb1643742d4cf97dcc69f6ef3f04556d0764126035a", size = 235788487, upload-time = "2026-05-10T07:38:37.747Z" }, + { url = "https://files.pythonhosted.org/packages/c5/aa/4488d49c481a2184e6e285b8d3f937905205f52cd5ac30fb348770494b6e/vllm-0.20.2-cp38-abi3-manylinux_2_35_x86_64.whl", hash = "sha256:22a7dd06eb03371298e13d6100f3dedbf307352342aaf08e87c929c60aae9b4d", size = 244425988, upload-time = "2026-05-10T07:39:04.327Z" }, +] + [[package]] name = "wandb" -version = "0.21.0" +version = "0.28.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -7455,18 +10032,134 @@ dependencies = [ { name = "sentry-sdk" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/73/09/c84264a219e20efd615e4d5d150cc7d359d57d51328d3fa94ee02d70ed9c/wandb-0.21.0.tar.gz", hash = "sha256:473e01ef200b59d780416062991effa7349a34e51425d4be5ff482af2dc39e02", size = 40085784, upload-time = "2025-07-02T00:24:15.516Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/a7/683bfbd6cbade3012bc90d3e9c4cfc72dd62566195bf4c30321946d64b77/wandb-0.28.0.tar.gz", hash = "sha256:b20e5af0fe80e2e2a466b0466a1d60cedcc578dce0f036eca04f4a0adcad95b6", size = 40558332, upload-time = "2026-06-23T00:38:50.115Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/38/dd/65eac086e1bc337bb5f0eed65ba1fe4a6dbc62c97f094e8e9df1ef83ffed/wandb-0.21.0-py3-none-any.whl", hash = "sha256:316e8cd4329738f7562f7369e6eabeeb28ef9d473203f7ead0d03e5dba01c90d", size = 6504284, upload-time = "2025-07-02T00:23:46.671Z" }, - { url = "https://files.pythonhosted.org/packages/17/a7/80556ce9097f59e10807aa68f4a9b29d736a90dca60852a9e2af1641baf8/wandb-0.21.0-py3-none-macosx_10_14_x86_64.whl", hash = "sha256:701d9cbdfcc8550a330c1b54a26f1585519180e0f19247867446593d34ace46b", size = 21717388, upload-time = "2025-07-02T00:23:49.348Z" }, - { url = "https://files.pythonhosted.org/packages/23/ae/660bc75aa37bd23409822ea5ed616177d94873172d34271693c80405c820/wandb-0.21.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:01689faa6b691df23ba2367e0a1ecf6e4d0be44474905840098eedd1fbcb8bdf", size = 21141465, upload-time = "2025-07-02T00:23:52.602Z" }, - { url = "https://files.pythonhosted.org/packages/23/ab/9861929530be56557c74002868c85d0d8ac57050cc21863afe909ae3d46f/wandb-0.21.0-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:55d3f42ddb7971d1699752dff2b85bcb5906ad098d18ab62846c82e9ce5a238d", size = 21793511, upload-time = "2025-07-02T00:23:55.447Z" }, - { url = "https://files.pythonhosted.org/packages/de/52/e5cad2eff6fbed1ac06f4a5b718457fa2fd437f84f5c8f0d31995a2ef046/wandb-0.21.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:893508f0c7da48917448daa5cd622c27ce7ce15119adaa861185034c2bd7b14c", size = 20704643, upload-time = "2025-07-02T00:23:58.255Z" }, - { url = "https://files.pythonhosted.org/packages/83/8f/6bed9358cc33767c877b221d4f565e1ddf00caf4bbbe54d2e3bbc932c6a7/wandb-0.21.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4e8245a8912247ddf7654f7b5330f583a6c56ab88fee65589158490d583c57d", size = 22243012, upload-time = "2025-07-02T00:24:01.423Z" }, - { url = "https://files.pythonhosted.org/packages/be/61/9048015412ea5ca916844af55add4fed7c21fe1ad70bb137951e70b550c5/wandb-0.21.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:2e4c4f951e0d02755e315679bfdcb5bc38c1b02e2e5abc5432b91a91bb0cf246", size = 20716440, upload-time = "2025-07-02T00:24:04.198Z" }, - { url = "https://files.pythonhosted.org/packages/02/d9/fcd2273d8ec3f79323e40a031aba5d32d6fa9065702010eb428b5ffbab62/wandb-0.21.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:873749966eeac0069e0e742e6210641b6227d454fb1dae2cf5c437c6ed42d3ca", size = 22320652, upload-time = "2025-07-02T00:24:07.175Z" }, - { url = "https://files.pythonhosted.org/packages/80/68/b8308db6b9c3c96dcd03be17c019aee105e1d7dc1e74d70756cdfb9241c6/wandb-0.21.0-py3-none-win32.whl", hash = "sha256:9d3cccfba658fa011d6cab9045fa4f070a444885e8902ae863802549106a5dab", size = 21484296, upload-time = "2025-07-02T00:24:10.147Z" }, - { url = "https://files.pythonhosted.org/packages/cf/96/71cc033e8abd00e54465e68764709ed945e2da2d66d764f72f4660262b22/wandb-0.21.0-py3-none-win_amd64.whl", hash = "sha256:28a0b2dad09d7c7344ac62b0276be18a2492a5578e4d7c84937a3e1991edaac7", size = 21484301, upload-time = "2025-07-02T00:24:12.658Z" }, + { url = "https://files.pythonhosted.org/packages/c0/47/1723605f76c5d6446b6d0db65b83eda1599721bc8c1e65bd76cc1682b1a7/wandb-0.28.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:c3dab1205a5aca4abbad1eca08902cdba86add0edfa83d8d61b4429d0e79fa87", size = 24335272, upload-time = "2026-06-23T00:38:26.002Z" }, + { url = "https://files.pythonhosted.org/packages/81/ff/42b539bc75bc48fc86981dccde89327ba9b71504b805b9ba42cba7c26de9/wandb-0.28.0-py3-none-macosx_12_0_x86_64.whl", hash = "sha256:ae255da18726ee8e731ef82cbc85035b901a28ae14cf91604c361b44b8d44ce0", size = 25557959, upload-time = "2026-06-23T00:38:28.993Z" }, + { url = "https://files.pythonhosted.org/packages/15/55/c3db03d04aeab3726066a418b2ef6a1f8119774ee510f4fbe992f52b7472/wandb-0.28.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:6dbcba12ab168aa37561f2f32dcdef8713495fc25fa7d30fdc9bfb37989694dd", size = 24878557, upload-time = "2026-06-23T00:38:31.417Z" }, + { url = "https://files.pythonhosted.org/packages/d8/5d/1385ce3c219cb5bd30d4027687e3f8d25969c7dfd09adad1cbd5080e1a72/wandb-0.28.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:325b2d0bd88be6eda5db10542499bad3710927f2569c81a84dc5eeaffc76825c", size = 26764727, upload-time = "2026-06-23T00:38:33.775Z" }, + { url = "https://files.pythonhosted.org/packages/00/58/23b6c17a6d3d5422b007707961c4496b2f6f892624d2910c9f7742fcc202/wandb-0.28.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:8954bc1c62ae43914dce2bebfd1d9957f72350f8fbb78e5cdfe2ca9b6be8a7b8", size = 25051656, upload-time = "2026-06-23T00:38:36.281Z" }, + { url = "https://files.pythonhosted.org/packages/89/67/9be00fb2db2281063af24a148636d2dd363d337317642ab5d8e93572c794/wandb-0.28.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:9fec6c908554c2dad33110c1312bc3028cc2e430f0679f16b84f82c8ea801e3b", size = 27074113, upload-time = "2026-06-23T00:38:38.737Z" }, + { url = "https://files.pythonhosted.org/packages/59/b1/f7a96c09cab0c5131b1e6466659b093b401e1653cbe6bb77b462fc1c361d/wandb-0.28.0-py3-none-win32.whl", hash = "sha256:8834ef3a7c8c43b701654162783caa7ad37af48a0ff06fc35d0d65a411f76ccd", size = 24525206, upload-time = "2026-06-23T00:38:42.041Z" }, + { url = "https://files.pythonhosted.org/packages/c6/c4/c7bed5e981679c74e9fbb22c03ff31c42e95f266199d03d8d325f4d0e6df/wandb-0.28.0-py3-none-win_amd64.whl", hash = "sha256:ac1f82292e2da4f98297b78c3a46726b3a6c5734ecb75fc39b8db2c8a4989159", size = 24525214, upload-time = "2026-06-23T00:38:44.549Z" }, + { url = "https://files.pythonhosted.org/packages/f0/77/b5ce9696c8cb955521a7941fbc443e78b2f504894c6ae1a2d0b1de6e12ae/wandb-0.28.0-py3-none-win_arm64.whl", hash = "sha256:c5b0faf1b84cf79ebabed77538c1940a4c6053e815f767a4004e877a1354bed1", size = 22378208, upload-time = "2026-06-23T00:38:47.148Z" }, +] + +[[package]] +name = "watchfiles" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/41/5e1a4bb12aac5f1493fa1bdc11154eca3b258ca4eba65d39c473fe19d8e9/watchfiles-1.2.0.tar.gz", hash = "sha256:c995fba777f1ea992f090f9236e9284cf7a5d1a0130dd5a3d82c598cacd76838", size = 108252, upload-time = "2026-05-18T04:32:04.251Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/5a/2bf22ecb24916983bf1cc0095e7dea2741d14d6553b0d6a2ac8bc96eca93/watchfiles-1.2.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:bb68bf4df85abebe5efddc53cf2075520f243a59868d9b3973278b23e76962a9", size = 400471, upload-time = "2026-05-18T04:31:08.908Z" }, + { url = "https://files.pythonhosted.org/packages/55/70/dea1f6a0e76607841a60fb51af150e70124864673f61704abb62b90cdcc7/watchfiles-1.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c16cb06dd17d43b9d185094268459eac92c9538356f050e55b54e82cf700e1d4", size = 394599, upload-time = "2026-05-18T04:30:19.845Z" }, + { url = "https://files.pythonhosted.org/packages/18/52/752dcc7dc817baef5e89518732925795ce52e36a683a9a3c9fb68b21504e/watchfiles-1.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77a0feab9af4c021c581f695258c642b3d10c5fd4c676e33a0d8606425d82631", size = 455458, upload-time = "2026-05-18T04:30:29.126Z" }, + { url = "https://files.pythonhosted.org/packages/12/48/366ebbb22fcc504c2f72b45f0b7e72f40a18795cc01752c16066d597b67a/watchfiles-1.2.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a16ffe19bf5cf9f5edaa1ad1dd830c5a816e8feec430c522302ab55483a4b994", size = 460513, upload-time = "2026-05-18T04:31:40.85Z" }, + { url = "https://files.pythonhosted.org/packages/ad/44/1f9e1b15e7a729062e0d0c3d0d7225ea4ab98b2267ef87287153be2495fc/watchfiles-1.2.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:204f299afcbd65918ab78dbc52626b0ae45e9d8cef403fdbf33ecf9e40eac66e", size = 493616, upload-time = "2026-05-18T04:30:58.47Z" }, + { url = "https://files.pythonhosted.org/packages/7e/55/8b1086dcc8a1d6a697a62767bd7ea368e74c61c6fd171683cfe24a3fe5d2/watchfiles-1.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11743adfa510bfffebe97659fb280182b5c9b238708f667e866f308c3430dc19", size = 573154, upload-time = "2026-05-18T04:30:37.903Z" }, + { url = "https://files.pythonhosted.org/packages/14/7a/242f400cc77fafa7b18d53d19d9cb64fc6a6f61f28c55913bae7c674d92a/watchfiles-1.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eb72919d93e3a16fc451d3aa3d4b1698423daca1b382d3d959c9ac51297c12a8", size = 467046, upload-time = "2026-05-18T04:30:41.869Z" }, + { url = "https://files.pythonhosted.org/packages/02/c8/79eee650c62d2c186598489814468e389b5def0ebe755399ff645b35b1b2/watchfiles-1.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62f042afde2dde21ec1d2c1a74361e804673df86f51e418a999c9acfe671b07", size = 457100, upload-time = "2026-05-18T04:31:13.064Z" }, + { url = "https://files.pythonhosted.org/packages/81/36/519f6dbb7a95e4fe7c1513ed25b1520295ef9905a27f1f2226a73892bfb7/watchfiles-1.2.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:027ae72bfdfd254862065d8b3e2a815c6ab9b1853ce41e6648ece84afd34a551", size = 467038, upload-time = "2026-05-18T04:30:32.915Z" }, + { url = "https://files.pythonhosted.org/packages/2f/12/951af6b9f89097e02511122258402cb3578443021930b70cf968d6310dc0/watchfiles-1.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e1cfd51e97e13ff3bd047c140764d277fc9b95b7cb5da59e46a47d167adab310", size = 632563, upload-time = "2026-05-18T04:30:11.539Z" }, + { url = "https://files.pythonhosted.org/packages/28/cc/0cba1f0a6117b7ec117271bdc3cb3a5a252005959755a2c09a745e0942cc/watchfiles-1.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:24b2405c0a46738dd9e1cf7135aa5dbdb9d42d024628651b3b13d5117e99f8df", size = 660851, upload-time = "2026-05-18T04:31:53.186Z" }, + { url = "https://files.pythonhosted.org/packages/d0/f2/26347558cc8bf6877845e66b315f644d03c173906aa09e233a3f4fd23928/watchfiles-1.2.0-cp310-cp310-win32.whl", hash = "sha256:8c520725602756229f045b032a1ff33d7ef0f7404189d62f6c2438cb6d8ef6a1", size = 277023, upload-time = "2026-05-18T04:30:18.825Z" }, + { url = "https://files.pythonhosted.org/packages/6d/68/a5e67b6b68e94f4c1511d61c46c55eba0737583620b6febf194c7b9cc23f/watchfiles-1.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:03b14855c6f35539e2d95c442ae9530a75762f1e26567152b9ed05f96534a74d", size = 290107, upload-time = "2026-05-18T04:32:09.677Z" }, + { url = "https://files.pythonhosted.org/packages/fc/3d/8024c801df84d1587740d0359e7fdd80afeae3d159011f3d5376dd82f18e/watchfiles-1.2.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:704fd259e332e01f9b9c178f4bce9e49027e5587cc2600eeeaf8e76e1c846201", size = 400242, upload-time = "2026-05-18T04:31:19.014Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/f4dfd45323e949984a3a7f9dc31d1cbb049921e7d98253488dda72ccdaa9/watchfiles-1.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6543cf55d170003296d185c0af981f3e1311564907e1f4e08671fc7693a890a5", size = 394562, upload-time = "2026-05-18T04:30:08.46Z" }, + { url = "https://files.pythonhosted.org/packages/98/d8/19483ef075d601c409bce8bcbb5c0f81a10876fff870400568f08ce484a1/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89d8c2394a065ca86f5d2910ff263ae67c127e1376ccc4f9fc35c71db879f80a", size = 456611, upload-time = "2026-05-18T04:30:45.723Z" }, + { url = "https://files.pythonhosted.org/packages/b1/6a/cc81fbe7ee42f2f22e661a6e12def7807e01b14b2f39e0ff83fd373fd307/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:772b80df316480d894a0e3165fdd19cf77f5d17f9a787f94029465ad0e3529d1", size = 461379, upload-time = "2026-05-18T04:31:29.292Z" }, + { url = "https://files.pythonhosted.org/packages/b1/57/7e669002082c0a0f4fb5113bb70125f7110124b846b0a11bc5ae8e90eac1/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d158cd89df6053823533e06fb1d73c549133bff5f0396170c0e53d9559340717", size = 493556, upload-time = "2026-05-18T04:30:05.44Z" }, + { url = "https://files.pythonhosted.org/packages/45/7d/f60a2b19807b21fe8281f3a8da4f59eef0d5f96825ac4680ba2d4f2ebf91/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d516b3283a758e087841aedb8031549fb41ced08f3db10aa6d2bf32dc042525b", size = 575255, upload-time = "2026-05-18T04:30:40.568Z" }, + { url = "https://files.pythonhosted.org/packages/bd/49/77f5b5e6efbcd57482f74948ebb1b97e5c0046d6b61475042d830c84b3ff/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:53b2290c92e0506d102cd448fbc610d87079553f86caa39d67440856a8b8bba5", size = 467052, upload-time = "2026-05-18T04:31:17.942Z" }, + { url = "https://files.pythonhosted.org/packages/ee/5a/73e2959af1b97fd5d556f9a8bdba017be23ceeef731869d5eaa0a753d5a3/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a711b51aec4370d0dcda5b6c09463206f133a5759341d7744b953a7b62e1100e", size = 456858, upload-time = "2026-05-18T04:30:30.182Z" }, + { url = "https://files.pythonhosted.org/packages/50/57/1bc8c27fad7e6c19bddee15d276dbb6ab72480ec01c127afff1673aee417/watchfiles-1.2.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:e2ca07fa7d89195ec0865d3d285666286740bfa83d83e5cee204043a31ecc165", size = 467579, upload-time = "2026-05-18T04:32:15.897Z" }, + { url = "https://files.pythonhosted.org/packages/09/6c/3c2e44edba3553c5e3c3b8c8a2a6dee6b9e12ae2cf4bd2378bebf9dc3038/watchfiles-1.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e0618518f282c4ebff60f5e5b1247b6d91bb8b9f4476947563a1e74acc66f3c6", size = 633253, upload-time = "2026-05-18T04:31:37.123Z" }, + { url = "https://files.pythonhosted.org/packages/30/c2/d8c84a882ab39bbefcc4915ab3e91830b7a7e990c5570b0b69075aba3faf/watchfiles-1.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0d191c054d0715c3c95c99df9b8dbf6fd096d8c1e021e8f212e1bd8bc444ccb5", size = 660713, upload-time = "2026-05-18T04:31:24.62Z" }, + { url = "https://files.pythonhosted.org/packages/a9/07/f97736a5fc605364fe67b25e9fa4a6965dfd4840d50c406ada507e9d735f/watchfiles-1.2.0-cp311-cp311-win32.whl", hash = "sha256:9342472aff9b093c5acd4f6d8f70ae0937964ab56542502bcf5579782da69ae8", size = 277222, upload-time = "2026-05-18T04:31:21.131Z" }, + { url = "https://files.pythonhosted.org/packages/cf/99/2b04981977fc2608afd60360d928c6aecf6b950292ca221d98f4005f6694/watchfiles-1.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:dbd6c97045dad81227c8d040173da044c1de08de64a5ea8b555da4aee1d5fa22", size = 290274, upload-time = "2026-05-18T04:31:45.966Z" }, + { url = "https://files.pythonhosted.org/packages/3c/74/f7f58a7075ee9cf612b0cfcddb78b8cd8234f0742d6f0075cf0da2dde1c6/watchfiles-1.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:57a2d9fa4fb4c2ecae57b13dfff2c7ab53e21a2ba674fe9f05506680fcdcc0d7", size = 283460, upload-time = "2026-05-18T04:31:39.126Z" }, + { url = "https://files.pythonhosted.org/packages/b8/2f/e42c992d2afda3108ea1c02acecc991b9f31d05c14adc2a7cee9ee211fc4/watchfiles-1.2.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bc13eb17538be00c874699dc0abe4ee2bc8d50bb1166a6b9e175ef3fd7eb8f26", size = 400115, upload-time = "2026-05-18T04:32:02.06Z" }, + { url = "https://files.pythonhosted.org/packages/5f/8f/6af2ea19065c91d8b0ea3516fdfc8c0d349f407e8e9fbf4e5a17360de8ad/watchfiles-1.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d95ddc1eb6914154253d239089900813f6a767e174b8e6a50e7fdacb7e4236c", size = 393659, upload-time = "2026-05-18T04:30:50.951Z" }, + { url = "https://files.pythonhosted.org/packages/13/01/b32a967c56fb3e3e5be3db52c3d3b87fa4513aa367d8ed1ad96d42952e5f/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f70d8b291ef6e88d19b1f297a6905ddb978888d9272b0d05e6f53309856bcfc", size = 453207, upload-time = "2026-05-18T04:31:04.231Z" }, + { url = "https://files.pythonhosted.org/packages/04/98/97557a812180338cb1abd32e1cffcc4588f59b5f23e0cb006b2ba95ba64a/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56d8641cf834c2836922899105bd3ce3d0dfc69291d52edf0b4d0436829b34c0", size = 459273, upload-time = "2026-05-18T04:31:50.377Z" }, + { url = "https://files.pythonhosted.org/packages/e8/a8/b4b08dcb7653b8087c6586f7ce649505900e866bbcfe40dc9587af02e686/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2581a94056e55d7d0a31a823ea92bf73749c489ca2285bfdc0fbe6b2bb49d50c", size = 489927, upload-time = "2026-05-18T04:31:42.485Z" }, + { url = "https://files.pythonhosted.org/packages/50/94/3dceea03545d2e5ddfd839f0ddd5e1cecbf1697b5a428d5ba11cef6af95d/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:41bc1199f7523b3f82843c88cbb979180c949caef0342cf90968f178e5d49b01", size = 570476, upload-time = "2026-05-18T04:31:03.071Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f2/d39a5450c3532092b91f81d274360e613c2371bc874a89c7a1a3c5e8d138/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7571e4464cb6e434958f867f7f730b8ab0b75e3f8e5eac0499168486ab3c33a8", size = 465650, upload-time = "2026-05-18T04:30:12.701Z" }, + { url = "https://files.pythonhosted.org/packages/22/24/ed72f68cbc1333ca9b9f2200aa048bb6658ae41709bc1caad4310f4bdffd/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e53a384f76b631c3ae5334ce6a52f0baa3a911eb94a4eac7f160079868b716d5", size = 456398, upload-time = "2026-05-18T04:30:13.784Z" }, + { url = "https://files.pythonhosted.org/packages/0d/64/982ef4a4e5bab5b6e5b6becc8cd5e732f6130a78b855f0abec6439a9a135/watchfiles-1.2.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d20029a60a71a052a24c4db7673bc4de39ab89adbaccbfb5d67987c5d73f424d", size = 465140, upload-time = "2026-05-18T04:31:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/a0/0c/95282abf4ed680b6096010bcfc30c5fa7a041fc5aa5a2ad17a2cc6c75bba/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2cb93af48550faf1cea04c303107c8b75833de7013e57ce27d3b8d21d8d0f58c", size = 630259, upload-time = "2026-05-18T04:31:25.676Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/607c1de1530c4bdcf2cf1d1ecc2505ddba5d96bd43ba9f2b0e79876f850f/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2995c176de7692b86a2e4c58d9ec718f753150a979cb4a754e2b4ffa38e70906", size = 659859, upload-time = "2026-05-18T04:30:24.333Z" }, + { url = "https://files.pythonhosted.org/packages/fa/08/d9e2e0f9e8e6791d33aefc694ad7eefa7f901f63caff84a81ded38692f9c/watchfiles-1.2.0-cp312-cp312-win32.whl", hash = "sha256:7a2cffd17d27d2ecbb310c2b1d8174f222a5495b1a721894afa88ec11e25b898", size = 275480, upload-time = "2026-05-18T04:30:31.307Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e6/9d42569c0102645cc8cea5d8c7d8a1e9d4ada2cb7f05f75e554b8aa2202a/watchfiles-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:f155b3a1b2a5fc89cdc70d47ee5d54e3b75e88efa34982028a35daef9ba00379", size = 288718, upload-time = "2026-05-18T04:32:10.745Z" }, + { url = "https://files.pythonhosted.org/packages/0a/26/88e0dc6ee3898169d7fa22bb6a69cabf2502d2ee25cb8c876d1262d204f8/watchfiles-1.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:8fa585ede612ee9f9e91b18bebf9ba11b9ae29a4e3a0d0cf6fca3e382133f0d5", size = 281026, upload-time = "2026-05-18T04:30:22.23Z" }, + { url = "https://files.pythonhosted.org/packages/d1/4d/70a7feced9f87e2ff26dba42667290f41694fc64646c67261fbb8cab5d5c/watchfiles-1.2.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:01ea8d66f0693b9b60a6541c8d10263091ca9a9060d242f3c1f3143f9aad2c98", size = 399730, upload-time = "2026-05-18T04:31:38.162Z" }, + { url = "https://files.pythonhosted.org/packages/31/3a/0da302f2307aee316922806ebd5726c542cbd787c938271cf14a074c7daf/watchfiles-1.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7ba0480b9a74af058f43b337e937a451e109295c420916d68ad24e3dc02f5e44", size = 392842, upload-time = "2026-05-18T04:30:27.051Z" }, + { url = "https://files.pythonhosted.org/packages/db/ef/d5bdb705c224dbc256aa0c1ec47bf4e61ec52558f2afb44a71a1fe4d7015/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f34e26a19f91f710c08e0183429f0d1d15df734e6bc78c31e77b9ea9c433658", size = 452989, upload-time = "2026-05-18T04:31:11.945Z" }, + { url = "https://files.pythonhosted.org/packages/71/29/5495f2c1661949ef7a35e4d71111d129cfe7606414a26887a919d0a55406/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b4e77f6a55f858504069abd35d336a637555c09bca453dde1ee1e5ada8a6a1fb", size = 458978, upload-time = "2026-05-18T04:30:52.606Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8c/7f9c07c433811c2fffd93e13fdfb7135de9aab5f2ae41be08960fa0047dc/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cb4d80e212f116474a545c21c912b445f16bb0cef9e6a73a498164223e14e2f", size = 490248, upload-time = "2026-05-18T04:31:36.003Z" }, + { url = "https://files.pythonhosted.org/packages/3c/11/d93632febc52fbc21be90231bb7c17fd5387f46c9076fd40a5f9c2ae6910/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b974946a10af379d425e2eef5b62f5c6ebeaccf91d45eaad6f5b27ecd4f91aa0", size = 571847, upload-time = "2026-05-18T04:31:10.862Z" }, + { url = "https://files.pythonhosted.org/packages/55/b4/383173e73aabb07ad1d9c7aa859d95437ac46a6d6a1e11005facda0c9d19/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86bc13c25a8d1fcd70b51d0ce7c9b65e90de5666fcbfd3e34957cc73ee19aeb5", size = 465974, upload-time = "2026-05-18T04:30:17.006Z" }, + { url = "https://files.pythonhosted.org/packages/a7/6c/89b1a230a78f57c52dd8893adb1f92f94411721b6ec12596c56d98c74356/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca148d73dea36c9763aaa351e4d7a51780ec1584217c45276f4fe8239c768b71", size = 454782, upload-time = "2026-05-18T04:30:35.656Z" }, + { url = "https://files.pythonhosted.org/packages/24/62/1732118367cfff0a9fce3bf62ff4bfded09ef5df21d9d446b858b3f70a96/watchfiles-1.2.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:c525543d91961c6955b2636b308569e84a1d1c5f5f2932041ab9ef46422f43e3", size = 465182, upload-time = "2026-05-18T04:30:20.846Z" }, + { url = "https://files.pythonhosted.org/packages/28/96/716f7e5f51339bf22963f3345f9f27d7f3b30e2eadc597e257c881dd3c53/watchfiles-1.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:a204794696ffb8f9b10fba6f7cb5216d42f3b2b71860ccac6b6e42f5f10973b0", size = 629841, upload-time = "2026-05-18T04:31:05.397Z" }, + { url = "https://files.pythonhosted.org/packages/4c/fe/c40783950fd771ccf66ab3ec2722d188a9af1c7f96c6e811f36e40c6e03f/watchfiles-1.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:10d86db20695afe7997ac9e1717637d6714a8d0220458c33f3d2061f54cec427", size = 658028, upload-time = "2026-05-18T04:31:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/71/72/4508db1856d1d87fcbb3b63f4839bab1b5682cb0e8d224d122263c09654a/watchfiles-1.2.0-cp313-cp313-win32.whl", hash = "sha256:eb283ee99e21ad6443c8cdb06ac5b34b1308c329cbdf03fa02b445363714c799", size = 275183, upload-time = "2026-05-18T04:30:59.57Z" }, + { url = "https://files.pythonhosted.org/packages/f9/36/14b76ca57652e5cc5fd1c11f32a261292c08a0d19a00351013c2549cbfb2/watchfiles-1.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:a0f27f01bee51861392bb6b7c4fdb290b27d1eb194e9e28788d68102a0e898d9", size = 288059, upload-time = "2026-05-18T04:32:07.937Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8d/0a85e395398d8d20fadfe5c5d32c726eee17a519e78fb356f2cf7531bffe/watchfiles-1.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:3651aa7058595e9cfb75d35dd5ada2bf9f48a5b8a0f3562821d3e210c507e077", size = 280186, upload-time = "2026-05-18T04:31:54.484Z" }, + { url = "https://files.pythonhosted.org/packages/37/68/36db056f1fdcc5f07302f56e631774d6835bcd6fa3ace402304621d5f9e5/watchfiles-1.2.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:faea288b6f0ab1902ef08f4ca6de005dccf856c4e0c4f21b8c5fce02d90a1b08", size = 399031, upload-time = "2026-05-18T04:30:44.576Z" }, + { url = "https://files.pythonhosted.org/packages/c1/64/01a9d6f66a82a5c101ce939274106cc72759d62427e153f01edd2b9f87c2/watchfiles-1.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:01859b11fd9fbca670f4d5da00fbac282cfea9bd67a2125d8b2833a3b5617ea9", size = 391205, upload-time = "2026-05-18T04:30:25.413Z" }, + { url = "https://files.pythonhosted.org/packages/84/2c/0a44fe058cb4bb7b8ede6b6670698bbb7c0400740e378d00022189b7b31d/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fff610d7bb2256a317bb1e96f0d7862c7aa8076733ee5df0fd41bbe76a24a4f4", size = 451892, upload-time = "2026-05-18T04:32:14.005Z" }, + { url = "https://files.pythonhosted.org/packages/67/a1/351e0d56cd35e6488b5c8b4fb11a809a5bc923e8fe8fed9faf8920be0c89/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b141a4891c995a039cd89e9a49e62df1dc8a559a5d1a6e4c7106d16c12777a55", size = 458867, upload-time = "2026-05-18T04:31:22.279Z" }, + { url = "https://files.pythonhosted.org/packages/d5/7d/9d09605187f1b838998624049fcf8bf47b73c1a3b76901fcac1782f62277/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f22943b7770483f6ea0721c6b11d022947a98eb0acae14694de034f4d0d38925", size = 490217, upload-time = "2026-05-18T04:31:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/60/5d/a17a16eccb182f04188cd308ec24b1a71a9b5c4e7098269cf35d9fa56d02/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1bc6195825b7dcd217968bb1f801a60fd4c16e8eeab5bedc7fe917d7d5995ab4", size = 571458, upload-time = "2026-05-18T04:32:11.875Z" }, + { url = "https://files.pythonhosted.org/packages/d3/3d/4dd457062083ab1938e5dfd45032eb425cee2ac817287ca8ff4356183e5d/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4a4b147f5dca2a5d325a06a832fb43f345751adfbc63204aec30e0d9ca965a2", size = 464707, upload-time = "2026-05-18T04:30:43.492Z" }, + { url = "https://files.pythonhosted.org/packages/c6/71/ea8c57b128f5383de74d0c7d2d9c57ad7c9a65a930c451bd25d524b295b7/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4543579a9bdb0c9560039b4ffddbdb39545707659fbc430ce4c10f3f68d557f9", size = 454663, upload-time = "2026-05-18T04:30:16.061Z" }, + { url = "https://files.pythonhosted.org/packages/53/fd/2e812bf938406d7db351f0703ddd3fc6c061cf30d96153a77bc79a943a44/watchfiles-1.2.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:20aa0e708b920bde876a4aa82dc7dd6ebea228a63a67cda6632c2fc87b787efa", size = 463537, upload-time = "2026-05-18T04:31:44.9Z" }, + { url = "https://files.pythonhosted.org/packages/86/56/d17a7f1dd1bc3035f1072694a551301272f1739c2d8e319c927cb9e29b38/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:d413349d565dab74297f2a63e84a097936be69bf8f3b3801f27f380e32040f44", size = 629194, upload-time = "2026-05-18T04:31:14.141Z" }, + { url = "https://files.pythonhosted.org/packages/be/06/f1ff66bf5cae50aa4062779a0ecd0bbaf15e466195719074078947d9a17d/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f28b2725eb8cce327b9b3ab02415c853011dc55c95832fe90de6bc56f5315f72", size = 656194, upload-time = "2026-05-18T04:31:47.14Z" }, + { url = "https://files.pythonhosted.org/packages/e7/54/a9c7ea9a82a4ac65e7004c0a03920b5cdd2f9c3b678757d9cd425aa51d53/watchfiles-1.2.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b8c8358484d5fa12ef34f05b7f4168eaf1932f408725ff6d023c33ec17bd79d4", size = 400205, upload-time = "2026-05-18T04:32:05.153Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5d/c9ab3534374a4a67450696905d6ef16a04405448b8dc52bd752ae50423d4/watchfiles-1.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9f04b092229ad2c50126dd3c922c8822e51e605993764a33058d4a791ab42281", size = 392508, upload-time = "2026-05-18T04:30:54.849Z" }, + { url = "https://files.pythonhosted.org/packages/26/ca/1ad30103535cf0cecd7b993e8d50edc5351b1820e38f2d22e3df58962feb/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a7ce236284f002a156f70add88efe5c70879cccbb658be0822c54b1306fc09d", size = 452448, upload-time = "2026-05-18T04:30:53.727Z" }, + { url = "https://files.pythonhosted.org/packages/37/a1/ceee2cdf2afbd715fa07758d39c9859513eae411b23196f7fd039e5feedd/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b9909cc2b48468b575eefa944919e1fe8a36c5849d5c7c168f80a8c1db69398e", size = 459605, upload-time = "2026-05-18T04:30:23.312Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f6/421e30fd1cb3907a84ed92ab3f1983e37ba2dca015e9a894a048418417a2/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a37faaed405c67e28e6be45a1fa4f206ef5a2860f27c237db9fa30704c38242", size = 490757, upload-time = "2026-05-18T04:30:47.358Z" }, + { url = "https://files.pythonhosted.org/packages/41/b0/55ed1b97ed08be7bba6f9a541cac15f2a858e1d74d2b07b6da70a82aab00/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9649193aa27bd9ff2e80ff29bfaa93085496c7a3a377592823cc58b77ee88add", size = 568672, upload-time = "2026-05-18T04:30:38.915Z" }, + { url = "https://files.pythonhosted.org/packages/d1/cf/d8ae8a80dd7bafab395ea7681c10237311bbf34d37704a8c744e7cf31fc7/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e4ff8e37f99cf1da89e255e07c9c4b37c214038c4283707bdec308cb1b0ea1f", size = 464197, upload-time = "2026-05-18T04:30:09.914Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/3076c496ca8dafe0e8cd03fcebdfc47be4b1174b4e5b24ff6e396e6b3af2/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:054dc20fd2e3132b4c3883b4a00d72fd6e1f56fdaf89fccd12e8057d74cd74d7", size = 453181, upload-time = "2026-05-18T04:30:14.829Z" }, + { url = "https://files.pythonhosted.org/packages/e5/10/9745e17c98e7b8a86454df0a3c7b5686bd650383f1e9f26e4ebcbd6cc0c0/watchfiles-1.2.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e140ed30ebde76796b686e67c182cff10ea2fbab186fafd1560f74bb5a473a6e", size = 465109, upload-time = "2026-05-18T04:30:28.123Z" }, + { url = "https://files.pythonhosted.org/packages/8f/95/8ef4a95481d3e0cb52d62a06fa6e972e81424be2d9698b91a2fecca9904c/watchfiles-1.2.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:bb7e52ecf68ba46d22df23467b87cffeb2146908aa523ebfe803019618cfda06", size = 630653, upload-time = "2026-05-18T04:31:49.304Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e4/3b3bf36b0f829b50c6ebcb8d031583863c59f923d6a6af3d485e470d0fac/watchfiles-1.2.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:23282a321c8baf9b3a3c4afff673f9fe65eb7fdc2338d765ccad9d3d1916a5ba", size = 657838, upload-time = "2026-05-18T04:31:06.497Z" }, + { url = "https://files.pythonhosted.org/packages/21/b1/6cbbb50c1f3002ab568777d44aa21206dfb8807a840990c4037523b51812/watchfiles-1.2.0-cp314-cp314-win32.whl", hash = "sha256:c0db965c5f79aa49fe672d297cf1febc5ad149b658594944f49a54a2b96270a7", size = 275108, upload-time = "2026-05-18T04:30:06.891Z" }, + { url = "https://files.pythonhosted.org/packages/92/45/190ce6db8dcb4536682cf75d3889ff1a27182a58cb519d343cb6d9ea63d8/watchfiles-1.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:71283b39fd17e5408eb123bd37aeecfd9d54c81fc184421943208aadb879d103", size = 288441, upload-time = "2026-05-18T04:32:12.901Z" }, + { url = "https://files.pythonhosted.org/packages/74/0d/3eae1c2313ab08378431d907c3f8095ecca00f3eda33111cf4f0f2591799/watchfiles-1.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:c5c19526f4e54a00f2666a6c0e9e40d582c09e865055ea7378bf0009aab857b3", size = 280684, upload-time = "2026-05-18T04:31:26.902Z" }, + { url = "https://files.pythonhosted.org/packages/b1/75/fb64e6c25d6b5ca636d03df34ffb1c6e9873303e76d27967e045f8df088f/watchfiles-1.2.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:d73a585accffa5ae39c17264c36ec3166d2fad7000c780f5ef83b2722afb9dd2", size = 398857, upload-time = "2026-05-18T04:32:17.108Z" }, + { url = "https://files.pythonhosted.org/packages/73/4e/9f7adf01754cbf81843722ccfec169d8f26c69778281a302855cecd2ee08/watchfiles-1.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ae99b14c5f21e026e0e9d96f40e07d8570ebee6cafd9d8fc318354606daa7a28", size = 392413, upload-time = "2026-05-18T04:31:07.911Z" }, + { url = "https://files.pythonhosted.org/packages/47/c8/bec626bcc2d69f44b9acb24ce7d60ed7b16b73628eea747fcbd169d8edda/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4429f3b105524a10b72c3a819b091c495d2811d419c1e1e8df773a5a5974f831", size = 452409, upload-time = "2026-05-18T04:31:20.142Z" }, + { url = "https://files.pythonhosted.org/packages/00/b7/b6362068e81e7c556d155a34c35d40ac3ef42d747b06d7f6e5bf58e359c2/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:43d818978d06062d9b22c4fab2ebe44cf5213d42dc8e62bda8c2760cfa2eeb33", size = 458827, upload-time = "2026-05-18T04:32:06.219Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/9a813fa42afb1e0b4625e75f0479826644d3ee8dc287e093799bc01f390c/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9f732dc58b2dbe69e464ccf8fff7a03b0dd0be439da4c0720d3558527d3d6b4", size = 490104, upload-time = "2026-05-18T04:31:56.034Z" }, + { url = "https://files.pythonhosted.org/packages/2f/bf/27dfb6094ca4c9aad21298b5525b6c53cb36121ee454331d05161e58d130/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f200104103feb097de4cab8fe4f5dd18a2026934c7dea98c55a2f5fd6d5a33b", size = 571360, upload-time = "2026-05-18T04:31:57.133Z" }, + { url = "https://files.pythonhosted.org/packages/fb/39/44a096d67270ea93df91d33877dbe91fbda3aa4f8ec2edf799d93eda8736/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:63ac26eefbf4af1741247d6fb68b11c49a25b2f7413fbd318a83a12aaa9cf666", size = 464644, upload-time = "2026-05-18T04:30:57.33Z" }, + { url = "https://files.pythonhosted.org/packages/0e/80/c7472203bad6268e3ef1ad260739704847898938ad7ea8b63a5131f46b50/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c4997d4e4a55f0d02b6cde327322daf3a0400e5df6c6b15948994bf72497925", size = 454771, upload-time = "2026-05-18T04:30:48.736Z" }, + { url = "https://files.pythonhosted.org/packages/51/cf/3b10b268b4b7f0fc26e9debb5eef1998b515887840f444cd3ec80c688755/watchfiles-1.2.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:4c887eba18b7945ac73067a8b4a66f21cd46c2539b2bc68588f7be6c7eb6d26b", size = 463494, upload-time = "2026-05-18T04:31:33.826Z" }, + { url = "https://files.pythonhosted.org/packages/3d/3e/a4302545cd589262a0dc7d140e86f7688eba3f9c72776c27f7e23b8864c4/watchfiles-1.2.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:3416ff151bb6b5a8d8d11664974fbef4d9305b9b2957839ab5a270468fd8df30", size = 629383, upload-time = "2026-05-18T04:31:15.596Z" }, + { url = "https://files.pythonhosted.org/packages/db/99/d5649df0a9a410d45b7c882304d0b790903ac9b6e8f2cfd12114e0c6b9f2/watchfiles-1.2.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:0e831a271c035d89789cffc386b6aa1375f39f1cd25eb7ca0997e4970d152fc5", size = 656093, upload-time = "2026-05-18T04:31:58.707Z" }, + { url = "https://files.pythonhosted.org/packages/92/b9/362702539275019a54dd2e94511b31a9b89c5f9e6a21966de7eb692549fc/watchfiles-1.2.0-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:37a6721cdf3f65dbb13aa9503510ccb4451603ac837e44d265d7992a597e1374", size = 400109, upload-time = "2026-05-18T04:31:16.879Z" }, + { url = "https://files.pythonhosted.org/packages/8f/75/71d5ba62db781e5587bded1d944c675374bc4aa37ff33d5018d98e8b6538/watchfiles-1.2.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2b37d10b5a63bd4d87e18472d80fa525bd670586fae62e5dd580452764879b65", size = 392167, upload-time = "2026-05-18T04:31:28.058Z" }, + { url = "https://files.pythonhosted.org/packages/3c/01/c66dd95d0423fe30d31820e2d1d5bda773764131bbb6ac0cb1cf303ac328/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a105bc2283f67e8fbec74253ec2d94925de92ed72c0393f1206bf326b7b7b69", size = 452372, upload-time = "2026-05-18T04:31:00.836Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/2fe99557e72f85627c6a8eed50d889e8d101623e060a22ad75b875cb932d/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5327989a465505f05cfe06f04fa9d0c2fd5432bb243e10e6f012b1bdca3c8579", size = 459596, upload-time = "2026-05-18T04:31:34.96Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/d4acfa0023367428ed48351b3b9b267893037b6cadae55620c61c24bcfd4/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ecb47f183a8025b2aa18b546725c3657e542112ae9c0613a2af79b4fa8d04ad7", size = 490869, upload-time = "2026-05-18T04:31:59.923Z" }, + { url = "https://files.pythonhosted.org/packages/a4/5f/3164cbdce06c9fb95c4f7b9e2f9760b5e2797af43a9ecc317ef42a23a278/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8520a4ab0e37f770afc34459c4f8f7019e153f9124dc101c15538365875d1ab2", size = 571641, upload-time = "2026-05-18T04:32:00.948Z" }, + { url = "https://files.pythonhosted.org/packages/41/e6/85d3731c55e65cd7690f3f803d24c139588aaf863e4bf2148fe7a7fa1a19/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71cd71740ed2c15211ebb237ced4e39a1cdf6f80566e5fe95428da1626f4fde6", size = 464444, upload-time = "2026-05-18T04:30:34.298Z" }, + { url = "https://files.pythonhosted.org/packages/f4/7d/562641012b8b09872742c3b8adf9629ec479fd78f8d68ae4a0c13da8add6/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f88af53d6ddaf72179ef613ddc905e6f4785f712b49b80b3bef9f3525e6194b4", size = 453593, upload-time = "2026-05-18T04:31:23.464Z" }, + { url = "https://files.pythonhosted.org/packages/56/fe/cb8ef3d6f929d14158fdaaad9925985b7310abc9384dcd4d82dd0016fb59/watchfiles-1.2.0-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:cee9d5efd929efdac5f7e58f72b3376f676b64050a91c5b99a7094c5b2317488", size = 465096, upload-time = "2026-05-18T04:31:30.384Z" }, + { url = "https://files.pythonhosted.org/packages/25/91/80908e835e100527a9267147b08c0eee1fa6ab0ffec15edc04d1d44885f7/watchfiles-1.2.0-cp315-cp315-musllinux_1_1_aarch64.whl", hash = "sha256:b718bf356bbc15e559bd8ef41782b573b8ae0e3f177ab244b440568d7ea02cfb", size = 630638, upload-time = "2026-05-18T04:30:49.89Z" }, + { url = "https://files.pythonhosted.org/packages/46/4b/95ab2f256bb4af3cb2eb23b9317bda984ee6e0f11733a5c004a6c95b06e3/watchfiles-1.2.0-cp315-cp315-musllinux_1_1_x86_64.whl", hash = "sha256:922c0e019fe68b3ae392965a766b02a71ba1168c932cebc3733cd52c5fe5b377", size = 657684, upload-time = "2026-05-18T04:31:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/23/f4/7513ef1e85fc4c6331b59479d6d72661fc391fbe543678052ac72c8b6c19/watchfiles-1.2.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4674d49eb94706dfe666c069fc0a1b646ffcf920473492e209f6d5f60d3f0cc2", size = 403050, upload-time = "2026-05-18T04:30:36.753Z" }, + { url = "https://files.pythonhosted.org/packages/27/0b/a54103cfd732bb703c7a749222011a0483ef3705948dae3b203158601119/watchfiles-1.2.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:094b9b70103d4e963499bdea001ee3c2697b144cd9ae6218a62c0f89ec9e31db", size = 396629, upload-time = "2026-05-18T04:32:03.268Z" }, + { url = "https://files.pythonhosted.org/packages/5e/2c/73f31a3b893886206c3f54d73e8ad8dee58cdb2f69ad2622e0a8a9e07f4e/watchfiles-1.2.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0ef001f8c25ad0fa9529f914c1600647ecd0f542d11c19b7894768c67b6acb7", size = 457318, upload-time = "2026-05-18T04:31:01.932Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f9/45d021e4a5cc7b9dd567f7cbb06d3b75f751a690063fb6cc7ec60f4e46b7/watchfiles-1.2.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a88fc94e647bc4eec523f1caa540258eb71d14278b9daf72fa1e2658a98df0f0", size = 457771, upload-time = "2026-05-18T04:30:56.331Z" }, ] [[package]] @@ -7505,6 +10198,122 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/84/44687a29792a70e111c5c477230a72c4b957d88d16141199bf9acb7537a3/websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526", size = 58826, upload-time = "2024-04-23T22:16:14.422Z" }, ] +[[package]] +name = "websockets" +version = "16.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/02/b9a097e1e16fee4e2fd1ec8c39f6a9c5d6257bae8fa12640caf869f54436/websockets-16.1.tar.gz", hash = "sha256:299468cbe42e2b9981134c7c51d99387d8a7bf562b00183b3eec53f882846dad", size = 182530, upload-time = "2026-07-10T06:32:57.734Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/31/cd11d2796b95c93645bac8e396b0f4bac0896a07a7b87d473bfc359f02c3/websockets-16.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:de72a9c611178b15557d98eabd3101c9663c4d68938510478a6d162f99afd213", size = 179772, upload-time = "2026-07-10T06:30:22.983Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9b/34306d802f9b599eab041688a2086318037560cfae616a860234cca575b6/websockets-16.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:37b0e4d726ffea3776670092d3d13e1cb605076f036a695fd1259de0d9b9fe02", size = 177457, upload-time = "2026-07-10T06:30:24.636Z" }, + { url = "https://files.pythonhosted.org/packages/06/3a/36ebbb978a7af70ff952afe5b22561264967164e9ad68b6734cae94efeb4/websockets-16.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:00d50c0a27098fcb7ab47b3d99a1b1159b534dbcd959fbf05113ebc37e5f927b", size = 177737, upload-time = "2026-07-10T06:30:25.954Z" }, + { url = "https://files.pythonhosted.org/packages/17/d7/944f341d0d3c0450ffd3d171479531df1818cb1df1623af4065113999c44/websockets-16.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1acb698bff1da1782b31aebd8d7a24d7d05453964abcd7d03dbf6e25893908e8", size = 186244, upload-time = "2026-07-10T06:30:27.235Z" }, + { url = "https://files.pythonhosted.org/packages/32/e5/a9b98fc49ef0214718a9c839c6c63856a921877256ec46f371be32decfa8/websockets-16.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc2c453f3b5f99c56b16e233aad5299860558487d26adb2ed27a00c14ca24b8c", size = 187484, upload-time = "2026-07-10T06:30:28.615Z" }, + { url = "https://files.pythonhosted.org/packages/ad/7a/a575b52ca090b1976ffbe4b5f0762d03f399dfcb48eab883101331be71a9/websockets-16.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1a9f08a0728b0835f1c6abe1d9b746ab3de49b7336a0e1919cf96be1e76273eb", size = 190143, upload-time = "2026-07-10T06:30:29.91Z" }, + { url = "https://files.pythonhosted.org/packages/7c/40/705fbbd5677242fd36f724e9a94103e6bbdcb7d71e8f4498bfc1a8a7d413/websockets-16.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a089979d6173b27af18026c8d8b0077f83669a9169174482c4651e9f5739a5b6", size = 188004, upload-time = "2026-07-10T06:30:31.357Z" }, + { url = "https://files.pythonhosted.org/packages/8e/0c/58227c8d66b1c4060c53bac8e066fb4fe2603060408e934f48660a448d72/websockets-16.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a3c18dba232ec2b92a68579c9fed8ff5a18f853d1e09fc0b6ca3159e94f689fe", size = 186689, upload-time = "2026-07-10T06:30:32.712Z" }, + { url = "https://files.pythonhosted.org/packages/d0/d0/5c1314782594aa347e0f18808ee277a61986a2a2f9f470df9893183995bd/websockets-16.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c1eb7df4170d5068892a8834fb5c07b9552353deb0dbeb0bff3820481ae4792", size = 184559, upload-time = "2026-07-10T06:30:34.127Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e6/109c6f16850fd674b7e3d0e58b8987f05d3881abaa25f42a9faf5e85f097/websockets-16.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c522bd48e625b6d557aa228967258d6d3da031c4cc21d3352fb302479aa9ba0a", size = 186997, upload-time = "2026-07-10T06:30:35.397Z" }, + { url = "https://files.pythonhosted.org/packages/ed/ec/6afa1aebc59426438b85cf7a3868c53a89005e2250a648c99e99943b90a4/websockets-16.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:d106396927a7f00b0f3a69215c3357f87bf0bca6844247121f7e8291e826a3b1", size = 185621, upload-time = "2026-07-10T06:30:36.88Z" }, + { url = "https://files.pythonhosted.org/packages/27/24/c038fe8682e9345bfa422d2cc5cc68b0491ab942c92e176bf8dfa6e8331f/websockets-16.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d71bed12909b8039955536e192867d02d76cd3797cedfd0facf822e7668636c3", size = 187384, upload-time = "2026-07-10T06:30:38.096Z" }, + { url = "https://files.pythonhosted.org/packages/30/ca/dc0ef2be39c67394e24bc982a0af59cd6249bf2f4e4272813c5c505d0da9/websockets-16.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:9c1cf6f9a936b030b5bed0e800c5ee32069338129084546baf5ff5014dc62fa9", size = 185258, upload-time = "2026-07-10T06:30:39.579Z" }, + { url = "https://files.pythonhosted.org/packages/17/6b/3ffecd83ca3404b41fbdf8e9b178e55b529cd59bf64ea08b5a37b616b568/websockets-16.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:3fd3e6a7af2c8fcdcf4ffbeaf7f54a567b91a83267204187797f31faaa2a4efa", size = 186050, upload-time = "2026-07-10T06:30:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/75/26/2e068497c78f31591a610ab7ef6d8d383ecadbe98f9121e1ebda77ef6d2b/websockets-16.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:dddd27175bf640acae5561fa79b77e8ec71fc445816200523e5c19b6a556fb72", size = 186273, upload-time = "2026-07-10T06:30:42.309Z" }, + { url = "https://files.pythonhosted.org/packages/44/ab/4dc049cb2c9e1be3a2c6fef77118f9c5049979e99cd56a97759d2f40f980/websockets-16.1-cp310-cp310-win32.whl", hash = "sha256:cce36c80b3f2fede7942f1756d3d885fa6fa086766c8c1bcf00695ab80f0d51a", size = 180157, upload-time = "2026-07-10T06:30:43.565Z" }, + { url = "https://files.pythonhosted.org/packages/6d/4f/5e010ce5f66a8e5df380843f704ada508195a021c0c8a0f933639c9ee1c0/websockets-16.1-cp310-cp310-win_amd64.whl", hash = "sha256:115fc4695b94bb855995b23fb1abcb66099a5995575d3d5bc5605a616c58d0eb", size = 180458, upload-time = "2026-07-10T06:30:45.01Z" }, + { url = "https://files.pythonhosted.org/packages/9e/13/d47429afcc2c28616c32640009c84ea3f95660dab805766345b9682468e0/websockets-16.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a9b1d7a63cba8e6b9b77e499a81eab29d31100298d090ad4507d1048c0b9cae0", size = 179770, upload-time = "2026-07-10T06:30:46.308Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c7/2f0a722039a1e0107be73ed672ba604449b4956e48733e8e6b8a005aea42/websockets-16.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bedbc5efeb96621aa2921d2d92608246691399418cac22acba427eb11877ea1f", size = 177455, upload-time = "2026-07-10T06:30:47.601Z" }, + { url = "https://files.pythonhosted.org/packages/43/6a/c26b0ae449e93d256ce5cdd50d5fe97b575a63e8dcd311a1faa972fd6bc6/websockets-16.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fd847ab82133015afe65d778e7966ab42dba16bd7ad2e5b8a7918db6539f3f94", size = 177731, upload-time = "2026-07-10T06:30:49.102Z" }, + { url = "https://files.pythonhosted.org/packages/cc/3f/381550b344a02f0d2f84cda25e79b54575291bc7022128a41163fe8ba5b0/websockets-16.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e2fb33ccb16ee40a95cc676d7b0ff451a9a2632f11a0dbc2e666326892b2e1de", size = 187066, upload-time = "2026-07-10T06:30:50.505Z" }, + { url = "https://files.pythonhosted.org/packages/4a/87/5ab1ec2086910f23cfb9ec0c1c29fbcc24a9d190b5198b1557c00ce4a47e/websockets-16.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f15b6d9ea9c2eaf6ccab964a082b09bfa6634a495bb0c2e9e7ee6943f58976", size = 188301, upload-time = "2026-07-10T06:30:51.835Z" }, + { url = "https://files.pythonhosted.org/packages/75/4b/bbbb8e6fac4cfc53d7aaa69a3d531bf10799354b0021f4b58914aced8c1a/websockets-16.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:638cf57c48b4ad8ac1ff1e453f4f97db2426b690ddc111e6da96b27b4a340bc3", size = 191594, upload-time = "2026-07-10T06:30:53.229Z" }, + { url = "https://files.pythonhosted.org/packages/5c/da/6c0c349443d6e999f481e3d9a0e57e7ac2956d75d6391bec24b92af3fe13/websockets-16.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c1c85f61bc9d5eac57ce705d848dc2d2ce3680638300bf4e1da7d749e2cf4ce", size = 188862, upload-time = "2026-07-10T06:30:54.744Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ea/a368d37c010425a5451f42052fe804e754e23333e8448aef5d55c8a8d64f/websockets-16.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:eeab6d27f51c7e579023c971f5e6dff200deadf01faf6831beaecd32052dfaef", size = 187633, upload-time = "2026-07-10T06:30:56.055Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4e/2ecd59add10d0855ec03dbdedfcdacdbd1aaabcd44b7dcbeda27538662e9/websockets-16.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2ed64e5a97b0b97a0b66e18bfe281317a75fbbd5afe692f939ea8d14a4292f2c", size = 185089, upload-time = "2026-07-10T06:30:57.444Z" }, + { url = "https://files.pythonhosted.org/packages/6f/eb/c6c3dcd7a01097bb0d42f4e9ef21a2c2a491d36b77cd0870ab59f9e8e77f/websockets-16.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9b3b021d0ed4bc16eea9775f62c9fa71acdacba0fc790b38581754dedf29ca60", size = 187790, upload-time = "2026-07-10T06:30:58.731Z" }, + { url = "https://files.pythonhosted.org/packages/9b/3e/775d36885d5e48ab8020aaf377de0ff5fbeb8bc2682a7e46419e4a14521c/websockets-16.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:6eb604a4167f0a0d53c2243dfc667a29f0b43c3436057184e070bb82a1000fa2", size = 186381, upload-time = "2026-07-10T06:31:00.355Z" }, + { url = "https://files.pythonhosted.org/packages/ad/90/6305c00812a92e47d0582604c02bd759db0118bbafc13f707d712dbcf898/websockets-16.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9a3f125e44c3e34d61d111652e608e0f5b85ce08c225c8d56ad0eb822fa40030", size = 188193, upload-time = "2026-07-10T06:31:01.677Z" }, + { url = "https://files.pythonhosted.org/packages/f6/32/96bf8302c81d961585b4d34a2ddd3f229782f9b8c57bc78bbf98f1b1a4ac/websockets-16.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8fdf0b00d0d1f30d1f06a92cab46fe542eec3eb302a7aee7163f142d0780f216", size = 185771, upload-time = "2026-07-10T06:31:03.062Z" }, + { url = "https://files.pythonhosted.org/packages/e8/1f/e8fe44b1d2dc417d740d9959d28fd2a846f268e7df38a686c04ac7dfe947/websockets-16.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:67b56828712f5fa7852de4c0265c28827311a657a4d275b7312ed0d1a918bee4", size = 186803, upload-time = "2026-07-10T06:31:04.34Z" }, + { url = "https://files.pythonhosted.org/packages/a5/29/b07d3a4e1eb2ab03e94e7f53f0c7a628e85fde6ad86011f7afd08f27b985/websockets-16.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:39c7e7730be33b8f0cd6f0aa8e8c82f9cdd1813f159765e073b2ece65f4824b5", size = 187041, upload-time = "2026-07-10T06:31:05.567Z" }, + { url = "https://files.pythonhosted.org/packages/a6/fd/e0abb8acc435642ac4a671490f6cf781c882f3fe682cdced9080ea455ab5/websockets-16.1-cp311-cp311-win32.whl", hash = "sha256:c54fe94fb2f11e11b48920c5f971e298cec73ac35db56efe57a49db63dfc95d4", size = 180158, upload-time = "2026-07-10T06:31:06.929Z" }, + { url = "https://files.pythonhosted.org/packages/81/06/85574d9458d3b913090087b817df0cc47b68e9a01dd0ab6ac04b77f49b0a/websockets-16.1-cp311-cp311-win_amd64.whl", hash = "sha256:f9f4fb9ae8b802e55609685db98382d48fd3feb1397804e1e774968dea0f28c7", size = 180456, upload-time = "2026-07-10T06:31:08.247Z" }, + { url = "https://files.pythonhosted.org/packages/a1/52/748c014f07f4e0e170c8932de7e647a1511d5ab3049cd978797136aee577/websockets-16.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b6aa3f7ad345cf3862c21f4fbf2ef5e14d911348476c2845e137c091fe3a3f0b", size = 179798, upload-time = "2026-07-10T06:31:09.664Z" }, + { url = "https://files.pythonhosted.org/packages/8b/5e/2a2e64d977d084e49d37c187c26c056daaff41965be7300cd5dbde6f8b07/websockets-16.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b43fcfb521ac2f34ba80b7b8ea16303e4ad82dd8af667bf40839ad3a5d37b164", size = 177478, upload-time = "2026-07-10T06:31:11.072Z" }, + { url = "https://files.pythonhosted.org/packages/aa/12/5b85b4e75d697e548a94962ce5c036b05dd21cb9545759d555c5586422fc/websockets-16.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2bd3e12cd9afbe2baedae0b1eeade8ba64329b60fe2f9abdc966bd10fd2c2ef5", size = 177746, upload-time = "2026-07-10T06:31:12.386Z" }, + { url = "https://files.pythonhosted.org/packages/9d/62/79b1c8f0cee0da648b4899e1c5b0dbd3aa59846985136a54854db6827ab4/websockets-16.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:35f41979c8623df9bd30d949d82010a8fda5c56ff12cd8508a5b7272b6d4b53a", size = 187345, upload-time = "2026-07-10T06:31:13.754Z" }, + { url = "https://files.pythonhosted.org/packages/25/34/b7c5c52c2f24280e1c017acb7ad491a566750a5cceca7f3cf999373bba21/websockets-16.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a24d1f35aef07d794a16c853c688e74956c50239bec37b4f2de080056046419b", size = 188581, upload-time = "2026-07-10T06:31:15.075Z" }, + { url = "https://files.pythonhosted.org/packages/bc/37/604193bebcbeffe96fdf795960b83a15d600880c64dc17ec9c31c5b3427d/websockets-16.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0c64c024ddf7a35331b21fcddb562a039c275d2c82e8c2d12939e7da23997270", size = 191362, upload-time = "2026-07-10T06:31:16.395Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b4/5ee27575b367d7110d4d13945e2a9de067ec84dc71e54b87f01e38550d9a/websockets-16.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c3e99757f5baafe20fc598e202ea6f5b0b265186ad38d0a17bd8beca16296955", size = 189216, upload-time = "2026-07-10T06:31:17.776Z" }, + { url = "https://files.pythonhosted.org/packages/7e/22/3e2dcc78d85fc5d9d814895ce6d07d0dfacc0f6aaa1d151f2b8c8d772299/websockets-16.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:353f3bc6e058ac1ccab4b3588e8598837a8c04cfc8351233e6d523be675d844c", size = 187971, upload-time = "2026-07-10T06:31:19.152Z" }, + { url = "https://files.pythonhosted.org/packages/9e/2f/cd271717b93d5ee19626cb5e38a85baab745c86e33db7c31a3ac729b31b8/websockets-16.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0352f5b38b40e857b6428d468fa21dbb4dd4a567d933c26d9831b4efe1b92f43", size = 185381, upload-time = "2026-07-10T06:31:20.665Z" }, + { url = "https://files.pythonhosted.org/packages/78/91/6ad6f2f1426317b5001bd490534208c7360636b35bac1dec2e0c22bfc40e/websockets-16.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:70bd789afab579602968c39f21cb925466505f3edff22f0ae852bca54978a4f9", size = 188015, upload-time = "2026-07-10T06:31:22.024Z" }, + { url = "https://files.pythonhosted.org/packages/c7/6d/533733132ab4c07540efd4a8f0b9a435d3a5059b2f26cc476ace1abf7f45/websockets-16.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d0fb4b46f121eccd539353baebd1083a8767a9a351109453d1d1caecd1ba40c2", size = 186619, upload-time = "2026-07-10T06:31:23.376Z" }, + { url = "https://files.pythonhosted.org/packages/08/73/16c059f3d73b3331eba10793704afa4faa9939234fb08ef7dca35794e8f0/websockets-16.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c14b6634af01541e4efe2954fd8f263386f7aa6d37c01e55dd8109fd17661452", size = 188497, upload-time = "2026-07-10T06:31:25.024Z" }, + { url = "https://files.pythonhosted.org/packages/4d/89/9a8fae7dd2acdcfb1a8844c29fe42b518a04b64fce38a0923b6290e452f1/websockets-16.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a58532c49a851bcb481e58c1be23b315c17fe2fbbed509d75aeea12f543d2c15", size = 186051, upload-time = "2026-07-10T06:31:26.291Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/b240c7dd6a0e0c59c1f68377cc3015263521080c327c15f5e753c1f6d378/websockets-16.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4e969170c3b08e1d8dabd990fef1fa702c4233aeaabec33f871806e444f6a0e4", size = 187029, upload-time = "2026-07-10T06:31:27.605Z" }, + { url = "https://files.pythonhosted.org/packages/50/35/524e3fac40e47d6fdcf6c4b2c95ef1bc8a97e01593c90eff86621df7b716/websockets-16.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ff9b000064b88787ba9f7a3cb2af2b68a658ca5aad76458a46469e7124b678a0", size = 187308, upload-time = "2026-07-10T06:31:28.927Z" }, + { url = "https://files.pythonhosted.org/packages/00/13/56840cf62c8859af6ba22b9529da937332468c80f32b598753e8a66d3990/websockets-16.1-cp312-cp312-win32.whl", hash = "sha256:b9f5d83f80f4d7c4bba6d97f3755ac05850c784dce0fd2ab371c4e41172f53ff", size = 180161, upload-time = "2026-07-10T06:31:30.316Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ff/87eb9eb44cb62424a8d729834f2b0515a47e2669fabec29820268f4d50a1/websockets-16.1-cp312-cp312-win_amd64.whl", hash = "sha256:6852c9f653966c16109d3b6f31181fd734f7914927e3f0fa1117af7a18c9aa21", size = 180462, upload-time = "2026-07-10T06:31:31.708Z" }, + { url = "https://files.pythonhosted.org/packages/d9/63/df158b155420b566f025e75613424ad9649a24bcb0e9f259321ab3d58bea/websockets-16.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:b0232ed141cec3df2af5a3959a071c51f40036336b0d37e17faf9ef52fc73e47", size = 179791, upload-time = "2026-07-10T06:31:33.108Z" }, + { url = "https://files.pythonhosted.org/packages/74/cf/00fe9414dfeafa6fe54eae9f5716c8c8e9ac59d192be3b893c096d395846/websockets-16.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a71b73d143991714144e159f767b698f03c4a70b8a65ae1733b650cff488045b", size = 177472, upload-time = "2026-07-10T06:31:34.522Z" }, + { url = "https://files.pythonhosted.org/packages/8b/76/b10633424d40681b4e892ffd08ca5226322b2426e62d4ab71eae484c3a32/websockets-16.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:187323204c3b2fc465e8fc2609e60437c521790cb9c1acb49c4c452a33e57f37", size = 177737, upload-time = "2026-07-10T06:31:35.964Z" }, + { url = "https://files.pythonhosted.org/packages/dc/61/d3bb03b2229bb1afd72008742d586cf1ea240dce64dd48c71c8c7fd3294c/websockets-16.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9dba74233c8c3ce368850818c98354dad2570f57231b3fd3bd00d7aa57628881", size = 187403, upload-time = "2026-07-10T06:31:37.496Z" }, + { url = "https://files.pythonhosted.org/packages/26/16/cc2e80478f688fc3c39c67dc1fac6a0783858058914ebc2489917462cb42/websockets-16.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63339bc8c63c86a463177775cb7c677691f5bcfac7b3b2f01b286d42acd41600", size = 188639, upload-time = "2026-07-10T06:31:38.86Z" }, + { url = "https://files.pythonhosted.org/packages/15/d6/ad87b2507e57de1cbf897a56c963f2925962ed5e85fbe06aaa83ced27acd/websockets-16.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:23e545ea8ae4263e37cdfd4e22a217f519e48e432728bc461185bbf585f38a83", size = 190078, upload-time = "2026-07-10T06:31:40.218Z" }, + { url = "https://files.pythonhosted.org/packages/9e/1a/5b37b3fd335d5811f29fc829f2646a3e6d1463a4bf09c3100708684c766e/websockets-16.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2237081454846fb40403a80ba86d82e2038b9c45865ab96af0abe7d002a91045", size = 189267, upload-time = "2026-07-10T06:31:41.523Z" }, + { url = "https://files.pythonhosted.org/packages/42/98/06afc33e9450d4230f94c664db78875d90f5f6a5fb77f0bc6ec15ae74e1c/websockets-16.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5f5218de1ed047385ca53744caba9435d65f75d008364970a3fae95a05812cf9", size = 188022, upload-time = "2026-07-10T06:31:42.838Z" }, + { url = "https://files.pythonhosted.org/packages/8c/bf/42fef5d5887c18cf2d148b02debf56cecb9cfbffc68027cde9b12c8f432c/websockets-16.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:75c98e3920039d0edff03b74478ada504b7ce3a1bc406db2cabfca84320f7baf", size = 185435, upload-time = "2026-07-10T06:31:44.219Z" }, + { url = "https://files.pythonhosted.org/packages/a0/9b/8021c133add5fe40ed40312553a6cd1408c069d7efe3444ad483d4973ed3/websockets-16.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1facd189d8190af30487a55b4c3688484dd50801628a3b5b2ccd26db08e67057", size = 188080, upload-time = "2026-07-10T06:31:45.986Z" }, + { url = "https://files.pythonhosted.org/packages/69/54/1e37384f395eaa127383aab15c1c45e200890a7d7b99db5c312233d193e0/websockets-16.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:cc0c6a6eef613c7da32d4fb068f82ef834b58134f6a16b54e6c1e5bf9529ab3d", size = 186678, upload-time = "2026-07-10T06:31:47.449Z" }, + { url = "https://files.pythonhosted.org/packages/68/79/1caeacab5bc2081e4519288d248bc8bd2de30652e6eaa94be6be09a1fe5b/websockets-16.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:ad9411eded8988b879be6038206698bf7106c85a78f642c004485bcb95be17eb", size = 188554, upload-time = "2026-07-10T06:31:48.886Z" }, + { url = "https://files.pythonhosted.org/packages/ee/83/b3dca5fad71487b726e31cb0acf56f226792c1cc34e6ab18cbf146bd2d74/websockets-16.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:cd68f0914f3b64694895bc5e9b14e8b447e41d7bf5ffaf989bb8dcb5e2dfdce7", size = 186109, upload-time = "2026-07-10T06:31:50.508Z" }, + { url = "https://files.pythonhosted.org/packages/5b/0b/8f246c3712f07f207b52ea5fb47f3b2b66fafec7303162644c74aed51c6a/websockets-16.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fef2debfe7f7ebdda12176f26166f95b7af17af05ba06150fcf889032e0213e9", size = 187061, upload-time = "2026-07-10T06:31:51.861Z" }, + { url = "https://files.pythonhosted.org/packages/47/eb/27d6c92a01696b6495386af4fc941d7d0a13f2eab2bf9c336111d7321491/websockets-16.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a3cd6c9b798218798f4bb7b2e71c38f0e744bb94ca537b13376f88019d46384d", size = 187347, upload-time = "2026-07-10T06:31:53.246Z" }, + { url = "https://files.pythonhosted.org/packages/6b/d5/eeee439921f55d5eaeabcea18d0f7ce32cdc39cb8fc1e185431a094c5c7b/websockets-16.1-cp313-cp313-win32.whl", hash = "sha256:84c170c6869633536921e4474b1cce7254c0c9b0053ef5725f966cee47e718e4", size = 180149, upload-time = "2026-07-10T06:31:55.058Z" }, + { url = "https://files.pythonhosted.org/packages/a3/03/971e98d4a4864cf263f9e94c5b2b7c9a9b7682d77bfbba4e732c55ee85a9/websockets-16.1-cp313-cp313-win_amd64.whl", hash = "sha256:bef52d327d70fa75dad93ee61ea2cb1d1489aca9f35c188833563f5a3b4df0a5", size = 180458, upload-time = "2026-07-10T06:31:56.767Z" }, + { url = "https://files.pythonhosted.org/packages/8d/e6/da1dc11507f8118145a81c751fe0c77e5e1c11b8554496addb39389e2dc2/websockets-16.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f881fca0a45dd6789939bd6637cd98169b92f1c3fdc78262f2cb9ec2cb1f324e", size = 179833, upload-time = "2026-07-10T06:31:58.19Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ac/c0d46f62e31e232487b2c123bc3cfd9a4e45684ca7dc0c37f0987f29baae/websockets-16.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:30c379d5b207d3a7f0ba4c2e4602a895b0bcc63fb5f5371a4ae7fbddb03b672b", size = 177524, upload-time = "2026-07-10T06:31:59.563Z" }, + { url = "https://files.pythonhosted.org/packages/4a/33/abd966074b34a51e4f134e0aaed80f5a4a0a35163ea5ac58a1bc5a076d23/websockets-16.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:98ab58a4faa72b46da0127ccc1931dcbfc0985b0778892300a092185910c4cbe", size = 177743, upload-time = "2026-07-10T06:32:00.959Z" }, + { url = "https://files.pythonhosted.org/packages/ea/30/646e47b8a8dff04e227bdab512e6dde60663a647eeac7bbd6edddd92bbc5/websockets-16.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8e9c4e369fc181b2d41a99e01477215cecdc8546a39f7d41a59cc0a7065a0b09", size = 187474, upload-time = "2026-07-10T06:32:02.54Z" }, + { url = "https://files.pythonhosted.org/packages/d2/72/890ab9d77494af93ea65268230bfbc0a90ba789401ed7a44356a44785644/websockets-16.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0704df094b2d5fa7f6f410925a594c2a5c9a09167731a76292e5410934208209", size = 188717, upload-time = "2026-07-10T06:32:04.156Z" }, + { url = "https://files.pythonhosted.org/packages/d5/aa/baedbbaa6bf9ed6029617ed5e8976535bd805f483ca9b3484e7ad9ee08bf/websockets-16.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b22b1f4950f6ab7126623329c3b47b3b90a14c05db517f2db2a026ad6c928352", size = 190090, upload-time = "2026-07-10T06:32:05.822Z" }, + { url = "https://files.pythonhosted.org/packages/52/4f/d813ec94e18002571ef4959d87a630eff6e01b72a51bcb0832b75ae8c51a/websockets-16.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1ae4a686a662964a6671069f84f7f908cc3475e782227726b0c622c715962105", size = 189320, upload-time = "2026-07-10T06:32:07.223Z" }, + { url = "https://files.pythonhosted.org/packages/b8/3c/8ec52a6662f3df64090fba28cd521d405d54759268d8e820477037e8c80d/websockets-16.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:856bdd638f8277f86465057bfdd4da097c73058fb0f9d2bd5baea29e2bf2d367", size = 188068, upload-time = "2026-07-10T06:32:08.586Z" }, + { url = "https://files.pythonhosted.org/packages/96/7f/f0ae6042b14f86fa5f996c6563ea4cf107adc036ccbedc9d4f418d0095f9/websockets-16.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9003a1fde1c21a322a3ca3fa0c4bda8c639da81dbc925162766086643b05ba87", size = 185493, upload-time = "2026-07-10T06:32:09.968Z" }, + { url = "https://files.pythonhosted.org/packages/89/ad/5ffc53af9939c49fd653d147fa5b8f78ced1f6bce6c49a7446860945b0ce/websockets-16.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:39e947b1f5fdab045174306e3916785bf3ed537648acc1549827c08c33b10953", size = 188141, upload-time = "2026-07-10T06:32:11.434Z" }, + { url = "https://files.pythonhosted.org/packages/67/62/729206c0ee577a4db8eae6dd06e0eef725a1287c6df11b2ef831d003df31/websockets-16.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5dd0e666b5931c0509cf65714686a1c5126771e663a79ac5d40da4f58b1f9502", size = 186653, upload-time = "2026-07-10T06:32:12.845Z" }, + { url = "https://files.pythonhosted.org/packages/1b/86/e8806a99ec4589914f255e6b658853fe537bf359c05e6ba5762ad9c27917/websockets-16.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:a0285df7925657ad65a65fb8dc330808bce082827538fd50ef45fa12d1fc5bca", size = 188614, upload-time = "2026-07-10T06:32:14.236Z" }, + { url = "https://files.pythonhosted.org/packages/89/38/ac554e2fc6ff0b8deeff9798b92e7abd8f99e2bd9731532e7033de208220/websockets-16.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:82d1c2cab3c133e9d059b3a5420bed9376bd30e21c185c63dda4ddadf6ddda47", size = 186165, upload-time = "2026-07-10T06:32:15.626Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c5/4ef4d8e53342f94f3c49e1ae089b32c1e8b3878e15e0022c7708c647f351/websockets-16.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:c39907f1eaf11f6277def65aa02d68f30576b693d0c1ca332aafa3caa723ac6d", size = 187119, upload-time = "2026-07-10T06:32:17.114Z" }, + { url = "https://files.pythonhosted.org/packages/3a/33/4788b1dd417bd97eeb2698af3b9df6775ac656f96e9987da0419a067602f/websockets-16.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:45c5ea55446171949eb99fd34b771ceddd511ca21958d40d0197ced33159e5ee", size = 187411, upload-time = "2026-07-10T06:32:18.629Z" }, + { url = "https://files.pythonhosted.org/packages/30/38/00d37aad6dc3244ce349e2864815362e50b3cfc00cac28d216db20efe40f/websockets-16.1-cp314-cp314-win32.whl", hash = "sha256:b8ef8b1c8d6bd029a475ac432e730fba2dfd456715d26c473e2a82291024b99c", size = 179822, upload-time = "2026-07-10T06:32:20.233Z" }, + { url = "https://files.pythonhosted.org/packages/9d/37/2a8cb0eaddee5eaebda47a90a3ba0898d1ce3d866b02a4857fea17d82e5b/websockets-16.1-cp314-cp314-win_amd64.whl", hash = "sha256:7358ff21632b5d062707f73e859c824f1c3807e73d8ca25e71caca7c4cdcf145", size = 180167, upload-time = "2026-07-10T06:32:21.749Z" }, + { url = "https://files.pythonhosted.org/packages/07/5a/262ad5fcaef4198997b165060f09a63f861e76939b1786ab546ccc3f8120/websockets-16.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d0f38f4c3e9b359e257c339c2cc1967ccaeedb102e57c1c986bdce4bf4f32268", size = 180166, upload-time = "2026-07-10T06:32:23.278Z" }, + { url = "https://files.pythonhosted.org/packages/1a/c7/36377db690f4292826e4501a6dec2801dc55fd1cf0405923b04937e478df/websockets-16.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:3c3d2cbd1602593bad49bd86fa3fbb25407d87a3b4bf8857c0ac5ac4914e1901", size = 177697, upload-time = "2026-07-10T06:32:25.164Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c7/07171abce1e39799a76f473608580fe98bd43a1230f5146159622c02bccf/websockets-16.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:36069b74671e7e667f48a7484249f84c45a825a134c8b1bdc01875d0daa10d79", size = 177902, upload-time = "2026-07-10T06:32:26.564Z" }, + { url = "https://files.pythonhosted.org/packages/14/17/c831f48e250bc4749f57c00dcce73337c41cd32f6d59a64567b84e782601/websockets-16.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:587f83c2ce8a5d628e166384d77fa7f0ac69b9007d515ab442123e6615aa8da3", size = 187766, upload-time = "2026-07-10T06:32:27.981Z" }, + { url = "https://files.pythonhosted.org/packages/2c/2e/4dfe63e245b0ecfaf470cf082d25c6ce35808159135fd88c82653a6b11ab/websockets-16.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a6db7972d52bc1b66cefe2246902e256cbaebc9ba8a45eac09343d7eb6671b2", size = 188939, upload-time = "2026-07-10T06:32:29.365Z" }, + { url = "https://files.pythonhosted.org/packages/ba/e5/5faf65aebd9562f6b4bc473d24ce38cc56f84eb5f5bee66ed9b86733f93c/websockets-16.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e7d6014888a0632e1ed7a4095248bb3095232999447f2d83bfb1900987dd9ed9", size = 191081, upload-time = "2026-07-10T06:32:30.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/cd/2634f2f2c0556c1aae6501ed6840019cc569dd6fdbcac6494378daea4dc0/websockets-16.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9cb074d150e4ad2a77aa8a332c2be85f3f64f2681519d2570c1225c12c9821ff", size = 189513, upload-time = "2026-07-10T06:32:32.399Z" }, + { url = "https://files.pythonhosted.org/packages/59/bb/2c700b51196104f09715b326b1f092ed25326bdf79a03e00a4842e503743/websockets-16.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d19c9067e1fe9490f974bffbc0e443b80a7674c5efb4980c429cc00771f07c5a", size = 188240, upload-time = "2026-07-10T06:32:33.897Z" }, + { url = "https://files.pythonhosted.org/packages/f1/20/86283636e499a1a357fa9441f690ba34f255e731f2fea174132b3b762b57/websockets-16.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d440ff0c6c7469ad59c0a412c383c235935b43635e89425e3f6a0c36de90c31b", size = 185955, upload-time = "2026-07-10T06:32:35.279Z" }, + { url = "https://files.pythonhosted.org/packages/91/23/d7fb734b0095d43bc7f1c9f68afd50adb4176e7e513403e8c70ad7daa4fa/websockets-16.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8613129a2533f08de24505e69a3e403cedaadae49abdb043c4d170ca71b7e4bd", size = 188491, upload-time = "2026-07-10T06:32:36.673Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5e/168a192689db468405ecf3b8e4a2c18811936b0724d017ad7e6d252734f0/websockets-16.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:a5bf9c23f197b4ec88290fd5463f33db67362a1bb10f85fc2e8e7627f0ddab97", size = 186983, upload-time = "2026-07-10T06:32:38.207Z" }, + { url = "https://files.pythonhosted.org/packages/7e/9b/66795fa91ebe49019ebe4fa910282172252e37046b80e08fc52e0c365150/websockets-16.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:520b0fd0395f075febb283c76755af724ab9fd19dffa4f3bfd18cb4e622790a3", size = 188890, upload-time = "2026-07-10T06:32:39.545Z" }, + { url = "https://files.pythonhosted.org/packages/5a/32/126bbc844be5afb3613fd43211dac10a9645f4cf39741d04acaa2ec7030c/websockets-16.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:7143aa09a67e1c013be44e81a88dfe90fc6244198ab86c7edd064152cf619805", size = 186583, upload-time = "2026-07-10T06:32:41.038Z" }, + { url = "https://files.pythonhosted.org/packages/22/b9/0b5db9cbcf6e4970db4496893244a8d92e07f71a8ef27cf34b08aa02fef1/websockets-16.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:7acb811fad08e611755800d1560e395c67e11a6bd563598ea6abb319afb86938", size = 187353, upload-time = "2026-07-10T06:32:42.501Z" }, + { url = "https://files.pythonhosted.org/packages/99/2e/254b2131a10d831b76e2c18dfe7add9729c6292c674a8085bf8de01ad151/websockets-16.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c5cf88e3faa2f7931bc6baeee7599c97656a3f6ac7f831f4fccba233e141783a", size = 187784, upload-time = "2026-07-10T06:32:43.929Z" }, + { url = "https://files.pythonhosted.org/packages/21/dc/e7288aa8e3ac5a88a0924619984d663c1abf2a87d0ea98290c66fdaee0ec/websockets-16.1-cp314-cp314t-win32.whl", hash = "sha256:589f8842521c8307684ce0b40ce4ad70c5e0aa46484c6f1225a94ef4b8970341", size = 179947, upload-time = "2026-07-10T06:32:45.495Z" }, + { url = "https://files.pythonhosted.org/packages/d3/de/37edf1260ff0fbbd2f82433489c4cfbe799ac2ff21355331609879329fe6/websockets-16.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c0e0857c30bbbc2bb5c30687508f0b7ec19aa026cd9f2ff8424d0fee42dcc07", size = 180291, upload-time = "2026-07-10T06:32:47.119Z" }, + { url = "https://files.pythonhosted.org/packages/4d/f4/84ef884775bbe77c46cce79bc7d705ea3bc6574cc00acf81af89754c077d/websockets-16.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7289d899c79e763e6221c8dcb8959361cb43274418538d7c7ad16a43b01d12f9", size = 177387, upload-time = "2026-07-10T06:32:48.574Z" }, + { url = "https://files.pythonhosted.org/packages/d3/d9/6831ec6f65e1eeac770375f4f4b604f23df9bafaa1b47004bc5f9488d513/websockets-16.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:e22e9e3719f5131bd62da4db63c8da63eb8c91cc99e16c1cbd122f130e1ae07a", size = 177663, upload-time = "2026-07-10T06:32:50.043Z" }, + { url = "https://files.pythonhosted.org/packages/9d/d4/21d4922fa7fe855813a8b38f181a0ecf02a586e16c1f095fd05471f78cc2/websockets-16.1-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:83bdabafef431247e6b11a9aab8a0893fd8e82e1ed95b32e0373625b03ffce4a", size = 178501, upload-time = "2026-07-10T06:32:51.439Z" }, + { url = "https://files.pythonhosted.org/packages/91/87/7a0320df854dacd09507ca972cb04a4dc5aae279583cc5b80ad5f5819533/websockets-16.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b8d13ceabc5c60995f201b5211d76876e17e68706ebf5d3bc666b32eefff1a6", size = 179397, upload-time = "2026-07-10T06:32:52.892Z" }, + { url = "https://files.pythonhosted.org/packages/31/6a/0da1eb8c8da2ace7b578c8523d32618af85e62a9ebad56051d4a14a38a1c/websockets-16.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:81495f9c0085361c582efbc3207fb877174cfe03370f17d9cd70624404aa526f", size = 180546, upload-time = "2026-07-10T06:32:54.619Z" }, + { url = "https://files.pythonhosted.org/packages/66/58/bd83247f39ddc26ffc2c24eb05087a3b749e00cb4509fc6d19daa23c8495/websockets-16.1-py3-none-any.whl", hash = "sha256:c5149dfe490ec7e5ee5dbf624c642fb725f93a5575c7f00ab594ca9eddb8dd81", size = 174031, upload-time = "2026-07-10T06:32:56.079Z" }, +] + [[package]] name = "werkzeug" version = "3.1.8" @@ -7601,6 +10410,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, ] +[[package]] +name = "xgrammar" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apache-tvm-ffi", marker = "sys_platform == 'linux'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version >= '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform == 'linux' and extra == 'extra-16-transformer-lens-vllm') or (python_full_version < '3.11' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (sys_platform != 'linux' and extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm')" }, + { name = "pydantic", marker = "sys_platform == 'linux'" }, + { name = "torch", marker = "sys_platform == 'linux'" }, + { name = "transformers", marker = "sys_platform == 'linux'" }, + { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/f4/e71693d8cec60b7e36dab660784ecc5a6aa51e478a83b556011645c58c87/xgrammar-0.2.3.tar.gz", hash = "sha256:f76423630ae3ac4e090cb38ce1e30e7bcc69b3dee4d22d94353944386a4c6f18", size = 2447704, upload-time = "2026-06-27T04:45:24.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/e2/54bf9a915665d380ca09ba86b2449ceaed2500e11bd3654e0df3f9621e19/xgrammar-0.2.3-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:5eec3987abb915b7182587cf8063ae16b9fcb51e8b5f70558d8d2f0bfb8e04c4", size = 23284501, upload-time = "2026-06-27T04:43:56.188Z" }, + { url = "https://files.pythonhosted.org/packages/a1/80/ee5e79e79a1ef19aec4ad26155ce7f3cb99b474d62dad2d5a795035896cb/xgrammar-0.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f1270736d74ca3276cfba593457c23b9ca37032bd8051614533c81ed892d2727", size = 23240063, upload-time = "2026-06-27T04:43:59.283Z" }, + { url = "https://files.pythonhosted.org/packages/2f/b4/d828e4d8b98b256609c18fd80b7b91c6b3c0b401b82830fb80b4ac36664b/xgrammar-0.2.3-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e983c6521ebac727e8513acd3f3eb76b9bfe9fedb3c528ddb01845c08f098652", size = 44314489, upload-time = "2026-06-27T04:44:02.863Z" }, + { url = "https://files.pythonhosted.org/packages/90/4d/2e8cf58db0ec4c31bf1a2370ec53987b1c5274b51003b777e2d0cd0e25b2/xgrammar-0.2.3-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b257973b2878bcb2c02f24057d174c0d9715cdc02e8ce29149a350184e619f86", size = 44855095, upload-time = "2026-06-27T04:44:06.242Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6b/e08774188ffaa04773048de4a326a228241a28d702a3db8ea104fe8f292c/xgrammar-0.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:ea79d53314d614a7beab09570a659ec72e4c79fe61b0a9ad153da95b4ee8fcd7", size = 15780258, upload-time = "2026-06-27T04:44:08.607Z" }, + { url = "https://files.pythonhosted.org/packages/6f/78/8664d2c92ffae29af09d98305432c5db276490f9ced87a4c4f054a60b606/xgrammar-0.2.3-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:00f6ba916fe84552f303b1b576061296ff8bea1e24d065efec49543489dd5217", size = 23284497, upload-time = "2026-06-27T04:44:10.736Z" }, + { url = "https://files.pythonhosted.org/packages/ae/24/e87485e32f9c1942dd56c66ce9e264784970eafc8e3c41ddd8f8daf916e0/xgrammar-0.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bec963362548e48b9a763de8f801d2b6c3f6dfb050cc88de2a83fe5e90dec357", size = 23240042, upload-time = "2026-06-27T04:44:13.618Z" }, + { url = "https://files.pythonhosted.org/packages/19/b5/f1b54a6f652cbd9dead9a6e31c8eedeea846abae020fb7eb08cc90b13786/xgrammar-0.2.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48d2c9d2bab9b60653204bf334663e06f6044d8f5c104aca68ee23526afb3161", size = 44314487, upload-time = "2026-06-27T04:44:16.182Z" }, + { url = "https://files.pythonhosted.org/packages/84/f3/4bd6bd3dd9a0450d78bcd9db3593b5df9ab5baf62a8f50d265a933156c47/xgrammar-0.2.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d38fb3ad3118b8f08b1da53fb6feb81a206e6eb9df6abb16bda47e2cb272deef", size = 44855068, upload-time = "2026-06-27T04:44:18.898Z" }, + { url = "https://files.pythonhosted.org/packages/9e/56/e7b1217befca273568efd6da80b2f1f7471633a5e7d6e2a9a1e87e13e490/xgrammar-0.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:8c262af85340825e6bda407293713582f95169fb098cb107751b55c0b3c309fb", size = 15780256, upload-time = "2026-06-27T04:44:21.562Z" }, + { url = "https://files.pythonhosted.org/packages/51/1c/0cdb22fc799e6d158b3243eeb895ae2e086825487b57767838c98d4864ee/xgrammar-0.2.3-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:173e167d43a5cf4171eee2be86097decff8803b0a0853d7baaf446c732a7d3a9", size = 23284489, upload-time = "2026-06-27T04:44:23.927Z" }, + { url = "https://files.pythonhosted.org/packages/cf/bc/994dc6f222189174840c29a1f5b4c175e69dfe13ed2e25b6dbbe9f200a29/xgrammar-0.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aa35f24835a59c822e249ecc80912eea4de03fc8b04afb2f82c8b950a56be6ef", size = 23240027, upload-time = "2026-06-27T04:44:26.255Z" }, + { url = "https://files.pythonhosted.org/packages/e4/79/0bb37937bf847c738c64b64dc50ddc12e7c526b34c5ab82cebe58da5ec8f/xgrammar-0.2.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11255f184971489fc72b948b096e2917f482ba2dca975177f5411562cedb9c6d", size = 44314481, upload-time = "2026-06-27T04:44:28.875Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fd/5ebd5d14b8993cb225151bbb8f2011742fc7a7d94a3bdbc3ec3954b9b62d/xgrammar-0.2.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fdf081fab29694302d41d61dcf52fad7d253879a718bc6afc68db0a0dabd7f19", size = 44855110, upload-time = "2026-06-27T04:44:31.586Z" }, + { url = "https://files.pythonhosted.org/packages/17/66/67239f43b0244f65aec4639f51ab95905db42eb66e532ee2a4e5cdce32de/xgrammar-0.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:e7787dd8321a04f86116b756aa3dadd622e3607a3559b1e986cc5f77da00d68e", size = 15780277, upload-time = "2026-06-27T04:44:34.081Z" }, + { url = "https://files.pythonhosted.org/packages/1b/06/a1a3c3a53cebb5f1bdaa1d1452f3327b6ca7413bdeff94ff639c3b9b8378/xgrammar-0.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11108010c54c8f12f0b14c239ef5bca217cf107fa3d0bf0db008bf594ab0a1ff", size = 23240039, upload-time = "2026-06-27T04:44:36.173Z" }, + { url = "https://files.pythonhosted.org/packages/4d/aa/1c74ff8bad6624c08a33924f7d051a9278a622b48d843e939d574a6b5bf8/xgrammar-0.2.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eeb5e46bd7d3230e5d8e6385793c48ec872a4fb377c19d341dbcaedc41f495e9", size = 44314496, upload-time = "2026-06-27T04:44:38.552Z" }, + { url = "https://files.pythonhosted.org/packages/28/f8/2407b44049416650c257e22399c32788d3497524a9a899addc74b5d27d1d/xgrammar-0.2.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d0fbd4709733b224e6e115d1001fb07124c71fe114a9087a2e3e53ce871517", size = 44855037, upload-time = "2026-06-27T04:44:41.122Z" }, + { url = "https://files.pythonhosted.org/packages/92/67/271dd31308c5a1d32d9c696044f9d44589f8ac4406faaddc3047765e0c71/xgrammar-0.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:3a718fa2c1bfc06951e1c480a656c28bde6d5f828199308fd577bd264e05b923", size = 15780261, upload-time = "2026-06-27T04:44:43.573Z" }, + { url = "https://files.pythonhosted.org/packages/45/f6/a71f8292cf7eeb0517e52e53f8f10edeb1ad6b306c206e76b9a63924dd50/xgrammar-0.2.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:11d5cd45891f311d7c487635fdee637c5ae375d8292601ad55483c89653749e6", size = 23284602, upload-time = "2026-06-27T04:44:46.105Z" }, + { url = "https://files.pythonhosted.org/packages/ad/5a/6e79d2c3fae530aacaffef7a0ea6af5499de4e6c43ec5c9b88a15ad1cebf/xgrammar-0.2.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07d4c238c4545741573653fbfa66072ae208a85926c3a3f4d57c99fd67fde886", size = 23240022, upload-time = "2026-06-27T04:44:48.997Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b5/983c482cad0a57230d16806b377710402644f7d4e6023fbf8897e0dc5256/xgrammar-0.2.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe71c621a18ec1fd0a74e755e72c5e521c4854b75d14ad52f1fc8c325d387124", size = 44314374, upload-time = "2026-06-27T04:44:51.526Z" }, + { url = "https://files.pythonhosted.org/packages/fb/9f/6c8601fa55545fdf9b9c95e289fc6db73b0c160759873f666a992741069d/xgrammar-0.2.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f26c8bb1845119856b09658bcf2ee525957dc618d954684e5c393d16bcc1f1da", size = 44855006, upload-time = "2026-06-27T04:44:54.545Z" }, + { url = "https://files.pythonhosted.org/packages/95/78/b313f8278ca2a52fa143067cc0e577a916418af64af367a28ff270b7fa65/xgrammar-0.2.3-cp314-cp314-win_amd64.whl", hash = "sha256:3fdcee5e375c8cfe83e41c4a396a861cafe7cda381c566c24d26628a970a9ec4", size = 15892265, upload-time = "2026-06-27T04:44:57.167Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c6/61fa7109b95ba504b8074904c9a2c4a4b466a564a96494371e496922d52d/xgrammar-0.2.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:355f21445ba361fa258d34106fa31852be9222f0fd3f48fa3fe089d5201679e7", size = 23284609, upload-time = "2026-06-27T04:44:59.459Z" }, + { url = "https://files.pythonhosted.org/packages/60/27/d410dee57ae641b2da28591602b0cdd81d8f2f6b4aea9fe60992c7a2764b/xgrammar-0.2.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1b6dc2657d9a1efbb770e0982ec0f2718c1de198cf24ccc8db2372eba01cb64", size = 23240041, upload-time = "2026-06-27T04:45:01.52Z" }, + { url = "https://files.pythonhosted.org/packages/7b/b4/1b7cfc9c8d1bd3d33fcfd79d87be0bd959cb3b7418ddcf51c53d2d6ce35e/xgrammar-0.2.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba69a13ae4cc94b7a3a0b5ac67865ce372e0eac7b6486f7e8ca0c0cbbbc3097", size = 44314501, upload-time = "2026-06-27T04:45:03.955Z" }, + { url = "https://files.pythonhosted.org/packages/b5/a1/a5cc64b42a183ec89aeae5d9455017c17c5e8cc4597bbc30bf76fdb7ec40/xgrammar-0.2.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:838f88cd74c00670e4b0797ffd7bec8399b67f90ca0f199c96a68333f70553b4", size = 44854955, upload-time = "2026-06-27T04:45:06.557Z" }, + { url = "https://files.pythonhosted.org/packages/b9/19/d80f30edcea594be5fd95b0a2849d110b389921aa72ecfa6380f8fb1714f/xgrammar-0.2.3-cp314-cp314t-win_amd64.whl", hash = "sha256:d5c03cdd7847e1281ef2189e600fe7b4fe39058e1f7b6946e58c7f0308dc11cf", size = 15892255, upload-time = "2026-06-27T04:45:09.455Z" }, +] + [[package]] name = "xxhash" version = "3.5.0" @@ -7773,12 +10629,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/2d/2345fce04cfd4bee161bf1e7d9cdc702e3e16109021035dbb24db654a622/yarl-1.20.1-py3-none-any.whl", hash = "sha256:83b8eb083fe4683c6115795d9fc1cfaf2cbbefb19b3a1cb68f6527460f483a77", size = 46542, upload-time = "2025-06-10T00:46:07.521Z" }, ] +[[package]] +name = "z3-solver" +version = "4.15.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/8e/0c8f17309549d2e5cde9a3ccefa6365437f1e7bafe71878eaf9478e47b18/z3_solver-4.15.4.0.tar.gz", hash = "sha256:928c29b58c4eb62106da51c1914f6a4a55d0441f8f48a81b9da07950434a8946", size = 5018600, upload-time = "2025-10-29T18:12:03.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/33/a3d5d2eaeb0f7b3174d57d405437eabb2075d4d50bd9ea0957696c435c7b/z3_solver-4.15.4.0-py3-none-macosx_13_0_arm64.whl", hash = "sha256:407e825cc9211f95ef46bdc8d151bf630e7ab2d62a21d24cd74c09cc5b73f3aa", size = 37052538, upload-time = "2025-10-29T18:11:46.233Z" }, + { url = "https://files.pythonhosted.org/packages/47/84/fd7ffac1551cd9f8d44fe41358f738be670fc4c24dfd514fab503f2cf3e7/z3_solver-4.15.4.0-py3-none-macosx_13_0_x86_64.whl", hash = "sha256:00bd10c5a6a5f6112d3a9a810d0799227e52f76caa860dafa5e00966bb47eb13", size = 39807925, upload-time = "2025-10-29T18:11:49.81Z" }, + { url = "https://files.pythonhosted.org/packages/21/c9/bb51a96af0091324c81b803f16c49f719f9f6ea0b0bb52200f5c97ec4892/z3_solver-4.15.4.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e103a6f203f505b8b8b8e5c931cc407c95b61556512d4921c1ddc0b3f41b08e", size = 29268352, upload-time = "2025-10-29T18:11:53.032Z" }, + { url = "https://files.pythonhosted.org/packages/bf/2e/0b49f7e4e53817cfb09a0f6585012b782dfe0b666e8abefcb4fac0570606/z3_solver-4.15.4.0-py3-none-manylinux_2_34_aarch64.whl", hash = "sha256:62c7e9cbdd711932301f29919ad9158de9b2f58b4d281dd259bbcd0a2f408ba1", size = 27226534, upload-time = "2025-10-29T18:11:55.59Z" }, + { url = "https://files.pythonhosted.org/packages/26/91/33de49538444d4aafbe47415c450c2f9abab1733e1226f276b496672f46c/z3_solver-4.15.4.0-py3-none-win32.whl", hash = "sha256:be3bc916545c96ffbf89e00d07104ff14f78336e55db069177a1bfbcc01b269d", size = 13191672, upload-time = "2025-10-29T18:11:58.424Z" }, + { url = "https://files.pythonhosted.org/packages/03/d6/a0b135e4419df475177ae78fc93c422430b0fd8875649486f9a5989772e6/z3_solver-4.15.4.0-py3-none-win_amd64.whl", hash = "sha256:00e35b02632ed085ea8199fb230f6015e6fc40554a6680c097bd5f060e827431", size = 16259597, upload-time = "2025-10-29T18:12:01.14Z" }, +] + [[package]] name = "zipfile-zstd" version = "0.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zstandard", marker = "python_full_version < '3.14'" }, + { name = "zstandard", marker = "python_full_version < '3.14' or (extra == 'extra-16-transformer-lens-lit' and extra == 'extra-16-transformer-lens-vllm') or (extra != 'extra-16-transformer-lens-lit' and extra != 'extra-16-transformer-lens-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f7/2a/2e0941bc0058d10ab37d8c578b94a19f611f6ae54f124140f2fb451f0932/zipfile-zstd-0.0.4.tar.gz", hash = "sha256:c1498e15b7922a3d1af0ea55df8b11b2af4e8f7e0e80e414e25d66899f7def89", size = 4603, upload-time = "2021-12-08T07:38:16.245Z" } wheels = [ From 21cd02e6409a2ecc32aa3808b65aff9d7ec0d064 Mon Sep 17 00:00:00 2001 From: Jonah Larson Date: Wed, 15 Jul 2026 21:33:06 -0500 Subject: [PATCH 11/87] Multi-GPU testing for vLLM systems (#1517) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * multi-gpu setup * Fixing typing errors * parity report improvements * improve variance * test splitting * disabled cache * adjusted tests to be divisible by 2 gpu * fixed bug * tensor decoding * Documented successfuly test * Add pipeline parallelism * bug fix for 0'ed layers * bug fix – dynamic ownership * Update cleanup --- docs/source/content/drivers.md | 2 + pyproject.toml | 9 + scripts/vllm_parity_report.py | 109 +++++++- tests/QUARANTINES.md | 8 + .../model_bridge/_vllm_multigpu_common.py | 80 ++++++ .../model_bridge/test_vllm_multigpu.py | 85 ++++++ .../model_bridge/test_vllm_multigpu_pos.py | 51 ++++ .../model_bridge/test_vllm_multigpu_pp.py | 85 ++++++ .../test_inspect_vllm_provider.py | 23 +- tests/unit/model_bridge/test_vllm_boot.py | 125 ++++++++- tests/unit/model_bridge/test_vllm_driver.py | 263 +++++++++++++++++- .../unit/model_bridge/test_vllm_internals.py | 37 ++- tests/unit/model_bridge/test_vllm_plugin.py | 119 ++++++++ .../test_vllm_worker_extension.py | 78 +++++- .../sources/inspect/vllm_provider.py | 64 ++--- .../model_bridge/sources/vllm/driver.py | 184 ++++++++++-- .../model_bridge/sources/vllm/internals.py | 28 ++ .../model_bridge/sources/vllm/plugin.py | 111 ++++++-- .../model_bridge/sources/vllm/source.py | 216 +++++++++----- .../sources/vllm/worker_extension.py | 118 +++++++- 20 files changed, 1575 insertions(+), 220 deletions(-) create mode 100644 tests/acceptance/model_bridge/_vllm_multigpu_common.py create mode 100644 tests/acceptance/model_bridge/test_vllm_multigpu.py create mode 100644 tests/acceptance/model_bridge/test_vllm_multigpu_pos.py create mode 100644 tests/acceptance/model_bridge/test_vllm_multigpu_pp.py diff --git a/docs/source/content/drivers.md b/docs/source/content/drivers.md index c3b66ac2f4..405188fa3c 100644 --- a/docs/source/content/drivers.md +++ b/docs/source/content/drivers.md @@ -76,6 +76,8 @@ Notes grounded in the source docstrings ([`sources/vllm/source.py`](https://gith - **Returned logits are reconstructed full-sequence logits**: vLLM's sampler bypasses `lm_head`, so the driver rebuilds them host-side as `ln_final @ lm_head.weight.T` (+ bias, + Gemma soft-cap) — valid at every position, so loss works. If the unembedding weight is unreachable it falls back to final-position log-probs and the bridge rejects `return_type="loss"`. - **Convention alignment**: vLLM materializes `ln_final` *post-weight*, but the driver un-folds the exposed capture (÷ weight, or ÷ (1 + weight) for Gemma) so `ln_final.hook_normalized` matches the pre-weight value `boot_transformers` serves. If the norm weight is unreachable it warns and serves the raw post-weight value. See [`sources/vllm/overlays/decoder_only.py`](https://github.com/TransformerLensOrg/TransformerLens/blob/dev-4.x/transformer_lens/model_bridge/sources/vllm/overlays/decoder_only.py) for which hooks diverge from HF conventions. - `enable_batching=True` switches to the eager batched path (`batch_size > 1`, chunked prefill) for data collection; `enable_position_interventions=True` lets a spec carry a `pos` field (int or list) to scope an edit to specific sequence positions. +- `tensor_parallel_size=2` enables single-node tensor parallelism (GPU-validated: capture/intervention/logit parity vs TP=1 within the standard band): every served hook point is post-all-reduce and replicated across a stage's TP ranks, with a first-forward cross-rank check that fails loud if that ever stops holding; vocab-sharded unembeddings are gathered for logit reconstruction. Incompatible with `enable_batching`; multi-node (Ray) remains unsupported. +- `pipeline_parallel_size=2` enables single-node pipeline parallelism (GPU-validated: capture/intervention/logit parity vs single-rank within the standard band): each stage owns a disjoint layer slice, so the driver merges per-rank capture reads (fail-loud if a requested hook is served by no rank) and each rank applies only the intervention specs whose hooks it owns. Ownership is determined by whether a hook actually fired, not by module presence — tied-embedding models alias `embed_tokens` onto the last stage and some builds instantiate the final norm everywhere. The first-forward layout check also verifies TP replicas of each stage fired identically under vLLM's PP microbatching. Composes with `tensor_parallel_size`; same `enable_batching` restriction. - Requires a CUDA GPU. Install with `pip install "transformer-lens[vllm]"` (or `uv sync --extra vllm`). The extra is Linux-only (vLLM ships no macOS/Windows wheels), pins the validated `vllm 0.20.x` band — which in turn pins its matching `torch` — and cannot co-install with the `[lit]` extra (numpy version conflict). See [`sources/vllm/internals.py`](https://github.com/TransformerLensOrg/TransformerLens/blob/dev-4.x/transformer_lens/model_bridge/sources/vllm/internals.py) before bumping the band. ### Inspect — interp inside `inspect_ai` evals diff --git a/pyproject.toml b/pyproject.toml index 771176d1ae..d78a8d1961 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,14 @@ tl_bridge="transformer_lens.model_bridge.sources.inspect.transformers_provider" tl_bridge_vllm="transformer_lens.model_bridge.sources.inspect.vllm_provider" + # vLLM loads general_plugins in EVERY process it owns — including spawned TP + # worker processes, which is the only way the Worker.load_model capture patch + # reaches them (boot_vllm's direct register() call only covers in-process + # workers). register() is idempotent and no-ops without a configured spec + # channel, so non-TL vllm usage is unaffected. + [project.entry-points."vllm.general_plugins"] + tl_vllm_capture="transformer_lens.model_bridge.sources.vllm.plugin:register" + [dependency-groups] demo=["gradio>=4.0.0", "orjson>=3.11.7,<4.0"] dev=[ @@ -128,6 +136,7 @@ markers=[ "slow: marks tests as slow (deselect with '-m \"not slow\"')", "inspect: requires inspect_ai (deselect with '-m \"not inspect\"')", + "multigpu: requires vllm and >= 2 CUDA devices (select with '-m multigpu')", ] pythonpath=["."] testpaths=["tests", "transformer_lens"] # Only test these directories diff --git a/scripts/vllm_parity_report.py b/scripts/vllm_parity_report.py index 1abf934774..21950a9589 100644 --- a/scripts/vllm_parity_report.py +++ b/scripts/vllm_parity_report.py @@ -21,10 +21,15 @@ Run: uv run python scripts/vllm_parity_report.py Env: TL_PARITY_MODELS="id1,id2,..." overrides the model list. TL_VLLM_ATOL / TL_VLLM_RTOL override tolerance (defaults 2e-2). + TL_PARITY_TP=2 boot vLLM with tensor_parallel_size=2 — + same PASS bar as TP=1 (needs >= TP GPUs). + TL_PARITY_PP=2 boot vLLM with pipeline_parallel_size=2 + (needs >= TP*PP GPUs). """ from __future__ import annotations import gc +import json import os import sys import warnings @@ -50,6 +55,14 @@ # A wrong hook mapping (e.g. un-un-folded ln_final) diverges by O(1), well outside this. ATOL = float(os.environ.get("TL_VLLM_ATOL", "2e-2")) RTOL = float(os.environ.get("TL_VLLM_RTOL", "2e-2")) +# Scale-aware band: final-layer residual streams reach O(1e3) magnitudes ("massive +# activations") and kernel-order noise grows with scale. Measured across 4 archs +# (2×A6000, fp32, 2026-07-15): worst relative diff 5.9e-4, always at the final +# layer, with downstream ln_final back in band — while a mapping bug is O(1) +# relative. 2e-3 keeps >3 orders of magnitude of discrimination. +REL_BAND = float(os.environ.get("TL_VLLM_REL", "2e-3")) +TP = int(os.environ.get("TL_PARITY_TP", "1")) +PP = int(os.environ.get("TL_PARITY_PP", "1")) # vLLM capture kind -> TransformerBridge-native hook name (per-layer uses {i}). DIRECT_KINDS = { @@ -67,12 +80,18 @@ def _to2d(t: torch.Tensor) -> torch.Tensor: return t.reshape(-1, t.shape[-1]) -def _diff(a: torch.Tensor, b: torch.Tensor) -> tuple[float, bool]: +def _diff(a: torch.Tensor, b: torch.Tensor) -> tuple[float, bool, str]: + """(max abs diff, in-band?, scale-aware profile line). The profile shows the diff + relative to the reference's own magnitude — distinguishes depth-accumulated kernel + noise (rel stays small as scale grows) from a structural capture error.""" a2, b2 = _to2d(a), _to2d(b) if a2.shape != b2.shape: - return float("inf"), False + return float("inf"), False, "shape mismatch" d = (a2 - b2).abs().max().item() - return d, torch.allclose(a2, b2, atol=ATOL, rtol=RTOL) + scale = b2.abs().max().item() + # Effective atol scales with the reference tensor's magnitude (see REL_BAND). + ok = torch.allclose(a2, b2, atol=max(ATOL, REL_BAND * scale), rtol=RTOL) + return d, ok, f"abs={d:.2e} rel={d / max(scale, 1e-9):.2e} scale={scale:.1f}" def _refold_lnf(vllm_val: torch.Tensor, weight: torch.Tensor, is_gemma: bool) -> torch.Tensor: @@ -100,7 +119,13 @@ def verify(model_id: str) -> dict: n_layers = int(hf.cfg.n_layers) toks = hf.to_tokens(PROMPT) - vllm = boot_vllm(model_id, dtype=torch.float32, max_model_len=2048) + vllm = boot_vllm( + model_id, + dtype=torch.float32, + max_model_len=2048, + tensor_parallel_size=TP, + pipeline_parallel_size=PP, + ) offered = vllm._driver.supported_hook_points hf_logits, hf_cache = hf.run_with_cache(toks) @@ -110,9 +135,13 @@ def verify(model_id: str) -> dict: mism: list[str] = [] notes: list[str] = [] - # embed + the three per-layer direct boundaries (first & last layer). + # embed + the three per-layer direct boundaries. Default checks first & last + # layer; TL_PARITY_ALL_LAYERS=1 diffs every layer and prints the depth profile — + # gradual growth = kernel-noise accumulation, a cliff at one layer = capture bug. + all_layers = bool(os.environ.get("TL_PARITY_ALL_LAYERS")) + layer_ids = range(n_layers) if all_layers else sorted({0, n_layers - 1}) checks = [(EMBED_HOOK, EMBED_HOOK)] - for i in sorted({0, n_layers - 1}): + for i in layer_ids: for name in DIRECT_KINDS.values(): hk = name.format(i=i) checks.append((hk, hk)) @@ -122,16 +151,25 @@ def verify(model_id: str) -> dict: if hk not in v_cache or hk not in hf_cache: mism.append(f"{hk} missing") continue - d, ok = _diff(hf_cache[hk], v_cache[hk]) + d, ok, profile = _diff(hf_cache[hk], v_cache[hk]) worst = max(worst, d if d != float("inf") else worst) if not ok: - mism.append(f"{hk} maxdiff={d:.2e}") + mism.append(f"{hk} {profile}") + if all_layers: + print(f" depth profile for {model_id} (rel = abs/tensor-scale):", flush=True) + for i in layer_ids: + cells = [] + for kind, name in DIRECT_KINDS.items(): + hk = name.format(i=i) + if hk in v_cache and hk in hf_cache: + cells.append(f"{kind} {_diff(hf_cache[hk], v_cache[hk])[2]}") + print(f" L{i:>2}: " + " | ".join(cells), flush=True) # ln_final: the driver already un-folds to the pre-weight convention — compare # directly. The refolded diff is diagnostic: "direct FAIL + refolded ok" means # the driver's un-fold regressed (serving raw post-weight values again). if LNF_HOOK in offered and LNF_HOOK in v_cache and LNF_HOOK in hf_cache: - d, ok = _diff(hf_cache[LNF_HOOK], v_cache[LNF_HOOK]) + d, ok, _profile = _diff(hf_cache[LNF_HOOK], v_cache[LNF_HOOK]) worst = max(worst, d if d != float("inf") else worst) if ok: notes.append(f"ln_final direct={d:.2e}") @@ -140,7 +178,7 @@ def verify(model_id: str) -> dict: detail = f"{LNF_HOOK} maxdiff={d:.2e}" if weight is not None: refolded = _refold_lnf(v_cache[LNF_HOOK], weight, is_gemma) - rf_d, rf_ok = _diff(hf_cache[LNF_HOOK], refolded) + rf_d, rf_ok, _rf = _diff(hf_cache[LNF_HOOK], refolded) detail += f" (refolded={rf_d:.2e}{', un-fold regressed' if rf_ok else ''})" mism.append(detail) @@ -183,6 +221,11 @@ def _preflight() -> str | None: """Return a human reason to abort (no GPU / no vllm), or None if runnable.""" if not torch.cuda.is_available(): return "no CUDA device — vLLM capture only materializes in a real GPU forward" + if TP * PP > torch.cuda.device_count(): + return ( + f"TL_PARITY_TP={TP} x TL_PARITY_PP={PP} needs {TP * PP} GPUs but only " + f"{torch.cuda.device_count()} visible" + ) try: import vllm # noqa: F401 except Exception as e: @@ -190,7 +233,49 @@ def _preflight() -> str | None: return None +# Child-process result marker (see main): the parent parses the child's last +# marker line back into a row dict. +_ROW_MARKER = "##TL_PARITY_ROW## " + + +def _verify_in_subprocess(model_id: str) -> dict: + """One engine per process: an in-process vLLM engine does not reliably release + GPU memory before exit, so sequential same-process boots leak until later + models can't reserve their budget (observed: 47→18 GiB free across 4 boots). + Child stdout streams through; the marker line carries the structured row.""" + import subprocess + + proc = subprocess.run( + [sys.executable, os.path.abspath(__file__), "--one", model_id], + capture_output=True, + text=True, + ) + row = None + for line in proc.stdout.splitlines(): + if line.startswith(_ROW_MARKER): + row = json.loads(line[len(_ROW_MARKER) :]) + else: + print(line, flush=True) + if proc.stderr: + tail = proc.stderr.strip().splitlines()[-3:] + for line in tail: + print(f" [child stderr] {line}", flush=True) + if row is None: + return { + "model": model_id, + "arch": "?", + "status": "SKIP", + "detail": f"child exited {proc.returncode} without a result row", + } + return row + + def main() -> None: + if len(sys.argv) > 2 and sys.argv[1] == "--one": + r = verify(sys.argv[2]) + print(_ROW_MARKER + json.dumps(r), flush=True) + sys.exit(0) + reason = _preflight() if reason is not None: print(f"SKIP ALL: {reason}", flush=True) @@ -201,11 +286,11 @@ def main() -> None: models = [m.strip() for m in ids.split(",")] if ids else DEFAULT_MODELS rows = [] for m in models: - r = verify(m) + r = _verify_in_subprocess(m) rows.append(r) print(f"[{r['status']:4}] {r['arch']:28} {r['model']:40} {r['detail']}", flush=True) - print("\n================ vLLM PARITY REPORT CARD ================") + print(f"\n================ vLLM PARITY REPORT CARD (TP={TP}, PP={PP}) ================") for status in ("PASS", "FAIL", "SKIP"): sel = [r for r in rows if r["status"] == status] print(f"\n{status} ({len(sel)}):") diff --git a/tests/QUARANTINES.md b/tests/QUARANTINES.md index 47f03ca50b..3de13899eb 100644 --- a/tests/QUARANTINES.md +++ b/tests/QUARANTINES.md @@ -44,6 +44,14 @@ A `[vllm]` extra exists (Linux-only marker; declared conflicting with `[lit]` in **Un-skip:** a GPU CI lane that installs vllm, or locally on a CUDA machine with `vllm==0.20.2` installed alongside the project env. +## Multi-GPU tier — `-m multigpu` (no automated lane) + +[`acceptance/model_bridge/test_vllm_multigpu.py`](acceptance/model_bridge/test_vllm_multigpu.py) validates the vLLM driver's `tensor_parallel_size=2` path (capture replication, sharded-unembed gather, intervention parity vs TP=1). Gated on `importorskip("vllm")` + `torch.cuda.device_count() >= 2`, so it skips everywhere except a provisioned multi-GPU box. **Validated 2026-07-15 on 2×A6000** (vllm 0.20.2, Qwen2.5-0.5B): both multigpu files pass, plus `TL_PARITY_TP=2` parity and the TP=1 parity regression. + +[`acceptance/model_bridge/test_vllm_multigpu_pp.py`](acceptance/model_bridge/test_vllm_multigpu_pp.py) validates the `pipeline_parallel_size=2` path (cross-stage capture merge, layout tripwire, first/last-stage intervention parity, logit reconstruction from last-stage gathers). Same gating. **Validated 2026-07-15 on 2×A6000** (vllm 0.20.2, Qwen2.5-0.5B): 6/6 pass, plus `TL_PARITY_PP=2` parity and the single-rank parity regression. + +**Un-skip:** run on a >= 2-GPU Linux box with the `vllm` extra: `uv run pytest tests/acceptance/model_bridge/test_vllm_multigpu.py -m multigpu -v` (and the `_pos` / `_pp` siblings, each in its own process), plus `TL_PARITY_TP=2` / `TL_PARITY_PP=2` runs of `scripts/vllm_parity_report.py`. + --- ## Permanent — hardware requirement diff --git a/tests/acceptance/model_bridge/_vllm_multigpu_common.py b/tests/acceptance/model_bridge/_vllm_multigpu_common.py new file mode 100644 index 0000000000..fc0800d84f --- /dev/null +++ b/tests/acceptance/model_bridge/_vllm_multigpu_common.py @@ -0,0 +1,80 @@ +"""Shared constants/helpers for the vLLM multi-GPU acceptance suites. + +The suites stay in separate files — each boots its own engine(s) and in-process +vLLM engines under-release GPU memory, so every file gets a fresh pytest process +(see each file's docstring for its run command). Everything they must agree on +lives here so the tolerance band and boot profile can't drift between them. +""" +from __future__ import annotations + +import pytest +import torch + +MULTIGPU_MARKS = [ + pytest.mark.multigpu, + pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.device_count() < 2, + reason="needs >= 2 CUDA devices", + ), +] + +# Qwen2.5-0.5B: 14 attention heads / 2 KV heads — both divisible by TP=2. +# (SmolLM2-135M has 9 heads and cannot tensor-parallelize; keep any +# replacement model even-headed.) 24 layers -> 12 per PP=2 stage. +MODEL = "Qwen/Qwen2.5-0.5B" +N_LAYERS = 24 +PROMPT_IDS = [504, 4674, 1442, 29892, 322] # fixed ids: no tokenizer variance in scope + +# vLLM kernel scheduling differs across parallel layouts (all-reduce / send-recv +# order), so exact equality is not expected; band matches scripts/vllm_parity_report.py, +# including its scale-aware atol (final-layer streams reach O(1e3) and noise grows +# with scale). +ATOL = RTOL = 2e-2 +REL_BAND = 2e-3 + + +def close(t1: torch.Tensor, t2: torch.Tensor) -> bool: + """Scale-aware allclose: atol widens with the reference tensor's magnitude.""" + atol_eff = max(ATOL, REL_BAND * t2.abs().max().item()) + return torch.allclose(t1, t2, atol=atol_eff, rtol=RTOL) + + +def boot_multigpu(**parallel_kwargs): + """Boot MODEL with the suites' shared fp32 profile; parallel sizes per test.""" + from transformer_lens.model_bridge.sources.vllm.source import boot_vllm + + return boot_vllm( + MODEL, + dtype=torch.float32, + max_model_len=2048, + gpu_memory_utilization=0.35, + **parallel_kwargs, + ) + + +def bridge_pair_fixture(**test_boot_kwargs): + """Module fixture: single-rank reference then the parallel bridge under test; + serial boots (never two engines booting at once), closed in reverse.""" + + @pytest.fixture(scope="module") + def bridges(): + b1 = boot_multigpu() + b2 = boot_multigpu(**test_boot_kwargs) + yield b1, b2 + b2.close() + b1.close() + + return bridges + + +def assert_caches_match(cache1, cache2) -> None: + """Full key-set equality plus scale-aware per-hook comparison — a missing key + means a rank/stage was dropped, a band miss means numerical drift.""" + assert set(cache1.keys()) == set(cache2.keys()) + for name in cache1: + t1, t2 = cache1[name].float(), cache2[name].float() + assert t1.shape == t2.shape, name + assert close(t1, t2), ( + f"{name}: max abs diff {(t1 - t2).abs().max().item():.3e} " + f"(scale {t2.abs().max().item():.1f})" + ) diff --git a/tests/acceptance/model_bridge/test_vllm_multigpu.py b/tests/acceptance/model_bridge/test_vllm_multigpu.py new file mode 100644 index 0000000000..8e9f79912d --- /dev/null +++ b/tests/acceptance/model_bridge/test_vllm_multigpu.py @@ -0,0 +1,85 @@ +"""TP=2 validation for the vLLM driver — real engine, real model, >= 2 GPUs. + +Run on a multi-GPU box with the ``vllm`` extra installed: + + uv run pytest tests/acceptance/model_bridge/test_vllm_multigpu.py -m multigpu -v + +Every test compares tensor_parallel_size=2 against the GPU-validated TP=1 path on +the same tiny model, so a pass means TP introduces no capture/intervention/logit +drift beyond kernel-order noise. Engines boot once per size (module fixtures); +position-scoped interventions run separately (test_vllm_multigpu_pos.py) so the +third engine gets a fresh process. +""" +from __future__ import annotations + +import pytest +import torch + +pytest.importorskip("vllm") + +from ._vllm_multigpu_common import ( + MULTIGPU_MARKS, + PROMPT_IDS, + assert_caches_match, + bridge_pair_fixture, + close, +) + +pytestmark = MULTIGPU_MARKS + +bridges = bridge_pair_fixture(tensor_parallel_size=2) + + +class TestTPCaptureParity: + def test_captures_match_tp1(self, bridges): + b1, b2 = bridges + toks = torch.tensor([PROMPT_IDS]) + _, cache1 = b1.run_with_cache(toks) + _, cache2 = b2.run_with_cache(toks) + assert_caches_match(cache1, cache2) + + def test_logits_argmax_matches_tp1(self, bridges): + b1, b2 = bridges + toks = torch.tensor([PROMPT_IDS]) + l1 = b1.forward(toks, return_type="logits") + l2 = b2.forward(toks, return_type="logits") + assert l1.shape == l2.shape + # Full-sequence argmax agreement — exercises the sharded-lm_head gather. + assert torch.equal(l1.argmax(dim=-1), l2.argmax(dim=-1)) + + def test_sequence_logits_and_loss_under_tp(self, bridges): + _, b2 = bridges + assert b2._driver.provides_sequence_logits is True # gather succeeded + loss = b2.forward(torch.tensor([PROMPT_IDS]), return_type="loss") + assert torch.isfinite(loss) + + def test_layout_check_ran_clean(self, bridges): + """The first capture-bearing forward cross-checked ranks without raising.""" + _, b2 = bridges + assert b2._driver._tp_size == 2 + assert b2._driver._layout_verified is True + + +class TestTPInterventionParity: + def test_suppress_matches_tp1(self, bridges): + b1, b2 = bridges + toks = torch.tensor([PROMPT_IDS]) + spec = {"blocks.0.mlp.hook_out": {"op": "suppress"}} + l1 = b1.forward(toks, return_type="logits", intervene=spec) + l2 = b2.forward(toks, return_type="logits", intervene=spec) + assert torch.equal(l1.argmax(dim=-1), l2.argmax(dim=-1)) + assert close(l1.float(), l2.float()) + + def test_intervention_actually_bites_under_tp(self, bridges): + """Guard against a TP no-op passing the parity test trivially.""" + _, b2 = bridges + toks = torch.tensor([PROMPT_IDS]) + clean = b2.forward(toks, return_type="logits") + edited = b2.forward( + toks, return_type="logits", intervene={"blocks.0.mlp.hook_out": {"op": "suppress"}} + ) + assert not torch.allclose(clean.float(), edited.float(), atol=1e-4, rtol=1e-4) + + +# Position-scoped interventions live in test_vllm_multigpu_pos.py — a third +# engine boot gets its own process (in-process engines under-release GPU memory). diff --git a/tests/acceptance/model_bridge/test_vllm_multigpu_pos.py b/tests/acceptance/model_bridge/test_vllm_multigpu_pos.py new file mode 100644 index 0000000000..44fd4db2a4 --- /dev/null +++ b/tests/acceptance/model_bridge/test_vllm_multigpu_pos.py @@ -0,0 +1,51 @@ +"""TP=2 position-scoped interventions — separate file so it runs in a fresh process. + +In-process vLLM engines don't reliably release GPU memory before process exit, so +this third engine boot (TP=2 + position-interventions buffers) gets its own pytest +invocation instead of stacking on test_vllm_multigpu.py's two resident engines: + + uv run pytest tests/acceptance/model_bridge/test_vllm_multigpu_pos.py -m multigpu -v +""" +from __future__ import annotations + +import pytest +import torch + +pytest.importorskip("vllm") + +from ._vllm_multigpu_common import MULTIGPU_MARKS, PROMPT_IDS, boot_multigpu + +pytestmark = MULTIGPU_MARKS + + +@pytest.fixture(scope="module") +def pos_bridge(): + bridge = boot_multigpu(tensor_parallel_size=2, enable_position_interventions=True) + yield bridge + bridge.close() + + +class TestTPPositionInterventions: + def test_pos_scoped_edit_is_row_scoped(self, pos_bridge): + toks = torch.tensor([PROMPT_IDS]) + _, clean = pos_bridge.run_with_cache(toks) + _, edited = pos_bridge.run_with_cache( + toks, intervene={"embed.hook_out": {"op": "suppress", "pos": 2}} + ) + name = "embed.hook_out" + assert torch.allclose( + edited[name][0, 2].float(), torch.zeros_like(edited[name][0, 2].float()) + ) + # Off-target rows untouched (row-scoped affine survived TP). + for row in (0, 1, 3, 4): + assert torch.allclose( + clean[name][0, row].float(), edited[name][0, row].float(), atol=1e-5, rtol=1e-5 + ) + + def test_pos_beyond_prompt_rejected_under_tp(self, pos_bridge): + """The prompt-length guard must hold with spawned TP workers too.""" + with pytest.raises(ValueError, match="beyond the prompt length"): + pos_bridge.forward( + torch.tensor([PROMPT_IDS]), + intervene={"embed.hook_out": {"op": "suppress", "pos": 50}}, + ) diff --git a/tests/acceptance/model_bridge/test_vllm_multigpu_pp.py b/tests/acceptance/model_bridge/test_vllm_multigpu_pp.py new file mode 100644 index 0000000000..7dfeacf5f7 --- /dev/null +++ b/tests/acceptance/model_bridge/test_vllm_multigpu_pp.py @@ -0,0 +1,85 @@ +"""PP=2 validation for the vLLM driver — real engine, real model, >= 2 GPUs. + +Run on a multi-GPU box with the ``vllm`` extra installed (own process — the +sibling TP suite boots its own engines): + + uv run pytest tests/acceptance/model_bridge/test_vllm_multigpu_pp.py -m multigpu -v + +Compares pipeline_parallel_size=2 against the GPU-validated single-rank path: +captures must merge across stages with no gaps, interventions must bite on +hooks owned by BOTH stages, and the fire-counter layout check must hold under +vLLM's PP microbatching. +""" +from __future__ import annotations + +import pytest +import torch + +pytest.importorskip("vllm") + +from ._vllm_multigpu_common import ( + MULTIGPU_MARKS, + N_LAYERS, + PROMPT_IDS, + assert_caches_match, + bridge_pair_fixture, + close, +) + +pytestmark = MULTIGPU_MARKS + +bridges = bridge_pair_fixture(pipeline_parallel_size=2) + + +class TestPPCaptureParity: + def test_captures_merge_across_stages_and_match(self, bridges): + b1, b2 = bridges + toks = torch.tensor([PROMPT_IDS]) + _, cache1 = b1.run_with_cache(toks) + _, cache2 = b2.run_with_cache(toks) + # Key-set equality doubles as the merge check: a missing later-stage hook + # means the per-rank read or the merge dropped a stage. + assert_caches_match(cache1, cache2) + + def test_layout_check_ran_clean(self, bridges): + """First forward verified per-stage ownership + replica fire-counter + agreement under PP microbatching without raising.""" + _, b2 = bridges + assert b2._driver._layout_verified is True + + def test_logits_argmax_matches(self, bridges): + b1, b2 = bridges + toks = torch.tensor([PROMPT_IDS]) + l1 = b1.forward(toks, return_type="logits") + l2 = b2.forward(toks, return_type="logits") + # Exercises the stage-local gathers: lm_head + final norm live on stage 1. + assert torch.equal(l1.argmax(dim=-1), l2.argmax(dim=-1)) + + def test_sequence_logits_and_loss_under_pp(self, bridges): + _, b2 = bridges + assert b2._driver.provides_sequence_logits is True + loss = b2.forward(torch.tensor([PROMPT_IDS]), return_type="loss") + assert torch.isfinite(loss) + + +class TestPPInterventionParity: + @pytest.mark.parametrize( + "hook", + [ + "blocks.0.mlp.hook_out", # first stage + f"blocks.{N_LAYERS - 1}.mlp.hook_out", # last stage + ], + ) + def test_suppress_bites_and_matches_single_rank(self, bridges, hook): + """Interventions must apply on whichever stage owns the hook.""" + b1, b2 = bridges + toks = torch.tensor([PROMPT_IDS]) + spec = {hook: {"op": "suppress"}} + clean2 = b2.forward(toks, return_type="logits") + l1 = b1.forward(toks, return_type="logits", intervene=spec) + l2 = b2.forward(toks, return_type="logits", intervene=spec) + assert not torch.allclose( + clean2.float(), l2.float(), atol=1e-4, rtol=1e-4 + ), f"{hook}: intervention was a no-op under PP" + assert torch.equal(l1.argmax(dim=-1), l2.argmax(dim=-1)) + assert close(l1.float(), l2.float()) diff --git a/tests/unit/model_bridge/test_inspect_vllm_provider.py b/tests/unit/model_bridge/test_inspect_vllm_provider.py index 995d23d199..2d9b1e81fd 100644 --- a/tests/unit/model_bridge/test_inspect_vllm_provider.py +++ b/tests/unit/model_bridge/test_inspect_vllm_provider.py @@ -24,6 +24,13 @@ def _install_vllm_mocks(monkeypatch, llm_class: Any) -> None: in ``sys.modules`` so the provider's lazy ``from vllm import …`` resolves to mocks.""" vllm = types.ModuleType("vllm") vllm.LLM = llm_class + # Provider __init__ runs the tl_absent_hooks coverage check; give bare mocks a + # passing default (side_effect-configured mocks override this). + if ( + isinstance(llm_class, MagicMock) + and llm_class.return_value.collective_rpc.side_effect is None + ): + llm_class.return_value.collective_rpc.return_value = [[]] vllm.SamplingParams = MagicMock(name="SamplingParams") vllm_inputs = types.ModuleType("vllm.inputs") vllm_inputs.TokensPrompt = MagicMock(name="TokensPrompt") @@ -108,7 +115,11 @@ def _make_provider(monkeypatch, llm_instance: Any, hf_config: Any | None = None) TransformerLensVLLMModelAPI, ) - return TransformerLensVLLMModelAPI("any/model", device="cpu"), llm_class + provider = TransformerLensVLLMModelAPI("any/model", device="cpu") + # Drop __init__-time RPC history (coverage check) so call_count assertions + # measure only the generate path. + llm_instance.collective_rpc.reset_mock() + return provider, llm_class class TestModuleSafety: @@ -344,7 +355,8 @@ def _make_capture_provider(self, monkeypatch, *, captures=None, next_token: int ) def rpc(method, args=()): - return [captures] if method == "tl_read_captures" else [None] + # [[]] default keeps the boot-time tl_absent_hooks coverage check green. + return [captures] if method == "tl_read_captures" else [[]] llm.collective_rpc.side_effect = rpc provider, llm_class = _make_provider(monkeypatch, llm) @@ -586,7 +598,8 @@ def _make_provider_with_capture(self, monkeypatch, capture: list[str]): captures = {"blocks.0.hook_out": torch.zeros(4, 4, dtype=torch.float32)} def rpc(method, args=()): - return [captures] if method == "tl_read_captures" else [None] + # [[]] default keeps the boot-time tl_absent_hooks coverage check green. + return [captures] if method == "tl_read_captures" else [[]] llm.collective_rpc.side_effect = rpc @@ -598,7 +611,9 @@ def rpc(method, args=()): TransformerLensVLLMModelAPI, ) - return TransformerLensVLLMModelAPI("any/model", device="cpu", capture=capture), llm + provider = TransformerLensVLLMModelAPI("any/model", device="cpu", capture=capture) + llm.collective_rpc.reset_mock() # drop the __init__-time coverage check call + return provider, llm def test_eval_capture_during_generate_no_extra_forward(self, monkeypatch): from inspect_ai.model import GenerateConfig diff --git a/tests/unit/model_bridge/test_vllm_boot.py b/tests/unit/model_bridge/test_vllm_boot.py index a54a2438b6..3592d56b0d 100644 --- a/tests/unit/model_bridge/test_vllm_boot.py +++ b/tests/unit/model_bridge/test_vllm_boot.py @@ -48,10 +48,23 @@ def _cfg() -> TransformerBridgeConfig: ) +def _fake_rpc(absent_per_rank): + """Method-dispatch collective_rpc stub: tl_absent_hooks feeds the boot-time + coverage check (list per rank); every other method returns a real tensor + (the reconstruction probe's get_param is beartype-enforced).""" + + def rpc(method, *args, **kwargs): + if method == "tl_absent_hooks": + return absent_per_rank + return [torch.ones(16, 4)] + + return MagicMock(side_effect=rpc) + + @pytest.fixture def mocked_boot(monkeypatch): """Mock every external boundary boot_vllm crosses; yield handles for assertions.""" - plugin._config.clear() + plugin.clear_config() hf_config = _hf_config() cfg = _cfg() adapter = ArchitectureAdapter(cfg) @@ -63,9 +76,7 @@ def mocked_boot(monkeypatch): monkeypatch.setattr("transformers.AutoTokenizer.from_pretrained", auto_tokenizer) fake_llm = MagicMock(name="llm") - # The boot-time reconstruction probe fetches the unembedding via get_param, whose - # return type is beartype-enforced — the RPC must yield a real tensor. - fake_llm.collective_rpc = MagicMock(return_value=[torch.ones(16, 4)]) + fake_llm.collective_rpc = _fake_rpc([[]]) fake_vllm = MagicMock() fake_vllm.LLM = MagicMock(return_value=fake_llm) monkeypatch.setitem(sys.modules, "vllm", fake_vllm) @@ -115,13 +126,13 @@ def _fake_configure_tokenizer(tokenizer, cfg_): "configure_tok": configure_tok, } - plugin._config.clear() + plugin.clear_config() def test_rejects_locked_kwarg_override(): - """tensor_parallel_size != 1 fails fast — before any I/O.""" - with pytest.raises(ValueError, match="tensor_parallel_size"): - boot_vllm("any-model", tensor_parallel_size=2) + """Locked kwargs fail fast — before any I/O.""" + with pytest.raises(ValueError, match="skip_tokenizer_init"): + boot_vllm("any-model", skip_tokenizer_init=False) def test_rejects_position_interventions_with_batching(): @@ -190,7 +201,7 @@ def test_env_var_zero_does_not_warn(mocked_boot, monkeypatch): def test_plugin_config_cleared_after_boot(mocked_boot): """No leak to non-TL vllm.LLM users in the same process.""" boot_vllm("any-model") - assert plugin._config == {} + assert plugin._ENV_CONFIG_KEY not in os.environ def test_plugin_config_cleared_when_llm_construction_fails(mocked_boot): @@ -199,7 +210,38 @@ def test_plugin_config_cleared_when_llm_construction_fails(mocked_boot): mocked_boot["vllm_llm"].side_effect = RuntimeError("CUDA out of memory") with pytest.raises(RuntimeError, match="CUDA out of memory"): boot_vllm("any-model") - assert plugin._config == {} + assert plugin._ENV_CONFIG_KEY not in os.environ + + +def test_env_channel_populated_during_llm_construction(mocked_boot): + """Spawned workers only see the env var — it must be live while LLM(...) runs.""" + seen: dict = {} + + def _capture_env(*args, **kwargs): + seen["env"] = os.environ.get(plugin._ENV_CONFIG_KEY) + return mocked_boot["vllm_llm"].return_value + + mocked_boot["vllm_llm"].side_effect = _capture_env + boot_vllm("any-model") + assert seen["env"], "env spec channel was empty during worker construction" + restored = plugin._deserialize_config(seen["env"]) + assert "embed.hook_out" in restored["capture_specs"] + + +def test_boot_fails_loud_when_hook_absent_on_all_ranks(mocked_boot): + """A spec that installed on no rank is a broken dot-path — silent zeros otherwise.""" + fake_llm = mocked_boot["vllm_llm"].return_value + fake_llm.collective_rpc = _fake_rpc([["blocks.0.hook_out"], ["blocks.0.hook_out"]]) + with pytest.raises(RuntimeError, match="blocks.0.hook_out"): + boot_vllm("any-model") + + +def test_boot_accepts_per_rank_absence(mocked_boot): + """PP shards legally lack some layers — only absent-everywhere is an error.""" + fake_llm = mocked_boot["vllm_llm"].return_value + # Disjoint per rank — union covers every spec. + fake_llm.collective_rpc = _fake_rpc([["blocks.1.hook_out"], ["blocks.0.hook_out"]]) + assert boot_vllm("any-model") is not None def test_rejects_prefix_caching_override(): @@ -256,4 +298,65 @@ def test_missing_vllm_raises_actionable_import_error(monkeypatch): monkeypatch.setitem(sys.modules, "vllm", None) # forces ImportError on import with pytest.raises(ImportError, match=r"transformer-lens\[vllm\]"): boot_vllm("any-model") - assert plugin._config == {} + assert plugin._ENV_CONFIG_KEY not in os.environ + + +class TestParallelBoot: + """TP/PP plumbing: kwarg validation, LLM wiring, env handling. GPU behavior is + validated by tests/acceptance/model_bridge/test_vllm_multigpu*.py.""" + + PARALLEL_KWARGS = pytest.mark.parametrize( + "kwarg", ["tensor_parallel_size", "pipeline_parallel_size"] + ) + + @PARALLEL_KWARGS + def test_size_passed_to_llm_and_layout_unverified(self, mocked_boot, kwarg): + bridge = boot_vllm("any-model", **{kwarg: 2}) + assert mocked_boot["vllm_llm"].call_args.kwargs[kwarg] == 2 + assert bridge._driver._layout_verified is False # first forward cross-checks + + @PARALLEL_KWARGS + def test_default_is_single_rank(self, mocked_boot, kwarg): + bridge = boot_vllm("any-model") + assert mocked_boot["vllm_llm"].call_args.kwargs[kwarg] == 1 + assert bridge._driver._layout_verified is True # nothing to cross-check + + @PARALLEL_KWARGS + def test_rejects_batching(self, kwarg): + with pytest.raises(ValueError, match="parallelism is unsupported"): + boot_vllm("any-model", enable_batching=True, **{kwarg: 2}) + + @PARALLEL_KWARGS + def test_invalid_value_rejected(self, kwarg): + with pytest.raises(ValueError, match="positive int"): + boot_vllm("any-model", **{kwarg: 0}) + + @PARALLEL_KWARGS + def test_parallel_boot_clears_stale_mp_zero(self, mocked_boot, monkeypatch, kwarg): + """A prior single-rank boot leaves '0' in the env; parallel boots must not + inherit it — it would force the uni-process executor and workers never spawn.""" + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") + boot_vllm("any-model", **{kwarg: 2}) + assert "VLLM_ENABLE_V1_MULTIPROCESSING" not in os.environ + + def test_single_rank_still_forces_in_process(self, mocked_boot, monkeypatch): + monkeypatch.delenv("VLLM_ENABLE_V1_MULTIPROCESSING", raising=False) + boot_vllm("any-model") + assert os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] == "0" + + def test_tp_and_pp_combine(self, mocked_boot): + bridge = boot_vllm("any-model", tensor_parallel_size=2, pipeline_parallel_size=2) + kwargs = mocked_boot["vllm_llm"].call_args.kwargs + assert kwargs["tensor_parallel_size"] == 2 + assert kwargs["pipeline_parallel_size"] == 2 + assert bridge._driver._tp_size == 2 + assert bridge._driver._layout_verified is False + + +def test_compile_cache_disabled_for_tl_boots(mocked_boot, monkeypatch): + """Hooks are traced into the compiled graph; a cross-process cached artifact + either crashes at AOT load (closure bytecode mismatch) or silently serves a + hookless graph — TL boots must never share vLLM's compile cache.""" + monkeypatch.delenv("VLLM_DISABLE_COMPILE_CACHE", raising=False) + boot_vllm("any-model") + assert os.environ["VLLM_DISABLE_COMPILE_CACHE"] == "1" diff --git a/tests/unit/model_bridge/test_vllm_driver.py b/tests/unit/model_bridge/test_vllm_driver.py index a5b174c128..223b47199d 100644 --- a/tests/unit/model_bridge/test_vllm_driver.py +++ b/tests/unit/model_bridge/test_vllm_driver.py @@ -313,11 +313,10 @@ def test_forward_pushes_interventions_before_generate(self): torch.tensor([[1, 2, 3]]), intervene={"embed.hook_out": {"op": "suppress"}}, ) - rpc_calls = [c.args for c in driver._llm.collective_rpc.call_args_list] assert any( - args[0] == "tl_set_interventions" - and args[1] == ({"embed.hook_out": {"op": "suppress"}},) - for args in rpc_calls + c.args[0] == "tl_set_interventions" + and c.kwargs.get("args") == ({"embed.hook_out": {"op": "suppress"}},) + for c in driver._llm.collective_rpc.call_args_list ) def test_forward_always_pushes_interventions_for_reset(self): @@ -325,8 +324,10 @@ def test_forward_always_pushes_interventions_for_reset(self): pytest.importorskip("vllm") driver = _driver(captures={"embed.hook_out": torch.zeros(3, 4)}) driver.forward(torch.tensor([[1, 2, 3]])) - rpc_calls = [c.args for c in driver._llm.collective_rpc.call_args_list] - assert any(args[0] == "tl_set_interventions" and args[1] == ({},) for args in rpc_calls) + assert any( + c.args[0] == "tl_set_interventions" and c.kwargs.get("args") == ({},) + for c in driver._llm.collective_rpc.call_args_list + ) def test_forward_rejects_max_new_tokens_gt_one(self): """Decode-step writes overwrite the prefill buffer — silent capture corruption.""" @@ -352,14 +353,25 @@ def test_zero_capture_skips_worker_read(self): ] assert reads == [], f"hookless forward still read the buffers: {reads}" - driver2 = _driver(captures={"ln_final.hook_normalized": torch.zeros(3, 4)}) + # Method-aware mock: a blanket return value would hand the reconstruction + # probe a captures dict as a "weight". + driver2 = _driver(captures={}) + lnf = {"ln_final.hook_normalized": torch.zeros(3, 4)} + + def rpc2(method, args=(), **kwargs): + if method == "tl_read_captures": + return [lnf] + if method == "tl_get_param": + return [None] # no unembedding -> sampler-logprob fallback + return [[]] + + driver2._llm.collective_rpc = MagicMock(side_effect=rpc2) driver2.forward(torch.tensor([[1, 2, 3]])) # return_logits=True default reads2 = [ - c.args - for c in driver2._llm.collective_rpc.call_args_list - if c.args[0] == "tl_read_captures" + c for c in driver2._llm.collective_rpc.call_args_list if c.args[0] == "tl_read_captures" ] - assert len(reads2) == 1 and reads2[0][1][1] == ["ln_final.hook_normalized"] + assert len(reads2) == 1 + assert reads2[0].kwargs.get("args")[1] == ["ln_final.hook_normalized"] def _batched_request_output(request_id, generated_token=None, top_logprobs=None): @@ -572,11 +584,10 @@ def test_batched_intervention_spec_pushed(self): captures_by_req={"r0": {"embed.hook_out": torch.ones(2, 4)}}, ) driver.forward([[1, 2]], intervene={"embed.hook_out": {"op": "suppress"}}) - calls = [c.args for c in driver._llm.collective_rpc.call_args_list] assert any( - a[0] == "tl_set_batched_interventions" - and a[1] == ({"embed.hook_out": {"op": "suppress"}},) - for a in calls + c.args[0] == "tl_set_batched_interventions" + and c.kwargs.get("args") == ({"embed.hook_out": {"op": "suppress"}},) + for c in driver._llm.collective_rpc.call_args_list ) @@ -873,3 +884,225 @@ def test_attention_mask_batch_mismatch_raises(self): attention_mask=torch.tensor([[1, 1]]), return_logits=False, ) + + +class TestGatherParam: + """Cross-rank param reads: replicated → rank 0; vocab-sharded → rank-order concat.""" + + def test_single_rank_returns_shard(self): + driver = _driver(captures={}) + driver._llm.collective_rpc = MagicMock(return_value=[torch.ones(16, 4)]) + assert torch.equal(driver._gather_param("lm_head.weight"), torch.ones(16, 4)) + + def test_replicated_shards_return_rank_zero(self): + """Norm weights are identical on every rank — must NOT concatenate.""" + driver = _driver(captures={}) + w = torch.full((4,), 2.0) + driver._llm.collective_rpc = MagicMock(return_value=[w, w.clone()]) + assert driver._gather_param("model.norm.weight").shape == (4,) + + def test_sharded_shards_concat_in_rank_order(self): + driver = _driver(captures={}) + shard0, shard1 = torch.zeros(8, 4), torch.ones(8, 4) + driver._llm.collective_rpc = MagicMock(return_value=[shard0, shard1]) + full = driver._gather_param("lm_head.weight") + assert full.shape == (16, 4) + assert torch.equal(full[:8], shard0) and torch.equal(full[8:], shard1) + + def test_missing_on_all_ranks_returns_none(self): + driver = _driver(captures={}) + driver._llm.collective_rpc = MagicMock(return_value=[None, None]) + assert driver._gather_param("lm_head.bias") is None + + def test_probe_reconstructs_across_sharded_vocab(self): + """End-to-end: a 2-rank sharded unembedding must reassemble so tokens in the + UPPER half of the vocab can win argmax (concat-order regression).""" + driver = _driver(captures={}) + # Rank 0 rows favor token 0; rank 1 rows favor the ln_final direction strongly. + shard0 = torch.zeros(8, 4) + shard1 = torch.zeros(8, 4) + shard1[3] = 10.0 # global token id 8 + 3 = 11 + + def rpc(method, args=(), **kwargs): + if method == "tl_get_param" and args[0] == "lm_head.weight": + return [shard0, shard1] + if method == "tl_get_param": + return [None, None] # no bias; no tied-embedding fallback needed + return [[]] + + driver._llm.collective_rpc = MagicMock(side_effect=rpc) + assert driver.probe_logit_reconstruction() is True + logits = driver._reconstruct_logits(torch.ones(3, 4)) + assert logits.shape == (3, 16) + assert int(logits[0].argmax()) == 11 + + +class TestRpcTensorCoercion: + """Multiproc collective_rpc (TP>1) serializes tensors to nested lists — + every RPC read must coerce back.""" + + def test_get_param_coerces_list_payload(self): + driver = _driver(captures={}) + driver._llm.collective_rpc = MagicMock(return_value=[[[1.0, 2.0], [3.0, 4.0]]]) + out = driver.get_param("model.norm.weight") + assert isinstance(out, torch.Tensor) and out.shape == (2, 2) + + def test_gather_param_concats_list_shards(self): + """List shards have no .shape — coercion must precede the replicated-vs-sharded check.""" + driver = _driver(captures={}) + shard0 = [[0.0] * 4] * 8 + shard1 = [[1.0] * 4] * 8 + driver._llm.collective_rpc = MagicMock(return_value=[shard0, shard1]) + full = driver._gather_param("lm_head.weight") + assert isinstance(full, torch.Tensor) and full.shape == (16, 4) + assert torch.equal(full[8:], torch.ones(8, 4)) + + def test_gather_param_detects_replicated_list_shards(self): + driver = _driver(captures={}) + weight = [2.0, 2.0, 2.0, 2.0] + driver._llm.collective_rpc = MagicMock(return_value=[weight, list(weight)]) + out = driver._gather_param("model.norm.weight") + assert out.shape == (4,) + + def test_rpc_captures_coerces_dict_values(self): + coerced = VLLMDriver._rpc_captures({"embed.hook_out": [[1.0, 2.0]]}) + assert isinstance(coerced["embed.hook_out"], torch.Tensor) + + def test_tensor_payloads_pass_through_unchanged(self): + t = torch.randn(3, 4) + assert VLLMDriver._rpc_tensor(t) is t + + def test_get_param_none_payload_stays_none(self): + """None (missing path) is filtered before coercion at the call sites.""" + driver = _driver(captures={}) + driver._llm.collective_rpc = MagicMock(return_value=[None]) + assert driver.get_param("no.such.param") is None + + +class TestRankLayoutVerification: + """First capture-bearing forward under TP/PP cross-checks the rank layout.""" + + def _multi_rank_driver(self, tp=2, pp=1): + driver = _driver(captures={}) + driver._tp_size = tp + driver._pp_size = pp + driver._layout_verified = False + return driver + + def _rpc(self, counters): + def rpc(method, *args, **kwargs): + if method == "tl_read_counter": + return counters + return [[]] + + return MagicMock(side_effect=rpc) + + def test_tp_replicated_captures_pass(self): + driver = self._multi_rank_driver(tp=2) + t = torch.randn(3, 4) + driver._llm.collective_rpc = self._rpc([4, 4]) + driver._verify_rank_layout([{"embed.hook_out": t}, {"embed.hook_out": t.clone()}]) + + def test_divergent_replicas_fail_loud(self): + driver = self._multi_rank_driver(tp=2) + with pytest.raises(RuntimeError, match="no longer replicated"): + driver._verify_rank_layout( + [ + {"embed.hook_out": torch.zeros(3, 4)}, + {"embed.hook_out": torch.ones(3, 4)}, + ] + ) + + def test_wrong_copy_count_fails_loud(self): + """tp=2 but a hook appears on one rank only — installation drifted.""" + driver = self._multi_rank_driver(tp=2) + with pytest.raises(RuntimeError, match="expected exactly tp_size"): + driver._verify_rank_layout([{"embed.hook_out": torch.zeros(3, 4)}, {}]) + + def test_pp_disjoint_ownership_passes(self): + """pp=2, tp=1: each hook on exactly one rank; stage counters differ freely + (different hook counts per stage) — only same-stage replicas must agree.""" + driver = self._multi_rank_driver(tp=1, pp=2) + driver._llm.collective_rpc = self._rpc([3, 1]) + driver._verify_rank_layout( + [ + {"embed.hook_out": torch.randn(3, 4)}, + {"blocks.1.hook_out": torch.randn(3, 4)}, + ] + ) + + def test_tp_replica_counter_mismatch_fails_loud(self): + """Ranks serving the same hook set are TP replicas of one stage; unequal + fire counters mean their forwards diverged.""" + driver = self._multi_rank_driver(tp=2, pp=1) + driver._llm.collective_rpc = self._rpc([4, 3]) + t = torch.randn(3, 4) + with pytest.raises(RuntimeError, match="Fire-counter mismatch"): + driver._verify_rank_layout([{"embed.hook_out": t}, {"embed.hook_out": t.clone()}]) + + def test_single_rank_driver_skips_verification(self): + driver = _driver(captures={}) + assert driver._tp_size == 1 and driver._layout_verified is True + + +class TestMergeRankCaptures: + """PP stages own disjoint hook subsets; the merge reassembles the full dict.""" + + def test_disjoint_ranks_merge(self): + driver = _driver(captures={}) + merged = driver._merge_rank_captures( + [ + {"embed.hook_out": torch.zeros(3, 4)}, + {"blocks.0.hook_out": torch.ones(3, 4)}, + ], + ["embed.hook_out", "blocks.0.hook_out"], + ) + assert set(merged) == {"embed.hook_out", "blocks.0.hook_out"} + assert torch.equal(merged["blocks.0.hook_out"], torch.ones(3, 4)) + + def test_tp_replicas_first_copy_wins(self): + driver = _driver(captures={}) + t = torch.randn(3, 4) + merged = driver._merge_rank_captures( + [{"embed.hook_out": t}, {"embed.hook_out": t.clone()}], ["embed.hook_out"] + ) + assert merged["embed.hook_out"] is t + + def test_supported_hook_served_by_no_rank_fails_loud(self): + driver = _driver(captures={}) + with pytest.raises(RuntimeError, match="returned by no rank"): + driver._merge_rank_captures([{}, {}], ["embed.hook_out"]) + + def test_unsupported_requested_name_does_not_false_alarm(self): + """The forced ln_final read may be legitimately absent on fallback paths.""" + driver = _driver(captures={}) + merged = driver._merge_rank_captures([{}], ["not.a.supported.hook"]) + assert merged == {} + + +class TestPPWorkerInterventionDispatch: + """Specs for hooks another PP stage owns are skipped, not KeyError'd.""" + + def test_absent_hook_spec_skipped(self): + from transformer_lens.model_bridge.sources.vllm.worker_extension import ( + TLWorkerExtension, + ) + + worker = TLWorkerExtension() + worker._tl_scale_buffers = {"embed.hook_out": torch.ones(4)} + worker._tl_bias_buffers = {"embed.hook_out": torch.zeros(4)} + worker._tl_absent_hooks = {"blocks.9.hook_out"} # owned by another stage + worker.tl_set_interventions({"blocks.9.hook_out": {"op": "suppress"}}) + assert torch.equal(worker._tl_scale_buffers["embed.hook_out"], torch.ones(4)) + + def test_truly_unknown_hook_still_raises(self): + from transformer_lens.model_bridge.sources.vllm.worker_extension import ( + TLWorkerExtension, + ) + + worker = TLWorkerExtension() + worker._tl_scale_buffers = {"embed.hook_out": torch.ones(4)} + worker._tl_bias_buffers = {"embed.hook_out": torch.zeros(4)} + worker._tl_absent_hooks = set() + with pytest.raises(KeyError, match="Unknown hook"): + worker.tl_set_interventions({"tpyo.hook_out": {"op": "suppress"}}) diff --git a/tests/unit/model_bridge/test_vllm_internals.py b/tests/unit/model_bridge/test_vllm_internals.py index cbca9315e8..7373398c92 100644 --- a/tests/unit/model_bridge/test_vllm_internals.py +++ b/tests/unit/model_bridge/test_vllm_internals.py @@ -9,9 +9,13 @@ from types import SimpleNamespace import numpy as np +import pytest import torch -from transformer_lens.model_bridge.sources.vllm.internals import segment_by_request +from transformer_lens.model_bridge.sources.vllm.internals import ( + segment_by_request, + verify_hook_coverage, +) def _model_runner(query_start_loc, req_ids): @@ -49,3 +53,34 @@ def test_single_request(self): offsets, req_ids = segment_by_request(mr) assert offsets.tolist() == [0, 5] assert req_ids == ["only"] + + +class TestVerifyHookCoverage: + """Boot-time fail-loud when a spec installed on no rank (broken dot-path).""" + + def _llm(self, absent_per_rank): + from unittest.mock import MagicMock + + llm = MagicMock() + llm.collective_rpc.return_value = absent_per_rank + return llm + + def test_all_installed_passes(self): + verify_hook_coverage(self._llm([[], []])) + + def test_disjoint_per_rank_absence_passes(self): + """PP shards each lack the other's layers — union coverage is what matters.""" + verify_hook_coverage(self._llm([["blocks.1.hook_out"], ["blocks.0.hook_out"]])) + + def test_installation_never_ran_raises(self): + """A rank returning None never ran installation (plugin patch absent) — + must not pass coverage vacuously.""" + + with pytest.raises(RuntimeError, match="never ran on rank"): + verify_hook_coverage(self._llm([[], None])) + + def test_absent_everywhere_raises(self): + with pytest.raises(RuntimeError, match="blocks.9.hook_out"): + verify_hook_coverage( + self._llm([["blocks.9.hook_out"], ["blocks.9.hook_out", "embed.hook_out"]]) + ) diff --git a/tests/unit/model_bridge/test_vllm_plugin.py b/tests/unit/model_bridge/test_vllm_plugin.py index 4de4383589..f8510c9296 100644 --- a/tests/unit/model_bridge/test_vllm_plugin.py +++ b/tests/unit/model_bridge/test_vllm_plugin.py @@ -6,11 +6,17 @@ """ from __future__ import annotations +import json +import os from types import SimpleNamespace +import pytest import torch +import torch.nn as nn +from transformer_lens.model_bridge.sources.vllm import plugin from transformer_lens.model_bridge.sources.vllm.plugin import _make_batched_hook +from transformer_lens.model_bridge.sources.vllm.worker_extension import resolve_dot_path def _worker(specs): @@ -54,3 +60,116 @@ def test_no_spec_leaves_output_unchanged(self): out = hook(None, None, torch.ones(2, 4) * 3) assert torch.equal(out, torch.ones(2, 4) * 3) + + +class TestSpecChannel: + """The driver→worker spec channel must survive process boundaries (env var).""" + + def _sample_config(self): + return { + "capture_specs": { + "embed.hook_out": ("model.embed_tokens", 4), + "blocks.0.hook_out": ("model.layers.0", 4), + }, + "max_num_batched_tokens": 2048, + "dtype": torch.bfloat16, + "enable_batching": False, + "enable_position_interventions": True, + } + + def test_serialize_round_trip(self): + config = self._sample_config() + restored = plugin._deserialize_config(plugin._serialize_config(config)) + assert restored == config + assert restored["dtype"] is torch.bfloat16 + + def test_deserialize_rejects_non_dtype(self): + raw = plugin._serialize_config(self._sample_config()) + data = json.loads(raw) + data["dtype"] = "nn" # torch.nn exists but is not a dtype + with pytest.raises(ValueError, match="not a torch dtype"): + plugin._deserialize_config(json.dumps(data)) + + def test_configure_sets_env_and_clear_config_removes_it(self): + config = self._sample_config() + try: + plugin.configure(**config) + assert plugin._ENV_CONFIG_KEY in os.environ + assert plugin._active_config()["capture_specs"] == config["capture_specs"] + finally: + plugin.clear_config() + assert plugin._ENV_CONFIG_KEY not in os.environ + + def test_active_config_reaches_spawned_workers(self, monkeypatch): + """A spawned worker re-imports the module fresh but inherits the env var.""" + config = self._sample_config() + monkeypatch.setenv(plugin._ENV_CONFIG_KEY, plugin._serialize_config(config)) + assert plugin._active_config() == config + + def test_active_config_none_when_no_channel(self, monkeypatch): + monkeypatch.delenv(plugin._ENV_CONFIG_KEY, raising=False) + assert plugin._active_config() is None + + +class TestResolveDotPath: + """Per-rank module resolution: missing segments mean 'not on this rank', not a crash.""" + + def _model(self): + model = nn.Module() + model.layers = nn.ModuleList([nn.Linear(2, 2)]) + model.norm = nn.LayerNorm(2) + return SimpleNamespace(model=model) + + def test_resolves_nested_and_indexed(self): + root = self._model() + assert resolve_dot_path(root, "model.norm") is root.model.norm + assert resolve_dot_path(root, "model.layers.0") is root.model.layers[0] + + def test_missing_attribute_returns_none(self): + assert resolve_dot_path(self._model(), "model.mlp") is None + + def test_out_of_range_index_returns_none(self): + """A PP rank owning layers [0..k) legally lacks the later indices.""" + assert resolve_dot_path(self._model(), "model.layers.7") is None + + def test_pp_missing_layer_stub_treated_as_absent(self): + """vLLM PP fills non-owned slots (layers, embed_tokens, norm) with + PPMissingLayer identity stubs the forward never calls — resolving one would + install a hook that serves its dead zero buffer as a real capture.""" + + class PPMissingLayer(nn.Module): # matched by name, as vLLM's real class is + pass + + root = self._model() + root.model.layers.append(PPMissingLayer()) + root.model.norm = PPMissingLayer() + assert resolve_dot_path(root, "model.layers.0") is root.model.layers[0] + assert resolve_dot_path(root, "model.layers.1") is None + assert resolve_dot_path(root, "model.norm") is None + + +class TestCompiledHookInstrumentationSafety: + """Compile-traced hook internals must stay annotation-free: pytest's jaxtyping + hook wraps annotated functions, and dynamo tracing the wrapper corrupts + jaxtyping's thread-local memo stack process-wide (GPU-verified).""" + + def test_gated_capture_has_no_annotations(self): + from transformer_lens.model_bridge.sources.vllm.plugin import _gated_capture + + assert ( + _gated_capture.__wrapped__.__annotations__ == {} + if hasattr(_gated_capture, "__wrapped__") + else _gated_capture.__annotations__ == {} + ) + + +class TestVllmPluginEntryPoint: + def test_general_plugin_entry_point_declared(self): + """Spawned TP workers only get the load_model patch via vllm.general_plugins — + without this entry point, workers boot hookless and captures are empty.""" + from importlib.metadata import entry_points + + eps = entry_points(group="vllm.general_plugins") + assert any( + ep.value == "transformer_lens.model_bridge.sources.vllm.plugin:register" for ep in eps + ) diff --git a/tests/unit/model_bridge/test_vllm_worker_extension.py b/tests/unit/model_bridge/test_vllm_worker_extension.py index 3d254347c2..ba96343a6a 100644 --- a/tests/unit/model_bridge/test_vllm_worker_extension.py +++ b/tests/unit/model_bridge/test_vllm_worker_extension.py @@ -8,12 +8,15 @@ from types import SimpleNamespace +import pytest import torch from transformer_lens.model_bridge.sources.vllm.worker_extension import ( TLWorkerExtension, _apply_intervention, _apply_op, + decode_tensor, + encode_tensor, ) @@ -26,9 +29,9 @@ def test_reads_named_tensor(self): ext.model_runner = SimpleNamespace( # type: ignore[attr-defined] model=SimpleNamespace(norm=SimpleNamespace(weight=weight)) ) - out = ext.tl_get_param("norm.weight") + out = decode_tensor(ext.tl_get_param("norm.weight")) assert torch.equal(out, weight) - assert out is not weight # cloned, not a live reference + assert out is not weight # cloned bytes, not a live reference def test_missing_path_returns_none(self): ext = TLWorkerExtension() @@ -54,12 +57,23 @@ def _ext(self): def test_names_filters(self): out = self._ext().tl_read_captures([2], names=["a"]) assert set(out) == {"a"} - assert tuple(out["a"].shape) == (2, 4) + assert tuple(decode_tensor(out["a"]).shape) == (2, 4) def test_none_reads_all(self): out = self._ext().tl_read_captures([2]) assert set(out) == {"a", "b"} + def test_unfired_hook_not_served(self): + """A still-open capture flag means the hook never ran this forward — under PP + that's an installed-but-not-owned module (tied-embedding alias, norm on a + non-final stage) whose buffer holds no data; serving it would hand dead zeros + to the rank merge. No flag entry keeps legacy single-rank behavior.""" + ext = self._ext() + ext._tl_capture_flags = {"a": torch.ones(1), "b": torch.zeros(1)} + assert set(ext.tl_read_captures([2])) == {"a"} + ext._tl_capture_flags = {"a": torch.ones(1)} # "b" flagless -> still served + assert set(ext.tl_read_captures([2])) == {"a", "b"} + def test_batched_names_filters(self): ext = TLWorkerExtension() ext._tl_accum = { @@ -68,6 +82,7 @@ def test_batched_names_filters(self): } out = ext.tl_read_batched_captures(names=["a"]) assert set(out["r"]) == {"a"} + assert torch.equal(decode_tensor(out["r"]["a"]), torch.ones(2, 4)) class TestFireCounter: @@ -252,9 +267,10 @@ def test_read_concatenates_chunks_in_order(self): } out = ext.tl_read_batched_captures() # req-A's two chunks cat to (3, 4); chunk order is token order. - assert tuple(out["req-A"]["embed.hook_out"].shape) == (3, 4) - assert torch.equal(out["req-A"]["embed.hook_out"][2], torch.full((4,), 2.0)) - assert tuple(out["req-B"]["embed.hook_out"].shape) == (3, 4) + req_a = decode_tensor(out["req-A"]["embed.hook_out"]) + assert tuple(req_a.shape) == (3, 4) + assert torch.equal(req_a[2], torch.full((4,), 2.0)) + assert tuple(decode_tensor(out["req-B"]["embed.hook_out"]).shape) == (3, 4) def test_reset_clears_accumulator(self): ext = TLWorkerExtension() @@ -273,3 +289,53 @@ def test_set_batched_interventions_stores_specs(self): ext = TLWorkerExtension() ext.tl_set_batched_interventions({"h": {"op": "suppress"}}) assert ext._tl_intervention_specs == {"h": {"op": "suppress"}} + + +class TestAbsentHooks: + """tl_absent_hooks reports per-rank installation gaps for the boot coverage check.""" + + def test_never_installed_returns_none(self): + """No _tl_absent_hooks attr = installation never ran — must NOT read as + 'nothing absent'.""" + worker = TLWorkerExtension() + assert worker.tl_absent_hooks() is None + + def test_reports_recorded_names_sorted(self): + worker = TLWorkerExtension() + worker._tl_absent_hooks = {"blocks.1.hook_out", "blocks.0.hook_out"} + assert worker.tl_absent_hooks() == ["blocks.0.hook_out", "blocks.1.hook_out"] + + def test_installed_with_no_gaps_returns_empty_list(self): + worker = TLWorkerExtension() + worker._tl_absent_hooks = set() + assert worker.tl_absent_hooks() == [] + + def test_remove_hooks_clears(self): + worker = TLWorkerExtension() + worker._tl_absent_hooks = {"blocks.0.hook_out"} + worker.tl_remove_hooks() + assert worker.tl_absent_hooks() == [] + + +class TestTensorWireFormat: + """encode/decode must round-trip exactly — vLLM's multiproc RPC can't carry + raw tensors from extension methods (GPU-verified header leak on 0.20.2).""" + + @pytest.mark.parametrize( + "dtype", + [torch.float32, torch.float16, torch.bfloat16, torch.int64], + ) + def test_round_trip_exact(self, dtype): + t = (torch.randn(3, 5) * 4).to(dtype) + payload = encode_tensor(t) + assert isinstance(payload["data"], bytes) + out = decode_tensor(payload) + assert out.dtype == dtype and out.shape == t.shape + assert torch.equal(out, t) + + def test_payload_is_msgpack_native(self): + """Only str/bool/int-list/bytes — nothing vLLM's serializer would mangle.""" + payload = encode_tensor(torch.ones(2, 2)) + assert set(payload) == {"__tl_tensor__", "dtype", "shape", "data"} + assert isinstance(payload["dtype"], str) + assert all(isinstance(d, int) for d in payload["shape"]) diff --git a/transformer_lens/model_bridge/sources/inspect/vllm_provider.py b/transformer_lens/model_bridge/sources/inspect/vllm_provider.py index 43f4c59843..b4d7f003e8 100644 --- a/transformer_lens/model_bridge/sources/inspect/vllm_provider.py +++ b/transformer_lens/model_bridge/sources/inspect/vllm_provider.py @@ -17,7 +17,6 @@ from __future__ import annotations import gc -import os from typing import Any, Mapping import numpy as np @@ -47,22 +46,17 @@ # Distinct from the HF ``tl_bridge`` provider and from inspect_ai's built-in ``vllm``. PROVIDER_NAME = "tl_bridge_vllm" -# Forced ``LLM(...)`` kwargs that the capture-hook design depends on (matches the same -# set in ``sources/vllm/source.py``). Multi-device / vLLM-owned tokenizer break the wire -# path; we own the tokenizer and the plugin assumes single-process workers. +# Forced ``LLM(...)`` kwargs the capture-hook design depends on. Multi-device and a +# vLLM-owned tokenizer break the wire path; prefix caching would misalign capture rows +# (see ``construct_instrumented_llm``, which supplies the caching/parallelism values). _LOCKED_VLLM_KWARGS = { "tensor_parallel_size": 1, "pipeline_parallel_size": 1, + "enable_prefix_caching": False, "skip_tokenizer_init": True, "disable_log_stats": True, } -# Dotted path to the worker extension whose ``tl_*`` methods this provider drives via -# ``collective_rpc`` (capture reads, intervention specs, hook teardown). -_WORKER_EXTENSION_CLS = ( - "transformer_lens.model_bridge.sources.vllm.worker_extension.TLWorkerExtension" -) - def _kinds_from_specs(specs: dict[str, Any]) -> frozenset[str]: """Boundary kinds (resid_post/attn_out/...) served by the overlay's ``capture_specs``. @@ -104,7 +98,7 @@ def __init__( from transformers import AutoConfig, AutoTokenizer try: - from vllm import LLM + import vllm # noqa: F401 — import check; construction is in construct_instrumented_llm except ImportError as exc: raise ImportError( "The tl_bridge_vllm provider requires vLLM (Linux + CUDA). Install with " @@ -114,9 +108,10 @@ def __init__( from transformer_lens.utilities.hf_utils import get_hf_token - from ..vllm import plugin from ..vllm.internals import extract_hf_config from ..vllm.overlays import get_overlay + from ..vllm.source import _dtype_from_hf_config, construct_instrumented_llm + from ..vllm.worker_extension import dtype_name # Caller-overridable LLM kwargs go through ``vllm_kwargs``; the locked set above # may not be overridden (multi-device / vLLM-owned tokenizer break our wire path). @@ -149,38 +144,25 @@ def __init__( overlay = get_overlay(architecture) resolved_dtype = dtype if dtype is not None else _dtype_from_hf_config(hf_config_preview) capture_specs = overlay.capture_specs(hf_config_preview) - plugin.configure( + self._llm = construct_instrumented_llm( + model_name, capture_specs=capture_specs, max_num_batched_tokens=max_num_batched_tokens, dtype=resolved_dtype, enable_batching=False, # eager batched path is a later increment - ) - plugin.register() - # Single-process workers are required — otherwise the plugin's _config singleton - # isn't visible to worker subprocesses and hooks silently fail to install. - os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0" - - try: - self._llm = LLM( - model=model_name, - gpu_memory_utilization=gpu_memory_utilization, - max_model_len=max_model_len, - max_num_batched_tokens=max_num_batched_tokens, - # Worker extension's tl_* methods are reachable via collective_rpc. - worker_extension_cls=_WORKER_EXTENSION_CLS, + llm_kwargs={ + "gpu_memory_utilization": gpu_memory_utilization, + "max_model_len": max_model_len, # Full-vocab logprobs so _generate_capture can synthesize logits at the # generated position (vLLM caps logprobs to this value; default 20 is too # small for mech interp). - max_logprobs=int(hf_config_preview.vocab_size), - dtype=str(resolved_dtype).replace("torch.", "") if dtype is not None else "auto", - **_LOCKED_VLLM_KWARGS, + "max_logprobs": int(hf_config_preview.vocab_size), + "dtype": dtype_name(resolved_dtype) if dtype is not None else "auto", + "skip_tokenizer_init": True, + "disable_log_stats": True, **vllm_kwargs, - ) - finally: - # Always clear, even on a failed boot — stale specs would make the next - # in-process vllm.LLM(...) walk our dot-paths on a foreign model. - plugin._config.clear() - # Capture-path validity was enforced inside patched_load_model during LLM(...). + }, + ) hf_config = extract_hf_config(self._llm) # Capture-relevant constants used by _generate_capture. @@ -441,16 +423,6 @@ def close(self) -> None: pass -def _dtype_from_hf_config(hf_config: Any) -> torch.dtype: - """Best-effort dtype from an HF config — vLLM prefers fp16 on GPU.""" - raw = getattr(hf_config, "torch_dtype", None) - if isinstance(raw, torch.dtype): - return raw - if isinstance(raw, str): - return getattr(torch, raw, torch.float16) - return torch.float16 - - def _synthesize_logits(request_output: Any, n_tokens: int, d_vocab: int) -> torch.Tensor: """Build a ``(1, n_tokens, d_vocab)`` logits-like tensor from vLLM's sampler output — log-probs (not raw logits), with earlier positions ``-inf`` (lm_head is bypassed so diff --git a/transformer_lens/model_bridge/sources/vllm/driver.py b/transformer_lens/model_bridge/sources/vllm/driver.py index aec008f45d..3fcb33ab81 100644 --- a/transformer_lens/model_bridge/sources/vllm/driver.py +++ b/transformer_lens/model_bridge/sources/vllm/driver.py @@ -5,7 +5,7 @@ import logging import threading import warnings -from typing import Any, Mapping +from typing import Any, Mapping, Optional, Sequence, Union import torch @@ -17,6 +17,7 @@ from transformer_lens.model_bridge.sources._driver_base import DriverBase from .intervention_specs import validate_spec +from .worker_extension import _TL_TENSOR_KEY, decode_tensor # vLLM's distributed teardown operates on process-wide globals, so close() may only run # it when no other VLLMDriver-owned engine is alive (notebook re-binding boots B before @@ -49,11 +50,20 @@ def __init__( max_num_batched_tokens: int, enable_batching: bool = False, enable_position_interventions: bool = False, + tensor_parallel_size: int = 1, + pipeline_parallel_size: int = 1, ) -> None: super().__init__(adapter.cfg, tokenizer) self._llm = llm self._max_num_batched_tokens = max_num_batched_tokens self._enable_batching = enable_batching + # Rank layout: PP stages own disjoint layer (hence hook) subsets; the TP + # ranks within a stage hold replicas of that stage's post-all-reduce hook + # points. Reads merge across stages and take the first replica within one; + # the first capture-bearing forward cross-checks the whole layout + # (see _verify_rank_layout) and then trusts it. + self._tp_size = tensor_parallel_size + self._layout_verified = tensor_parallel_size == 1 and pipeline_parallel_size == 1 # Position-scoped 'pos' interventions need (max_n, width) affine buffers, # allocated at boot only when this is set (see plugin.patched_load_model). self._enable_position_interventions = enable_position_interventions @@ -95,7 +105,9 @@ def __init__( def forward( self, - input_ids: TensorLike | None = None, + # Wider than the protocol's TensorLike: the batched path documents plain + # (ragged) list[int] / list[list[int]] prompts, which have no shape/dtype. + input_ids: Optional[Union[TensorLike, Sequence[Any]]] = None, *, capture: tuple[str, ...] = (), intervene: Mapping[str, Intervention] | None = None, @@ -162,13 +174,23 @@ def forward( n_tokens = len(ids_list) read_names = self._read_names(names, return_logits) - # collective_rpc returns one result per worker; single-rank, so [0]. Nothing to - # read (no captures, logits off) → skip the crossing altogether. - worker_captures = ( - self._llm.collective_rpc("tl_read_captures", args=([n_tokens], read_names))[0] - if read_names - else {} - ) + # collective_rpc returns one result per worker; each returns only the hooks + # it owns (PP shards) with TP replicas within a stage — merge across ranks. + # Nothing to read (no captures, logits off) → skip the crossing altogether. + if read_names: + per_rank_raw = self._llm.collective_rpc( + "tl_read_captures", args=([n_tokens], read_names) + ) + if not self._layout_verified: + self._verify_rank_layout([self._rpc_captures(caps) for caps in per_rank_raw]) + self._layout_verified = True + # Merge on the wire dicts, decode only the survivors — decoding every TP + # replica just to discard all but the first is wasted CPU per forward. + worker_captures = self._rpc_captures( + self._merge_rank_captures(per_rank_raw, read_names) + ) + else: + worker_captures = {} logits: torch.Tensor | None = None if return_logits: @@ -210,7 +232,7 @@ def _expose_captured( def _forward_batched( self, - input_ids: TensorLike, + input_ids: Union[TensorLike, Sequence[Any]], intervene_specs: dict, return_logits: bool, names: list[str], @@ -242,7 +264,6 @@ def _forward_batched( # Reset accumulators so prior-forward chunks don't leak into the cat. self._llm.collective_rpc("tl_reset_accumulators") - self._llm.collective_rpc("tl_reset_counter") self._llm.collective_rpc("tl_set_batched_interventions", args=(intervene_specs,)) d_vocab = self.bridge_config.d_vocab @@ -264,6 +285,11 @@ def _forward_batched( worker_captures = self._llm.collective_rpc( "tl_read_batched_captures", args=(read_names,) )[0] + # Batched runs single-rank today, but coerce anyway — multiproc RPC + # serializes tensors to lists (see _rpc_tensor). + worker_captures = { + req_id: self._rpc_captures(caps) for req_id, caps in worker_captures.items() + } captured = self._assemble_padded(outputs, worker_captures, prompt_lens) else: captured = {} @@ -360,12 +386,129 @@ def _synthesize_logits_batched( return logits def get_param(self, dotted_name: str) -> torch.Tensor | None: - """Fetch a named worker tensor (e.g. ``model.norm.weight``) for conversions + """Fetch a named model tensor (e.g. ``model.norm.weight``) for conversions the bridge can't otherwise do (ln_final post→pre-weight; see the overlay). - None if closed or the path is missing.""" + Gathered across ranks: replicated params return one copy, vocab-sharded + weights concatenate, stage-local params (PP) come from whichever rank owns + them. None if closed or the path resolves nowhere.""" + return self._gather_param(dotted_name) + + @staticmethod + def _rpc_tensor(value: Any) -> torch.Tensor: + """Decode a non-None collective_rpc payload back to a tensor. + + Worker methods return the explicit wire format (see + ``worker_extension.encode_tensor``) because vLLM's multiproc RPC can't + round-trip raw tensors. Raw tensors still pass through for mocks and any + legacy in-process payloads; anything else is best-effort coerced.""" + if isinstance(value, torch.Tensor): + return value + if isinstance(value, Mapping) and value.get(_TL_TENSOR_KEY): + return decode_tensor(dict(value)) + return torch.as_tensor(value) + + @classmethod + def _rpc_captures(cls, captures: Mapping[str, Any]) -> dict[str, torch.Tensor]: + return {name: cls._rpc_tensor(value) for name, value in captures.items()} + + def _gather_param(self, dotted_name: str, dim: int = 0) -> torch.Tensor | None: + """Fetch a param across all TP ranks: replicated → rank 0; sharded → concat. + + Vocab-parallel weights (``lm_head.weight``, ``embed_tokens.weight``) hold + contiguous per-rank slices in rank order, which is also collective_rpc's + result order — concatenation along ``dim`` reassembles the full tensor + (vocab padding lands in the tail rows, sliced off by reconstruction). + """ if self._llm is None: return None - return self._llm.collective_rpc("tl_get_param", args=(dotted_name,))[0] + shards = [ + self._rpc_tensor(s) + for s in self._llm.collective_rpc("tl_get_param", args=(dotted_name,)) + if s is not None + ] + if not shards: + return None + if len(shards) == 1: + return shards[0] + if all(s.shape == shards[0].shape and torch.equal(s, shards[0]) for s in shards[1:]): + return shards[0] # replicated (norm weights, biases on some archs) + return torch.cat(shards, dim=dim) + + def _merge_rank_captures(self, per_rank_captures: list, requested: list[str]) -> dict[str, Any]: + """Merge per-rank capture dicts into one (values stay opaque — callers decode): + PP stages own disjoint hook subsets; TP replicas within a stage agree (verified + once by _verify_rank_layout), so the first copy wins. A requested, supported + hook served by NO rank fails loud — zero-filling would be silent data loss + (same policy as the batched join).""" + merged: dict[str, Any] = {} + for captures in per_rank_captures: + for name, tensor in captures.items(): + if name not in merged: + merged[name] = tensor + missing = [ + name for name in requested if name not in merged and name in self.supported_hook_points + ] + if missing: + raise RuntimeError( + f"Capture(s) {missing} returned by no rank — a hook fired nowhere " + "despite passing the boot-time coverage check. Report this." + ) + return merged + + def _verify_rank_layout(self, per_rank_captures: list) -> None: + """One-time cross-rank check of the capture layout under TP/PP. + + Invariant: each hook lives on exactly one PP stage and is replicated across + that stage's ``tp_size`` ranks (every overlay hook point is post-all-reduce). + A wrong copy count means installation drift; divergent replicas mean a hook + point moved pre-all-reduce — either way rank-merged reads would be silently + wrong, so fail loud on the first capture-bearing forward. Ranks serving the + same hook set are TP replicas of one stage and must report identical fire + counters; counters are NOT comparable across stages — per-rank installed + counts overcount modules that exist but never run (tied-embedding aliases, + archs that instantiate norm on every rank). + """ + all_names: set[str] = set() + for captures in per_rank_captures: + all_names |= set(captures.keys()) + for name in sorted(all_names): + copies = [ + (rank, caps[name]) for rank, caps in enumerate(per_rank_captures) if name in caps + ] + if len(copies) != self._tp_size: + raise RuntimeError( + f"Rank-layout check failed for {name!r}: found on {len(copies)} rank(s) " + f"but expected exactly tp_size={self._tp_size} (one PP stage × its TP " + "replicas). Per-rank installation drifted — report this." + ) + rank0, t0 = copies[0] + for rank, t_r in copies[1:]: + if t0.shape != t_r.shape or not torch.allclose( + t0.float(), t_r.float(), atol=1e-5, rtol=1e-5 + ): + diff = ( + (t0.float() - t_r.float()).abs().max().item() + if t0.shape == t_r.shape + else float("inf") + ) + raise RuntimeError( + f"TP replication check failed for {name!r}: rank {rank0} vs rank " + f"{rank} max abs diff {diff:.3e}. This hook point is no longer " + "replicated across ranks (vLLM may have moved it pre-all-reduce) — " + "merged capture reads would be silently wrong. Report this." + ) + counters = [int(c) for c in self._llm.collective_rpc("tl_read_counter")] + stages: dict = {} + for rank, captures in enumerate(per_rank_captures): + stages.setdefault(frozenset(captures.keys()), []).append(rank) + for ranks in stages.values(): + group = [counters[r] for r in ranks] + if len(set(group)) > 1: + raise RuntimeError( + f"Fire-counter mismatch within a TP replica group: ranks {ranks} " + f"serve the same hooks but fired {group} times. Replicated forwards " + "diverged — merged capture reads would be silently wrong. Report this." + ) def probe_logit_reconstruction(self) -> bool: """One-time unembedding fetch + cache; downgrades ``provides_sequence_logits`` @@ -375,13 +518,14 @@ def probe_logit_reconstruction(self) -> bool: if self._unembed_probed: return self._unembed is not None self._unembed_probed = True - weight = self.get_param("lm_head.weight") + # Gathered reads: under TP the unembedding is vocab-sharded per rank. + weight = self._gather_param("lm_head.weight") if weight is None: # tied embeddings expose no separate lm_head - weight = self.get_param("model.embed_tokens.weight") + weight = self._gather_param("model.embed_tokens.weight") if weight is None: self.provides_sequence_logits = False return False - bias = self.get_param("lm_head.bias") + bias = self._gather_param("lm_head.bias") d_vocab = int(self.bridge_config.d_vocab) # Slice vLLM's vocab-pad rows at cache time; fp32 residency (~2× checkpoint # dtype on CPU) trades memory for skipping a full-matrix upcast per forward. @@ -432,9 +576,11 @@ def _unfold_ln_final(self, t: torch.Tensor) -> torch.Tensor: beats silent cross-backend mismatch.""" if not self._lnf_probed: self._lnf_probed = True - # The overlay's capture spec owns the module path; derive the weight from it. + # The overlay's capture spec owns the module path; derive the weight from + # it. Gathered read: under PP the final norm lives only on the last stage, + # so a rank-0 get_param would miss it. path = self._capture_paths.get(self._LN_FINAL, "model.norm") - weight = self.get_param(f"{path}.weight") + weight = self._gather_param(f"{path}.weight") if weight is None: warnings.warn( "ln_final.hook_normalized: norm weight unreachable — the captured " diff --git a/transformer_lens/model_bridge/sources/vllm/internals.py b/transformer_lens/model_bridge/sources/vllm/internals.py index 185a5a9a4b..4ec1efc621 100644 --- a/transformer_lens/model_bridge/sources/vllm/internals.py +++ b/transformer_lens/model_bridge/sources/vllm/internals.py @@ -24,6 +24,34 @@ def extract_hf_config(llm: Any) -> Any: ) from e +def verify_hook_coverage(llm: Any) -> None: + """Raise if any configured capture hook installed on NO rank. + + Per-rank absence is legal (pipeline-parallel ranks own layer subsets), so + hook installation skips missing modules instead of raising — this boot-time + check restores the fail-loud contract: a hook absent everywhere is a broken + overlay dot-path and would otherwise read back as silent zeros. + """ + absent_per_rank = llm.collective_rpc("tl_absent_hooks") + if not absent_per_rank: + return + never_ran = [rank for rank, absent in enumerate(absent_per_rank) if absent is None] + if never_ran: + raise RuntimeError( + f"Capture-hook installation never ran on rank(s) {never_ran}: the TL vLLM " + "plugin did not execute in those worker processes. The vllm.general_plugins " + "entry point registers it — if this install predates that, rerun `uv sync`." + ) + absent_everywhere = set(absent_per_rank[0]) + for rank_absent in absent_per_rank[1:]: + absent_everywhere &= set(rank_absent) + if absent_everywhere: + raise RuntimeError( + f"Capture hooks failed to install on every worker: {sorted(absent_everywhere)}. " + "The overlay's dot-paths don't match this model's module tree." + ) + + # Cumulative per-request query offsets: FlashAttention/Triton name them # query_start_loc, FlashInfer names them qo_indptr. Request i = rows i:i+1. _QUERY_OFFSET_ATTRS = ("query_start_loc", "qo_indptr") diff --git a/transformer_lens/model_bridge/sources/vllm/plugin.py b/transformer_lens/model_bridge/sources/vllm/plugin.py index 51e5d70fb5..dcd7b238e7 100644 --- a/transformer_lens/model_bridge/sources/vllm/plugin.py +++ b/transformer_lens/model_bridge/sources/vllm/plugin.py @@ -25,21 +25,27 @@ """ from __future__ import annotations +import json +import os import re -from typing import Any, Dict, Tuple +from typing import Any, Dict, Optional, Tuple import torch from .internals import segment_by_request -from .worker_extension import _apply_op +from .worker_extension import _apply_op, dtype_name, resolve_dot_path # Decoder layers return the fused-residual (mlp_delta, residual) 2-tuple, so # their hooks materialize the sum (see _make_capture_hook); other modules don't. _DECODER_LAYER_PATH = re.compile(r"^model\.layers\.\d+$") -# Transient signal driver → worker during LLM construction. Per-Worker buffers -# live on Worker instances so concurrent boot_vllm calls don't collide. -_config: Dict[str, Any] = {} +# Transient signal driver → worker during LLM construction: an env var, because +# spawned worker processes (TP/PP > 1) re-import this module and would see any +# module global empty — vLLM runs ``register()`` in every worker, so the patch +# itself propagates; only this payload needs a cross-process channel. Per-Worker +# buffers live on Worker instances so concurrent boot_vllm calls don't collide. +# Single-node only: env vars don't reach Ray remote workers. +_ENV_CONFIG_KEY = "TL_VLLM_PLUGIN_CONFIG" _install_patched = False _orig_load_model = None @@ -52,11 +58,56 @@ def configure( enable_position_interventions: bool = False, ) -> None: """Set capture specs, buffer length, dtype, and hook flavor before ``LLM(...)``.""" - _config["capture_specs"] = capture_specs - _config["max_num_batched_tokens"] = max_num_batched_tokens - _config["dtype"] = dtype - _config["enable_batching"] = enable_batching - _config["enable_position_interventions"] = enable_position_interventions + os.environ[_ENV_CONFIG_KEY] = _serialize_config( + { + "capture_specs": capture_specs, + "max_num_batched_tokens": max_num_batched_tokens, + "dtype": dtype, + "enable_batching": enable_batching, + "enable_position_interventions": enable_position_interventions, + } + ) + + +def clear_config() -> None: + """Unset the spec channel. Boot sites call this in a ``finally`` after + ``LLM(...)`` so a later non-TL engine can't inherit the specs.""" + os.environ.pop(_ENV_CONFIG_KEY, None) + + +def _serialize_config(config: Dict[str, Any]) -> str: + return json.dumps( + { + "capture_specs": { + name: [path, width] for name, (path, width) in config["capture_specs"].items() + }, + "max_num_batched_tokens": config["max_num_batched_tokens"], + "dtype": dtype_name(config["dtype"]), + "enable_batching": config["enable_batching"], + "enable_position_interventions": config["enable_position_interventions"], + } + ) + + +def _deserialize_config(raw: str) -> Dict[str, Any]: + data = json.loads(raw) + dtype = getattr(torch, data["dtype"], None) + if not isinstance(dtype, torch.dtype): + raise ValueError(f"{_ENV_CONFIG_KEY}: {data['dtype']!r} is not a torch dtype.") + return { + "capture_specs": { + name: (path, int(width)) for name, (path, width) in data["capture_specs"].items() + }, + "max_num_batched_tokens": int(data["max_num_batched_tokens"]), + "dtype": dtype, + "enable_batching": bool(data["enable_batching"]), + "enable_position_interventions": bool(data["enable_position_interventions"]), + } + + +def _active_config() -> Optional[Dict[str, Any]]: + raw = os.environ.get(_ENV_CONFIG_KEY) + return _deserialize_config(raw) if raw else None def register() -> None: @@ -67,9 +118,9 @@ def register() -> None: don't double-wrap. No ``unregister()`` symmetry: the patch stays for process lifetime. Benign - because ``patched_load_model`` no-ops when ``_config["capture_specs"]`` is - absent — and ``boot_vllm`` clears ``_config`` after each ``LLM(...)``, so - any subsequent non-TL ``LLM(...)`` in the same process hits the no-op path. + because ``patched_load_model`` no-ops when no spec channel is populated — and + boot sites call ``clear_config()`` after each ``LLM(...)``, so any subsequent + non-TL ``LLM(...)`` in the same process hits the no-op path. """ global _install_patched, _orig_load_model if _install_patched: @@ -80,15 +131,16 @@ def register() -> None: def patched_load_model(self): _orig_load_model(self) - specs = _config.get("capture_specs") - if not specs: + config = _active_config() + if config is None: return # not a TL-driven LLM; no hooks to install - max_n = _config["max_num_batched_tokens"] - dtype = _config["dtype"] - enable_batching = _config.get("enable_batching", False) + specs = config["capture_specs"] + max_n = config["max_num_batched_tokens"] + dtype = config["dtype"] + enable_batching = config.get("enable_batching", False) # Per-position affine buffers are (max_n, width) instead of (width,), so each # sequence row can carry a distinct scale/bias (position-scoped patching). - per_position = _config.get("enable_position_interventions", False) + per_position = config.get("enable_position_interventions", False) device = next(self.model_runner.model.parameters()).device # Detach prior handles before reassigning — vLLM doesn't double-load @@ -107,12 +159,16 @@ def patched_load_model(self): # Batched-mode per-(req_id, hook) accumulators + global spec dict. self._tl_accum = {} self._tl_intervention_specs = {} + # Specs whose module isn't on this rank (PP layer shards); the boot site + # verifies via tl_absent_hooks that every spec landed somewhere. + self._tl_absent_hooks = set() # Shared counter — surfaces hook double-fire under compile via tl_read_counter. self._tl_fire_counter = torch.zeros(1, device=device, dtype=torch.int64) for canonical_name, (dot_path, width) in specs.items(): - target = self.model_runner.model - for seg in dot_path.split("."): - target = target[int(seg)] if seg.isdigit() else getattr(target, seg) + target = resolve_dot_path(self.model_runner.model, dot_path) + if target is None: + self._tl_absent_hooks.add(canonical_name) + continue # Decoder layers return vLLM's (mlp_delta, residual) tuple; the # hook materializes their sum so the capture semantically matches # HF's full residual stream. Other modules use the default path. @@ -192,7 +248,11 @@ def _make_capture_hook( ``fire_counter`` is incremented per call for the fire-once check. """ - def _affine(t: torch.Tensor, n: Any) -> torch.Tensor: + # NO type annotations on _affine: the pytest jaxtyping hook wraps annotated + # transformer_lens functions, and dynamo tracing that wrapper into the compiled + # graph corrupts jaxtyping's thread-local memo stack for the whole process + # (GPU-verified: unrelated outer calls die with 'pop from empty list'). + def _affine(t, n): # per_position is a closure constant → torch.compile specializes this branch. # narrow(0, 0, n) keeps the SymInt dynamic shape (same trick as _gated_capture); # the (width,) path broadcasts across all rows. @@ -234,9 +294,8 @@ def hook(_module, _inputs, output): return hook -def _gated_capture( - capture_buf: torch.Tensor, n: Any, modified: torch.Tensor, capture_flag: torch.Tensor -) -> None: +# NO type annotations: this is traced into the compiled graph — see _affine's note. +def _gated_capture(capture_buf, n, modified, capture_flag): """First-write-wins via torch.where, compile-safe (no Python branching). When ``capture_flag == 0`` (open), writes ``modified`` to ``capture_buf[:n]``. diff --git a/transformer_lens/model_bridge/sources/vllm/source.py b/transformer_lens/model_bridge/sources/vllm/source.py index e8661c4681..7901028eaf 100644 --- a/transformer_lens/model_bridge/sources/vllm/source.py +++ b/transformer_lens/model_bridge/sources/vllm/source.py @@ -21,8 +21,9 @@ from . import plugin from .driver import VLLMDriver -from .internals import extract_hf_config +from .internals import extract_hf_config, verify_hook_coverage from .overlays import get_overlay +from .worker_extension import dtype_name # Forced LLM(...) kwargs that the capture-hook design depends on. Caller override → ValueError. # enable_prefix_caching MUST stay off: a prefix-cache hit makes the prefill compute only the @@ -30,17 +31,105 @@ # interventions skip cached positions, and an intervened forward writes poisoned K/V that a # later clean forward on the same prompt would silently reuse. _LOCKED_KWARGS = { - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, "skip_tokenizer_init": True, "disable_log_stats": True, "enable_prefix_caching": False, } -# Serializes the configure() → LLM(...) → clear() handoff: the spec channel is a module -# global, so interleaved boots would cross-wire capture specs between engines. +# Serializes the configure() → LLM(...) → clear() handoff: the spec channel is a +# process-wide env var, so interleaved boots would cross-wire capture specs between engines. _BOOT_LOCK = threading.Lock() +_WORKER_EXTENSION_CLS = ( + "transformer_lens.model_bridge.sources.vllm.worker_extension.TLWorkerExtension" +) + + +def construct_instrumented_llm( + model_name: str, + *, + capture_specs: Dict[str, Any], + max_num_batched_tokens: int, + dtype: torch.dtype, + enable_batching: bool = False, + enable_position_interventions: bool = False, + tensor_parallel_size: int = 1, + pipeline_parallel_size: int = 1, + llm_kwargs: Dict[str, Any], +) -> Any: + """The one place a TL-instrumented ``vllm.LLM`` is constructed — shared by + ``boot_vllm`` and the Inspect vLLM provider so the capture contract can't drift + between them. Owns everything hook correctness depends on: + + - ``VLLM_DISABLE_COMPILE_CACHE=1``: our hooks are traced INTO vLLM's compiled + graph, but its compile cache is keyed only on its own config — a cached + artifact from a differently-instrumented process either crashes at AOT load + (bytecode binds the hook closures) or silently serves a hookless graph. + - ``VLLM_ENABLE_V1_MULTIPROCESSING``: forced ``"0"`` single-rank (in-process + worker, the historical GPU-validated path); parallel boots spawn workers, + which read the specs via the plugin's env channel — and must not inherit a + stale ``"0"`` that would force the uni-process executor. + - ``enable_prefix_caching=False``: a prefix-cache hit computes only the uncached + suffix, so captures land row-misaligned, interventions skip cached positions, + and an intervened forward writes poisoned K/V a later clean forward reuses. + - configure → register → ``LLM(...)`` → clear under ``_BOOT_LOCK``, then a + hook-coverage check so a spec that landed on no rank fails here, not as zeros. + + ``llm_kwargs`` carries the caller's remaining ``LLM(...)`` arguments; restating a + contract kwarg is allowed only at the same value (callers validate overrides). + """ + from vllm import LLM + + os.environ["VLLM_DISABLE_COMPILE_CACHE"] = "1" + if tensor_parallel_size == 1 and pipeline_parallel_size == 1: + existing_mp = os.environ.get("VLLM_ENABLE_V1_MULTIPROCESSING") + if existing_mp not in (None, "0"): + warnings.warn( + f"VLLM_ENABLE_V1_MULTIPROCESSING={existing_mp!r} overridden to '0' — " + "single-rank TL boots keep the worker in-process.", + UserWarning, + stacklevel=3, + ) + os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0" + elif os.environ.get("VLLM_ENABLE_V1_MULTIPROCESSING") == "0": + os.environ.pop("VLLM_ENABLE_V1_MULTIPROCESSING") + + with _BOOT_LOCK: + plugin.configure( + capture_specs=capture_specs, + max_num_batched_tokens=max_num_batched_tokens, + dtype=dtype, + enable_batching=enable_batching, + enable_position_interventions=enable_position_interventions, + ) + plugin.register() + try: + llm = LLM( + **{ + "model": model_name, + # vLLM defaults this to 8192 for chunked prefill; the capture buffers + # are sized to it, so the compiled dynamic-shape range must match — + # otherwise Dynamo's symbolic hint exceeds the buffer dim at compile. + "max_num_batched_tokens": max_num_batched_tokens, + # Mixes tl_* methods into the Worker for collective_rpc (multiple + # inheritance; the tl_ prefix avoids attribute collisions). + "worker_extension_cls": _WORKER_EXTENSION_CLS, + "enable_prefix_caching": False, + "tensor_parallel_size": tensor_parallel_size, + "pipeline_parallel_size": pipeline_parallel_size, + **llm_kwargs, + } + ) + finally: + # Always clear, even on a failed boot: stale specs would make the next + # in-process vllm.LLM(...) walk our dot-paths on a foreign model. + plugin.clear_config() + + # Hook installation skips modules absent on a rank (PP shards); a spec that + # landed on NO rank is a broken dot-path and must fail here, not read zeros. + verify_hook_coverage(llm) + return llm + def boot_vllm( model_name: str, @@ -51,6 +140,8 @@ def boot_vllm( max_num_batched_tokens: int = 2048, enable_batching: bool = False, enable_position_interventions: bool = False, + tensor_parallel_size: int = 1, + pipeline_parallel_size: int = 1, **vllm_kwargs: Any, ) -> RemoteBridge: """Boot a model via vLLM and wrap it in a :class:`RemoteBridge` via :class:`VLLMDriver`. @@ -108,12 +199,35 @@ def boot_vllm( Costs ~2× extra resident GPU memory across all hooks (the scale and bias buffers join the already-``(max_n, width)`` capture buffer), so it is opt-in and defaults ``False``. Compiled-path only — incompatible with ``enable_batching``. + + ``tensor_parallel_size`` > 1 enables single-node tensor parallelism. Every + capture point the overlay hooks is post-all-reduce and therefore replicated + across a stage's TP ranks; ``pipeline_parallel_size`` > 1 shards layers across + stages, each owning the hooks that actually fire on it. + Capture reads merge across ranks with a first-forward layout check that fails + loud on installation drift or non-replicated hook points; sharded/stage-local + weights (``lm_head``, final norm) are gathered for logit reconstruction and the + ln_final un-fold. Single-node only (the spec channel is an env var, which never + reaches Ray remote workers); both are incompatible with ``enable_batching`` + (per-rank chunk boundaries are unvalidated). """ if enable_position_interventions and enable_batching: raise ValueError( "enable_position_interventions requires the compiled path and is incompatible " "with enable_batching=True (the batched/eager path has no affine buffers)." ) + for kwarg_name, size in ( + ("tensor_parallel_size", tensor_parallel_size), + ("pipeline_parallel_size", pipeline_parallel_size), + ): + if not isinstance(size, int) or size < 1: + raise ValueError(f"{kwarg_name} must be a positive int; got {size!r}.") + if (tensor_parallel_size > 1 or pipeline_parallel_size > 1) and enable_batching: + raise ValueError( + "enable_batching=True with tensor/pipeline parallelism is unsupported: the " + "eager batched path's per-rank chunk boundaries are unvalidated. " + "Use the compiled single-prompt path." + ) _reject_locked_overrides(vllm_kwargs) # Import-check first: fail with an actionable message before any network I/O # or plugin state mutation. @@ -137,68 +251,35 @@ def boot_vllm( resolved_dtype = dtype or _dtype_from_hf_config(hf_config_preview) - # The plugin's _config singleton must be visible to the worker, which only - # holds with single-process execution. Multi-GPU is unsupported. Override - # any user setting — silent capture failure otherwise. - existing_mp = os.environ.get("VLLM_ENABLE_V1_MULTIPROCESSING") - if existing_mp not in (None, "0"): - warnings.warn( - f"VLLM_ENABLE_V1_MULTIPROCESSING={existing_mp!r} overridden to '0' — " - "boot_vllm needs single-process execution for capture hooks to install. " - "Multi-GPU vLLM is unsupported.", - UserWarning, - stacklevel=2, - ) - os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0" - # Batched capture reads query_start_loc from the forward context, untraceable # under torch.compile — so the batched path must run eager. eager_kwargs: Dict[str, Any] = {"enforce_eager": True} if enable_batching else {} - with _BOOT_LOCK: - plugin.configure( - capture_specs=overlay.capture_specs(hf_config_preview), - max_num_batched_tokens=max_num_batched_tokens, - dtype=resolved_dtype, - enable_batching=enable_batching, - enable_position_interventions=enable_position_interventions, - ) - plugin.register() - try: - llm = LLM( - model=model_name, - gpu_memory_utilization=gpu_memory_utilization, - max_model_len=max_model_len, - # Critical: vLLM defaults max_num_batched_tokens to 8192 for chunked prefill. - # We size our capture buffer to this same value, so vLLM must compile for - # the matching dynamic-shape range — otherwise Dynamo's symbolic-shape - # hint exceeds the buffer dim and narrow() fails at compile time. - max_num_batched_tokens=max_num_batched_tokens, - # Register TLWorkerExtension so its tl_* methods are reachable via - # collective_rpc. Passed as a dotted path; vLLM imports the class at - # worker construction and mixes it into the Worker via multiple - # inheritance (asserts no attribute name collisions — hence the tl_ prefix). - worker_extension_cls="transformer_lens.model_bridge.sources.vllm.worker_extension.TLWorkerExtension", - # Allow full-vocab logprobs so the fallback path can synthesize logits when - # host-side reconstruction is unavailable (vLLM caps logprobs to this value; - # default 20 is too small for mech-interp). - max_logprobs=hf_config_preview.vocab_size, - # Always explicit — vLLM's "auto" downcasts fp32 checkpoints to fp16, which - # would leave the capture/affine buffers (allocated at resolved_dtype) at a - # different dtype than the engine's activations. - dtype=str(resolved_dtype).replace("torch.", ""), - **eager_kwargs, - **_LOCKED_KWARGS, - **vllm_kwargs, - ) - finally: - # Always clear, even on a failed boot: a stale populated _config would make the - # next in-process vllm.LLM(...) walk our dot-paths on a foreign model. - plugin._config.clear() - - # Capture-path validity is enforced inside patched_load_model during - # LLM(...) above — any missing dot-path raises AttributeError there. By - # the time we reach this line every spec has already been walked successfully. + llm = construct_instrumented_llm( + model_name, + capture_specs=overlay.capture_specs(hf_config_preview), + max_num_batched_tokens=max_num_batched_tokens, + dtype=resolved_dtype, + enable_batching=enable_batching, + enable_position_interventions=enable_position_interventions, + tensor_parallel_size=tensor_parallel_size, + pipeline_parallel_size=pipeline_parallel_size, + llm_kwargs={ + "gpu_memory_utilization": gpu_memory_utilization, + "max_model_len": max_model_len, + # Allow full-vocab logprobs so the fallback path can synthesize logits when + # host-side reconstruction is unavailable (vLLM caps logprobs to this value; + # default 20 is too small for mech-interp). + "max_logprobs": hf_config_preview.vocab_size, + # Always explicit — vLLM's "auto" downcasts fp32 checkpoints to fp16, which + # would leave the capture/affine buffers (allocated at resolved_dtype) at a + # different dtype than the engine's activations. + "dtype": dtype_name(resolved_dtype), + **eager_kwargs, + **_LOCKED_KWARGS, + **vllm_kwargs, + }, + ) hf_config = extract_hf_config(llm) if tokenizer is None: tokenizer = AutoTokenizer.from_pretrained(model_name, token=hf_token) @@ -223,6 +304,8 @@ def boot_vllm( max_num_batched_tokens=max_num_batched_tokens, enable_batching=enable_batching, enable_position_interventions=enable_position_interventions, + tensor_parallel_size=tensor_parallel_size, + pipeline_parallel_size=pipeline_parallel_size, ) # One-time unembedding fetch: caches the weight for per-forward reconstruction and # downgrades provides_sequence_logits honestly if no unembedding is reachable. @@ -237,8 +320,9 @@ def _reject_locked_overrides(vllm_kwargs: Dict[str, Any]) -> None: if key in vllm_kwargs and vllm_kwargs[key] != locked: raise ValueError( f"boot_vllm forces {key}={locked}; caller passed {key}={vllm_kwargs[key]}. " - "Multi-device, prefix caching, continuous batching, and vLLM-owned " - "tokenizers are unsupported — each breaks the row=position capture invariant." + "Prefix caching, continuous batching, and vLLM-owned tokenizers are " + "unsupported — each breaks the capture-read invariants. (TP/PP are " + "supported via the tensor_parallel_size/pipeline_parallel_size kwargs.)" ) @@ -274,4 +358,4 @@ def _log_hook_summary(model_name: str, architecture: str, driver: VLLMDriver) -> ) -__all__ = ["boot_vllm"] +__all__ = ["boot_vllm", "construct_instrumented_llm"] diff --git a/transformer_lens/model_bridge/sources/vllm/worker_extension.py b/transformer_lens/model_bridge/sources/vllm/worker_extension.py index 036ee36645..9622e1eceb 100644 --- a/transformer_lens/model_bridge/sources/vllm/worker_extension.py +++ b/transformer_lens/model_bridge/sources/vllm/worker_extension.py @@ -16,12 +16,79 @@ """ from __future__ import annotations -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Set import torch from .intervention_specs import SUPPORTED_OPS, validate_spec +# Explicit tensor wire format for collective_rpc returns. vLLM's multiproc RPC +# cannot round-trip raw tensors from extension methods — its tensor-aware msgpack +# encoder's header (['float32', [rows, cols], buf_idx]) leaks through undecoded on +# the response path (GPU-verified on vllm 0.20.2). dtype/shape/bytes are all +# msgpack-native, so this survives any executor topology; the in-process executor +# pays one extra CPU copy. +_TL_TENSOR_KEY = "__tl_tensor__" + + +def dtype_name(dtype: torch.dtype) -> str: + """``torch.float32`` → ``"float32"`` — the one spelling for every dtype-string site.""" + return str(dtype).removeprefix("torch.") + + +def is_pp_missing_layer(module: Any) -> bool: + """vLLM PP keeps full-length module lists on every rank, filling non-owned slots + (layers, embed_tokens, norm, lm_head) with PPMissingLayer identity stubs that the + forward never calls — a hook installed there would serve its dead zero buffer as + a real capture. Matched by name so non-vllm test doubles work; the pinned band's + class is named exactly this, and a rename fails loud via the rank-layout check.""" + return type(module).__name__ == "PPMissingLayer" + + +def resolve_dot_path(root: Any, dot_path: str) -> Any: + """Walk a dot-path; ``None`` when any segment is missing or the target is a + PPMissingLayer stub. Per-rank absence is legal under pipeline parallelism (each + rank owns a layer subset) — the boot site verifies every hook landed on at least + one rank.""" + target = root + for seg in dot_path.split("."): + if seg.isdigit(): + try: + target = target[int(seg)] + except (IndexError, KeyError, TypeError): + return None + else: + target = getattr(target, seg, None) + if target is None: + return None + return None if is_pp_missing_layer(target) else target + + +def encode_tensor(t: torch.Tensor) -> Dict[str, Any]: + """Encode a CPU tensor for the RPC wire. bf16 rides as fp32 bytes (exact).""" + t = t.detach().cpu().contiguous() + name = dtype_name(t.dtype) + if t.dtype == torch.bfloat16: + t = t.to(torch.float32) + return { + _TL_TENSOR_KEY: True, + "dtype": name, + "shape": list(t.shape), + "data": t.numpy().tobytes(), + } + + +def decode_tensor(payload: Dict[str, Any]) -> torch.Tensor: + """Inverse of :func:`encode_tensor`; used driver-side.""" + import numpy as np + + dtype = getattr(torch, payload["dtype"]) + wire_dtype = torch.float32 if dtype == torch.bfloat16 else dtype + np_dtype = torch.empty(0, dtype=wire_dtype).numpy().dtype + array = np.frombuffer(payload["data"], dtype=np_dtype).copy() + tensor = torch.from_numpy(array).reshape(payload["shape"]) + return tensor.to(torch.bfloat16) if dtype == torch.bfloat16 else tensor + class TLWorkerExtension: """Mixed into vLLM's ``Worker`` via ``worker_extension_cls``.""" @@ -36,11 +103,22 @@ class TLWorkerExtension: # Batched-mode state (eager). _tl_accum: Dict[tuple, List[torch.Tensor]] _tl_intervention_specs: Dict[str, Dict[str, Any]] + # Specs whose module doesn't exist on this rank (PP layer shards). + _tl_absent_hooks: set + + def tl_absent_hooks(self) -> Optional[List[str]]: + """Spec names that installed no hook on this rank — the boot site verifies + their union across ranks covers every spec. ``None`` means installation + never ran at all (plugin patch absent or spec channel empty), which the + coverage check must treat as fatal rather than vacuously complete.""" + if not hasattr(self, "_tl_absent_hooks"): + return None + return sorted(self._tl_absent_hooks) def tl_read_captures( self, prompt_lens: List[int], names: Optional[List[str]] = None - ) -> Dict[str, torch.Tensor]: - """Slice each capture buffer to ``sum(prompt_lens)`` rows; CPU copies. + ) -> Dict[str, Dict[str, Any]]: + """Slice each capture buffer to ``sum(prompt_lens)`` rows; wire-encoded CPU copies. ``names`` restricts the read (``None`` = all) — this is the only GPU→CPU crossing, so it's where a names_filtered run saves bandwidth. Caller gates @@ -48,8 +126,17 @@ def tl_read_captures( """ total = sum(prompt_lens) buffers: Dict[str, torch.Tensor] = getattr(self, "_tl_buffers", {}) + # Ownership is dynamic, not structural: a hook can be installed on a module + # this rank holds but never runs (PP stages share tied-embedding aliases and + # some archs instantiate norm on every rank), so a still-open first-write + # flag means the buffer holds no data from this forward — don't serve it. + flags: Dict[str, torch.Tensor] = getattr(self, "_tl_capture_flags", {}) wanted = buffers.keys() if names is None else [n for n in names if n in buffers] - return {name: buffers[name][:total].detach().cpu().clone() for name in wanted} + return { + name: encode_tensor(buffers[name][:total]) + for name in wanted + if name not in flags or bool(flags[name].item()) + } def tl_set_interventions(self, specs: Dict[str, Dict[str, Any]]) -> None: """Reset all affine buffers to identity, then apply each spec. @@ -66,7 +153,12 @@ def tl_set_interventions(self, specs: Dict[str, Dict[str, Any]]) -> None: for name, sb in scale_bufs.items(): sb.fill_(1.0) bias_bufs[name].zero_() + absent: Set[str] = getattr(self, "_tl_absent_hooks", set()) for hook_name, spec in specs.items(): + if hook_name in absent: + # Another PP stage owns this hook; its rank applies the spec. The + # driver's supported_hook_points check already caught real typos. + continue if hook_name not in scale_bufs: raise KeyError(f"Unknown hook for intervention: {hook_name!r}") # Authoritative validation: producers that bypass VLLMDriver (the Inspect @@ -74,18 +166,15 @@ def tl_set_interventions(self, specs: Dict[str, Dict[str, Any]]) -> None: validate_spec(hook_name, spec, width=scale_bufs[hook_name].shape[-1]) _apply_intervention(scale_bufs[hook_name], bias_bufs[hook_name], spec) - def tl_get_param(self, dotted_name: str) -> Optional[torch.Tensor]: - """Read a named model tensor (e.g. ``model.norm.weight``) as a CPU clone. + def tl_get_param(self, dotted_name: str) -> Optional[Dict[str, Any]]: + """Read a named model tensor (e.g. ``model.norm.weight``) as a wire-encoded + CPU clone. ``None`` if the path doesn't resolve to a tensor. The bridge has no general weight surface, so this is how callers reach e.g. the ln_final weight. """ - target = getattr(self, "model_runner").model - for seg in dotted_name.split("."): - target = target[int(seg)] if seg.isdigit() else getattr(target, seg, None) - if target is None: - return None - return target.detach().cpu().clone() if isinstance(target, torch.Tensor) else None + target = resolve_dot_path(getattr(self, "model_runner").model, dotted_name) + return encode_tensor(target) if isinstance(target, torch.Tensor) else None def tl_reset_counter(self) -> None: """Zero the shared hook-fire counter before a forward.""" @@ -123,11 +212,11 @@ def tl_read_batched_captures( """ accum: Dict[tuple, List[torch.Tensor]] = getattr(self, "_tl_accum", {}) nameset = None if names is None else set(names) - out: Dict[str, Dict[str, torch.Tensor]] = {} + out: Dict[str, Dict[str, Any]] = {} for (req_id, name), chunks in accum.items(): if nameset is not None and name not in nameset: continue - out.setdefault(req_id, {})[name] = torch.cat(chunks, dim=0) + out.setdefault(req_id, {})[name] = encode_tensor(torch.cat(chunks, dim=0)) return out def tl_set_batched_interventions(self, specs: Dict[str, Dict[str, Any]]) -> None: @@ -147,6 +236,7 @@ def tl_remove_hooks(self) -> None: self._tl_capture_flags = {} self._tl_accum = {} self._tl_intervention_specs = {} + self._tl_absent_hooks = set() def _apply_op(t: torch.Tensor, spec: Dict[str, Any]) -> torch.Tensor: From 003b20f0ce6c5da3091ad693ceeb9a8fdd5bc707 Mon Sep 17 00:00:00 2001 From: jlarson4 Date: Mon, 20 Jul 2026 10:05:31 -0500 Subject: [PATCH 12/87] lock fixed --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 07df9b9792..0da72f87bf 100644 --- a/uv.lock +++ b/uv.lock @@ -9645,7 +9645,7 @@ quantization = [ [[package]] name = "transformers" -version = "5.14.1" +version = "5.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, @@ -9660,9 +9660,9 @@ dependencies = [ { name = "tqdm" }, { name = "typer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5a/fb/2a2ba88f325e68a921d8b69ff63b477830b2e73ade9a3c8c8cab2f06d741/transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173", size = 9295927, upload-time = "2026-07-16T09:41:57.773Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/e1/720ff7ff666b04279fea5bb7ac3ef8675e98f0ddbc1b8cb8bc9f3889d62e/transformers-5.13.0.tar.gz", hash = "sha256:940c1428e42a4238f9ccf0cd41e63c590701aa63c19fd2ce3d7d602222d68495", size = 9195801, upload-time = "2026-07-03T16:05:39.362Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/67/8d85ca2323233ae3c0365a659c4e52ee1f587b440e4bc577e7d8e4416d0f/transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725", size = 11625234, upload-time = "2026-07-16T09:41:54.143Z" }, + { url = "https://files.pythonhosted.org/packages/a5/3a/d99704c5effe10c6339c98cb236259161103e159bb99a78468b6729572ec/transformers-5.13.0-py3-none-any.whl", hash = "sha256:8adbc1d20bd5463cd6876b2eb7cb31971e1065788e7dc6bc12bab597a7c504b7", size = 11503730, upload-time = "2026-07-03T16:05:35.569Z" }, ] [[package]] From 44a26ab6d362da53bf7d16cf98e00bb0ad6162c7 Mon Sep 17 00:00:00 2001 From: jlarson4 Date: Mon, 20 Jul 2026 13:44:10 -0500 Subject: [PATCH 13/87] fix re-export --- .../model_bridge/sources/transformers/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/transformer_lens/model_bridge/sources/transformers/__init__.py b/transformer_lens/model_bridge/sources/transformers/__init__.py index 1da07bc4b6..942a65e1c7 100644 --- a/transformer_lens/model_bridge/sources/transformers/__init__.py +++ b/transformer_lens/model_bridge/sources/transformers/__init__.py @@ -2,10 +2,11 @@ from __future__ import annotations # Re-exported so external code that patches ``AutoConfig.from_pretrained`` / -# ``AutoTokenizer.from_pretrained`` via this module path keeps working after the -# package split. Class-method monkey-patches reach the same class objects that -# ``source.py`` imports directly, so this re-export keeps tests stable. -from transformers import AutoConfig, AutoTokenizer +# ``AutoModelForCausalLM.from_pretrained`` / ``AutoTokenizer.from_pretrained`` via this +# module path keeps working after the package split. Class-method monkey-patches reach +# the same class objects that ``source.py`` / ``helpers.py`` use directly, so this +# re-export keeps tests stable. +from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer from transformer_lens.model_bridge.bridge import TransformerBridge @@ -34,6 +35,7 @@ __all__ = [ "AutoConfig", + "AutoModelForCausalLM", "AutoTokenizer", "boot", "check_model_support", From d92683d00b4a1cc650fc98c428fef95186017ba5 Mon Sep 17 00:00:00 2001 From: Jonah Larson Date: Fri, 24 Jul 2026 15:55:26 -0500 Subject: [PATCH 14/87] Initial Setup for HookedTransformers Deprecation (#1538) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add dedicated GraniteMoe adapter tests (#1302) (#1524) granite_moe had substantive test coverage, but embedded inside test_granite_adapter.py rather than in its own file — inconsistent with every other architecture listed in #1302 (including granite_moe_hybrid), which each get a dedicated test__adapter.py. Moves that coverage into test_granite_moe_adapter.py and adds the one piece that was missing: an explicit config-flag test class (GraniteMoe doesn't override config setup, so it inherits dense Granite's flags, but nothing asserted that directly before). test_granite_adapter.py goes back to covering only GraniteArchitectureAdapter, matching its own docstring. * Updating documentation for future deprecation (#1523) * feat: add RWKV-7 (Goose) TransformerBridge adapter (RWKV7ForCausalLM) (#1521) * feat: add RWKV-7 (Goose) TransformerBridge adapter (RWKV7ForCausalLM) Signed-off-by: Mukund Pandey * refactor: use OpaqueBlockBridge for RWKV-7; introduce OpaqueBlockBridge base class Signed-off-by: Mukund Pandey * added hook_out_is_single_residual_stream default to OpaqueBlockBridge * style: apply black formatting to opaque_block, ssm_block, __init__ Signed-off-by: Mukund Pandey * style: fix import order in generalized_components/__init__.py (isort) Signed-off-by: Mukund Pandey * style: fix import order in generalized_components/__init__.py (isort) Signed-off-by: Mukund Pandey * style: fix import order in generalized_components/__init__.py (isort) Signed-off-by: Mukund Pandey * refactor: replace SSM2MixerBridge with GeneralizedComponent for RWKV-7 attn/ffn sublayers Signed-off-by: Mukund Pandey * style: fix black formatting in generate_report.py Signed-off-by: Mukund Pandey --------- Signed-off-by: Mukund Pandey Co-authored-by: jlarson4 * feat: add RavenForCausalLM (Huginn) depth-recurrent adapter (#1520) * Add RavenForCausalLM (Huginn) TransformerBridge adapter Adds a TransformerBridge architecture adapter for RavenForCausalLM (tomg-group-umd/huginn-0125), a depth-recurrent decoder with a prelude / weight-tied recurrent core / coda structure, resolving #1469. - raven.py delegates the recurrence to the remote-code HF forward and maps the three physical block lists (prelude / core_block / coda) via SSMBlockBridge, with combined-QKV native attention and a gated MLP. Sets applicable_phases=[] (a random initial latent state and post-residual sandwich norms diverge from the verify_models phases) and supports_fold_ln=False (ln_f is reused mid-network). prepare_loading patches Huginn's remote code for transformers v5 (tied-weights-keys dict form + a weight re-init guard). - Registers in the adapter factory, the model registry (canonical author and description) and supported_models.json; surfaces the recurrence-shape config via both _HF_PASSTHROUGH_ATTRS lists. - Adds synthetic-config unit tests and CI-gated integration tests. * fix: pass num_steps as int to iterate_forward (0-d tensor has no len) * fix: remove unused torch import and use setattr for dynamic cfg attrs * style: apply black formatting to raven.py Signed-off-by: Mukund Pandey * refactor: introduce OpaqueBlockBridge; use it for Raven instead of SSMBlockBridge Signed-off-by: Mukund Pandey * style: apply black formatting to opaque_block, ssm_block, __init__ Signed-off-by: Mukund Pandey * style: fix import order in generalized_components/__init__.py (isort) Signed-off-by: Mukund Pandey * fix: strip dead hook_q/k/v aliases from Raven AttentionBridge Raven uses a combined Wqkv projection (no separate q/k/v submodules), so AttentionBridge's default hook_q/hook_k/hook_v aliases (which target q.hook_out / k.hook_out / v.hook_out) are unresolvable. The upstream test_every_hook_alias_resolves_to_hookpoint audit catches these as 9 dead aliases across prelude / core_block / coda. Strip them by setting an instance-level hook_aliases that omits those three keys, leaving only the aliases that have real HookPoint targets. Signed-off-by: Mukund Pandey * chore: merge upstream/dev — add RWKV-7 passthrough attrs and registry entry Brings in the RWKV-7 additions from #1521 that landed in dev: - _bridge_builder.py: added RWKV-7 passthrough attrs (num_heads, value_dim, decay/gate/a/v_low_rank_dim, norm_first, norm_bias, fuse_norm, attn_mode, hidden_act) - generate_report.py: added RWKV7ForCausalLM description entry Signed-off-by: Mukund Pandey --------- Signed-off-by: Mukund Pandey * Fix Tracr demo output_label: compiled labels are auto-numbered (#1525) The Tracr demo passes output_label="reverse", but Tracr builds labels as f"{name}_{unique_id}", so the compiled expression's residual labels are reverse_1:* and the demo raises ValueError at the state-dict cell. Pass reverse.label instead of hardcoding the name, and correct the same wrong label shape in the unit-test fixture (real Tracr never emits "reverse:1") and in the infer_tracr_output_label docstring example. Co-authored-by: Claude Fable 5 * Add TransformerBridge adapter for lightweight decoder-only pretraining models (#1519) * Add TransformerLens pretrain bridge adapter * Add TransformerLens pretrain bridge adapter * Setup registry for hookedtransformer deprecation * bug fixes and rebase to 4.x * Hook management improvements * implemented `stop_at_layer` on TransformerBridge * input_to_embed and pos_slice added to bridge * prevent corruption, check_hooks_to_add placed in all locations that need it * Adjusted head detector to be HookedTransformer agnostic, added key value injection to bridge * Add specific migration differences section * Attempting to let pytest run in parallel * Attempting to fix the OOM on the multi-device Full Coverage run --------- Signed-off-by: Mukund Pandey Co-authored-by: Delaida Muminovic Co-authored-by: Mukund Pandey Co-authored-by: Joseph Quevedo <41499530+dewstend@users.noreply.github.com> Co-authored-by: Claude Fable 5 Co-authored-by: Cacapice --- .github/workflows/checks.yml | 22 +- demos/Tracr_to_Transformer_Lens_Demo.ipynb | 6 +- docs/source/content/migrating_to_v3.md | 24 +- makefile | 7 +- .../compatibility/test_hook_duplication.py | 5 +- .../test_bridge_hook_behavior.py | 4 +- .../model_bridge/test_bridge_caching_hooks.py | 102 ++ .../test_bridge_input_to_embed.py | 52 ++ .../test_bridge_manual_kv_cache.py | 71 ++ .../test_bridge_run_with_cache_pos_slice.py | 70 ++ .../test_bridge_start_at_layer.py | 134 +++ .../model_bridge/test_pretrain_adapter.py | 312 +++++++ .../model_bridge/test_raven_adapter.py | 251 +++++ .../model_bridge/test_rwkv7_adapter.py | 161 ++++ tests/mocks/tiny_pretrain_model.py | 193 ++++ .../test_moe_bridge_tuple_output.py | 88 ++ .../_pretrain_mocks.py | 214 +++++ .../test_granite_adapter.py | 60 +- .../test_granite_moe_adapter.py | 212 +++++ .../test_hook_orchestration_parity.py | 180 ++++ .../test_pretrain_adapter.py | 453 +++++++++ .../test_pretrain_model_container.py | 537 +++++++++++ .../test_raven_adapter.py | 335 +++++++ .../test_rwkv7_adapter.py | 301 ++++++ .../test_bridge_train_mode_propagation.py | 100 ++ .../model_bridge/test_checkpoint_revision.py | 12 +- tests/unit/test_tracr_conversion.py | 12 +- tests/unit/tools/test_model_registry.py | 112 +++ transformer_lens/ActivationCache.py | 7 +- transformer_lens/HookedAudioEncoder.py | 11 + transformer_lens/HookedEncoderDecoder.py | 11 + transformer_lens/HookedTransformer.py | 11 + .../conversion_utils/hook_conversion_utils.py | 4 +- transformer_lens/evals.py | 4 +- .../factories/architecture_adapter_factory.py | 6 + transformer_lens/head_detector.py | 4 +- transformer_lens/hook_points.py | 2 +- transformer_lens/loading_from_pretrained.py | 23 +- transformer_lens/model_bridge/bridge_core.py | 387 ++++++-- .../generalized_components/__init__.py | 55 +- .../generalized_components/block.py | 67 +- .../generalized_components/moe.py | 17 +- .../generalized_components/opaque_block.py | 112 +++ .../generalized_components/ssm_block.py | 98 +- .../model_bridge/remote_bridge.py | 15 +- .../model_bridge/sources/_bridge_builder.py | 22 + .../sources/transformers/helpers.py | 4 +- .../sources/transformers/source.py | 15 +- .../supported_architectures/__init__.py | 12 + .../supported_architectures/pretrain.py | 442 +++++++++ .../supported_architectures/raven.py | 303 ++++++ .../supported_architectures/rwkv7.py | 296 ++++++ .../model_bridge/transformer_bridge.py | 137 ++- transformer_lens/model_protocol.py | 65 ++ transformer_lens/patching.py | 8 +- transformer_lens/supported_models.py | 400 +------- transformer_lens/tools/__init__.py | 19 +- .../tools/model_registry/__init__.py | 23 + .../tools/model_registry/checkpoints.py | 45 + .../model_registry/data/model_aliases.json | 883 ++++++++++++++++++ .../model_registry/data/supported_models.json | 14 + .../tools/model_registry/generate_report.py | 2 + .../tools/model_registry/registry_io.py | 17 + .../tools/model_registry/verify_models.py | 23 +- transformer_lens/utilities/tracr.py | 2 +- 65 files changed, 6868 insertions(+), 728 deletions(-) create mode 100644 tests/integration/model_bridge/test_bridge_caching_hooks.py create mode 100644 tests/integration/model_bridge/test_bridge_input_to_embed.py create mode 100644 tests/integration/model_bridge/test_bridge_manual_kv_cache.py create mode 100644 tests/integration/model_bridge/test_bridge_run_with_cache_pos_slice.py create mode 100644 tests/integration/model_bridge/test_bridge_start_at_layer.py create mode 100644 tests/integration/model_bridge/test_pretrain_adapter.py create mode 100644 tests/integration/model_bridge/test_raven_adapter.py create mode 100644 tests/integration/model_bridge/test_rwkv7_adapter.py create mode 100644 tests/mocks/tiny_pretrain_model.py create mode 100644 tests/unit/model_bridge/generalized_components/test_moe_bridge_tuple_output.py create mode 100644 tests/unit/model_bridge/supported_architectures/_pretrain_mocks.py create mode 100644 tests/unit/model_bridge/supported_architectures/test_granite_moe_adapter.py create mode 100644 tests/unit/model_bridge/supported_architectures/test_hook_orchestration_parity.py create mode 100644 tests/unit/model_bridge/supported_architectures/test_pretrain_adapter.py create mode 100644 tests/unit/model_bridge/supported_architectures/test_pretrain_model_container.py create mode 100644 tests/unit/model_bridge/supported_architectures/test_raven_adapter.py create mode 100644 tests/unit/model_bridge/supported_architectures/test_rwkv7_adapter.py create mode 100644 tests/unit/model_bridge/test_bridge_train_mode_propagation.py create mode 100644 transformer_lens/model_bridge/generalized_components/opaque_block.py create mode 100644 transformer_lens/model_bridge/supported_architectures/pretrain.py create mode 100644 transformer_lens/model_bridge/supported_architectures/raven.py create mode 100644 transformer_lens/model_bridge/supported_architectures/rwkv7.py create mode 100644 transformer_lens/model_protocol.py create mode 100644 transformer_lens/tools/model_registry/checkpoints.py create mode 100644 transformer_lens/tools/model_registry/data/model_aliases.json diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index f792c1af15..62329c3765 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -247,9 +247,22 @@ jobs: coverage-test: name: Full Code Coverage Test runs-on: ubuntu-latest - timeout-minutes: 90 + # Suite runs under pytest-xdist on a 2-vCPU runner (~40 min with swap); + # 60 catches a real hang well below the old 90. + timeout-minutes: 60 steps: - uses: actions/checkout@v4 + - name: Add swap space + # The 2-vCPU runner has ~7GB RAM; two xdist workers each holding + # torch + resident models overflow it (OOM-killed the runner at ~83%). + # Swap absorbs the model-load spikes. Mirrors the notebook-checks job. + run: | + sudo swapoff /swapfile 2>/dev/null || true + sudo rm -f /swapfile + sudo fallocate -l 8G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile - name: Install uv uses: astral-sh/setup-uv@v7 with: @@ -293,6 +306,13 @@ jobs: run: make coverage-report-test env: HF_TOKEN: ${{ secrets.HF_TOKEN }} + # One BLAS thread per worker: N xdist workers each spawning ncores + # OMP threads would oversubscribe the 4-vCPU runner and erase the gain. + OMP_NUM_THREADS: "1" + MKL_NUM_THREADS: "1" + # Worker count knob (this runner is 2-vCPU, so -n auto = 2). If swap + # still can't hold peak memory, drop to "-n 1" or grow the swapfile. + XDIST_ARGS: "-n auto --dist loadscope" - name: Build check run: uv build - name: Upload Coverage Report Artifact diff --git a/demos/Tracr_to_Transformer_Lens_Demo.ipynb b/demos/Tracr_to_Transformer_Lens_Demo.ipynb index 30fd67eced..25d9cc450d 100644 --- a/demos/Tracr_to_Transformer_Lens_Demo.ipynb +++ b/demos/Tracr_to_Transformer_Lens_Demo.ipynb @@ -147,7 +147,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Extract the state dict. The helper reconstructs Tracr's categorical unembed from the output basis labels, so it still works when the final RASP expression is explicitly named and its residual coordinates are not first." + "Extract the state dict. The helper reconstructs Tracr's categorical unembed from the output basis labels, so it still works when the final RASP expression is explicitly named and its residual coordinates are not first.\n", + "\n", + "Note that Tracr suffixes every label with a unique id — the expression named `reverse` compiles to residual labels like `reverse_1:1` — so we pass `reverse.label` rather than hardcoding the name. (Omitting `output_label` entirely also works: the helper infers it from the output encoder's value set.)" ] }, { @@ -157,7 +159,7 @@ "outputs": [], "source": [ "\n", - "sd = make_tracr_transformer_bridge_state_dict(model, output_label=\"reverse\")\n", + "sd = make_tracr_transformer_bridge_state_dict(model, output_label=reverse.label)\n", "print(sd.keys())" ] }, diff --git a/docs/source/content/migrating_to_v3.md b/docs/source/content/migrating_to_v3.md index f6d2ed0058..a275c2ddc7 100644 --- a/docs/source/content/migrating_to_v3.md +++ b/docs/source/content/migrating_to_v3.md @@ -4,6 +4,8 @@ TransformerLens 3 introduces **TransformerBridge**, a new way of loading and ins This page explains the differences and gives side-by-side migration recipes for the most common patterns. +> **Deprecation status.** `HookedTransformer.from_pretrained` — along with the `HookedEncoderDecoder` and `HookedAudioEncoder` load paths — now emits a `DeprecationWarning`. `HookedTransformer` and the other `Hooked*` classes are slated for removal in a future major release; every feature is being migrated to `TransformerBridge` and the driver system (features that aren't a fit for a driver, such as train-from-scratch, are moving to bridge-based homes rather than staying on `HookedTransformer`). New code should use `TransformerBridge.boot_transformers(...)`. Follow the migration progress in the deprecation plan. + ## Why the change? `HookedTransformer` was a single unified implementation that every supported architecture had to be mapped into. That was beautiful in theory — interpretability code written once worked everywhere — but in practice it meant that adding a new architecture required reimplementing its forward pass inside TransformerLens, and any divergence from the HuggingFace version was a latent source of bugs. @@ -36,7 +38,9 @@ Weight-processing flags (`fold_ln`, `center_writing_weights`, `center_unembed`, ### Parameters that were removed -`n_devices`, `move_to_device`, and `first_n_layers` are not part of `boot_transformers`. If you relied on any of these, file an issue describing your use case — the right pattern for multi-GPU loads under the bridge is still being worked out. +`move_to_device` and `first_n_layers` are not part of `boot_transformers`. If you relied on either, file an issue describing your use case. + +> **Multi-GPU update.** The 3.0 release notes listed `n_devices` among the removed parameters. This feature has since been restored alongside `device_map` and `max_memory`. See [Parameters that are new](#parameters-that-are-new) below. ### Parameters that are new @@ -44,6 +48,8 @@ Weight-processing flags (`fold_ln`, `center_writing_weights`, `center_unembed`, - `trust_remote_code: bool = False` — pass through to HuggingFace for models that ship custom modeling code. - `hf_config_overrides: dict | None = None` — override specific fields of the HF config before the model is constructed. - `n_ctx: int | None = None` — override the model's context length. The bridge writes to whichever HF config field this architecture uses (`n_positions` / `max_position_embeddings` / etc.) so callers don't need to know the field name. Warns if larger than the model's default. +- `n_devices` / `device_map` / `max_memory` — multi-GPU loading. `device_map` takes a HuggingFace-style map (`"auto"`, `"balanced"`, or an explicit dict) and is passed straight to `from_pretrained`; `n_devices=N` is a convenience that splits the model across `N` visible CUDA devices (translated to a `max_memory` dict internally); `max_memory` sets a per-device budget. `device` and `device_map` are mutually exclusive. +- `revision` / `checkpoint_index` / `checkpoint_value` — load a specific HF revision, or a training checkpoint for checkpointed families (`EleutherAI/pythia*`, `stanford-crfm/*`). `checkpoint_index` / `checkpoint_value` resolve to a revision string, mirroring the old `HookedTransformer.from_pretrained` checkpoint arguments. - `hf_model` / `model_class` — advanced: pass in a pre-loaded HF model or a specific model class. ## Weight processing is now opt-in @@ -202,6 +208,18 @@ pattern = cache["blocks.0.attn.hook_pattern"] The cache, hook, and config APIs are the same. The only lines that had to change are the import, the load call, and — if you want the old weight-processing behavior — one extra call to `enable_compatibility_mode`. -## When to stay on HookedTransformer +## Migrating specific `HookedTransformer` APIs + +`HookedTransformer` is deprecated and will be removed in a future major release. The compatibility layer keeps existing code running in the meantime, but new work should target `TransformerBridge`, and migrating existing projects is the long-term supported path. + +Most `HookedTransformer` methods and properties exist on `TransformerBridge` under the same name — see [APIs that are unchanged](#apis-that-are-unchanged). The table below covers the cases where the name or access path differs. + +> If you hit a `HookedTransformer` API whose bridge equivalent isn't obvious and isn't listed here, [open an issue](https://github.com/TransformerLensOrg/TransformerLens/issues); when you (or we) work out the equivalent, add a row below. + +Weight-matrix rows return **raw** HuggingFace weights by default. `HookedTransformer.from_pretrained` applies weight processing (LayerNorm folding, `center_writing_weights`, `center_unembed`) at load, so those properties differ numerically from the bridge's unless the bridge is in compatibility mode — see [Will my numbers match HookedTransformer?](#will-my-numbers-match-hookedtransformer). -If your code runs unchanged on TransformerLens 3 via the compatibility layer and you don't need architectures beyond what `HookedTransformer` already supported, there is no hard deadline to migrate. But new architectures, weight-processing controls, and hook refinements are landing on the bridge side — new work should target the bridge, and migrating existing projects is the long-term supported path. +| `HookedTransformer` | `TransformerBridge` equivalent | Notes | +|---|---|---| +| `model.W_pos` | `bridge.pos_embed.W_pos` | Raw weight (also `bridge.pos_embed.weight`). `center_writing_weights` centers `W_pos` in default HT loads, so it matches HT's only under matching processing (`enable_compatibility_mode()`, or HT loaded with no processing). | +| `model.W_E_pos` | `torch.cat([bridge.W_E, bridge.pos_embed.W_pos], dim=0)` | No single accessor — concatenate the token + positional matrices. Same weight-processing caveat as `W_pos` (both `W_E` and `W_pos` are centered writing-weights). | +| `HookedTransformer.from_pretrained_no_processing(name)` | `TransformerBridge.boot_transformers(name, no_processing=True)` | Both load raw weights, so these match. | diff --git a/makefile b/makefile index 9f9ea9044d..d9655e7d08 100644 --- a/makefile +++ b/makefile @@ -7,6 +7,11 @@ UV_SYNC := uv sync$(ACTIVE_FLAG) # Remove this line when no longer needed RERUN_ARGS := --reruns 2 --reruns-delay 5 +# Parallelism for the coverage run. loadscope keeps a module's tests + its +# module/session fixtures on one worker (models load once per worker, not +# re-scattered). Override to -n 2 (or -n 0) if a runner OOMs. +XDIST_ARGS ?= -n auto --dist loadscope + dep: $(UV_SYNC) @@ -33,7 +38,7 @@ benchmark-test: $(RUN) pytest tests/benchmarks $(RERUN_ARGS) coverage-report-test: - $(RUN) pytest --cov=transformer_lens/ --cov-report=html --cov-branch -m "not slow" tests/integration tests/unit tests/acceptance $(RERUN_ARGS) + $(RUN) pytest -o "addopts=--jaxtyping-packages=transformer_lens,beartype.beartype -W ignore::beartype.roar.BeartypeDecorHintPep585DeprecationWarning" --cov=transformer_lens/ --cov-report=html --cov-branch $(XDIST_ARGS) -m "not slow" tests/integration tests/unit tests/acceptance $(RERUN_ARGS) docstring-test: $(RUN) pytest transformer_lens/ $(RERUN_ARGS) diff --git a/tests/acceptance/model_bridge/compatibility/test_hook_duplication.py b/tests/acceptance/model_bridge/compatibility/test_hook_duplication.py index 254b5a7414..bc85f20a8d 100644 --- a/tests/acceptance/model_bridge/compatibility/test_hook_duplication.py +++ b/tests/acceptance/model_bridge/compatibility/test_hook_duplication.py @@ -21,7 +21,8 @@ def count_hooked_calls(acts, hook): hooked_model.blocks[0].hook_mlp_out.add_hook(count_hooked_calls, is_permanent=True) _ = hooked_model(test_input) - hooked_model.reset_hooks() + # Session-scoped fixture: clear the permanent hook too, or it leaks into later tests. + hooked_model.reset_hooks(including_permanent=True) bridge_call_count = 0 @@ -88,7 +89,7 @@ def use_cached_activations(acts, hook): except KeyError: success = False finally: - bridge_model.reset_hooks() + bridge_model.reset_hooks(including_permanent=True) assert success, ( "Stateful hook pattern failed - hook was likely called multiple times, " diff --git a/tests/integration/model_bridge/compatibility/test_bridge_hook_behavior.py b/tests/integration/model_bridge/compatibility/test_bridge_hook_behavior.py index 5e4e580f4c..8d6daa7c6f 100644 --- a/tests/integration/model_bridge/compatibility/test_bridge_hook_behavior.py +++ b/tests/integration/model_bridge/compatibility/test_bridge_hook_behavior.py @@ -101,7 +101,9 @@ def hook_fn(tensor, hook): bridge("World") assert count == 2 finally: - bridge.reset_hooks() + # Session-scoped fixture: plain reset_hooks() keeps permanent hooks, + # leaking this one onto blocks.0.hook_in for every later test. + bridge.reset_hooks(including_permanent=True) class TestHookModification: diff --git a/tests/integration/model_bridge/test_bridge_caching_hooks.py b/tests/integration/model_bridge/test_bridge_caching_hooks.py new file mode 100644 index 0000000000..aeb2773382 --- /dev/null +++ b/tests/integration/model_bridge/test_bridge_caching_hooks.py @@ -0,0 +1,102 @@ +"""HookedRootModule caching-hook API on TransformerBridge. + +get_caching_hooks / add_caching_hooks / cache_all / cache_some, plus +run_with_cache(incl_bwd=...). The correctness anchor is that caches built via +these paths match run_with_cache, and that incl_bwd captures nonzero gradients +under "_grad". +""" + +import warnings + +import pytest +import torch + + +@pytest.fixture() +def bridge(distilgpt2_bridge): + return distilgpt2_bridge + + +def _tokens(): + return torch.randint(0, 100, (1, 6)) + + +def test_get_caching_hooks_matches_run_with_cache(bridge): + tokens = _tokens() + with torch.no_grad(): + _, ref = bridge.run_with_cache(tokens) + + cache, fwd_hooks, bwd_hooks = bridge.get_caching_hooks( + names_filter=lambda n: n.endswith("hook_out") + ) + assert bwd_hooks == [] + with torch.no_grad(), bridge.hooks(fwd_hooks=fwd_hooks): + bridge.forward(tokens) + + assert cache, "nothing cached" + for name in cache: + assert torch.allclose(cache[name], ref[name], atol=1e-5), name + + +def test_add_caching_hooks_persists_until_reset(bridge): + tokens = _tokens() + with torch.no_grad(): + _, ref = bridge.run_with_cache(tokens) + + cache = bridge.add_caching_hooks(names_filter="blocks.2.hook_out") + try: + with torch.no_grad(): + bridge.forward(tokens) + assert torch.allclose(cache["blocks.2.hook_out"], ref["blocks.2.hook_out"], atol=1e-5) + finally: + bridge.reset_hooks() + + cache.clear() + with torch.no_grad(): + bridge.forward(tokens) + assert cache == {}, "hooks should be gone after reset_hooks" + + +def test_cache_all_and_cache_some_are_deprecated(bridge): + tokens = _tokens() + for call in ( + lambda c: bridge.cache_all(c), + lambda c: bridge.cache_some(c, names=lambda n: n.endswith("hook_out")), + ): + cache: dict = {} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + call(cache) + assert any(issubclass(w.category, DeprecationWarning) for w in caught) + try: + with torch.no_grad(): + bridge.forward(tokens) + assert cache, "deprecated helper cached nothing" + finally: + bridge.reset_hooks() + + +def test_run_with_cache_incl_bwd_caches_gradients(bridge): + tokens = _tokens() + output, cache = bridge.run_with_cache(tokens, incl_bwd=True, return_type="loss") + + assert output.dim() == 0, "incl_bwd needs a scalar output (return_type='loss')" + grad_key = "blocks.4.hook_out_grad" + assert grad_key in cache + grad = cache[grad_key] + assert grad.shape == (1, 6, bridge.cfg.d_model) + assert grad.abs().sum() > 0, "gradient should be nonzero" + # forward activation is also present and detached + assert "blocks.4.hook_out" in cache + assert not cache["blocks.4.hook_out"].requires_grad + + +def test_incl_bwd_leaves_no_hooks_attached(bridge): + tokens = _tokens() + bridge.run_with_cache(tokens, incl_bwd=True, return_type="loss") + leftover = [hp.name for hp in bridge.hook_points() if hp.has_hooks(dir="both")] + assert leftover == [], leftover + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/integration/model_bridge/test_bridge_input_to_embed.py b/tests/integration/model_bridge/test_bridge_input_to_embed.py new file mode 100644 index 0000000000..4a3fd0176c --- /dev/null +++ b/tests/integration/model_bridge/test_bridge_input_to_embed.py @@ -0,0 +1,52 @@ +"""input_to_embed on TransformerBridge — the residual entering block 0. + +Bridge analog of HookedTransformer.input_to_embed. The correctness anchor is +that feeding its residual to forward(..., start_at_layer=0) reproduces the full +forward exactly. +""" + +import pytest +import torch + + +@pytest.fixture() +def bridge(distilgpt2_bridge): + return distilgpt2_bridge + + +def test_returns_residual_tokens_and_none_shortformer(bridge): + tokens = torch.randint(0, 100, (2, 7)) + residual, out_tokens, shortformer_pos_embed, attention_mask = bridge.input_to_embed(tokens) + + assert residual.shape == (2, 7, bridge.cfg.d_model) + assert torch.equal(out_tokens, tokens) + assert shortformer_pos_embed is None + + +def test_round_trips_through_start_at_layer_0(bridge): + tokens = torch.randint(0, 100, (2, 7)) + full = bridge.forward(tokens) + + residual, _, _, _ = bridge.input_to_embed(tokens) + resumed = bridge.forward(residual, start_at_layer=0) + + assert torch.allclose(resumed, full, atol=1e-4) + + +def test_residual_matches_resid_pre_of_block_0(bridge): + tokens = torch.randint(0, 100, (2, 7)) + _, cache = bridge.run_with_cache(tokens) + residual, _, _, _ = bridge.input_to_embed(tokens) + assert torch.allclose(residual, cache["blocks.0.hook_in"], atol=1e-4) + + +def test_string_input(bridge): + residual, tokens, _, _ = bridge.input_to_embed("Hello, world!") + assert residual.ndim == 3 + assert residual.shape[0] == 1 + assert residual.shape[-1] == bridge.cfg.d_model + assert tokens.shape[1] == residual.shape[1] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/integration/model_bridge/test_bridge_manual_kv_cache.py b/tests/integration/model_bridge/test_bridge_manual_kv_cache.py new file mode 100644 index 0000000000..08ef745c39 --- /dev/null +++ b/tests/integration/model_bridge/test_bridge_manual_kv_cache.py @@ -0,0 +1,71 @@ +"""Manual KV-cache injection into TransformerBridge.forward (HF-native). + +forward(past_key_values=...) hands HF back its own cache object and computes +only the new tokens' keys/values. The correctness anchor is that incremental +decoding (token-by-token or chunked) matches a single full forward, and that +use_cache is enabled automatically when a cache is passed. +""" + +import pytest +import torch + + +@pytest.fixture() +def bridge(distilgpt2_bridge): + return distilgpt2_bridge + + +def _seq(): + return torch.randint(0, 100, (1, 7)) + + +def test_token_by_token_matches_full_forward(bridge): + seq = _seq() + with torch.no_grad(): + ref = bridge.forward(seq) + + cache = None + step_logits = [] + with torch.no_grad(): + for i in range(seq.shape[1]): + logits, cache = bridge.forward( + seq[:, i : i + 1], past_key_values=cache, return_type="logits_and_cache" + ) + step_logits.append(logits[:, -1, :]) + incremental = torch.stack(step_logits, dim=1) + + assert torch.allclose(ref, incremental, atol=1e-3) + + +def test_chunked_prefill_matches_full_forward(bridge): + seq = _seq() + with torch.no_grad(): + ref = bridge.forward(seq) + _, cache = bridge.forward(seq[:, :4], return_type="logits_and_cache") + tail_logits, _ = bridge.forward( + seq[:, 4:], past_key_values=cache, return_type="logits_and_cache" + ) + assert torch.allclose(ref[:, -1], tail_logits[:, -1], atol=1e-3) + + +def test_use_cache_enabled_automatically(bridge): + """Passing a cache alone (no explicit use_cache) must return a grown cache.""" + seq = _seq() + with torch.no_grad(): + _, cache = bridge.forward(seq[:, :3], return_type="logits_and_cache") + assert cache is not None + _, cache2 = bridge.forward( + seq[:, 3:5], past_key_values=cache, return_type="logits_and_cache" + ) + assert cache2 is not None + + +def test_normal_forward_unaffected(bridge): + seq = _seq() + with torch.no_grad(): + out = bridge.forward(seq) + assert out.shape == (1, seq.shape[1], bridge.cfg.d_vocab) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/integration/model_bridge/test_bridge_run_with_cache_pos_slice.py b/tests/integration/model_bridge/test_bridge_run_with_cache_pos_slice.py new file mode 100644 index 0000000000..6d5e92b492 --- /dev/null +++ b/tests/integration/model_bridge/test_bridge_run_with_cache_pos_slice.py @@ -0,0 +1,70 @@ +"""run_with_cache(pos_slice=...) on TransformerBridge. + +Mirrors HookedTransformer's pos_slice: each cached activation is sliced along +its position dimension (dim 1 for resid/per-head/token-id activations; the +query position -2 for attention patterns/scores). An int slice keeps the +position dim at size 1, matching HookedTransformer. +""" + +import pytest +import torch + + +@pytest.fixture() +def bridge(distilgpt2_bridge): + return distilgpt2_bridge + + +def _pos_dim(name: str) -> int: + return -2 if name.endswith(("hook_pattern", "hook_attn_scores")) else 1 + + +def test_int_slice_keeps_pos_dim_and_matches_full_cache(bridge): + tokens = torch.randint(0, 100, (1, 8)) + _, full = bridge.run_with_cache(tokens) + + for i in (0, 4, 7, -1): + _, sliced = bridge.run_with_cache(tokens, pos_slice=i) + for name, tensor in full.items(): + if not isinstance(tensor, torch.Tensor) or tensor.dim() < 2: + continue + dim = _pos_dim(name) + abs_dim = dim if dim >= 0 else tensor.dim() + dim + expected = tensor.index_select(abs_dim, torch.tensor([i % tensor.shape[abs_dim]])) + assert sliced[name].shape == expected.shape, name + assert torch.allclose(sliced[name], expected, atol=1e-5), name + + +def test_per_head_and_pattern_shapes(bridge): + tokens = torch.randint(0, 100, (1, 8)) + _, sliced = bridge.run_with_cache(tokens, pos_slice=2) + n_heads, d_head = bridge.cfg.n_heads, bridge.cfg.d_head + + # per-head projection: [batch, pos, head, d_head] -> pos sliced to 1 + per_head = sliced["blocks.0.attn.q.hook_out"] + assert per_head.shape == (1, 1, n_heads, d_head) + + # attention pattern: [batch, head, q_pos, k_pos] -> query pos sliced to 1 + pattern = sliced["blocks.0.attn.hook_pattern"] + assert pattern.shape == (1, n_heads, 1, 8) + + +def test_tuple_range_slice(bridge): + tokens = torch.randint(0, 100, (1, 8)) + _, full = bridge.run_with_cache(tokens) + _, sliced = bridge.run_with_cache(tokens, pos_slice=(1, 4)) + + resid = sliced["blocks.1.hook_out"] + assert resid.shape == (1, 3, bridge.cfg.d_model) + assert torch.allclose(resid, full["blocks.1.hook_out"][:, 1:4], atol=1e-5) + + +def test_none_pos_slice_is_unchanged(bridge): + tokens = torch.randint(0, 100, (1, 8)) + _, full = bridge.run_with_cache(tokens) + _, no_slice = bridge.run_with_cache(tokens, pos_slice=None) + assert full["blocks.0.hook_out"].shape == no_slice["blocks.0.hook_out"].shape + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/integration/model_bridge/test_bridge_start_at_layer.py b/tests/integration/model_bridge/test_bridge_start_at_layer.py new file mode 100644 index 0000000000..62c80a14d4 --- /dev/null +++ b/tests/integration/model_bridge/test_bridge_start_at_layer.py @@ -0,0 +1,134 @@ +"""start_at_layer (residual-stream input) on TransformerBridge. + +The correctness anchor is a round trip: cache the residual entering block k +from a full run, resume from it with start_at_layer=k, and require the logits +and the downstream cache to match the full run bit-for-bit. Blocks below k +execute internally (the bridge delegates to HF) but must not appear in the +cache or fire hooks, matching HookedTransformer. +""" + +import pytest +import torch + + +@pytest.fixture() +def bridge(distilgpt2_bridge): + return distilgpt2_bridge + + +def _tokens(): + return torch.randint(0, 100, (2, 8)) + + +def test_resume_matches_full_run_logits_and_downstream_cache(bridge): + tokens = _tokens() + full_logits, full_cache = bridge.run_with_cache(tokens) + k = 3 + resid = full_cache[f"blocks.{k}.hook_in"] + + logits, cache = bridge.run_with_cache(resid, start_at_layer=k) + + assert torch.allclose(logits, full_logits, atol=1e-4) + # Every downstream key present in the full run must match exactly. + for name, value in full_cache.items(): + if name.startswith("blocks.") and int(name.split(".")[1]) >= k: + assert name in cache, f"{name} missing from resumed cache" + assert torch.allclose(cache[name], value, atol=1e-4), name + + +def test_upstream_hooks_absent_downstream_present(bridge): + tokens = _tokens() + _, full_cache = bridge.run_with_cache(tokens) + k = 3 + _, cache = bridge.run_with_cache(full_cache[f"blocks.{k}.hook_in"], start_at_layer=k) + + keys = set(cache.keys()) + embedding_stage = [ + n + for n in keys + if n in ("hook_embed", "hook_pos_embed", "hook_tokens") + or n.startswith(("embed.", "pos_embed.")) + ] + assert embedding_stage == [], f"embedding-stage hooks should be bypassed: {embedding_stage}" + for below in range(k): + assert not any(n.startswith(f"blocks.{below}.") for n in keys), f"block {below} leaked" + assert any(n.startswith(f"blocks.{k}.") for n in keys), "start block missing" + assert any("ln_final" in n for n in keys), "final norm missing" + assert any("unembed" in n for n in keys), "unembed (output stage) should be present" + + +def test_negative_start_at_layer(bridge): + tokens = _tokens() + full_logits, full_cache = bridge.run_with_cache(tokens) + last = bridge.cfg.n_layers - 1 + resid = full_cache[f"blocks.{last}.hook_in"] + + logits, cache = bridge.run_with_cache(resid, start_at_layer=-1) + + assert torch.allclose(logits, full_logits, atol=1e-4) + assert not any(n.startswith(f"blocks.{last - 1}.") for n in cache.keys()) + + +def test_start_and_stop_bound_a_subrange(bridge): + tokens = _tokens() + _, full_cache = bridge.run_with_cache(tokens) + _, cache = bridge.run_with_cache( + full_cache["blocks.1.hook_in"], start_at_layer=1, stop_at_layer=3 + ) + keys = set(cache.keys()) + assert any(n.startswith("blocks.1.") for n in keys) + assert any(n.startswith("blocks.2.") for n in keys) + assert not any(n.startswith("blocks.0.") for n in keys) + assert not any(n.startswith("blocks.3.") for n in keys) + assert not any("ln_final" in n for n in keys) + + +def test_run_with_hooks_skips_sub_start_layers(bridge): + tokens = _tokens() + _, full_cache = bridge.run_with_cache(tokens) + k = 3 + fired = [] + + def rec(activation, hook): + fired.append(hook.name) + return None + + logits = bridge.run_with_hooks( + full_cache[f"blocks.{k}.hook_in"], + start_at_layer=k, + fwd_hooks=[ + ("blocks.0.attn.hook_out", rec), + ("blocks.2.attn.hook_out", rec), + ("blocks.3.attn.hook_out", rec), + ("blocks.5.attn.hook_out", rec), + ], + ) + assert logits is not None + assert all(int(n.split(".")[1]) >= k for n in fired), fired + assert any(n.startswith("blocks.3.") for n in fired) + assert any(n.startswith("blocks.5.") for n in fired) + + +def test_token_input_with_start_at_layer_raises(bridge): + with pytest.raises(ValueError, match="residual-stream tensor"): + bridge.forward(_tokens(), start_at_layer=1) + + +def test_out_of_range_start_at_layer_raises(bridge): + tokens = _tokens() + _, full_cache = bridge.run_with_cache(tokens) + resid = full_cache["blocks.0.hook_in"] + with pytest.raises(ValueError, match="out of range"): + bridge.forward(resid, start_at_layer=bridge.cfg.n_layers) + + +def test_state_cleaned_up_after_start_at_layer(bridge): + tokens = _tokens() + full_logits, full_cache = bridge.run_with_cache(tokens) + bridge.forward(full_cache["blocks.2.hook_in"], start_at_layer=2) + # A subsequent normal forward must be unaffected by the injection state. + assert torch.allclose(bridge.forward(tokens), full_logits, atol=1e-4) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/integration/model_bridge/test_pretrain_adapter.py b/tests/integration/model_bridge/test_pretrain_adapter.py new file mode 100644 index 0000000000..5878532e38 --- /dev/null +++ b/tests/integration/model_bridge/test_pretrain_adapter.py @@ -0,0 +1,312 @@ +"""Integration tests for PretrainArchitectureAdapter -- numerical parity +against a real (not structural-mock) reference model. + +Uses `tests/mocks/tiny_pretrain_model.py`'s `TinyPretrainModel`, which +implements genuine adjacent-pair RoPE and RMSNorm math (unlike +`_pretrain_mocks.py`'s unit-test mocks, which stub RoPE with constant +cos/sin and only need to get attribute names and call-signatures right). + +Covers: source/bridge logit parity (dense, full-MoE, mixed dense/MoE, +untied embeddings); that this file's fixtures supply a `cfg` truthfully +describing the wrapped model's head/vocab dimensions (a fixture property, +not something `build_pretrain_bridge` itself enforces -- see +`TestIntegrationFixturesSupplyATruthfulConfig`); residual-stream +decomposition via `run_with_cache`; a forward-hook intervention +cross-checked against an independent way of expressing the same edit; and +float64 construction-time preservation. + +Note on scope: since the adapter delegates the whole forward to the +source model rather than reimplementing attention/RoPE, exact logit +parity mainly demonstrates that wrapping and output normalization don't +alter an unhooked forward, not that the RoPE convention itself is +correct (the same implementation runs on both sides of the comparison). +`TestResidualStreamDecomposition` and `TestHookIntervention` are the +stronger evidence for correct intervention points. +""" + +from __future__ import annotations + +import torch + +from tests.mocks.tiny_pretrain_model import TinyPretrainModel +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge.supported_architectures.pretrain import ( + ARCHITECTURE_NAME, + build_pretrain_bridge, +) + + +def _make_cfg( + *, + d_model: int, + n_layers: int, + n_heads: int = 2, + d_vocab: int = 64, +) -> TransformerBridgeConfig: + """Build a bridge config matching this file's TinyPretrainModel + fixtures (n_heads=2, vocab_size=64 by default). See + `TestIntegrationFixturesSupplyATruthfulConfig` for why this matching + matters and what happens when it doesn't.""" + return TransformerBridgeConfig( + d_model=d_model, + d_head=d_model // n_heads, + n_layers=n_layers, + n_ctx=128, + n_heads=n_heads, + d_vocab=d_vocab, + d_mlp=d_model * 2, + architecture=ARCHITECTURE_NAME, + ) + + +class TestSourceBridgeLogitParity: + """Bridge output must equal the source model's own `"logits"` output + exactly -- not within tolerance. The adapter never translates weights + into a second parameter layout, so there is no floating-point drift + to tolerate; any mismatch here is a real bug, not numerical noise.""" + + def test_dense_model_logit_parity_exact(self) -> None: + torch.manual_seed(0) + model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=2, d_ff=32, vocab_size=64) + model.eval() + tokens = torch.randint(0, 64, (1, 5)) + with torch.no_grad(): + source_logits = model(tokens)["logits"] + + bridge = build_pretrain_bridge(model, _make_cfg(d_model=16, n_layers=2)) + with torch.no_grad(): + bridge_logits = bridge(tokens) + + torch.testing.assert_close(bridge_logits, source_logits, atol=0, rtol=0) + + def test_full_moe_model_logit_parity_exact(self) -> None: + """Every block's MLP is MoE (moe_layer_indices covers all layers).""" + torch.manual_seed(0) + model = TinyPretrainModel( + d_model=16, + n_heads=2, + n_layers=2, + d_ff=32, + vocab_size=64, + n_experts=4, + top_k=2, + moe_layer_indices=frozenset({0, 1}), + ) + model.eval() + tokens = torch.randint(0, 64, (1, 5)) + with torch.no_grad(): + source_logits = model(tokens)["logits"] + + bridge = build_pretrain_bridge(model, _make_cfg(d_model=16, n_layers=2)) + with torch.no_grad(): + bridge_logits = bridge(tokens) + + torch.testing.assert_close(bridge_logits, source_logits, atol=0, rtol=0) + + def test_mixed_dense_moe_model_logit_parity_exact(self) -> None: + """One block dense, one block MoE -- the dispatcher must route each + block's MLP correctly within a single model, not just across + separately-constructed all-dense/all-MoE models.""" + torch.manual_seed(0) + model = TinyPretrainModel( + d_model=16, + n_heads=2, + n_layers=2, + d_ff=32, + vocab_size=64, + n_experts=4, + top_k=2, + moe_layer_indices=frozenset({1}), + ) + model.eval() + tokens = torch.randint(0, 64, (1, 5)) + with torch.no_grad(): + source_logits = model(tokens)["logits"] + + bridge = build_pretrain_bridge(model, _make_cfg(d_model=16, n_layers=2)) + with torch.no_grad(): + bridge_logits = bridge(tokens) + + torch.testing.assert_close(bridge_logits, source_logits, atol=0, rtol=0) + + def test_untied_embeddings_logit_parity_exact(self) -> None: + torch.manual_seed(0) + model = TinyPretrainModel( + d_model=16, + n_heads=2, + n_layers=1, + d_ff=32, + vocab_size=64, + tie_embeddings=False, + ) + model.eval() + tokens = torch.randint(0, 64, (1, 5)) + with torch.no_grad(): + source_logits = model(tokens)["logits"] + + bridge = build_pretrain_bridge(model, _make_cfg(d_model=16, n_layers=1)) + with torch.no_grad(): + bridge_logits = bridge(tokens) + + torch.testing.assert_close(bridge_logits, source_logits, atol=0, rtol=0) + + +class TestIntegrationFixturesSupplyATruthfulConfig: + """Integration fixtures must supply a config matching the source + model. `build_pretrain_bridge` does not itself validate this -- `cfg` + is caller-owned, per ordinary TransformerBridge convention -- so a + mismatch shows up only in `bridge.cfg`, silently, never in the + logits (see the second test).""" + + def test_cfg_matches_model_head_and_vocab_dimensions(self) -> None: + torch.manual_seed(0) + n_heads = 2 + vocab_size = 64 + model = TinyPretrainModel( + d_model=16, n_heads=n_heads, n_layers=2, d_ff=32, vocab_size=vocab_size + ) + model.eval() + cfg = _make_cfg(d_model=16, n_layers=2, n_heads=n_heads, d_vocab=vocab_size) + bridge = build_pretrain_bridge(model, cfg) + + assert bridge.cfg.n_heads == n_heads + assert bridge.cfg.d_head == 16 // n_heads + assert bridge.cfg.d_vocab == model.embed.num_embeddings + + def test_build_pretrain_bridge_does_not_validate_cfg_against_model(self) -> None: + """A cfg describing a different architecture than the wrapped + model still builds and still produces exact logit parity + (delegation never reads cfg) -- only bridge.cfg is wrong. + + Current framework convention; not an adapter guarantee. This test + documents present behavior, not a requirement -- if + build_pretrain_bridge later gains cfg-vs-model validation, this + test should be updated (or removed) rather than treated as a + regression to preserve.""" + torch.manual_seed(0) + model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64) + model.eval() + tokens = torch.randint(0, 64, (1, 5)) + with torch.no_grad(): + source_logits = model(tokens)["logits"] + + wrong_cfg = _make_cfg(d_model=16, n_layers=1, n_heads=4, d_vocab=256) + bridge = build_pretrain_bridge(model, wrong_cfg) + with torch.no_grad(): + bridge_logits = bridge(tokens) + + torch.testing.assert_close(bridge_logits, source_logits, atol=0, rtol=0) + assert bridge.cfg.n_heads != model.blocks[0].attn.n_heads + assert bridge.cfg.d_vocab != model.embed.num_embeddings + + +class TestResidualStreamDecomposition: + """`run_with_cache`'s resid hooks must decompose the way the residual + stream actually works, not just be present under the right names + (attribute/key presence is already covered by the structural unit + tests) -- resid_pre + attn_out == resid_mid, resid_mid + mlp_out == + resid_post, and one block's resid_post is the next block's resid_pre.""" + + def test_resid_stream_decomposes_exactly_via_run_with_cache(self) -> None: + torch.manual_seed(0) + model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=2, d_ff=32, vocab_size=64) + model.eval() + bridge = build_pretrain_bridge(model, _make_cfg(d_model=16, n_layers=2)) + tokens = torch.randint(0, 64, (1, 5)) + + with torch.no_grad(): + _, cache = bridge.run_with_cache(tokens) + + torch.testing.assert_close( + cache["blocks.0.hook_resid_pre"], cache["hook_embed"], atol=0, rtol=0 + ) + torch.testing.assert_close( + cache["blocks.0.hook_resid_mid"], + cache["blocks.0.hook_resid_pre"] + cache["blocks.0.hook_attn_out"], + ) + torch.testing.assert_close( + cache["blocks.0.hook_resid_post"], + cache["blocks.0.hook_resid_mid"] + cache["blocks.0.hook_mlp_out"], + ) + torch.testing.assert_close( + cache["blocks.1.hook_resid_pre"], + cache["blocks.0.hook_resid_post"], + atol=0, + rtol=0, + ) + torch.testing.assert_close( + cache["blocks.1.hook_resid_post"], + cache["blocks.1.hook_resid_mid"] + cache["blocks.1.hook_mlp_out"], + ) + + +class TestHookIntervention: + """A forward-hook edit at `blocks.{i}.mlp.hook_out` must land at the + point the residual-stream decomposition says it should -- cross- + checked against an independently-expressed version of the same edit + (forcing resid_post to equal resid_mid), rather than only checking + that *some* change occurred.""" + + def test_mlp_hook_ablation_matches_independent_resid_post_override(self) -> None: + torch.manual_seed(0) + model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=2, d_ff=32, vocab_size=64) + model.eval() + bridge = build_pretrain_bridge(model, _make_cfg(d_model=16, n_layers=2)) + tokens = torch.randint(0, 64, (1, 5)) + + with torch.no_grad(): + baseline_logits, baseline_cache = bridge.run_with_cache(tokens) + resid_mid0 = baseline_cache["blocks.0.hook_resid_mid"].clone() + + def zero_mlp_out(value: torch.Tensor, hook) -> torch.Tensor: + return torch.zeros_like(value) + + def force_resid_post_to_resid_mid(value: torch.Tensor, hook) -> torch.Tensor: + return resid_mid0 + + with torch.no_grad(): + via_mlp_ablation = bridge.run_with_hooks( + tokens, fwd_hooks=[("blocks.0.mlp.hook_out", zero_mlp_out)] + ) + via_resid_override = bridge.run_with_hooks( + tokens, + fwd_hooks=[("blocks.0.hook_resid_post", force_resid_post_to_resid_mid)], + ) + + # Sanity: the intervention actually changed something. + assert not torch.equal(via_mlp_ablation, baseline_logits) + # The real check: two different-looking edits that are + # mathematically the same edit must produce identical output. + torch.testing.assert_close(via_mlp_ablation, via_resid_override, atol=0, rtol=0) + + +class TestDtypePreservation: + """float64 is checked for preservation through construction only, not + full numerical parity at that dtype -- RoPE's cos/sin computation + involves trig functions whose float32-vs-float64 rounding can + legitimately differ from the bridge's own dtype-casting path in ways + unrelated to correctness, so exact-equality parity isn't a meaningful + check at non-default dtypes the way it is at float32.""" + + def test_float64_construction_preserves_dtype_and_tied_weights(self) -> None: + torch.manual_seed(0) + model = TinyPretrainModel( + d_model=16, + n_heads=2, + n_layers=1, + d_ff=32, + vocab_size=64, + tie_embeddings=True, + ).to(torch.float64) + assert model.lm_head.weight is model.embed.weight + + bridge = build_pretrain_bridge(model, _make_cfg(d_model=16, n_layers=1)) + + assert next(model.parameters()).dtype == torch.float64 + assert model.lm_head.weight is model.embed.weight + + tokens = torch.randint(0, 64, (1, 4)) + with torch.no_grad(): + output = bridge(tokens) + assert output.dtype == torch.float64 + assert torch.isfinite(output).all() diff --git a/tests/integration/model_bridge/test_raven_adapter.py b/tests/integration/model_bridge/test_raven_adapter.py new file mode 100644 index 0000000000..9a3d5801a0 --- /dev/null +++ b/tests/integration/model_bridge/test_raven_adapter.py @@ -0,0 +1,251 @@ +"""Integration tests for the Raven / Huginn architecture adapter (RavenForCausalLM). + +Model: tomg-group-umd/huginn-0125 + - Remote-code (auto_map → raven_modeling_minimal.RavenForCausalLM), so no + tiny random checkpoint exists; this loads the real ~3.5B weights (~14GB + download, ~28GB RAM for the two fp32 copies). Gated out of CI on + network/memory budget; run locally with: + uv run pytest tests/integration/model_bridge/test_raven_adapter.py -v -m slow + +Huginn is a depth-recurrent decoder: prelude (2 blocks) → weight-tied +recurrent core (4 blocks applied N times, default N = mean_recurrence = 32) → +coda (2 blocks). The recurrence, the per-step prelude re-injection, the +sandwich norms and RoPE all live inside the remote-code forward, which the +bridge delegates to, so logit parity holds unchanged. + +Two behaviours make Huginn different from a flat decoder and shape these tests: + +1. Random initial latent state. ``iterate_forward`` seeds the recurrence with + ``torch.randn_like`` (``initialize_state``), so the forward is + non-deterministic across calls. Parity is only meaningful with the RNG + pinned identically before the bridge and HF calls; ``_seeded`` does that. + +2. Weight-tied recurrent core. The four ``core_block`` blocks run once per + recurrence step, so their hooks fire N times per forward and + ``run_with_cache`` keeps the final step (pinned by TestRavenRecurrentCore). + +Comparing against an independent HF load (never ``bridge.original_model``, +whose modules are hook-wrapped) keeps the parity check honest. +""" + +import os +import platform + +import pytest +import torch +from transformers import AutoConfig, AutoModelForCausalLM + +from transformer_lens.model_bridge import TransformerBridge + +MODEL = "tomg-group-umd/huginn-0125" + +pytestmark = [ + pytest.mark.slow, + pytest.mark.skipif( + bool(os.getenv("CI")), + reason="tomg-group-umd/huginn-0125: ~14GB download + ~28GB RAM, too large for CI", + ), +] + +# Depth-recurrent sandwich-norm model with 100+ effective layer applications at +# fp32; allow a wider op-order noise floor on GH Actions macOS-arm64 (same idiom +# as the Ouro and SmolLM3 bridge-vs-HF parity tests). +_MACOS_ARM64 = platform.system() == "Darwin" and platform.machine() == "arm64" +FP32_NOISE_TOL = 1e-2 if _MACOS_ARM64 else 1e-3 + +# Reduced recurrence for a tractable local run; passed identically to both the +# bridge and HF forward so parity is exact regardless of the value. +NUM_STEPS = 8 + + +def _seeded(fn, seed: int = 0): + """Run ``fn`` with the global torch RNG pinned. + + Huginn's ``initialize_state`` draws the initial latent from + ``torch.randn_like``; pinning the seed makes the bridge and HF forwards + start from the same latent so their logits are comparable. + """ + torch.manual_seed(seed) + return fn() + + +@pytest.fixture(scope="module") +def bridge() -> TransformerBridge: + return TransformerBridge.boot_transformers( + MODEL, device="cpu", dtype=torch.float32, trust_remote_code=True + ) + + +@pytest.fixture(scope="module") +def hf_eager(bridge: TransformerBridge) -> torch.nn.Module: + """HF model loaded independently of the bridge's wrapped instance. + + Depends on the bridge fixture only for ordering: booting the bridge runs + the adapter's prepare_loading patch (guarding transformers v5 weight + re-init) on the cached raven modeling module, and this plain + from_pretrained load benefits from that patch too. The weights are still a + separate copy. + """ + config = AutoConfig.from_pretrained(MODEL, trust_remote_code=True) + return AutoModelForCausalLM.from_pretrained( + MODEL, + config=config, + torch_dtype=torch.float32, + trust_remote_code=True, + ).eval() + + +@pytest.fixture(scope="module") +def tokens(hf_eager: torch.nn.Module) -> torch.Tensor: + """A short, deterministic random token sequence.""" + generator = torch.Generator().manual_seed(0) + return torch.randint(0, hf_eager.config.vocab_size, (1, 16), generator=generator) + + +class TestRavenBridgeCreation: + """The bridge loads the remote-code model and wires the three phases.""" + + def test_boot_transformers_succeeds(self, bridge: TransformerBridge) -> None: + assert bridge is not None + + def test_phase_block_lists_present(self, bridge: TransformerBridge) -> None: + """Prelude / core_block / coda are exposed as separate block lists.""" + assert len(bridge.prelude) == bridge.cfg.n_layers_in_prelude + assert len(bridge.core_block) == bridge.cfg.n_layers_in_recurrent_block + assert len(bridge.coda) == bridge.cfg.n_layers_in_coda + + def test_physical_layer_split( + self, bridge: TransformerBridge, hf_eager: torch.nn.Module + ) -> None: + """The three lists sum to num_hidden_layers (the physical block count).""" + physical = len(bridge.prelude) + len(bridge.core_block) + len(bridge.coda) + assert physical == hf_eager.config.num_hidden_layers + + def test_config_flags(self, bridge: TransformerBridge) -> None: + assert bridge.cfg.normalization_type == "RMS" + assert bridge.cfg.positional_embedding_type == "rotary" + assert bridge.cfg.gated_mlp is True + assert bridge.cfg.final_rms is True + + def test_sandwich_norms_wired(self, bridge: TransformerBridge) -> None: + """All four per-block RMSNorms are bridged on each phase.""" + for phase in (bridge.prelude, bridge.core_block, bridge.coda): + block = phase[0] + for ln in ("norm_1", "norm_2", "norm_3", "norm_4"): + assert hasattr(block, ln), f"missing {ln}" + + def test_recurrence_config_propagates( + self, bridge: TransformerBridge, hf_eager: torch.nn.Module + ) -> None: + assert bridge.cfg.mean_recurrence == hf_eager.config.mean_recurrence + + +class TestRavenHFDelegation: + """The bridge wraps the live remote-code modules in place (no copies).""" + + def test_core_block_is_the_hf_tree_module(self, bridge: TransformerBridge) -> None: + assert bridge.core_block[0] is bridge.original_model.transformer.core_block[0] + assert type(bridge.core_block[0].original_component).__name__ == "SandwichBlock" + + def test_attention_combined_qkv_wraps_live_hf_modules(self, bridge: TransformerBridge) -> None: + attn = bridge.core_block[0].attn + assert type(attn.original_component).__name__ == "CausalSelfAttention" + # Huginn uses a combined Wqkv projection (not split q/k/v). + assert attn.qkv is attn.original_component.Wqkv + assert attn.o is attn.original_component.proj + + def test_mlp_combined_fc_wraps_live_hf_modules(self, bridge: TransformerBridge) -> None: + mlp = bridge.core_block[0].mlp + assert type(mlp.original_component).__name__ == "GatedMLP" + # Combined gate+up "fc" and output "proj". + assert getattr(mlp, "in") is mlp.original_component.fc + assert mlp.out is mlp.original_component.proj + + def test_sandwich_norms_wrap_live_hf_modules(self, bridge: TransformerBridge) -> None: + block = bridge.core_block[0] + layer = block.original_component + assert block.norm_1 is layer.norm_1 + assert block.norm_2 is layer.norm_2 + assert block.norm_3 is layer.norm_3 + assert block.norm_4 is layer.norm_4 + + +class TestRavenForwardEquivalence: + """Bridge output reproduces the HF eager reference through the recurrence. + + Both forwards are given the same reduced ``num_steps`` and the same pinned + RNG seed, so the random initial latent state and recurrence depth match and + the logits are directly comparable. + """ + + def test_forward_logits_match_hf_eager( + self, bridge: TransformerBridge, hf_eager: torch.nn.Module, tokens: torch.Tensor + ) -> None: + with torch.inference_mode(): + bridge_logits = _seeded(lambda: bridge(tokens, num_steps=NUM_STEPS)) + hf_logits = _seeded(lambda: hf_eager(tokens, num_steps=NUM_STEPS).logits) + max_diff = (bridge_logits - hf_logits).abs().max().item() + assert max_diff < FP32_NOISE_TOL, ( + f"Raven bridge vs HF eager logit drift={max_diff:.2e} exceeds the " + f"fp32-noise tolerance {FP32_NOISE_TOL:.0e}." + ) + + +class TestRavenRecurrentCore: + """Pin the weight-tied recurrent-core semantics the adapter documents.""" + + def test_core_block_hook_fires_once_per_recurrence_step( + self, bridge: TransformerBridge, tokens: torch.Tensor + ) -> None: + """The same physical core block executes num_steps times per forward. + + This is the load-bearing difference from a flat decoder: hooks on + core_block.{i} fire once per recurrence step and run_with_cache keeps + the final step. If HF's remote code changes the loop (or the bridge + stops delegating it), this fails. + """ + fired: list[bool] = [] + _seeded( + lambda: bridge.run_with_hooks( + tokens, + num_steps=NUM_STEPS, + fwd_hooks=[("core_block.0.hook_out", lambda value, hook: fired.append(True))], + ) + ) + assert len(fired) == NUM_STEPS + + def test_prelude_hook_fires_once(self, bridge: TransformerBridge, tokens: torch.Tensor) -> None: + """Prelude blocks are non-recurrent: their hooks fire exactly once.""" + fired: list[bool] = [] + _seeded( + lambda: bridge.run_with_hooks( + tokens, + num_steps=NUM_STEPS, + fwd_hooks=[("prelude.0.hook_out", lambda value, hook: fired.append(True))], + ) + ) + assert len(fired) == 1 + + +class TestRavenHookShapes: + """Residual-stream hooks fire with the expected shape on each phase.""" + + @pytest.mark.parametrize("phase", ["prelude", "core_block", "coda"]) + def test_hook_out_shape_matches_residual_stream( + self, bridge: TransformerBridge, tokens: torch.Tensor, phase: str + ) -> None: + captured: list[torch.Tensor] = [] + + def _capture(value, hook): + captured.append(value[0] if isinstance(value, tuple) else value) + return value + + _seeded( + lambda: bridge.run_with_hooks( + tokens, + num_steps=NUM_STEPS, + fwd_hooks=[(f"{phase}.0.hook_out", _capture)], + ) + ) + assert captured + assert captured[-1].shape == (1, tokens.shape[1], bridge.cfg.d_model) diff --git a/tests/integration/model_bridge/test_rwkv7_adapter.py b/tests/integration/model_bridge/test_rwkv7_adapter.py new file mode 100644 index 0000000000..4c47889670 --- /dev/null +++ b/tests/integration/model_bridge/test_rwkv7_adapter.py @@ -0,0 +1,161 @@ +"""Integration tests for the RWKV-7 architecture adapter (RWKV7ForCausalLM). + +Model: fla-hub/rwkv7-0.1B-g1 + - Remote-code (``trust_remote_code=True``); the modeling classes live in the + flash-linear-attention (``fla``) library, which must be installed: + pip install flash-linear-attention + - ~191M params (~0.8GB download). Gated out of CI on the network/dependency + budget; run locally with: + uv run pytest tests/integration/model_bridge/test_rwkv7_adapter.py -v -m slow + +RWKV-7 is an attention-free recurrent decoder: a flat stack of pre-norm blocks, +each a generalized-delta-rule time-mixing sublayer (``attn``) plus a +token-shifted squared-ReLU channel-mixing sublayer (``ffn``), wrapped by standard +biased LayerNorm. The recurrence, the token shift, and the cross-block +``v_first`` threading all live inside the ``fla`` remote-code forward, which the +bridge delegates to, so logit parity holds unchanged. + +Comparing against an independent HF load (never ``bridge.original_model``, whose +modules are hook-wrapped) keeps the parity check honest. +""" + +import os +import platform + +import pytest +import torch + +from transformer_lens.model_bridge import TransformerBridge + +pytest.importorskip("fla", reason="RWKV-7 needs flash-linear-attention (pip install it)") + +from transformers import AutoConfig, AutoModelForCausalLM # noqa: E402 + +MODEL = "fla-hub/rwkv7-0.1B-g1" + +pytestmark = [ + pytest.mark.slow, + pytest.mark.skipif( + bool(os.getenv("CI")), + reason="fla-hub/rwkv7-0.1B-g1: remote code + flash-linear-attention dep, skipped in CI", + ), +] + +# Recurrent fp32 model; allow a wider op-order noise floor on GH Actions +# macOS-arm64 (same idiom as the Ouro / Raven bridge-vs-HF parity tests). +_MACOS_ARM64 = platform.system() == "Darwin" and platform.machine() == "arm64" +FP32_NOISE_TOL = 1e-2 if _MACOS_ARM64 else 1e-3 + + +@pytest.fixture(scope="module") +def bridge() -> TransformerBridge: + return TransformerBridge.boot_transformers( + MODEL, device="cpu", dtype=torch.float32, trust_remote_code=True + ) + + +@pytest.fixture(scope="module") +def hf_eager(bridge: TransformerBridge) -> torch.nn.Module: + """HF model loaded independently of the bridge's wrapped instance. + + Depends on the bridge fixture only for ordering: booting the bridge runs the + adapter's prepare_loading patch (guarding transformers v5 weight re-init) on + the cached rwkv7 modeling module, and this plain from_pretrained load benefits + from that patch too. The weights are still a separate copy. + """ + config = AutoConfig.from_pretrained(MODEL, trust_remote_code=True) + return AutoModelForCausalLM.from_pretrained( + MODEL, + config=config, + torch_dtype=torch.float32, + trust_remote_code=True, + ).eval() + + +@pytest.fixture(scope="module") +def tokens(hf_eager: torch.nn.Module) -> torch.Tensor: + """A short, deterministic random token sequence.""" + generator = torch.Generator().manual_seed(0) + return torch.randint(0, hf_eager.config.vocab_size, (1, 16), generator=generator) + + +class TestRWKV7BridgeCreation: + """The bridge loads the remote-code model and wires the flat block stack.""" + + def test_boot_transformers_succeeds(self, bridge: TransformerBridge) -> None: + assert bridge is not None + + def test_block_count_matches_config( + self, bridge: TransformerBridge, hf_eager: torch.nn.Module + ) -> None: + assert len(bridge.blocks) == hf_eager.config.num_hidden_layers + + def test_config_flags(self, bridge: TransformerBridge) -> None: + assert bridge.cfg.normalization_type == "LN" + assert bridge.cfg.positional_embedding_type == "none" + assert bridge.cfg.gated_mlp is False + assert bridge.cfg.final_rms is False + + def test_block_submodules_wired(self, bridge: TransformerBridge) -> None: + """attn_norm / attn / ffn_norm / ffn are bridged on each block.""" + block = bridge.blocks[0] + for sub in ("attn_norm", "attn", "ffn_norm", "ffn"): + assert hasattr(block, sub), f"missing {sub}" + + def test_attn_projections_wrap_live_hf_modules(self, bridge: TransformerBridge) -> None: + attn = bridge.blocks[0].attn + hf_attn = bridge.original_model.model.layers[0].attn + assert attn.r_proj is hf_attn.r_proj + assert attn.k_proj is hf_attn.k_proj + assert attn.v_proj is hf_attn.v_proj + assert attn.o_proj is hf_attn.o_proj + + def test_ffn_projections_wrap_live_hf_modules(self, bridge: TransformerBridge) -> None: + ffn = bridge.blocks[0].ffn + hf_ffn = bridge.original_model.model.layers[0].ffn + assert ffn.key is hf_ffn.key + assert ffn.value is hf_ffn.value + + +class TestRWKV7ForwardEquivalence: + """Bridge output reproduces the HF eager reference through the recurrence.""" + + def test_forward_logits_match_hf_eager( + self, bridge: TransformerBridge, hf_eager: torch.nn.Module, tokens: torch.Tensor + ) -> None: + with torch.inference_mode(): + bridge_logits = bridge(tokens) + hf_logits = hf_eager(tokens).logits + max_diff = (bridge_logits - hf_logits).abs().max().item() + assert max_diff < FP32_NOISE_TOL, ( + f"RWKV-7 bridge vs HF eager logit drift={max_diff:.2e} exceeds the " + f"fp32-noise tolerance {FP32_NOISE_TOL:.0e}." + ) + + +class TestRWKV7Hooks: + """Residual-stream and sublayer hooks fire with the expected shape.""" + + @pytest.mark.parametrize( + "hook_name", + ["blocks.0.hook_out", "blocks.0.attn.hook_out", "blocks.0.ffn.hook_out"], + ) + def test_hook_fires_with_residual_shape( + self, bridge: TransformerBridge, tokens: torch.Tensor, hook_name: str + ) -> None: + captured: list[torch.Tensor] = [] + + def _capture(value, hook): + captured.append(value[0] if isinstance(value, tuple) else value) + return value + + bridge.run_with_hooks(tokens, fwd_hooks=[(hook_name, _capture)]) + assert captured, f"{hook_name} did not fire" + assert captured[-1].shape == (1, tokens.shape[1], bridge.cfg.d_model) + + def test_run_with_cache_has_block_hooks( + self, bridge: TransformerBridge, tokens: torch.Tensor + ) -> None: + _, cache = bridge.run_with_cache(tokens) + for key in ("blocks.0.hook_out", "blocks.0.attn.hook_out", "blocks.0.ffn.hook_out"): + assert key in cache, f"{key} missing from cache" diff --git a/tests/mocks/tiny_pretrain_model.py b/tests/mocks/tiny_pretrain_model.py new file mode 100644 index 0000000000..2c3d9434ca --- /dev/null +++ b/tests/mocks/tiny_pretrain_model.py @@ -0,0 +1,193 @@ +"""A tiny, self-contained decoder-only reference model (RoPE, RMSNorm, +gated SwiGLU MLP, optional MoE) used only by the pretrain-adapter +integration tests, to exercise real numerical parity rather than a purely +structural mock. + +Deliberately uses the *adjacent-pair* RoPE convention +(`[x0, x1] -> [-x1, x0]`), not HuggingFace's rotate-half (contiguous-half) +convention -- this is the concrete case `PretrainArchitectureAdapter`'s +docstring is about: an existing HF-oriented attention bridge would silently +apply the wrong rotation to a model using this convention. + +Also deliberately computes RoPE `cos`/`sin` once per forward pass and +passes them into every block call as extra positional args +(`block(x, cos, sin)`), matching the target architecture's convention. +This matters beyond efficiency: `BlockBridge` wraps a bare-tensor output +in a 1-tuple for single-positional-argument "standalone hidden_states" +calls (an HF-compatibility convention). A loop calling `block(x)` alone +would need to unwrap `[0]` from each result -- modifying the source +model's forward specifically because it's being bridged. Passing +`cos`/`sin` positionally avoids that, so the forward loop needs no +changes at all. +""" + +from __future__ import annotations + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def _rotate_adjacent_pairs(x: torch.Tensor) -> torch.Tensor: + x1 = x[..., 0::2] + x2 = x[..., 1::2] + return torch.stack((-x2, x1), dim=-1).flatten(-2) + + +def _apply_rotary_adjacent_pairs( + x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor +) -> torch.Tensor: + cos = cos.repeat_interleave(2, dim=-1) + sin = sin.repeat_interleave(2, dim=-1) + return x * cos + _rotate_adjacent_pairs(x) * sin + + +class RMSNorm(nn.Module): + def __init__(self, d_model: int, eps: float = 1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(d_model)) + self.eps = eps + + def forward(self, x: torch.Tensor) -> torch.Tensor: + variance = x.pow(2).mean(-1, keepdim=True) + x = x * torch.rsqrt(variance + self.eps) + return x * self.weight + + +class Attention(nn.Module): + """Receives precomputed `cos`/`sin` rather than computing them + internally -- matches the real target architecture, where rotary + tables are computed once per forward pass and shared across blocks.""" + + def __init__(self, d_model: int, n_heads: int): + super().__init__() + self.n_heads = n_heads + self.d_head = d_model // n_heads + self.qkv = nn.Linear(d_model, 3 * d_model, bias=False) + self.o = nn.Linear(d_model, d_model, bias=False) + + def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + b, s, d = x.shape + qkv = self.qkv(x).view(b, s, 3, self.n_heads, self.d_head).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # each: [b, n_heads, s, d_head] + + q = _apply_rotary_adjacent_pairs(q, cos, sin) + k = _apply_rotary_adjacent_pairs(k, cos, sin) + + attn = torch.softmax( + (q @ k.transpose(-2, -1)) / (self.d_head**0.5) + + torch.triu(torch.full((s, s), float("-inf"), device=x.device), diagonal=1), + dim=-1, + ) + out = (attn @ v).transpose(1, 2).reshape(b, s, d) + return self.o(out) + + +class DenseMLP(nn.Module): + def __init__(self, d_model: int, d_ff: int): + super().__init__() + self.gate = nn.Linear(d_model, d_ff, bias=False) + self.up = nn.Linear(d_model, d_ff, bias=False) + self.down = nn.Linear(d_ff, d_model, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.down(F.silu(self.gate(x)) * self.up(x)) + + +class MoEMLP(nn.Module): + def __init__(self, d_model: int, d_ff: int, n_experts: int, top_k: int): + super().__init__() + self.top_k = top_k + self.router = nn.Linear(d_model, n_experts, bias=False) + self.experts = nn.ModuleList([DenseMLP(d_model, d_ff) for _ in range(n_experts)]) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + weights = torch.softmax(self.router(x), dim=-1) + topk_weights, topk_idx = weights.topk(self.top_k, dim=-1) + out = torch.zeros_like(x) + for e in range(len(self.experts)): + mask = topk_idx == e + if not mask.any(): + continue + gate = (mask.float() * topk_weights).sum(dim=-1, keepdim=True) + out = out + gate * self.experts[e](x) + return out + + +class Block(nn.Module): + """Takes `cos`/`sin` as extra positional args alongside the hidden + state -- see the module docstring for why this matters beyond RoPE + plumbing.""" + + def __init__(self, d_model: int, n_heads: int, d_ff: int, mlp: nn.Module): + super().__init__() + self.norm1 = RMSNorm(d_model) + self.attn = Attention(d_model, n_heads) + self.norm2 = RMSNorm(d_model) + self.mlp = mlp + + def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.norm1(x), cos, sin) + x = x + self.mlp(self.norm2(x)) + return x + + +class TinyPretrainModel(nn.Module): + """Full tiny reference model for numerical-parity integration tests. + Matches `PretrainArchitectureAdapter`'s expected attribute paths: + `embed`, `blocks`, `norm_f`, `lm_head`. + """ + + def __init__( + self, + d_model: int = 32, + n_heads: int = 4, + n_layers: int = 2, + d_ff: int = 64, + vocab_size: int = 256, + n_experts: int = 4, + top_k: int = 2, + moe_layer_indices: frozenset[int] = frozenset(), + tie_embeddings: bool = True, + rope_base: float = 10000.0, + ): + super().__init__() + self.embed = nn.Embedding(vocab_size, d_model) + self.blocks = nn.ModuleList( + [ + Block( + d_model, + n_heads, + d_ff, + mlp=( + MoEMLP(d_model, d_ff, n_experts, top_k) + if i in moe_layer_indices + else DenseMLP(d_model, d_ff) + ), + ) + for i in range(n_layers) + ] + ) + self.norm_f = RMSNorm(d_model) + self.lm_head = nn.Linear(d_model, vocab_size, bias=False) + if tie_embeddings: + self.lm_head.weight = self.embed.weight + + d_head = d_model // n_heads + inv_freq = 1.0 / (rope_base ** (torch.arange(0, d_head, 2).float() / d_head)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def forward(self, tokens: torch.Tensor) -> dict: + b, s = tokens.shape + x = self.embed(tokens) + + # Computed once per forward pass, passed into every block -- the + # real target architecture's convention (see module docstring). + pos = torch.arange(s, device=tokens.device, dtype=self.inv_freq.dtype) + freqs = torch.outer(pos, self.inv_freq) + cos, sin = freqs.cos(), freqs.sin() + + for block in self.blocks: + x = block(x, cos, sin) + x = self.norm_f(x) + return {"logits": self.lm_head(x)} diff --git a/tests/unit/model_bridge/generalized_components/test_moe_bridge_tuple_output.py b/tests/unit/model_bridge/generalized_components/test_moe_bridge_tuple_output.py new file mode 100644 index 0000000000..3b644497bc --- /dev/null +++ b/tests/unit/model_bridge/generalized_components/test_moe_bridge_tuple_output.py @@ -0,0 +1,88 @@ +"""Tests for MoEBridge tuple-output validation and router-score hooks. + +Unlike the pretrain-adapter's TestDenseOrMoEFeedForwardBridgeTupleOutput +(which patches _delegate.forward and therefore exercises +DenseOrMoEFeedForwardBridge.forward in isolation), these tests call +MoEBridge.forward() directly via a stub original_component, so they cover +MoEBridge's own tuple-validation contract rather than the dispatcher's. +""" + +from __future__ import annotations + +import pytest +import torch +import torch.nn as nn + +from transformer_lens.model_bridge.generalized_components import MoEBridge + + +class _StubMoEModule(nn.Module): + """Minimal nn.Module whose forward is swapped in per-test, standing in + for a real MoE layer wrapped by MoEBridge.""" + + def __init__(self, fake_forward): + super().__init__() + # Give the module at least one parameter so + # `next(self.parameters())` in MoEBridge.forward doesn't raise + # StopIteration and skip the dtype-cast branch entirely. + self.dummy = nn.Linear(4, 4) + self._fake_forward = fake_forward + + def forward(self, *args, **kwargs): + return self._fake_forward(*args, **kwargs) + + +def _bridge_with_stub(fake_forward) -> MoEBridge: + bridge = MoEBridge(name="mlp") + bridge.set_original_component(_StubMoEModule(fake_forward)) + return bridge + + +class TestMoEBridgeTupleOutput: + def test_empty_tuple_raises_clear_type_error(self) -> None: + bridge = _bridge_with_stub(lambda *a, **kw: ()) + with pytest.raises(TypeError, match="torch.Tensor"): + bridge(torch.ones(1, 3, 4)) + + def test_non_tensor_metadata_preserved_without_router_hook(self) -> None: + bridge = _bridge_with_stub(lambda *a, **kw: (torch.zeros(1, 3, 4), "not_router_scores")) + fired = {"called": False} + + def mark_fired(value, hook): + fired["called"] = True + + bridge.hook_router_scores.add_hook(mark_fired) + + output = bridge(torch.ones(1, 3, 4)) + + assert isinstance(output, tuple) + assert output[1] == "not_router_scores" + torch.testing.assert_close(output[0], torch.zeros(1, 3, 4)) + assert fired["called"] is False + + def test_tensor_router_scores_still_fire_hook(self) -> None: + router_scores = torch.rand(1, 3, 8) + bridge = _bridge_with_stub(lambda *a, **kw: (torch.zeros(1, 3, 4), router_scores)) + captured = {} + + def capture_router_scores(value, hook): + captured["router_scores"] = value + return value + + bridge.hook_router_scores.add_hook(capture_router_scores) + + output = bridge(torch.ones(1, 3, 4)) + + assert isinstance(output, tuple) + torch.testing.assert_close(output[1], router_scores) + assert "router_scores" in captured + torch.testing.assert_close(captured["router_scores"], router_scores) + + def test_non_tensor_first_element_raises_clear_type_error(self) -> None: + """Exercises the output[0]-validation added to MoEBridge itself + (not just the dispatcher) -- a malformed first element should + raise a clear TypeError here rather than failing inside + HookPoint's own type checking.""" + bridge = _bridge_with_stub(lambda *a, **kw: ("not_a_tensor", torch.zeros(1, 3, 4))) + with pytest.raises(TypeError, match="torch.Tensor"): + bridge(torch.ones(1, 3, 4)) diff --git a/tests/unit/model_bridge/supported_architectures/_pretrain_mocks.py b/tests/unit/model_bridge/supported_architectures/_pretrain_mocks.py new file mode 100644 index 0000000000..28c4f94f44 --- /dev/null +++ b/tests/unit/model_bridge/supported_architectures/_pretrain_mocks.py @@ -0,0 +1,214 @@ +"""Shared mock modules and fixtures for PretrainArchitectureAdapter and +PretrainModelContainer unit tests. Not a test file itself (no `test_` +prefix, so pytest won't collect it) -- imported by both +test_pretrain_adapter.py and test_pretrain_model_container.py. + +Fully self-contained: builds tiny mock `nn.Module`s that reproduce the +relevant module tree (embed / blocks / norm1 / attn / norm2 / mlp / norm_f / +lm_head) rather than depending on any external training-framework package. +No weight loading, no network access. +""" + +from __future__ import annotations + +import torch +import torch.nn as nn + +from transformer_lens.config.transformer_bridge_config import TransformerBridgeConfig +from transformer_lens.model_bridge.supported_architectures.pretrain import ( + ARCHITECTURE_NAME, +) + +# -------------------------------------------------------------------------- +# Tiny mock module tree, matching the real target architecture's +# block-calling convention (cos/sin as extra positional args -- see +# tiny_pretrain_model.py's module docstring). Real RoPE/RMSNorm math isn't +# needed here (covered by integration parity tests); only correct +# attribute names/shapes/call-signatures for component_mapping to wire up. +# -------------------------------------------------------------------------- + + +class TinyRMSNorm(nn.Module): + def __init__(self, d_model: int): + super().__init__() + self.weight = nn.Parameter(torch.ones(d_model)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x * self.weight + + +class TinyAttention(nn.Module): + """Opaque attention stand-in: linear-in, linear-out, no real RoPE math. + Takes (and ignores) cos/sin positionally, matching the real target + architecture's block-calling convention.""" + + def __init__(self, d_model: int, n_heads: int): + super().__init__() + self.n_heads = n_heads + self.qkv = nn.Linear(d_model, 3 * d_model) + self.out = nn.Linear(d_model, d_model) + + def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + b, s, d = x.shape + qkv = self.qkv(x) + q, k, v = qkv.chunk(3, dim=-1) + attn = torch.softmax(q @ k.transpose(-2, -1) / (d**0.5), dim=-1) + return self.out(attn @ v) + + +class TinyDenseMLP(nn.Module): + def __init__(self, d_model: int, d_ff: int): + super().__init__() + self.gate = nn.Linear(d_model, d_ff, bias=False) + self.up = nn.Linear(d_model, d_ff, bias=False) + self.down = nn.Linear(d_ff, d_model, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.down(torch.nn.functional.silu(self.gate(x)) * self.up(x)) + + +class TinyMoE(nn.Module): + def __init__(self, d_model: int, d_ff: int, n_experts: int, top_k: int): + super().__init__() + self.top_k = top_k + self.router = nn.Linear(d_model, n_experts, bias=False) + self.experts = nn.ModuleList([TinyDenseMLP(d_model, d_ff) for _ in range(n_experts)]) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + weights = torch.softmax(self.router(x), dim=-1) + topk_weights, topk_idx = weights.topk(self.top_k, dim=-1) + out = torch.zeros_like(x) + for expert_idx in range(len(self.experts)): + mask = (topk_idx == expert_idx).any(dim=-1, keepdim=True) + if mask.any(): + out = out + mask * self.experts[expert_idx](x) + return out + + +class TinyBlock(nn.Module): + def __init__(self, d_model: int, n_heads: int, d_ff: int, mlp: nn.Module): + super().__init__() + self.norm1 = TinyRMSNorm(d_model) + self.attn = TinyAttention(d_model, n_heads) + self.norm2 = TinyRMSNorm(d_model) + self.mlp = mlp + + def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.norm1(x), cos, sin) + x = x + self.mlp(self.norm2(x)) + return x + + +class TinyPretrainModel(nn.Module): + """Minimal decoder-only model matching the adapter's expected paths: + embed / blocks / norm_f / lm_head. `moe_layer_indices` (possibly empty) + controls which blocks get a `TinyMoE` mlp instead of a dense one. + """ + + def __init__( + self, + d_model: int = 32, + n_heads: int = 4, + n_layers: int = 2, + d_ff: int = 64, + vocab_size: int = 256, + n_experts: int = 4, + top_k: int = 2, + moe_layer_indices: frozenset[int] = frozenset(), + tie_embeddings: bool = True, + ): + super().__init__() + self.embed = nn.Embedding(vocab_size, d_model) + self.blocks = nn.ModuleList( + [ + TinyBlock( + d_model, + n_heads, + d_ff, + mlp=( + TinyMoE(d_model, d_ff, n_experts, top_k) + if i in moe_layer_indices + else TinyDenseMLP(d_model, d_ff) + ), + ) + for i in range(n_layers) + ] + ) + self.norm_f = TinyRMSNorm(d_model) + self.lm_head = nn.Linear(d_model, vocab_size, bias=False) + if tie_embeddings: + self.lm_head.weight = self.embed.weight + self.d_head = d_model // n_heads + + def _forward_impl(self, tokens: torch.Tensor) -> dict: + b, s = tokens.shape + x = self.embed(tokens) + # Dummy cos/sin (no real RoPE math needed for these structural + # tests) but real tensors, passed positionally into every block -- + # matches the real target architecture's convention. + cos = torch.ones(s, self.d_head // 2) + sin = torch.zeros(s, self.d_head // 2) + for block in self.blocks: + x = block(x, cos, sin) + x = self.norm_f(x) + return {"logits": self.lm_head(x)} + + def forward(self, tokens: torch.Tensor, **kwargs) -> dict: + # **kwargs declared so pytest fixtures can pass through + # run_with_cache's force-injected kwargs -- the actual kwarg- + # forwarding contracts (strict vs. **kwargs-accepting source + # forwards) are exercised directly against ForwardStrict/ + # ForwardVarKwargs below instead. + return self._forward_impl(tokens) + + +class ForwardStrict(nn.Module): + """Declares no **kwargs catch-all -- exercises the "must filter" + branch of PretrainModelContainer's kwarg handling.""" + + def __init__(self): + super().__init__() + self.seen_kwargs: dict = {} + + def forward(self, tokens: torch.Tensor, targets: torch.Tensor | None = None) -> dict: + self.seen_kwargs = {"targets": targets} + return {"logits": tokens.float()} + + +class ForwardVarKwargs(nn.Module): + """Declares **kwargs -- exercises the "pass everything through" branch + of PretrainModelContainer's kwarg handling.""" + + def __init__(self): + super().__init__() + self.seen_kwargs: dict = {} + + def forward(self, tokens: torch.Tensor, **kwargs) -> dict: + self.seen_kwargs = kwargs + return {"logits": tokens.float()} + + +class MalformedMLP(nn.Module): + """Has neither a dense MLP's (gate, up, down) nor an MoE layer's + (router, experts) attributes -- used to test the adapter's failure + path.""" + + def __init__(self, d_model: int): + super().__init__() + self.some_other_layer = nn.Linear(d_model, d_model) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.some_other_layer(x) + + +def make_cfg(d_model: int = 32, n_layers: int = 2) -> TransformerBridgeConfig: + return TransformerBridgeConfig( + d_model=d_model, + d_head=d_model // 4, + n_layers=n_layers, + n_ctx=128, + n_heads=4, + d_vocab=256, + d_mlp=d_model * 2, + architecture=ARCHITECTURE_NAME, + ) diff --git a/tests/unit/model_bridge/supported_architectures/test_granite_adapter.py b/tests/unit/model_bridge/supported_architectures/test_granite_adapter.py index b5b3f848b6..2603d0f56a 100644 --- a/tests/unit/model_bridge/supported_architectures/test_granite_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_granite_adapter.py @@ -1,9 +1,12 @@ -"""Unit tests for GraniteArchitectureAdapter and GraniteMoeArchitectureAdapter. +"""Unit tests for GraniteArchitectureAdapter. Tests cover: - Component mapping structure (bridge types and HF module names) - Weight conversion key set - Config flags set by the adapter + +GraniteMoeArchitectureAdapter has its own dedicated test file, +test_granite_moe_adapter.py. """ import pytest @@ -14,7 +17,6 @@ EmbeddingBridge, GatedMLPBridge, LinearBridge, - MoEBridge, PositionEmbeddingsAttentionBridge, RMSNormalizationBridge, RotaryEmbeddingBridge, @@ -23,9 +25,6 @@ from transformer_lens.model_bridge.supported_architectures.granite import ( GraniteArchitectureAdapter, ) -from transformer_lens.model_bridge.supported_architectures.granite_moe import ( - GraniteMoeArchitectureAdapter, -) # --------------------------------------------------------------------------- # Helpers / fixtures @@ -74,16 +73,6 @@ def adapter(cfg: TransformerBridgeConfig) -> GraniteArchitectureAdapter: return GraniteArchitectureAdapter(cfg) -@pytest.fixture -def moe_cfg() -> TransformerBridgeConfig: - return _make_cfg() - - -@pytest.fixture -def moe_adapter(moe_cfg: TransformerBridgeConfig) -> GraniteMoeArchitectureAdapter: - return GraniteMoeArchitectureAdapter(moe_cfg) - - # --------------------------------------------------------------------------- # Config flag tests # --------------------------------------------------------------------------- @@ -212,44 +201,3 @@ def test_exact_conversion_key_set(self, adapter: GraniteArchitectureAdapter) -> "blocks.{i}.attn.v.weight", "blocks.{i}.attn.o.weight", } - - -# --------------------------------------------------------------------------- -# GraniteMoe component mapping tests -# --------------------------------------------------------------------------- - - -class TestGraniteMoeAdapterComponentMapping: - """GraniteMoe replaces dense MLP with MoE; everything else is identical to Granite.""" - - def test_top_level_keys(self, moe_adapter: GraniteMoeArchitectureAdapter) -> None: - assert set(moe_adapter.component_mapping.keys()) == { - "embed", - "rotary_emb", - "blocks", - "ln_final", - "unembed", - } - - def test_mlp_is_moe_bridge(self, moe_adapter: GraniteMoeArchitectureAdapter) -> None: - mlp = moe_adapter.component_mapping["blocks"].submodules["mlp"] - assert isinstance(mlp, MoEBridge) - - def test_moe_hf_path(self, moe_adapter: GraniteMoeArchitectureAdapter) -> None: - mlp = moe_adapter.component_mapping["blocks"].submodules["mlp"] - assert mlp.name == "block_sparse_moe" - - def test_non_mlp_components_match_dense( - self, - adapter: GraniteArchitectureAdapter, - moe_adapter: GraniteMoeArchitectureAdapter, - ) -> None: - """Embed, rotary_emb, ln_final, unembed, and attention are shared with dense Granite.""" - for key in ("embed", "rotary_emb", "ln_final", "unembed"): - assert type(moe_adapter.component_mapping[key]) is type(adapter.component_mapping[key]) - assert moe_adapter.component_mapping[key].name == adapter.component_mapping[key].name - - def test_attention_unchanged_in_moe(self, moe_adapter: GraniteMoeArchitectureAdapter) -> None: - attn = moe_adapter.component_mapping["blocks"].submodules["attn"] - assert isinstance(attn, PositionEmbeddingsAttentionBridge) - assert set(attn.submodules.keys()) == {"q", "k", "v", "o"} diff --git a/tests/unit/model_bridge/supported_architectures/test_granite_moe_adapter.py b/tests/unit/model_bridge/supported_architectures/test_granite_moe_adapter.py new file mode 100644 index 0000000000..0f96d26f83 --- /dev/null +++ b/tests/unit/model_bridge/supported_architectures/test_granite_moe_adapter.py @@ -0,0 +1,212 @@ +"""Unit tests for GraniteMoeArchitectureAdapter. + +GraniteMoe is dense Granite with the gated MLP replaced by a Sparse Mixture +of Experts block (``block_sparse_moe``); everything else (embed, rotary +embeddings, attention, final norm, unembed, and the common Granite config +flags) is inherited unchanged from ``GraniteArchitectureAdapter``. +""" + +import pytest + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge.generalized_components import ( + BlockBridge, + EmbeddingBridge, + LinearBridge, + MoEBridge, + PositionEmbeddingsAttentionBridge, + RMSNormalizationBridge, + RotaryEmbeddingBridge, + UnembeddingBridge, +) +from transformer_lens.model_bridge.supported_architectures.granite import ( + GraniteArchitectureAdapter, +) +from transformer_lens.model_bridge.supported_architectures.granite_moe import ( + GraniteMoeArchitectureAdapter, +) + +# --------------------------------------------------------------------------- +# Helpers / fixtures +# --------------------------------------------------------------------------- + +N_HEADS = 8 +N_KV_HEADS = 2 +D_MODEL = 64 +D_MLP = 256 +N_LAYERS = 2 +N_CTX = 256 +D_VOCAB = 1000 + + +def _make_dense_cfg() -> TransformerBridgeConfig: + """Same shape as the MoE config, but for dense Granite (comparison baseline).""" + return TransformerBridgeConfig( + d_model=D_MODEL, + d_head=D_MODEL // N_HEADS, + n_layers=N_LAYERS, + n_ctx=N_CTX, + n_heads=N_HEADS, + d_vocab=D_VOCAB, + d_mlp=D_MLP, + n_key_value_heads=N_KV_HEADS, + default_prepend_bos=False, + architecture="GraniteForCausalLM", + ) + + +def _make_moe_cfg() -> TransformerBridgeConfig: + """Return a minimal synthetic config for GraniteMoe adapter tests.""" + return TransformerBridgeConfig( + d_model=D_MODEL, + d_head=D_MODEL // N_HEADS, + n_layers=N_LAYERS, + n_ctx=N_CTX, + n_heads=N_HEADS, + d_vocab=D_VOCAB, + d_mlp=D_MLP, + n_key_value_heads=N_KV_HEADS, + num_experts=4, + experts_per_token=2, + default_prepend_bos=False, + architecture="GraniteMoeForCausalLM", + ) + + +@pytest.fixture +def dense_adapter() -> GraniteArchitectureAdapter: + return GraniteArchitectureAdapter(_make_dense_cfg()) + + +@pytest.fixture +def cfg() -> TransformerBridgeConfig: + return _make_moe_cfg() + + +@pytest.fixture +def adapter(cfg: TransformerBridgeConfig) -> GraniteMoeArchitectureAdapter: + return GraniteMoeArchitectureAdapter(cfg) + + +# --------------------------------------------------------------------------- +# Config flag tests +# --------------------------------------------------------------------------- + + +class TestGraniteMoeAdapterConfig: + """GraniteMoe doesn't override config setup, so it should inherit the same + flags as dense Granite -- checked explicitly here rather than only + implied by inheritance.""" + + def test_normalization_type(self, adapter: GraniteMoeArchitectureAdapter) -> None: + assert adapter.cfg.normalization_type == "RMS" + + def test_positional_embedding_type(self, adapter: GraniteMoeArchitectureAdapter) -> None: + assert adapter.cfg.positional_embedding_type == "rotary" + + def test_final_rms(self, adapter: GraniteMoeArchitectureAdapter) -> None: + assert adapter.cfg.final_rms is True + + def test_gated_mlp(self, adapter: GraniteMoeArchitectureAdapter) -> None: + assert adapter.cfg.gated_mlp is True + + def test_attn_only_false(self, adapter: GraniteMoeArchitectureAdapter) -> None: + assert adapter.cfg.attn_only is False + + def test_uses_rms_norm(self, adapter: GraniteMoeArchitectureAdapter) -> None: + assert adapter.cfg.uses_rms_norm is True + + def test_default_prepend_bos_false(self, adapter: GraniteMoeArchitectureAdapter) -> None: + assert adapter.cfg.default_prepend_bos is False + + def test_n_key_value_heads_propagated(self, adapter: GraniteMoeArchitectureAdapter) -> None: + assert adapter.cfg.n_key_value_heads == N_KV_HEADS + + +# --------------------------------------------------------------------------- +# Component mapping tests +# --------------------------------------------------------------------------- + + +class TestGraniteMoeAdapterComponentMapping: + """GraniteMoe replaces the dense MLP with MoE; everything else is + identical to dense Granite.""" + + def test_top_level_keys(self, adapter: GraniteMoeArchitectureAdapter) -> None: + assert set(adapter.component_mapping.keys()) == { + "embed", + "rotary_emb", + "blocks", + "ln_final", + "unembed", + } + + def test_top_level_bridge_types(self, adapter: GraniteMoeArchitectureAdapter) -> None: + mapping = adapter.component_mapping + assert isinstance(mapping["embed"], EmbeddingBridge) + assert isinstance(mapping["rotary_emb"], RotaryEmbeddingBridge) + assert isinstance(mapping["blocks"], BlockBridge) + assert isinstance(mapping["ln_final"], RMSNormalizationBridge) + assert isinstance(mapping["unembed"], UnembeddingBridge) + + def test_top_level_hf_paths(self, adapter: GraniteMoeArchitectureAdapter) -> None: + mapping = adapter.component_mapping + assert mapping["embed"].name == "model.embed_tokens" + assert mapping["rotary_emb"].name == "model.rotary_emb" + assert mapping["blocks"].name == "model.layers" + assert mapping["ln_final"].name == "model.norm" + assert mapping["unembed"].name == "lm_head" + + def test_block_submodule_keys(self, adapter: GraniteMoeArchitectureAdapter) -> None: + blocks = adapter.component_mapping["blocks"] + assert set(blocks.submodules.keys()) == {"ln1", "ln2", "attn", "mlp"} + + def test_mlp_is_moe_bridge(self, adapter: GraniteMoeArchitectureAdapter) -> None: + mlp = adapter.component_mapping["blocks"].submodules["mlp"] + assert isinstance(mlp, MoEBridge) + + def test_moe_hf_path(self, adapter: GraniteMoeArchitectureAdapter) -> None: + mlp = adapter.component_mapping["blocks"].submodules["mlp"] + assert mlp.name == "block_sparse_moe" + + def test_attention_unchanged_from_dense(self, adapter: GraniteMoeArchitectureAdapter) -> None: + attn = adapter.component_mapping["blocks"].submodules["attn"] + assert isinstance(attn, PositionEmbeddingsAttentionBridge) + assert set(attn.submodules.keys()) == {"q", "k", "v", "o"} + assert attn.submodules["q"].name == "q_proj" + assert attn.submodules["k"].name == "k_proj" + assert attn.submodules["v"].name == "v_proj" + assert attn.submodules["o"].name == "o_proj" + for submodule in attn.submodules.values(): + assert isinstance(submodule, LinearBridge) + + def test_non_mlp_components_match_dense( + self, + dense_adapter: GraniteArchitectureAdapter, + adapter: GraniteMoeArchitectureAdapter, + ) -> None: + """Embed, rotary_emb, ln_final, and unembed are shared with dense Granite.""" + for key in ("embed", "rotary_emb", "ln_final", "unembed"): + assert type(adapter.component_mapping[key]) is type( + dense_adapter.component_mapping[key] + ) + assert adapter.component_mapping[key].name == dense_adapter.component_mapping[key].name + + +# --------------------------------------------------------------------------- +# Weight conversion key tests +# --------------------------------------------------------------------------- + + +class TestGraniteMoeAdapterWeightConversions: + """GraniteMoe doesn't override weight_processing_conversions, so it should + carry the same qkvo conversion set as dense Granite (there is no + MoE-specific weight remapping).""" + + def test_exact_conversion_key_set(self, adapter: GraniteMoeArchitectureAdapter) -> None: + assert set(adapter.weight_processing_conversions.keys()) == { + "blocks.{i}.attn.q.weight", + "blocks.{i}.attn.k.weight", + "blocks.{i}.attn.v.weight", + "blocks.{i}.attn.o.weight", + } diff --git a/tests/unit/model_bridge/supported_architectures/test_hook_orchestration_parity.py b/tests/unit/model_bridge/supported_architectures/test_hook_orchestration_parity.py new file mode 100644 index 0000000000..de3aa2c809 --- /dev/null +++ b/tests/unit/model_bridge/supported_architectures/test_hook_orchestration_parity.py @@ -0,0 +1,180 @@ +"""HookedRootModule-parity for BridgeCore hook orchestration. + +`reset_hooks` direction/permanence/level/clear-contexts selectivity, the +`mod_dict` accessor, and the `check_hooks_to_add` extension point -- the +surface migrated hook code relies on when moving off HookedTransformer. + +Uses the network-free pretrain mock bridge (real block tree, HT-style +hook aliases like `blocks.0.hook_mlp_out`) rather than a thin stub, so the +"backward hook on a non-io hook point" case -- which the old registry walk +missed -- is actually exercised. +""" +from __future__ import annotations + +import pytest +import torch + +from transformer_lens.model_bridge.supported_architectures.pretrain import ( + build_pretrain_bridge, +) + +from ._pretrain_mocks import TinyPretrainModel, make_cfg + + +def _build(): + cfg = make_cfg(n_layers=2) + model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=2, d_ff=32, vocab_size=64) + return build_pretrain_bridge(model, cfg) + + +def _noop(value, hook=None): + return value + + +class TestResetHooks: + def test_removes_forward_and_backward_from_every_point(self): + """Default direction='both' clears bwd hooks even on non-io points + (block-level hook_mlp_in etc.), which the old fwd-only walk left behind.""" + bridge = _build() + for hp in bridge.hook_points(): + hp.add_hook(_noop, dir="fwd") + hp.add_hook(_noop, dir="bwd") + + bridge.reset_hooks() + + leftover = [ + hp.name + for hp in bridge.hook_points() + if hp.has_hooks(dir="both", including_permanent=False) + ] + assert leftover == [] + + def test_direction_is_selective(self): + bridge = _build() + hp = bridge._hook_registry["blocks.0.hook_out"] + hp.add_hook(_noop, dir="fwd") + hp.add_hook(_noop, dir="bwd") + + bridge.reset_hooks(direction="fwd") + assert not hp.has_hooks(dir="fwd") + assert hp.has_hooks(dir="bwd") + + bridge.reset_hooks(direction="bwd") + assert not hp.has_hooks(dir="bwd") + + def test_permanent_hooks_survive_default_reset(self): + bridge = _build() + hp = bridge._hook_registry["blocks.0.hook_out"] + bridge.add_perma_hook("blocks.0.hook_out", _noop) + + bridge.reset_hooks() + + assert hp.has_hooks(dir="fwd", including_permanent=True) + + def test_including_permanent_removes_permanent_hooks(self): + bridge = _build() + hp = bridge._hook_registry["blocks.0.hook_out"] + bridge.add_perma_hook("blocks.0.hook_out", _noop) + + bridge.reset_hooks(including_permanent=True) + + assert not hp.has_hooks(dir="fwd", including_permanent=True) + + def test_level_is_selective(self): + bridge = _build() + hp = bridge._hook_registry["blocks.0.hook_out"] + hp.add_hook(_noop, dir="fwd", level=0) + hp.add_hook(_noop, dir="fwd", level=1) + + bridge.reset_hooks(level=0) + + assert not hp.has_hooks(dir="fwd", level=0) + assert hp.has_hooks(dir="fwd", level=1) + + def test_clear_contexts_flag(self): + bridge = _build() + hp = bridge._hook_registry["blocks.0.hook_out"] + + hp.ctx["scratch"] = 1 + bridge.reset_hooks(clear_contexts=False) + assert hp.ctx == {"scratch": 1} + + bridge.reset_hooks(clear_contexts=True) + assert hp.ctx == {} + + +class TestModDict: + def test_canonical_and_alias_names_resolve_to_same_hook_point(self): + bridge = _build() + mod_dict = bridge.mod_dict + assert mod_dict["blocks.0.hook_mlp_out"] is mod_dict["blocks.0.mlp.hook_out"] + + def test_includes_non_hook_modules(self): + """mod_dict is a superset of hook_dict -- it also exposes container + modules (blocks, block 0) that hook_dict omits.""" + bridge = _build() + mod_dict = bridge.mod_dict + assert "blocks" in mod_dict + assert "blocks.0" in mod_dict + for name, hook_point in bridge.hook_dict.items(): + assert mod_dict[name] is hook_point + + +class TestCheckHooksToAdd: + def test_invoked_before_add(self, monkeypatch): + bridge = _build() + seen: list[str] = [] + + def recording( + self, hook_point, hook_point_name, hook, dir="fwd", is_permanent=False, prepend=False + ): + seen.append(hook_point_name) + + monkeypatch.setattr(type(bridge), "check_hooks_to_add", recording) + bridge.add_hook("blocks.0.hook_out", _noop) + assert seen == ["blocks.0.hook_out"] + + def test_can_veto_hook_addition(self, monkeypatch): + """A raising override blocks the add -- the hook must not land.""" + bridge = _build() + hp = bridge._hook_registry["blocks.0.hook_out"] + + def rejecting( + self, hook_point, hook_point_name, hook, dir="fwd", is_permanent=False, prepend=False + ): + raise AssertionError(f"hook {hook_point_name} rejected") + + monkeypatch.setattr(type(bridge), "check_hooks_to_add", rejecting) + with pytest.raises(AssertionError, match="rejected"): + bridge.add_hook("blocks.0.hook_out", _noop) + assert not hp.has_hooks(dir="fwd") + + def test_invoked_by_run_with_hooks(self, monkeypatch): + """The extension point must fire for run_with_hooks too, not just + add_hook -- run_with_hooks is the common hooking path (matches HT).""" + bridge = _build() + seen: list[str] = [] + + def recording( + self, hook_point, hook_point_name, hook, dir="fwd", is_permanent=False, prepend=False + ): + seen.append(hook_point_name) + + monkeypatch.setattr(type(bridge), "check_hooks_to_add", recording) + tokens = torch.randint(0, 64, (1, 4)) + bridge.run_with_hooks(tokens, fwd_hooks=[("blocks.0.hook_out", _noop)]) + assert "blocks.0.hook_out" in seen + + def test_invoked_by_hooks_context_manager(self, monkeypatch): + bridge = _build() + seen: list[str] = [] + + def recording( + self, hook_point, hook_point_name, hook, dir="fwd", is_permanent=False, prepend=False + ): + seen.append(hook_point_name) + + monkeypatch.setattr(type(bridge), "check_hooks_to_add", recording) + with bridge.hooks(fwd_hooks=[("blocks.0.hook_out", _noop)]): + pass + assert "blocks.0.hook_out" in seen diff --git a/tests/unit/model_bridge/supported_architectures/test_pretrain_adapter.py b/tests/unit/model_bridge/supported_architectures/test_pretrain_adapter.py new file mode 100644 index 0000000000..d2fa03f24e --- /dev/null +++ b/tests/unit/model_bridge/supported_architectures/test_pretrain_adapter.py @@ -0,0 +1,453 @@ +"""Unit tests for PretrainArchitectureAdapter and DenseOrMoEFeedForwardBridge +-- component mapping, structural dispatch, and bridge-construction-level +behavior. Container, builder, and wrapped-model lifecycle behavior (kwarg +filtering, output-contract normalization, hook registration for the hidden +MLP delegate, train/eval, dtype/device) lives in +test_pretrain_model_container.py. + +Fully self-contained (see _pretrain_mocks.py): tiny mock `nn.Module`s, no +external package dependency, no network access. +""" + +from __future__ import annotations + +import pytest +import torch + +from transformer_lens.model_bridge.generalized_components import ( + DelegatedAttentionBlockBridge, +) +from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_from_module, +) +from transformer_lens.model_bridge.supported_architectures.pretrain import ( + ARCHITECTURE_NAME, + DenseOrMoEFeedForwardBridge, + NativeForwardAttentionBridge, + PretrainArchitectureAdapter, + PretrainModelContainer, + build_pretrain_bridge, +) + +from ._pretrain_mocks import ( + MalformedMLP, + TinyDenseMLP, + TinyMoE, + TinyPretrainModel, + make_cfg, +) + +_make_cfg = make_cfg # local alias, matches call sites below + + +class TestPretrainAdapterConstruction: + def test_adapter_sets_required_config_flags(self) -> None: + adapter = PretrainArchitectureAdapter(_make_cfg()) + assert adapter.cfg.normalization_type == "RMS" + assert adapter.cfg.positional_embedding_type == "rotary" + assert adapter.cfg.final_rms is True + assert adapter.cfg.gated_mlp is True + assert adapter.cfg.attn_only is False + + def test_adapter_mutates_the_passed_in_cfg_object_in_place(self) -> None: + """Documented, intentional behavior (matches nanogpt.py's adapter + convention) -- not an accidental side effect. This test guards the + specific claim: the object passed in is the object mutated, not a + copy.""" + cfg = _make_cfg() + adapter = PretrainArchitectureAdapter(cfg) + assert adapter.cfg is cfg + assert cfg.normalization_type == "RMS" + + def test_component_mapping_has_expected_top_level_keys(self) -> None: + adapter = PretrainArchitectureAdapter(_make_cfg()) + mapping = adapter.get_component_mapping() + assert set(mapping.keys()) == {"embed", "blocks", "ln_final", "unembed"} + + def test_mlp_mapping_always_uses_the_dispatcher(self) -> None: + """Unconditional now -- no cfg.num_experts branch to depend on.""" + adapter = PretrainArchitectureAdapter(_make_cfg()) + mlp_component = adapter.component_mapping["blocks"].submodules["mlp"] + assert isinstance(mlp_component, DenseOrMoEFeedForwardBridge) + + +class TestNativeForwardAttentionBridge: + def test_opaque_attention_bridge_does_not_advertise_per_head_aliases(self) -> None: + """The opaque attention wrap has no Q/K/V/O submodules, so it must + not advertise AttentionBridge's class-level hook_aliases/ + property_aliases (which assume those submodules exist) or the + split-QKV-fork machinery -- see NativeForwardAttentionBridge's + docstring.""" + adapter = PretrainArchitectureAdapter(_make_cfg()) + attn = adapter.component_mapping["blocks"].submodules["attn"] + + assert isinstance(attn, NativeForwardAttentionBridge) + assert attn.hook_aliases == {} + assert attn.property_aliases == {} + assert attn.supports_split_qkv_fork is False + + # Also check the actual constructed runtime bridge, not just the + # adapter's own component_mapping -- proves bridge construction + # didn't reintroduce the aliases along the way. + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel(d_model=32, n_heads=4, n_layers=1, d_ff=64, vocab_size=256) + bridge = build_pretrain_bridge(model, cfg) + assert bridge.blocks[0].attn.hook_aliases == {} + + def test_block_level_attention_output_still_reaches_hook_out(self) -> None: + """Clearing the per-head aliases must not also silence the block's + own attn.hook_out -- that's the actual hook point this adapter's + block-level hook coverage relies on.""" + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel(d_model=32, n_heads=4, n_layers=1, d_ff=64, vocab_size=256) + bridge = build_pretrain_bridge(model, cfg) + tokens = torch.randint(0, 256, (1, 5)) + _, cache = bridge.run_with_cache(tokens) + assert "blocks.0.attn.hook_out" in cache + + def test_blocks_use_delegated_attention_block_bridge(self) -> None: + """The adapter reuses DelegatedAttentionBlockBridge rather than + plain BlockBridge, since NativeForwardAttentionBridge's + supports_split_qkv_fork = False means the block-level + hook_attn_in/hook_q_input/hook_k_input/hook_v_input aliases would + otherwise dangle (point at HookPoints that are never created).""" + adapter = PretrainArchitectureAdapter(_make_cfg()) + assert isinstance(adapter.component_mapping["blocks"], DelegatedAttentionBlockBridge) + + def test_split_qkv_hook_names_are_absent_but_attention_output_remains_available( + self, + ) -> None: + """hook_attn_in/hook_q_input/hook_k_input/hook_v_input must not + appear as resolvable hook names anywhere in the built bridge -- + DelegatedAttentionBlockBridge strips the block-level aliases, and + NativeForwardAttentionBridge never creates the underlying + attention-level HookPoints in the first place. hook_attn_out is + untouched by either change and must still resolve, both as the + block-level alias and as the attention component's own hook_out.""" + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel(d_model=32, n_heads=4, n_layers=1, d_ff=64, vocab_size=256) + bridge = build_pretrain_bridge(model, cfg) + hook_dict = bridge.hook_dict + for dangling in ( + "blocks.0.hook_attn_in", + "blocks.0.hook_q_input", + "blocks.0.hook_k_input", + "blocks.0.hook_v_input", + "blocks.0.attn.hook_attn_in", + "blocks.0.attn.hook_q_input", + "blocks.0.attn.hook_k_input", + "blocks.0.attn.hook_v_input", + ): + assert dangling not in hook_dict, f"{dangling} should not resolve" + assert "blocks.0.hook_attn_out" in hook_dict + assert "blocks.0.attn.hook_out" in hook_dict + + +class TestDenseOrMoEFeedForwardBridgeDispatch: + def test_dispatches_to_moe_delegate_for_moe_module(self) -> None: + cfg = _make_cfg() + bridge = DenseOrMoEFeedForwardBridge(name="mlp", config=cfg) + moe_module = TinyMoE(d_model=32, d_ff=64, n_experts=4, top_k=2) + bridge.set_original_component(moe_module) + + from transformer_lens.model_bridge.generalized_components import MoEBridge + + assert isinstance(bridge._delegate, MoEBridge) + + def test_dispatches_to_dense_delegate_for_dense_module(self) -> None: + cfg = _make_cfg() + bridge = DenseOrMoEFeedForwardBridge(name="mlp", config=cfg) + dense_module = TinyDenseMLP(d_model=32, d_ff=64) + bridge.set_original_component(dense_module) + + from transformer_lens.model_bridge.generalized_components import GatedMLPBridge + + assert isinstance(bridge._delegate, GatedMLPBridge) + + def test_raises_clear_error_for_malformed_module(self) -> None: + cfg = _make_cfg() + bridge = DenseOrMoEFeedForwardBridge(name="mlp", config=cfg) + malformed = MalformedMLP(d_model=32) + + with pytest.raises(ValueError, match="doesn't know how to wrap it"): + bridge.set_original_component(malformed) + + +class TestDenseOrMoEFeedForwardBridgeDispatchValidatesTypes: + """hasattr alone would let a module win a branch by attribute-name + coincidence even if the attributes are the wrong type. These tests + exercise the basic type checks added to set_original_component so + such a mismatch is caught here, with a clear message naming the + actual field and type, rather than failing later inside MoEBridge/ + GatedMLPBridge or during forward.""" + + def test_router_of_wrong_type_raises_type_error(self) -> None: + class BadRouterMoE(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.router = "not_a_module" # right name, wrong type + self.experts = torch.nn.ModuleList( + [TinyDenseMLP(d_model=32, d_ff=64) for _ in range(4)] + ) + + cfg = _make_cfg() + bridge = DenseOrMoEFeedForwardBridge(name="mlp", config=cfg) + with pytest.raises(TypeError, match="router"): + bridge.set_original_component(BadRouterMoE()) + + def test_experts_of_wrong_type_raises_type_error(self) -> None: + class BadExpertsMoE(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.router = torch.nn.Linear(32, 4) + self.experts = "not_a_module_collection" # right name, wrong type + + cfg = _make_cfg() + bridge = DenseOrMoEFeedForwardBridge(name="mlp", config=cfg) + with pytest.raises(TypeError, match="experts"): + bridge.set_original_component(BadExpertsMoE()) + + def test_plain_list_of_experts_is_rejected(self) -> None: + """Modules held in an ordinary Python list are not registered as + children -- their parameters would silently drop out of + parameters()/state_dict()/.to(...)/train()/eval(), contradicting + this adapter's lifecycle guarantees. Rejected even though every + element is itself a valid nn.Module.""" + + class ListExpertsMoE(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.router = torch.nn.Linear(32, 4) + self.experts = [TinyDenseMLP(d_model=32, d_ff=64) for _ in range(4)] + + cfg = _make_cfg() + bridge = DenseOrMoEFeedForwardBridge(name="mlp", config=cfg) + with pytest.raises(TypeError, match="registered module collection"): + bridge.set_original_component(ListExpertsMoE()) + + def test_gate_of_wrong_type_raises_type_error(self) -> None: + class BadGateDense(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.gate = "not_a_module" # right name, wrong type + self.up = torch.nn.Linear(32, 64, bias=False) + self.down = torch.nn.Linear(64, 32, bias=False) + + cfg = _make_cfg() + bridge = DenseOrMoEFeedForwardBridge(name="mlp", config=cfg) + with pytest.raises(TypeError, match="gate"): + bridge.set_original_component(BadGateDense()) + + +class TestDenseOrMoEFeedForwardBridgeTupleOutput: + """Exercises the dispatcher's own tuple-output handling directly, by + patching an already-wired delegate's forward -- independent of + whether GatedMLPBridge/MoEBridge happen to return a tuple today. The + dispatcher explicitly supports and validates this shape, so it's + tested as its own contract rather than assumed from what the real + delegates currently do.""" + + def _dispatcher_with_patched_delegate(self, fake_forward): + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel( + d_model=16, + n_heads=2, + n_layers=1, + d_ff=32, + vocab_size=64, + n_experts=4, + top_k=2, + moe_layer_indices=frozenset({0}), + ) + bridge = build_pretrain_bridge(model, cfg) + dispatcher = bridge.blocks[0].mlp + dispatcher._delegate.forward = fake_forward + return dispatcher + + def test_tensor_first_tuple_passes_through_with_hook_applied(self) -> None: + dispatcher = self._dispatcher_with_patched_delegate( + lambda *a, **kw: (torch.zeros(1, 3, 16), "aux") + ) + output = dispatcher(torch.ones(1, 3, 16)) + assert isinstance(output, tuple) + assert output[1] == "aux" + torch.testing.assert_close(output[0], torch.zeros(1, 3, 16)) + + def test_empty_tuple_raises_clear_type_error(self) -> None: + dispatcher = self._dispatcher_with_patched_delegate(lambda *a, **kw: ()) + with pytest.raises(TypeError, match="torch.Tensor"): + dispatcher(torch.ones(1, 3, 16)) + + def test_non_tensor_first_element_raises_clear_type_error(self) -> None: + dispatcher = self._dispatcher_with_patched_delegate( + lambda *a, **kw: ("not_a_tensor", torch.zeros(1)) + ) + with pytest.raises(TypeError, match="torch.Tensor"): + dispatcher(torch.ones(1, 3, 16)) + + +class TestPretrainAdapterBridgeConstruction: + @pytest.fixture + def dense_bridge(self): + cfg = _make_cfg(n_layers=2) + model = TinyPretrainModel(d_model=32, n_heads=4, n_layers=2, d_ff=64, vocab_size=256) + return build_pretrain_bridge(model, cfg) + + @pytest.fixture + def moe_bridge(self): + cfg = _make_cfg(n_layers=2) + model = TinyPretrainModel( + d_model=32, + n_heads=4, + n_layers=2, + d_ff=64, + vocab_size=256, + n_experts=4, + top_k=2, + moe_layer_indices=frozenset({1}), + ) + return build_pretrain_bridge(model, cfg) + + def test_bridge_has_correct_block_count(self, dense_bridge) -> None: + assert len(dense_bridge.blocks) == 2 + + def test_bridge_has_embed_unembed_and_final_norm(self, dense_bridge) -> None: + assert hasattr(dense_bridge, "embed") + assert hasattr(dense_bridge, "unembed") + assert hasattr(dense_bridge, "ln_final") + + def test_forward_returns_logits_of_expected_shape(self, dense_bridge) -> None: + tokens = torch.randint(0, 256, (1, 5)) + with torch.no_grad(): + output = dense_bridge(tokens) + assert output.shape == (1, 5, 256) + + def test_run_with_cache_exposes_resid_hooks(self, dense_bridge) -> None: + tokens = torch.randint(0, 256, (1, 5)) + _, cache = dense_bridge.run_with_cache(tokens) + assert any("resid_pre" in k for k in cache.keys()) + assert any("resid_post" in k for k in cache.keys()) + + def test_no_per_head_attention_hooks(self, dense_bridge) -> None: + """This adapter deliberately does not expose hook_q/hook_k/hook_v -- + see PretrainArchitectureAdapter's class docstring.""" + tokens = torch.randint(0, 256, (1, 5)) + _, cache = dense_bridge.run_with_cache(tokens) + assert not any("hook_q" in k for k in cache.keys()) + assert not any("hook_pattern" in k for k in cache.keys()) + + def test_moe_bridge_forward_runs(self, moe_bridge) -> None: + tokens = torch.randint(0, 256, (1, 5)) + with torch.no_grad(): + output = moe_bridge(tokens) + assert output.shape == (1, 5, 256) + assert not torch.isnan(output).any() + + def test_moe_block_uses_dispatcher(self, moe_bridge) -> None: + assert isinstance(moe_bridge.blocks[1].mlp, DenseOrMoEFeedForwardBridge) + + def test_moe_layer_hooks_are_registered_at_the_dispatcher_path(self, moe_bridge) -> None: + """Verifies actual cache keys, not just isinstance -- the claim in + DenseOrMoEFeedForwardBridge's docstring that hooks fire at + blocks.{i}.mlp.hook_in/hook_out regardless of dense-vs-MoE.""" + tokens = torch.randint(0, 256, (1, 5)) + _, cache = moe_bridge.run_with_cache(tokens) + assert "blocks.1.mlp.hook_in" in cache + assert "blocks.1.mlp.hook_out" in cache + # Layer 0 is dense in this fixture -- same path convention applies. + assert "blocks.0.mlp.hook_in" in cache + assert "blocks.0.mlp.hook_out" in cache + + +class TestMoEConfigFieldIndependence: + def test_bridge_behaves_identically_with_moe_config_fields_populated(self) -> None: + """MoEBridge itself (transformer_lens/model_bridge/generalized_ + components/moe.py) never reads cfg.num_experts or + cfg.experts_per_token. Populating them with realistic values must + not change behavior. + Uses an actual MoE layer (moe_layer_indices={0}) -- both models + being dense would let this pass without ever constructing + MoEBridge, so the dispatch type is asserted directly too.""" + from transformer_lens.model_bridge.generalized_components import MoEBridge + + torch.manual_seed(0) + model_a = TinyPretrainModel( + d_model=16, + n_heads=2, + n_layers=1, + d_ff=32, + vocab_size=64, + n_experts=4, + top_k=2, + moe_layer_indices=frozenset({0}), + ) + cfg_without = _make_cfg(n_layers=1) + bridge_without = build_pretrain_bridge(model_a, cfg_without) + + torch.manual_seed(0) + model_b = TinyPretrainModel( + d_model=16, + n_heads=2, + n_layers=1, + d_ff=32, + vocab_size=64, + n_experts=4, + top_k=2, + moe_layer_indices=frozenset({0}), + ) + cfg_with = _make_cfg(n_layers=1) + cfg_with.num_experts = 4 + cfg_with.experts_per_token = 2 + bridge_with = build_pretrain_bridge(model_b, cfg_with) + + assert isinstance(bridge_without.blocks[0].mlp._delegate, MoEBridge) + assert isinstance(bridge_with.blocks[0].mlp._delegate, MoEBridge) + + tokens = torch.randint(0, 64, (1, 3)) + with torch.no_grad(): + out_without = bridge_without(tokens) + out_with = bridge_with(tokens) + torch.testing.assert_close(out_without, out_with, atol=0, rtol=0) + + +class TestPretrainAdapterTiedEmbeddings: + def test_tied_embedding_bridge_construction_succeeds(self) -> None: + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel( + d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64, tie_embeddings=True + ) + bridge = build_pretrain_bridge(model, cfg) + tokens = torch.randint(0, 64, (1, 3)) + with torch.no_grad(): + output = bridge(tokens) + assert output.shape == (1, 3, 64) + + def test_untied_embedding_bridge_construction_succeeds(self) -> None: + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel( + d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64, tie_embeddings=False + ) + bridge = build_pretrain_bridge(model, cfg) + tokens = torch.randint(0, 64, (1, 3)) + with torch.no_grad(): + output = bridge(tokens) + assert output.shape == (1, 3, 64) + + +class TestBuildBridgeFromModuleDirectlyStillWorks: + """Advanced/internal path -- build_pretrain_bridge is the intended + public entry point, but direct build_bridge_from_module use (with the + container applied manually) must keep working too.""" + + def test_direct_build_bridge_from_module_with_manual_container(self) -> None: + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64) + bridge = build_bridge_from_module( + PretrainModelContainer(model), + architecture=ARCHITECTURE_NAME, + tl_config=cfg, + ) + tokens = torch.randint(0, 64, (1, 3)) + with torch.no_grad(): + output = bridge(tokens) + assert output.shape == (1, 3, 64) diff --git a/tests/unit/model_bridge/supported_architectures/test_pretrain_model_container.py b/tests/unit/model_bridge/supported_architectures/test_pretrain_model_container.py new file mode 100644 index 0000000000..43c40901ea --- /dev/null +++ b/tests/unit/model_bridge/supported_architectures/test_pretrain_model_container.py @@ -0,0 +1,537 @@ +"""Container, builder, and wrapped-model lifecycle behavior for +PretrainModelContainer and build_pretrain_bridge -- kwarg filtering, +output-contract normalization, train/eval mode propagation, and +dtype/tied-weight preservation. Hook registration tests for the hidden +MLP delegate +(implemented by DenseOrMoEFeedForwardBridge) live here too, since what +they verify -- real parameters staying reachable through +`original_model.inner` -- is a lifecycle concern, not a mapping one. +Adapter-mapping-level behavior lives in test_pretrain_adapter.py. + +Fully self-contained (see _pretrain_mocks.py): tiny mock `nn.Module`s, no +external package dependency, no network access. +""" + +from __future__ import annotations + +import pytest +import torch +import torch.nn as nn + +from transformer_lens.model_bridge.supported_architectures.pretrain import ( + PretrainModelContainer, + _LogitsAttrDict, + build_pretrain_bridge, +) + +from ._pretrain_mocks import ( + ForwardStrict, + ForwardVarKwargs, + TinyPretrainModel, + make_cfg, +) + +_make_cfg = make_cfg # local alias, matches call sites below + + +class TestPretrainModelContainerKwargFiltering: + def test_output_attentions_is_stripped_for_strict_signature(self) -> None: + """The one TransformerBridge-injected kwarg (run_with_cache + forces output_attentions=True) must not reach a source forward with + no **kwargs catch-all.""" + model = ForwardStrict() + container = PretrainModelContainer(model) + container(torch.tensor([[1, 2, 3]]), targets=None, output_attentions=True) + assert model.seen_kwargs == {"targets": None} + + def test_output_attentions_is_stripped_even_for_var_kwargs_signature(self) -> None: + """Stripped unconditionally, regardless of whether the source would + have accepted it anyway -- keeps behavior uniform across source + forward signatures.""" + model = ForwardVarKwargs() + container = PretrainModelContainer(model) + container(torch.tensor([[1, 2, 3]]), output_attentions=True, some_other_kwarg=42) + assert model.seen_kwargs == {"some_other_kwarg": 42} + + def test_unrelated_kwarg_typo_is_not_silently_swallowed(self) -> None: + """A genuine caller mistake (e.g. `target=` instead of `targets=`) + must raise, not be silently discarded alongside the one kwarg this + container is actually responsible for stripping.""" + model = ForwardStrict() + container = PretrainModelContainer(model) + with pytest.raises(TypeError): + container(torch.tensor([[1, 2, 3]]), target=torch.tensor([1])) + + +class TestPretrainModelContainerHookRegistration: + """The delegate is hidden from nn.Module registration (see + DenseOrMoEFeedForwardBridge), so blocks.{i}.mlp.hook_in/out are the + only publicly registered MLP hook points. Forward, gradients, dtype + conversion, and state_dict() still reach the hidden delegate's real + weights through the raw wrapped model.""" + + def _build(self): + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64) + return build_pretrain_bridge(model, cfg) + + def test_only_the_dispatcher_hook_out_is_registered(self) -> None: + bridge = self._build() + tokens = torch.randint(0, 64, (1, 4)) + with torch.no_grad(): + _, cache = bridge.run_with_cache(tokens) + assert "blocks.0.mlp.hook_out" in cache + assert "blocks.0.mlp._delegate.hook_out" not in cache + + def test_broad_hook_selector_applies_intervention_exactly_once(self) -> None: + bridge = self._build() + tokens = torch.randint(0, 64, (1, 4)) + calls: list[str] = [] + + def recording_add_one(value: torch.Tensor, hook) -> torch.Tensor: + calls.append(hook.name) + return value + 1.0 + + def broad_filter(name: str) -> bool: + return "mlp" in name and "hook_out" in name + + with torch.no_grad(): + bridge.run_with_hooks(tokens, fwd_hooks=[(broad_filter, recording_add_one)]) + + assert calls == ["blocks.0.mlp.hook_out"] + + def test_gradients_still_reach_the_hidden_delegates_weights(self) -> None: + bridge = self._build() + tokens = torch.randint(0, 64, (1, 4)) + bridge(tokens).sum().backward() + mlp = bridge.original_model.inner.blocks[0].mlp + assert mlp.gate.weight.grad is not None + assert mlp.gate.weight.grad.abs().sum().item() > 0 + + def test_bridge_to_dtype_reaches_the_hidden_delegates_weights(self) -> None: + bridge = self._build() + bridge.to(dtype=torch.float64) + + assert bridge.original_model.inner.embed.weight.dtype == torch.float64 + assert bridge.original_model.inner.blocks[0].mlp.gate.weight.dtype == torch.float64 + + tokens = torch.randint(0, 64, (1, 4)) + with torch.no_grad(): + output = bridge(tokens) + assert output.dtype == torch.float64 + + def test_state_dict_includes_the_hidden_delegates_weights(self) -> None: + bridge = self._build() + source_weight = bridge.original_model.inner.blocks[0].mlp.gate.weight + + state = bridge.state_dict() + matching = [v for k, v in state.items() if k.endswith("blocks.0.mlp.gate.weight")] + + assert len(matching) == 1 + torch.testing.assert_close(matching[0], source_weight) + + def test_repeated_hook_lifecycle_still_works(self) -> None: + """Guards against a future assumption that every internally-used + GeneralizedComponent must be registered -- the dispatcher's own + hooks are, and that's what reset_hooks()/run_with_cache rely on.""" + bridge = self._build() + tokens = torch.randint(0, 64, (1, 4)) + with torch.no_grad(): + bridge.run_with_cache(tokens) + bridge.reset_hooks() + _, cache = bridge.run_with_cache(tokens) + assert "blocks.0.mlp.hook_out" in cache + + +class TestTrainEvalModePropagation: + """bridge.train()/.eval() propagate to the wrapped model: + build_pretrain_bridge reassigns the returned bridge's class to a small + generated TransformerBridge subclass (see + _get_mode_propagating_bridge_class) whose overridden train() also sets + mode on original_model; inherited eval() calls train(False), so it + reaches the override too without needing its own override. This + closes a gap in TransformerBridge's own train()/eval(), which only + walk component_mapping and never reach original_model on their own.""" + + def test_bridge_eval_propagates_to_wrapped_model(self) -> None: + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64) + bridge = build_pretrain_bridge(model, cfg) + + assert model.training is True + bridge.eval() + assert model.training is False + + def test_bridge_train_propagates_to_wrapped_model(self) -> None: + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64) + bridge = build_pretrain_bridge(model, cfg) + + model.eval() + assert model.training is False + bridge.train() + assert model.training is True + + def test_caller_can_still_set_mode_directly_on_their_own_model_reference(self) -> None: + """Setting mode via the raw model reference still works and stays + in sync -- both paths write to the same underlying module.""" + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64) + build_pretrain_bridge(model, cfg) + + model.eval() + assert model.training is False + model.train() + assert model.training is True + + def test_train_and_eval_return_the_bridge_itself(self) -> None: + """nn.Module convention: train()/eval() return self, so callers can + chain (`model.train().to(device)`, etc). The generated subclass + must preserve this, not just the mode-propagation side effect.""" + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64) + bridge = build_pretrain_bridge(model, cfg) + + assert bridge.train(False) is bridge + assert bridge.train(True) is bridge + assert bridge.eval() is bridge + + def test_bridge_can_be_reconstructed_from_supported_state(self) -> None: + """The adapter's actual persistence contract is: rebuild the + bridge from the source model class + cfg + the source model's + *raw* state_dict, then confirm behavior (outputs, train/eval + propagation) survives the round trip. + + The state_dict must be snapshotted BEFORE `build_pretrain_bridge` + is called. Building the bridge wraps the model's submodules in + place (each one gains an `_original_component` wrapper for + hooking), so `model.state_dict()` taken *after* wrapping has + keys like `embed._original_component.weight` instead of + `embed.weight` -- a fresh, unwrapped `TinyPretrainModel` can't + load that. Capturing the state_dict first, then wrapping a + second fresh model and loading the pre-wrap snapshot into it, + avoids that trap. + + Only the source model's state_dict is used -- not + `bridge.state_dict()` -- since the bridge doesn't own an + independent set of learned weights; it exposes the wrapped + source model's parameters under bridge-shaped names. + + This is deliberately weaker than whole-object `pickle.dumps`, + which would also require every nested `TransformerBridge` + implementation detail (e.g. `AttentionBridge`'s hook + conversions) to be picklable -- a guarantee unrelated to this + adapter and not part of what it needs to promise. If a stronger + whole-object-pickle guarantee is later required, it belongs in a + `TransformerBridge`-level test, not here. + """ + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64) + + # Snapshot BEFORE wrapping -- see docstring. + raw_model_state = {key: value.detach().clone() for key, value in model.state_dict().items()} + + bridge = build_pretrain_bridge(model, cfg) + bridge.eval() + + restored_model = TinyPretrainModel( + d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64 + ) + restored_model.load_state_dict(raw_model_state) + + restored_bridge = build_pretrain_bridge(restored_model, cfg) + restored_bridge.eval() + + tokens = torch.randint(0, 64, (1, 4)) + with torch.no_grad(): + expected = bridge(tokens) + actual = restored_bridge(tokens) + + expected_logits = expected.logits if hasattr(expected, "logits") else expected + actual_logits = actual.logits if hasattr(actual, "logits") else actual + torch.testing.assert_close(actual_logits, expected_logits, atol=0, rtol=0) + + assert restored_bridge.training is False + assert restored_model.training is False + + restored_bridge.train() + assert restored_model.training is True + + def test_repeated_construction_reuses_the_same_generated_class(self) -> None: + """Guards the cache's stated purpose: build_pretrain_bridge should + not generate a new mode-propagating subclass on every call -- two + independently-built bridges from the same base bridge class must + share one generated class.""" + cfg_a = _make_cfg(n_layers=1) + model_a = TinyPretrainModel(d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64) + bridge_a = build_pretrain_bridge(model_a, cfg_a) + + cfg_b = _make_cfg(n_layers=1) + model_b = TinyPretrainModel(d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64) + bridge_b = build_pretrain_bridge(model_b, cfg_b) + + assert type(bridge_a) is type(bridge_b) + + +class TestDtypeAndTiedWeightPreservation: + def test_non_default_dtype_and_tied_weights_survive_construction(self) -> None: + torch.manual_seed(0) + model = TinyPretrainModel( + d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64, tie_embeddings=True + ).to(torch.float64) + assert model.lm_head.weight is model.embed.weight + + cfg = _make_cfg(n_layers=1) + bridge = build_pretrain_bridge(model, cfg) + + assert next(model.parameters()).dtype == torch.float64 + assert model.lm_head.weight is model.embed.weight + + tokens = torch.randint(0, 64, (1, 3)) + with torch.no_grad(): + output = bridge(tokens) + assert output.dtype == torch.float64 + + +class TestBuildPretrainBridgeErgonomics: + def test_explicit_model_name_is_forwarded(self) -> None: + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64) + bridge = build_pretrain_bridge(model, cfg, model_name="my-custom-name") + assert bridge.cfg.model_name == "my-custom-name" + + def test_omitting_optional_kwargs_still_works(self) -> None: + """The default (no device/dtype/model_name passed) path, exercised + everywhere else in this file, must keep working unchanged.""" + cfg = _make_cfg(n_layers=1) + model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64) + bridge = build_pretrain_bridge(model, cfg) + tokens = torch.randint(0, 64, (1, 3)) + with torch.no_grad(): + output = bridge(tokens) + assert output.shape == (1, 3, 64) + + +class TestPretrainModelContainerOutputContract: + def test_container_normalizes_dict_output_to_logits_attr_dict(self) -> None: + model = TinyPretrainModel( + d_model=16, + n_heads=2, + n_layers=1, + d_ff=32, + vocab_size=64, + ) + container = PretrainModelContainer(model) + tokens = torch.randint(0, 64, (1, 3)) + + with torch.no_grad(): + output = container(tokens) + + assert isinstance(output, _LogitsAttrDict) + assert output.logits is output["logits"] + assert output.logits.shape == (1, 3, 64) + + def test_container_returns_a_fresh_wrapper_on_each_call(self) -> None: + model = TinyPretrainModel( + d_model=16, + n_heads=2, + n_layers=1, + d_ff=32, + vocab_size=64, + ) + container = PretrainModelContainer(model) + tokens = torch.randint(0, 64, (1, 3)) + + with torch.no_grad(): + out1 = container(tokens) + out2 = container(tokens) + + assert isinstance(out1, _LogitsAttrDict) + assert isinstance(out2, _LogitsAttrDict) + assert out1 is not out2 + assert out1["logits"] is not out2["logits"] + torch.testing.assert_close(out1["logits"], out2["logits"]) + + def test_container_does_not_reuse_or_mutate_output_mapping(self) -> None: + model = TinyPretrainModel( + d_model=16, + n_heads=2, + n_layers=1, + d_ff=32, + vocab_size=64, + ).eval() + container = PretrainModelContainer(model) + tokens = torch.randint(0, 64, (1, 3)) + + with torch.no_grad(): + out1 = container(tokens) + + out1["sentinel"] = True + + with torch.no_grad(): + out2 = container(tokens) + + assert out1 is not out2 + assert "sentinel" not in out2 + torch.testing.assert_close(out1["logits"], out2["logits"]) + + def test_dict_missing_logits_key_raises_value_error(self) -> None: + class ForwardWrongKey(nn.Module): + def forward(self, tokens: torch.Tensor) -> dict: + return {"scores": tokens.float()} + + container = PretrainModelContainer(ForwardWrongKey()) + with pytest.raises(ValueError, match="'logits'"): + container(torch.tensor([[1, 2, 3]])) + + def test_dict_with_heterogeneous_keys_still_raises_value_error(self) -> None: + """sorted(output.keys()) would raise TypeError on non-comparable + mixed key types (e.g. str and int); the error message must use + list() instead, so an invalid model output produces the intended + ValueError, not an unrelated sorting error.""" + + class ForwardHeterogeneousKeys(nn.Module): + def forward(self, tokens: torch.Tensor) -> dict: + return {"scores": tokens.float(), 1: "metadata"} + + container = PretrainModelContainer(ForwardHeterogeneousKeys()) + with pytest.raises(ValueError, match="'logits'"): + container(torch.tensor([[1, 2, 3]])) + + def test_dict_with_non_tensor_logits_raises_type_error(self) -> None: + class ForwardBadLogitsType(nn.Module): + def forward(self, tokens: torch.Tensor) -> dict: + return {"logits": "not a tensor"} + + container = PretrainModelContainer(ForwardBadLogitsType()) + with pytest.raises(TypeError, match="torch.Tensor"): + container(torch.tensor([[1, 2, 3]])) + + def test_already_logits_attr_dict_passes_through_by_identity(self) -> None: + """An already-wrapped output must not be re-wrapped.""" + already_wrapped = _LogitsAttrDict({"logits": torch.zeros(1)}) + + class ForwardAlreadyWrapped(nn.Module): + def forward(self, tokens: torch.Tensor) -> _LogitsAttrDict: + return already_wrapped + + container = PretrainModelContainer(ForwardAlreadyWrapped()) + output = container(torch.tensor([[1, 2, 3]])) + assert output is already_wrapped + + def test_malformed_logits_attr_dict_raises_value_error_not_keyerror(self) -> None: + """Not a realistic target-model failure (_LogitsAttrDict is + private), but closes the contract: a malformed instance gets the + same clear ValueError an ordinary dict missing 'logits' gets, + rather than a raw KeyError leaking out.""" + malformed = _LogitsAttrDict({"scores": torch.zeros(1)}) + + class ForwardMalformedWrapped(nn.Module): + def forward(self, tokens: torch.Tensor) -> _LogitsAttrDict: + return malformed + + container = PretrainModelContainer(ForwardMalformedWrapped()) + with pytest.raises(ValueError, match="'logits'"): + container(torch.tensor([[1, 2, 3]])) + + def test_hf_style_object_with_logits_attribute_passes_through_unchanged(self) -> None: + """A source model that already satisfies hasattr(output, 'logits') + on its own (an HF-ModelOutput-like object, not a dict) needs no + normalization at all.""" + + class FakeModelOutput: + def __init__(self, logits: torch.Tensor): + self.logits = logits + + already_hf_style = FakeModelOutput(torch.zeros(1)) + + class ForwardHFStyle(nn.Module): + def forward(self, tokens: torch.Tensor) -> FakeModelOutput: + return already_hf_style + + container = PretrainModelContainer(ForwardHFStyle()) + output = container(torch.tensor([[1, 2, 3]])) + assert output is already_hf_style + + def test_property_backed_logits_is_only_evaluated_once(self) -> None: + """.logits must be read once and cached locally, not re-evaluated + on every access -- matters for a property-backed .logits, which + could otherwise do redundant (or side-effecting) work per access.""" + access_count = 0 + + class PropertyBackedOutput: + @property + def logits(self) -> torch.Tensor: + nonlocal access_count + access_count += 1 + return torch.zeros(1) + + class ForwardPropertyBacked(nn.Module): + def forward(self, tokens: torch.Tensor) -> PropertyBackedOutput: + return PropertyBackedOutput() + + container = PretrainModelContainer(ForwardPropertyBacked()) + container(torch.tensor([[1, 2, 3]])) + assert access_count == 1 + + def test_object_with_non_tensor_logits_attribute_raises_type_error(self) -> None: + class BadModelOutput: + logits = "not a tensor" + + class ForwardBadModelOutput(nn.Module): + def forward(self, tokens: torch.Tensor) -> BadModelOutput: + return BadModelOutput() + + container = PretrainModelContainer(ForwardBadModelOutput()) + with pytest.raises(TypeError, match=r"\.logits"): + container(torch.tensor([[1, 2, 3]])) + + def test_bare_tensor_output_passes_through_unchanged(self) -> None: + class ForwardBareTensor(nn.Module): + def forward(self, tokens: torch.Tensor) -> torch.Tensor: + return tokens.float() + + container = PretrainModelContainer(ForwardBareTensor()) + tokens = torch.tensor([[1, 2, 3]]) + output = container(tokens) + assert torch.equal(output, tokens.float()) + + def test_tuple_with_tensor_first_element_passes_through_unchanged(self) -> None: + """TransformerBridge extracts output[0] as logits for tuple + returns -- no normalization needed.""" + + class ForwardTuple(nn.Module): + def forward(self, tokens: torch.Tensor) -> tuple: + return (tokens.float(), "some_aux_value") + + container = PretrainModelContainer(ForwardTuple()) + output = container(torch.tensor([[1, 2, 3]])) + assert isinstance(output, tuple) + assert output[1] == "some_aux_value" + + def test_tuple_with_non_tensor_first_element_raises_type_error(self) -> None: + class ForwardBadTuple(nn.Module): + def forward(self, tokens: torch.Tensor) -> tuple: + return ("not a tensor", tokens) + + container = PretrainModelContainer(ForwardBadTuple()) + with pytest.raises(TypeError, match="torch.Tensor"): + container(torch.tensor([[1, 2, 3]])) + + def test_list_output_raises_type_error(self) -> None: + class ForwardList(nn.Module): + def forward(self, tokens: torch.Tensor) -> list: + return [tokens.float()] + + container = PretrainModelContainer(ForwardList()) + with pytest.raises(TypeError, match="must return"): + container(torch.tensor([[1, 2, 3]])) + + def test_string_output_raises_type_error(self) -> None: + class ForwardString(nn.Module): + def forward(self, tokens: torch.Tensor) -> str: + return "wrong" + + container = PretrainModelContainer(ForwardString()) + with pytest.raises(TypeError, match="must return"): + container(torch.tensor([[1, 2, 3]])) diff --git a/tests/unit/model_bridge/supported_architectures/test_raven_adapter.py b/tests/unit/model_bridge/supported_architectures/test_raven_adapter.py new file mode 100644 index 0000000000..3005b4dbbd --- /dev/null +++ b/tests/unit/model_bridge/supported_architectures/test_raven_adapter.py @@ -0,0 +1,335 @@ +"""Unit tests for RavenArchitectureAdapter (Huginn depth-recurrent decoder). + +Synthetic-config only — no HF Hub access, no weight loading. Covers the +architecture quirks: RMS/rotary/gated flags, ``supports_fold_ln = False``, +the empty ``applicable_phases`` (off the transformer-shaped verify path), +recurrence-config propagation, the three separate prelude / core_block / coda +block lists mapped via ``OpaqueBlockBridge``, the combined-QKV native attention +and combined gate+up MLP submodules, and four-place registration. +""" + +import pytest + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.factories.architecture_adapter_factory import ( + ArchitectureAdapterFactory, +) +from transformer_lens.model_bridge.generalized_components import ( + AttentionBridge, + EmbeddingBridge, + LinearBridge, + MLPBridge, + OpaqueBlockBridge, + RMSNormalizationBridge, + UnembeddingBridge, +) +from transformer_lens.model_bridge.supported_architectures.raven import ( + RavenArchitectureAdapter, +) + +# The three physical block lists Huginn stores under model.transformer. +BLOCK_LISTS = ("prelude", "core_block", "coda") + + +def _make_cfg( + n_embd: int = 55, + n_heads: int = 55, + n_layers: int = 8, + n_ctx: int = 128, + d_vocab: int = 256, + mean_recurrence: int = 32, + n_layers_in_prelude: int = 2, + n_layers_in_recurrent_block: int = 4, + n_layers_in_coda: int = 2, + injection_type: str = "linear", + qk_bias: bool = True, +) -> TransformerBridgeConfig: + """Minimal TransformerBridgeConfig for Raven adapter tests. + + n_heads == n_embd keeps d_head == 1 so the tiny synthetic dims are legal + (Huginn is MHA: num_key_value_heads == num_attention_heads). + """ + cfg = TransformerBridgeConfig( + d_model=n_embd, + d_head=n_embd // n_heads, + n_layers=n_layers, + n_ctx=n_ctx, + n_heads=n_heads, + d_vocab=d_vocab, + d_mlp=4 * n_embd, + default_prepend_bos=False, + architecture="RavenForCausalLM", + ) + # Inject Huginn recurrence-shape fields the adapter reads via getattr. + cfg.mean_recurrence = mean_recurrence # type: ignore[attr-defined] + cfg.n_layers_in_prelude = n_layers_in_prelude # type: ignore[attr-defined] + cfg.n_layers_in_recurrent_block = n_layers_in_recurrent_block # type: ignore[attr-defined] + cfg.n_layers_in_coda = n_layers_in_coda # type: ignore[attr-defined] + cfg.injection_type = injection_type # type: ignore[attr-defined] + cfg.qk_bias = qk_bias # type: ignore[attr-defined] + return cfg + + +@pytest.fixture(scope="class") +def cfg() -> TransformerBridgeConfig: + return _make_cfg() + + +@pytest.fixture(scope="class") +def adapter(cfg: TransformerBridgeConfig) -> RavenArchitectureAdapter: + return RavenArchitectureAdapter(cfg) + + +# --------------------------------------------------------------------------- +# Config attributes +# --------------------------------------------------------------------------- + + +class TestRavenAdapterConfig: + """Adapter sets the required norm / positional / MLP flags.""" + + def test_normalization_type_rms(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.cfg.normalization_type == "RMS" + + def test_uses_rms_norm(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.cfg.uses_rms_norm is True + + def test_positional_embedding_type_rotary(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.cfg.positional_embedding_type == "rotary" + + def test_final_rms_true(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.cfg.final_rms is True + + def test_gated_mlp_true(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.cfg.gated_mlp is True + + def test_attn_only_false(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.cfg.attn_only is False + + def test_supports_fold_ln_false(self, adapter: RavenArchitectureAdapter) -> None: + # ln_f is reused mid-network (after recurrence, feeding the coda), so + # it must not be folded into W_U. + assert adapter.supports_fold_ln is False + + def test_applicable_phases_empty(self) -> None: + # Off the transformer-shaped verify_models path; correctness lives in + # the integration tests. + assert RavenArchitectureAdapter.applicable_phases == [] + + def test_weight_processing_conversions_empty(self, adapter: RavenArchitectureAdapter) -> None: + # Full delegation to the HF forward — no HT-format reshaping. + assert adapter.weight_processing_conversions == {} + + +class TestRavenRecurrenceConfigPropagation: + """Recurrence-shape attributes are surfaced on cfg.""" + + def test_mean_recurrence(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.cfg.mean_recurrence == 32 + + def test_n_layers_in_prelude(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.cfg.n_layers_in_prelude == 2 + + def test_n_layers_in_recurrent_block(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.cfg.n_layers_in_recurrent_block == 4 + + def test_n_layers_in_coda(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.cfg.n_layers_in_coda == 2 + + def test_injection_type(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.cfg.injection_type == "linear" + + def test_qk_bias(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.cfg.qk_bias is True + + def test_physical_layer_split_sums_to_n_layers(self, adapter: RavenArchitectureAdapter) -> None: + """prelude + recurrent-block + coda counts the 8 physical layers.""" + total = ( + adapter.cfg.n_layers_in_prelude + + adapter.cfg.n_layers_in_recurrent_block + + adapter.cfg.n_layers_in_coda + ) + assert total == adapter.cfg.n_layers + + def test_recurrence_defaults_when_absent(self) -> None: + """Adapter falls back to Huginn defaults if the attrs are missing.""" + bare = TransformerBridgeConfig( + d_model=55, + d_head=1, + n_layers=8, + n_ctx=128, + n_heads=55, + d_vocab=256, + architecture="RavenForCausalLM", + ) + a = RavenArchitectureAdapter(bare) + assert a.cfg.mean_recurrence == 32 + assert a.cfg.n_layers_in_prelude == 2 + assert a.cfg.n_layers_in_recurrent_block == 4 + assert a.cfg.n_layers_in_coda == 2 + + +# --------------------------------------------------------------------------- +# Top-level component mapping +# --------------------------------------------------------------------------- + + +class TestRavenTopLevelComponents: + """component_mapping exposes prelude / core_block / coda as three lists.""" + + def test_required_keys(self, adapter: RavenArchitectureAdapter) -> None: + assert set(adapter.component_mapping.keys()) == { + "embed", + "prelude", + "core_block", + "coda", + "ln_final", + "unembed", + } + + def test_embed_is_embedding_bridge(self, adapter: RavenArchitectureAdapter) -> None: + assert isinstance(adapter.component_mapping["embed"], EmbeddingBridge) + + def test_embed_name(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.component_mapping["embed"].name == "transformer.wte" + + @pytest.mark.parametrize("key", BLOCK_LISTS) + def test_block_list_is_opaque_block_bridge( + self, adapter: RavenArchitectureAdapter, key: str + ) -> None: + # OpaqueBlockBridge delegates the whole SandwichBlock so the post-residual + # norm placement is preserved (a standard BlockBridge would assume a + # pre-norm flow). + assert isinstance(adapter.component_mapping[key], OpaqueBlockBridge) + + @pytest.mark.parametrize("key", BLOCK_LISTS) + def test_block_list_name(self, adapter: RavenArchitectureAdapter, key: str) -> None: + assert adapter.component_mapping[key].name == f"transformer.{key}" + + def test_ln_final_is_rms_normalization_bridge(self, adapter: RavenArchitectureAdapter) -> None: + assert isinstance(adapter.component_mapping["ln_final"], RMSNormalizationBridge) + + def test_ln_final_name(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.component_mapping["ln_final"].name == "transformer.ln_f" + + def test_unembed_is_unembedding_bridge(self, adapter: RavenArchitectureAdapter) -> None: + assert isinstance(adapter.component_mapping["unembed"], UnembeddingBridge) + + def test_unembed_name(self, adapter: RavenArchitectureAdapter) -> None: + assert adapter.component_mapping["unembed"].name == "lm_head" + + +# --------------------------------------------------------------------------- +# SandwichBlock submodules (shared shape across all three block lists) +# --------------------------------------------------------------------------- + + +class TestRavenBlockSubmodules: + """Each block list wraps a SandwichBlock: four norms, native attn, gated MLP.""" + + @pytest.mark.parametrize("key", BLOCK_LISTS) + def test_submodule_keys(self, adapter: RavenArchitectureAdapter, key: str) -> None: + block = adapter.component_mapping[key] + assert set(block.submodules.keys()) == { + "norm_1", + "attn", + "norm_2", + "norm_3", + "mlp", + "norm_4", + } + + @pytest.mark.parametrize("key", BLOCK_LISTS) + @pytest.mark.parametrize("norm", ["norm_1", "norm_2", "norm_3", "norm_4"]) + def test_norms_are_rms(self, adapter: RavenArchitectureAdapter, key: str, norm: str) -> None: + block = adapter.component_mapping[key] + assert isinstance(block.submodules[norm], RMSNormalizationBridge) + assert block.submodules[norm].name == norm + + @pytest.mark.parametrize("key", BLOCK_LISTS) + def test_attn_is_native_attention_with_combined_qkv( + self, adapter: RavenArchitectureAdapter, key: str + ) -> None: + attn = adapter.component_mapping[key].submodules["attn"] + assert isinstance(attn, AttentionBridge) + assert attn.name == "attn" + # Combined Wqkv projection + output proj (no split q/k/v). + assert isinstance(attn.submodules["qkv"], LinearBridge) + assert attn.submodules["qkv"].name == "Wqkv" + assert isinstance(attn.submodules["o"], LinearBridge) + assert attn.submodules["o"].name == "proj" + assert set(attn.submodules.keys()) == {"qkv", "o"} + + @pytest.mark.parametrize("key", BLOCK_LISTS) + def test_mlp_is_gated_with_combined_fc( + self, adapter: RavenArchitectureAdapter, key: str + ) -> None: + mlp = adapter.component_mapping[key].submodules["mlp"] + assert isinstance(mlp, MLPBridge) + assert mlp.name == "mlp" + # Combined gate+up "fc" and output "proj". + assert isinstance(mlp.submodules["in"], LinearBridge) + assert mlp.submodules["in"].name == "fc" + assert isinstance(mlp.submodules["out"], LinearBridge) + assert mlp.submodules["out"].name == "proj" + + +class TestRavenBlockListsAreIndependent: + """The three block lists must not share bridge instances. + + Each OpaqueBlockBridge binds to a distinct HF ModuleList, so reusing a single + submodule bridge object across lists would cross-wire them. + """ + + def test_top_level_bridges_distinct(self, adapter: RavenArchitectureAdapter) -> None: + bridges = [adapter.component_mapping[k] for k in BLOCK_LISTS] + assert len({id(b) for b in bridges}) == len(BLOCK_LISTS) + + def test_submodule_bridges_distinct(self, adapter: RavenArchitectureAdapter) -> None: + attns = [adapter.component_mapping[k].submodules["attn"] for k in BLOCK_LISTS] + assert len({id(a) for a in attns}) == len(BLOCK_LISTS) + norms = [adapter.component_mapping[k].submodules["norm_1"] for k in BLOCK_LISTS] + assert len({id(n) for n in norms}) == len(BLOCK_LISTS) + + +# --------------------------------------------------------------------------- +# Factory registration + model registry +# --------------------------------------------------------------------------- + + +class TestRavenFactoryRegistration: + """RavenForCausalLM is wired into the adapter factory.""" + + def test_factory_returns_raven_adapter(self) -> None: + cfg = _make_cfg() + adapter = ArchitectureAdapterFactory.select_architecture_adapter(cfg) + assert isinstance(adapter, RavenArchitectureAdapter) + + def test_architecture_key_present(self) -> None: + from transformer_lens.factories.architecture_adapter_factory import ( + SUPPORTED_ARCHITECTURES, + ) + + assert "RavenForCausalLM" in SUPPORTED_ARCHITECTURES + assert SUPPORTED_ARCHITECTURES["RavenForCausalLM"] is RavenArchitectureAdapter + + +class TestRavenModelRegistry: + """RavenForCausalLM is listed in the model registry constants.""" + + def test_in_hf_supported(self) -> None: + from transformer_lens.tools.model_registry import HF_SUPPORTED_ARCHITECTURES + + assert "RavenForCausalLM" in HF_SUPPORTED_ARCHITECTURES + + def test_canonical_author_is_tomg_group(self) -> None: + from transformer_lens.tools.model_registry import CANONICAL_AUTHORS_BY_ARCH + + assert CANONICAL_AUTHORS_BY_ARCH.get("RavenForCausalLM") == ["tomg-group-umd"] + + def test_has_architecture_description(self) -> None: + from transformer_lens.tools.model_registry.generate_report import ( + ARCHITECTURE_DESCRIPTIONS, + ) + + assert "RavenForCausalLM" in ARCHITECTURE_DESCRIPTIONS diff --git a/tests/unit/model_bridge/supported_architectures/test_rwkv7_adapter.py b/tests/unit/model_bridge/supported_architectures/test_rwkv7_adapter.py new file mode 100644 index 0000000000..13844765e5 --- /dev/null +++ b/tests/unit/model_bridge/supported_architectures/test_rwkv7_adapter.py @@ -0,0 +1,301 @@ +"""Unit tests for RWKV7ArchitectureAdapter (RWKV-7 "Goose" recurrent decoder). + +Synthetic-config only — no HF Hub access, no weight loading. Covers the +architecture quirks: standard biased LayerNorm (``normalization_type == "LN"``), +no positional embeddings, ungated FFN, the empty ``applicable_phases`` (off the +transformer-shaped verify path), shape-attribute propagation, the single +``model.layers`` block list mapped via ``OpaqueBlockBridge``, the delegated +time-mixing (``attn``) and channel-mixing (``ffn``) sublayers wrapped by +``GeneralizedComponent`` with their projection submodules, the fused-signature +``ffn_norm`` likewise delegated via a plain ``GeneralizedComponent``, and four-place +registration. +""" + +import pytest + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.factories.architecture_adapter_factory import ( + ArchitectureAdapterFactory, +) +from transformer_lens.model_bridge.generalized_components import ( + EmbeddingBridge, + LinearBridge, + NormalizationBridge, + OpaqueBlockBridge, + UnembeddingBridge, +) +from transformer_lens.model_bridge.generalized_components.base import ( + GeneralizedComponent, +) +from transformer_lens.model_bridge.supported_architectures.rwkv7 import ( + RWKV7ArchitectureAdapter, +) + + +def _make_cfg( + d_model: int = 64, + head_dim: int = 64, + num_heads: int = 1, + n_layers: int = 4, + n_ctx: int = 128, + d_vocab: int = 256, + inject_shape_attrs: bool = True, +) -> TransformerBridgeConfig: + """Minimal TransformerBridgeConfig for RWKV-7 adapter tests. + + ``head_dim == d_model`` keeps ``num_heads == 1`` so the tiny synthetic dims + are legal. + """ + cfg = TransformerBridgeConfig( + d_model=d_model, + d_head=head_dim, + n_layers=n_layers, + n_ctx=n_ctx, + n_heads=num_heads, + d_vocab=d_vocab, + d_mlp=4 * d_model, + default_prepend_bos=False, + architecture="RWKV7ForCausalLM", + ) + if inject_shape_attrs: + # Inject RWKV-7 shape fields the adapter reads via getattr. head_dim is a + # read-only alias of d_head on the config, so it is not set here. + cfg.num_heads = num_heads # type: ignore[attr-defined] + cfg.value_dim = [d_model] * n_layers # type: ignore[attr-defined] + cfg.decay_low_rank_dim = 64 # type: ignore[attr-defined] + cfg.gate_low_rank_dim = 128 # type: ignore[attr-defined] + cfg.a_low_rank_dim = 64 # type: ignore[attr-defined] + cfg.v_low_rank_dim = 16 # type: ignore[attr-defined] + cfg.norm_first = True # type: ignore[attr-defined] + cfg.norm_bias = True # type: ignore[attr-defined] + cfg.fuse_norm = True # type: ignore[attr-defined] + cfg.attn_mode = "chunk" # type: ignore[attr-defined] + cfg.hidden_act = "sqrelu" # type: ignore[attr-defined] + cfg.norm_eps = 1e-5 # type: ignore[attr-defined] + return cfg + + +@pytest.fixture(scope="class") +def cfg() -> TransformerBridgeConfig: + return _make_cfg() + + +@pytest.fixture(scope="class") +def adapter(cfg: TransformerBridgeConfig) -> RWKV7ArchitectureAdapter: + return RWKV7ArchitectureAdapter(cfg) + + +# --------------------------------------------------------------------------- +# Config attributes +# --------------------------------------------------------------------------- + + +class TestRWKV7AdapterConfig: + """Adapter sets the required norm / positional / MLP flags.""" + + def test_normalization_type_ln(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert adapter.cfg.normalization_type == "LN" + + def test_uses_rms_norm_false(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert adapter.cfg.uses_rms_norm is False + + def test_positional_embedding_type_none(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert adapter.cfg.positional_embedding_type == "none" + + def test_final_rms_false(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert adapter.cfg.final_rms is False + + def test_gated_mlp_false(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert adapter.cfg.gated_mlp is False + + def test_attn_only_false(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert adapter.cfg.attn_only is False + + def test_is_stateful_false(self, adapter: RWKV7ArchitectureAdapter) -> None: + # fla drives decode state through its own Cache, not cache_params. + assert adapter.cfg.is_stateful is False + + def test_applicable_phases_empty(self) -> None: + # Off the transformer-shaped verify_models path; correctness lives in + # the integration tests. + assert RWKV7ArchitectureAdapter.applicable_phases == [] + + def test_weight_processing_conversions_empty(self, adapter: RWKV7ArchitectureAdapter) -> None: + # Full delegation to the fla forward — no HT-format reshaping. + assert adapter.weight_processing_conversions == {} + + +class TestRWKV7ShapeAttrPropagation: + """RWKV-7 shape attributes are surfaced on cfg.""" + + def test_head_dim(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert adapter.cfg.head_dim == 64 + + def test_num_heads(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert adapter.cfg.num_heads == 1 + + def test_value_dim(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert adapter.cfg.value_dim == [64, 64, 64, 64] + + def test_low_rank_dims(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert adapter.cfg.decay_low_rank_dim == 64 + assert adapter.cfg.gate_low_rank_dim == 128 + assert adapter.cfg.a_low_rank_dim == 64 + assert adapter.cfg.v_low_rank_dim == 16 + + def test_norm_flags(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert adapter.cfg.norm_first is True + assert adapter.cfg.norm_bias is True + assert adapter.cfg.fuse_norm is True + + def test_attn_mode_and_hidden_act(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert adapter.cfg.attn_mode == "chunk" + assert adapter.cfg.hidden_act == "sqrelu" + + def test_defaults_when_absent(self) -> None: + """Adapter falls back to RWKV-7 defaults if the attrs are missing.""" + bare = _make_cfg(inject_shape_attrs=False) + a = RWKV7ArchitectureAdapter(bare) + assert a.cfg.head_dim == 64 + # num_heads defaults to d_model // head_dim. + assert a.cfg.num_heads == 1 + # value_dim defaults to [d_model] * n_layers. + assert a.cfg.value_dim == [64, 64, 64, 64] + assert a.cfg.decay_low_rank_dim == 64 + assert a.cfg.gate_low_rank_dim == 128 + assert a.cfg.a_low_rank_dim == 64 + assert a.cfg.v_low_rank_dim == 16 + assert a.cfg.norm_first is True + assert a.cfg.fuse_norm is True + assert a.cfg.hidden_act == "sqrelu" + + +# --------------------------------------------------------------------------- +# Top-level component mapping +# --------------------------------------------------------------------------- + + +class TestRWKV7TopLevelComponents: + """component_mapping exposes embed / blocks / ln_final / unembed.""" + + def test_required_keys(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert set(adapter.component_mapping.keys()) == { + "embed", + "blocks", + "ln_final", + "unembed", + } + + def test_embed_is_embedding_bridge(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert isinstance(adapter.component_mapping["embed"], EmbeddingBridge) + assert adapter.component_mapping["embed"].name == "model.embeddings" + + def test_blocks_is_ssm_block_bridge(self, adapter: RWKV7ArchitectureAdapter) -> None: + # OpaqueBlockBridge delegates the whole recurrent block so the internal + # mixing is preserved (a standard BlockBridge assumes a pre-norm attn flow). + blocks = adapter.component_mapping["blocks"] + assert isinstance(blocks, OpaqueBlockBridge) + assert blocks.name == "model.layers" + + def test_ln_final_is_normalization_bridge(self, adapter: RWKV7ArchitectureAdapter) -> None: + ln_final = adapter.component_mapping["ln_final"] + assert isinstance(ln_final, NormalizationBridge) + assert ln_final.name == "model.norm" + + def test_unembed_is_unembedding_bridge(self, adapter: RWKV7ArchitectureAdapter) -> None: + assert isinstance(adapter.component_mapping["unembed"], UnembeddingBridge) + assert adapter.component_mapping["unembed"].name == "lm_head" + + +# --------------------------------------------------------------------------- +# Block submodules +# --------------------------------------------------------------------------- + + +class TestRWKV7BlockSubmodules: + """Each block wraps attn_norm / attn / ffn_norm / ffn.""" + + def test_submodule_keys(self, adapter: RWKV7ArchitectureAdapter) -> None: + block = adapter.component_mapping["blocks"] + assert set(block.submodules.keys()) == {"attn_norm", "attn", "ffn_norm", "ffn"} + + def test_attn_norm_is_normalization_bridge(self, adapter: RWKV7ArchitectureAdapter) -> None: + attn_norm = adapter.component_mapping["blocks"].submodules["attn_norm"] + assert isinstance(attn_norm, NormalizationBridge) + assert attn_norm.name == "attn_norm" + + def test_ffn_norm_is_delegating_passthrough(self, adapter: RWKV7ArchitectureAdapter) -> None: + # Under config.fuse_norm the block calls ffn_norm(x, residual, True) -> + # (normed, residual); the reimplementing NormalizationBridge can't express + # that, so it is a plain delegating GeneralizedComponent, not a norm bridge. + ffn_norm = adapter.component_mapping["blocks"].submodules["ffn_norm"] + assert type(ffn_norm) is GeneralizedComponent + assert not isinstance(ffn_norm, NormalizationBridge) + assert ffn_norm.name == "ffn_norm" + + def test_attn_is_generalized_component_with_projections( + self, adapter: RWKV7ArchitectureAdapter + ) -> None: + attn = adapter.component_mapping["blocks"].submodules["attn"] + assert isinstance(attn, GeneralizedComponent) + assert attn.name == "attn" + assert set(attn.submodules.keys()) == {"r_proj", "k_proj", "v_proj", "o_proj"} + for proj in ("r_proj", "k_proj", "v_proj", "o_proj"): + assert isinstance(attn.submodules[proj], LinearBridge) + assert attn.submodules[proj].name == proj + + def test_ffn_is_generalized_component_with_projections( + self, adapter: RWKV7ArchitectureAdapter + ) -> None: + ffn = adapter.component_mapping["blocks"].submodules["ffn"] + assert isinstance(ffn, GeneralizedComponent) + assert ffn.name == "ffn" + # HF names the up-projection "key" and the (down) output projection "value". + assert set(ffn.submodules.keys()) == {"key", "value"} + assert isinstance(ffn.submodules["key"], LinearBridge) + assert ffn.submodules["key"].name == "key" + assert isinstance(ffn.submodules["value"], LinearBridge) + assert ffn.submodules["value"].name == "value" + + +# --------------------------------------------------------------------------- +# Factory registration + model registry +# --------------------------------------------------------------------------- + + +class TestRWKV7FactoryRegistration: + """RWKV7ForCausalLM is wired into the adapter factory.""" + + def test_factory_returns_rwkv7_adapter(self) -> None: + cfg = _make_cfg() + adapter = ArchitectureAdapterFactory.select_architecture_adapter(cfg) + assert isinstance(adapter, RWKV7ArchitectureAdapter) + + def test_architecture_key_present(self) -> None: + from transformer_lens.factories.architecture_adapter_factory import ( + SUPPORTED_ARCHITECTURES, + ) + + assert "RWKV7ForCausalLM" in SUPPORTED_ARCHITECTURES + assert SUPPORTED_ARCHITECTURES["RWKV7ForCausalLM"] is RWKV7ArchitectureAdapter + + +class TestRWKV7ModelRegistry: + """RWKV7ForCausalLM is listed in the model registry constants.""" + + def test_in_hf_supported(self) -> None: + from transformer_lens.tools.model_registry import HF_SUPPORTED_ARCHITECTURES + + assert "RWKV7ForCausalLM" in HF_SUPPORTED_ARCHITECTURES + + def test_canonical_author_is_fla_hub(self) -> None: + from transformer_lens.tools.model_registry import CANONICAL_AUTHORS_BY_ARCH + + assert CANONICAL_AUTHORS_BY_ARCH.get("RWKV7ForCausalLM") == ["fla-hub"] + + def test_has_architecture_description(self) -> None: + from transformer_lens.tools.model_registry.generate_report import ( + ARCHITECTURE_DESCRIPTIONS, + ) + + assert "RWKV7ForCausalLM" in ARCHITECTURE_DESCRIPTIONS diff --git a/tests/unit/model_bridge/test_bridge_train_mode_propagation.py b/tests/unit/model_bridge/test_bridge_train_mode_propagation.py new file mode 100644 index 0000000000..4d08d0c71e --- /dev/null +++ b/tests/unit/model_bridge/test_bridge_train_mode_propagation.py @@ -0,0 +1,100 @@ +"""Regression tests for TransformerBridge train/eval mode propagation. + +`original_model` is stored outside the registered module tree, so +inherited `nn.Module.train()` does not reach it. These tests verify that +bridge mode changes propagate to the wrapped model, whose mode-dependent +layers (Dropout) rely on that flag. Architecture-independent stub bridge; +no HF Hub access, no weights. +""" +from __future__ import annotations + +import torch +import torch.nn as nn + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.bridge import TransformerBridge +from transformer_lens.model_bridge.generalized_components import LinearBridge + + +class _StubModel(nn.Module): + """Minimal source model with a mode-dependent layer (Dropout), so + train/eval propagation has an observable behavioral consequence, not + just a flag.""" + + def __init__(self) -> None: + super().__init__() + self.proj = nn.Linear(4, 4) + self.drop = nn.Dropout(0.5) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.drop(self.proj(x)) + + +class _StubAdapter(ArchitectureAdapter): + """Smallest adapter TransformerBridge will accept: a one-entry + component_mapping. The key is `stub_proj` (not `proj` or `embed`) + because TransformerBridge reserves several canonical attribute names + on itself; a colliding key fails at add_module time.""" + + def __init__(self, cfg: TransformerBridgeConfig) -> None: + super().__init__(cfg) + self.component_mapping = {"stub_proj": LinearBridge(name="proj")} + + +def _make_stub_bridge() -> tuple[TransformerBridge, _StubModel]: + cfg = TransformerBridgeConfig( + d_model=4, + d_head=2, + n_layers=1, + n_ctx=8, + n_heads=2, + d_vocab=8, + architecture="StubForTest", + ) + model = _StubModel() + bridge = TransformerBridge(model, _StubAdapter(cfg), tokenizer=None) + return bridge, model + + +class TestTrainEvalModePropagation: + """bridge.train()/.eval() must reach original_model. + + Without the TransformerBridge.train() override these fail with + original_model.training stuck at its wrap-time value (bridge.training + flips, the wrapped model's does not) -- i.e. bridge.eval() would leave + dropout active.""" + + def test_eval_propagates_to_original_model(self) -> None: + bridge, model = _make_stub_bridge() + assert model.training is True # nn.Module default at wrap time + bridge.eval() + assert bridge.training is False + assert model.training is False + + def test_train_propagates_to_original_model(self) -> None: + bridge, model = _make_stub_bridge() + bridge.eval() + assert model.training is False + bridge.train() + assert bridge.training is True + assert model.training is True + + def test_train_and_eval_return_the_bridge_itself(self) -> None: + """nn.Module convention: train()/eval() return self so callers can + chain (`bridge.eval().to(device)`, etc.).""" + bridge, _ = _make_stub_bridge() + assert bridge.train(False) is bridge + assert bridge.train(True) is bridge + assert bridge.eval() is bridge + + def test_direct_mode_on_original_model_stays_in_sync(self) -> None: + """Setting mode via the caller's own reference to the source model + still works -- both paths write the same underlying flags, so + neither clobbers the other.""" + bridge, model = _make_stub_bridge() + bridge.eval() + model.train() + assert model.training is True + bridge.eval() + assert model.training is False diff --git a/tests/unit/model_bridge/test_checkpoint_revision.py b/tests/unit/model_bridge/test_checkpoint_revision.py index dedcb8a99e..f24e428e66 100644 --- a/tests/unit/model_bridge/test_checkpoint_revision.py +++ b/tests/unit/model_bridge/test_checkpoint_revision.py @@ -13,7 +13,7 @@ class TestResolveCheckpointToRevision: def test_pythia_index_resolves_to_step_revision(self): labels = [0, 1000, 3000, 10000] with patch( - "transformer_lens.loading_from_pretrained.get_checkpoint_labels", + "transformer_lens.model_bridge.sources.transformers.helpers.get_checkpoint_labels", return_value=(labels, "step"), ): revision = _resolve_checkpoint_to_revision( @@ -24,7 +24,7 @@ def test_pythia_index_resolves_to_step_revision(self): def test_pythia_value_resolves_to_step_revision(self): labels = [0, 1000, 3000, 10000] with patch( - "transformer_lens.loading_from_pretrained.get_checkpoint_labels", + "transformer_lens.model_bridge.sources.transformers.helpers.get_checkpoint_labels", return_value=(labels, "step"), ): revision = _resolve_checkpoint_to_revision( @@ -35,7 +35,7 @@ def test_pythia_value_resolves_to_step_revision(self): def test_stanford_crfm_uses_checkpoint_prefix(self): labels = [100, 200, 400] with patch( - "transformer_lens.loading_from_pretrained.get_checkpoint_labels", + "transformer_lens.model_bridge.sources.transformers.helpers.get_checkpoint_labels", return_value=(labels, "step"), ): revision = _resolve_checkpoint_to_revision( @@ -50,7 +50,7 @@ def test_unknown_family_raises(self): def test_index_out_of_range_raises(self): labels = [0, 1000] with patch( - "transformer_lens.loading_from_pretrained.get_checkpoint_labels", + "transformer_lens.model_bridge.sources.transformers.helpers.get_checkpoint_labels", return_value=(labels, "step"), ): with pytest.raises(ValueError, match="out of range"): @@ -61,7 +61,7 @@ def test_index_out_of_range_raises(self): def test_unknown_value_raises(self): labels = [0, 1000] with patch( - "transformer_lens.loading_from_pretrained.get_checkpoint_labels", + "transformer_lens.model_bridge.sources.transformers.helpers.get_checkpoint_labels", return_value=(labels, "step"), ): with pytest.raises(ValueError, match="not in available checkpoints"): @@ -125,7 +125,7 @@ def test_revision_forwarded_to_model_load(self): def test_checkpoint_index_resolves_to_revision(self): labels = [0, 1000, 3000, 10000] with patch( - "transformer_lens.loading_from_pretrained.get_checkpoint_labels", + "transformer_lens.model_bridge.sources.transformers.helpers.get_checkpoint_labels", return_value=(labels, "step"), ): captured = self._patched_boot(checkpoint_index=2) diff --git a/tests/unit/test_tracr_conversion.py b/tests/unit/test_tracr_conversion.py index b8f00b3b6a..0b9eb34de8 100644 --- a/tests/unit/test_tracr_conversion.py +++ b/tests/unit/test_tracr_conversion.py @@ -74,9 +74,9 @@ def _fake_tracr_model() -> SimpleNamespace: "aggregate_1:2", "aggregate_1:3", "indices:0", - "reverse:1", - "reverse:2", - "reverse:3", + "reverse_1:1", + "reverse_1:2", + "reverse_1:3", "tokens:BOS", ], output_encoder=_CategoricalEncoder({1: 0, 2: 1, 3: 2}), @@ -94,7 +94,7 @@ def _fake_tracr_model() -> SimpleNamespace: def test_categorical_unembed_uses_named_output_basis(): model = _fake_tracr_model() - unembed = make_tracr_categorical_unembed(model, output_label="reverse") + unembed = make_tracr_categorical_unembed(model, output_label="reverse_1") assert unembed.shape == (8, 3) np.testing.assert_array_equal(unembed[:4], np.zeros((4, 3))) @@ -125,7 +125,7 @@ def test_bridge_state_dict_transposes_tracr_weights_and_reconstructs_unembed(): model = _fake_tracr_model() state_dict = make_tracr_transformer_bridge_state_dict( - model, output_label="reverse", dtype=torch.float64 + model, output_label="reverse_1", dtype=torch.float64 ) assert state_dict["tok_embed.weight"].shape == (5, 8) @@ -161,4 +161,4 @@ def test_bridge_state_dict_rejects_layer_norm_tracr_models(): model.model_config.layer_norm = True with pytest.raises(NotImplementedError, match="layer_norm=True"): - make_tracr_transformer_bridge_state_dict(model, output_label="reverse") + make_tracr_transformer_bridge_state_dict(model, output_label="reverse_1") diff --git a/tests/unit/tools/test_model_registry.py b/tests/unit/tools/test_model_registry.py index 8fbb12c014..4c8b66b883 100644 --- a/tests/unit/tools/test_model_registry.py +++ b/tests/unit/tools/test_model_registry.py @@ -807,6 +807,7 @@ class TestRegistrySyncedWithFactory: "NeelSoluOldForCausalLM", "GPT2LMHeadCustomModel", "TransformerLensNative", + "TransformerLensPretrain", # Group 2: factory-internal alias casings (HF emits the canonical name). "Gemma1ForCausalLM", # HF emits: GemmaForCausalLM "NeoForCausalLM", # HF emits: GPTNeoForCausalLM @@ -873,3 +874,114 @@ def test_canonical_authors_keys_have_a_factory_adapter(self): assert ( not orphaned ), f"CANONICAL_AUTHORS_BY_ARCH entries with no factory adapter: {orphaned}" + + +class TestModelAliases: + """Registry-canonical alias table (data/model_aliases.json).""" + + def test_alias_resolves_to_official_name(self): + from transformer_lens.tools.model_registry.registry_io import ( + resolve_model_alias, + ) + + assert resolve_model_alias("gpt2-small") == "gpt2" + + def test_official_name_and_unknown_return_none(self): + from transformer_lens.tools.model_registry.registry_io import ( + resolve_model_alias, + ) + + assert resolve_model_alias("gpt2") is None + assert resolve_model_alias("no-such-model-xyz") is None + + def test_no_alias_maps_to_two_officials(self): + """Duplicate aliases would make resolution depend on table order.""" + from transformer_lens.tools.model_registry.registry_io import load_model_aliases + + seen: dict[str, str] = {} + for official, aliases in load_model_aliases().items(): + assert aliases, f"{official} has an empty alias list" + for alias in aliases: + assert ( + alias not in seen + ), f"alias {alias!r} under both {seen[alias]!r} and {official!r}" + seen[alias] = official + + def test_legacy_loader_agrees_with_registry(self): + """HookedTransformer and the bridge must resolve aliases identically until 4.0.""" + from transformer_lens.loading_from_pretrained import get_official_model_name + from transformer_lens.tools.model_registry.registry_io import ( + resolve_model_alias, + ) + + assert get_official_model_name("gpt2-small") == resolve_model_alias("gpt2-small") + + +class TestCheckpointLabels: + """Registry-canonical checkpoint schedules (checkpoints.py).""" + + def test_pythia_routes_to_v1_schedule(self): + from transformer_lens.tools.model_registry.checkpoints import ( + PYTHIA_CHECKPOINTS, + get_checkpoint_labels, + ) + + labels, label_type = get_checkpoint_labels("EleutherAI/pythia-70m") + assert labels == PYTHIA_CHECKPOINTS + assert label_type == "step" + + def test_pythia_v0_routes_to_v0_schedule(self): + from transformer_lens.tools.model_registry.checkpoints import ( + PYTHIA_V0_CHECKPOINTS, + get_checkpoint_labels, + ) + + labels, _ = get_checkpoint_labels("EleutherAI/pythia-70m-v0") + assert labels == PYTHIA_V0_CHECKPOINTS + + def test_stanford_crfm_routes_to_crfm_schedule(self): + from transformer_lens.tools.model_registry.checkpoints import ( + STANFORD_CRFM_CHECKPOINTS, + get_checkpoint_labels, + ) + + labels, label_type = get_checkpoint_labels("stanford-crfm/alias-gpt2-small-x21") + assert labels == STANFORD_CRFM_CHECKPOINTS + assert label_type == "step" + + def test_alias_input_is_resolved(self): + from transformer_lens.tools.model_registry.checkpoints import ( + get_checkpoint_labels, + ) + + assert get_checkpoint_labels("pythia-70m") == get_checkpoint_labels("EleutherAI/pythia-70m") + + def test_non_checkpointed_model_raises(self): + from transformer_lens.tools.model_registry.checkpoints import ( + get_checkpoint_labels, + ) + + with pytest.raises(ValueError, match="not checkpointed"): + get_checkpoint_labels("gpt2") + + def test_matches_legacy_loader(self): + """Bridge checkpoint resolution must not drift from the legacy HT path until 4.0.""" + from transformer_lens.loading_from_pretrained import ( + get_checkpoint_labels as legacy_labels, + ) + from transformer_lens.tools.model_registry.checkpoints import ( + get_checkpoint_labels, + ) + + for name in ("EleutherAI/pythia-70m", "stanford-crfm/alias-gpt2-small-x21"): + assert get_checkpoint_labels(name) == legacy_labels(name) + + +class TestRemoteCodePrefixes: + def test_covers_legacy_remote_code_models(self): + """Registry tooling loads legacy-listed models too; prefixes must stay a superset until 4.0.""" + from transformer_lens.loading_from_pretrained import NEED_REMOTE_CODE_MODELS + from transformer_lens.tools.model_registry import REMOTE_CODE_MODEL_PREFIXES + + missing = set(NEED_REMOTE_CODE_MODELS) - set(REMOTE_CODE_MODEL_PREFIXES) + assert not missing, f"legacy remote-code prefixes not in registry list: {missing}" diff --git a/transformer_lens/ActivationCache.py b/transformer_lens/ActivationCache.py index 52c4c11c76..d4454e4c27 100644 --- a/transformer_lens/ActivationCache.py +++ b/transformer_lens/ActivationCache.py @@ -37,7 +37,7 @@ class first, including the examples, and then skimming the available methods. Yo from transformer_lens.utilities import Slice, SliceInput, warn_if_mps if TYPE_CHECKING: - from transformer_lens.HookedTransformer import HookedTransformer + from transformer_lens.model_protocol import TransformerLensModelWithWeights def _normalize_projection_to_2d( @@ -144,8 +144,9 @@ def __init__( has_batch_dim: bool = True, ): self.cache_dict = cache_dict - # Helper methods require HT-internal structure; bridge users only use cache_dict. - self.model = cast("HookedTransformer", model) + # Advanced helpers (LN folding, residual-direction projection) need the + # weight-processing surface; both HookedTransformer and TransformerBridge expose it. + self.model = cast("TransformerLensModelWithWeights", model) self.has_batch_dim = has_batch_dim self.has_embed = "hook_embed" in self.cache_dict self.has_pos_embed = "hook_pos_embed" in self.cache_dict diff --git a/transformer_lens/HookedAudioEncoder.py b/transformer_lens/HookedAudioEncoder.py index a1cbd47eb0..5b58351588 100644 --- a/transformer_lens/HookedAudioEncoder.py +++ b/transformer_lens/HookedAudioEncoder.py @@ -362,6 +362,17 @@ def from_pretrained( **from_pretrained_kwargs: Any, ) -> "HookedAudioEncoder": """Loads in the pretrained weights from huggingface. Currently supports loading weight from HuggingFace BertForMaskedLM. Unlike HookedTransformer, this does not yet do any preprocessing on the model.""" + import warnings + + warnings.warn( + "HookedAudioEncoder.from_pretrained is deprecated and will be removed in a " + "future major release. Use TransformerBridge.boot_transformers(...) instead — " + "HuBERT/Wav2Vec2 are supported via the bridge's audio adapter. See " + "docs/source/content/migrating_to_v3.md.", + DeprecationWarning, + stacklevel=2, + ) + logging.warning( "Support for HuBERT in TransformerLens is currently experimental, until such a time when it has feature " "parity with HookedTransformer and has been tested on real research tasks. Until then, backward " diff --git a/transformer_lens/HookedEncoderDecoder.py b/transformer_lens/HookedEncoderDecoder.py index df466aaa0e..aa08483914 100644 --- a/transformer_lens/HookedEncoderDecoder.py +++ b/transformer_lens/HookedEncoderDecoder.py @@ -551,6 +551,17 @@ def from_pretrained( **from_pretrained_kwargs: Any, ) -> T: """Loads in the pretrained weights from huggingface. Currently supports loading weight from HuggingFace BertForMaskedLM. Unlike HookedTransformer, this does not yet do any preprocessing on the model.""" + import warnings + + warnings.warn( + "HookedEncoderDecoder.from_pretrained is deprecated and will be removed in a " + "future major release. Use TransformerBridge.boot_transformers(...) instead — " + "the bridge supports T5-style encoder-decoder models. See " + "docs/source/content/migrating_to_v3.md.", + DeprecationWarning, + stacklevel=2, + ) + logging.warning( "Support for T5 in TransformerLens is currently experimental, until such a time when it has feature " "parity with HookedTransformer and has been tested on real research tasks. Until then, backward " diff --git a/transformer_lens/HookedTransformer.py b/transformer_lens/HookedTransformer.py index 8d96ce2e09..133eb5dae9 100644 --- a/transformer_lens/HookedTransformer.py +++ b/transformer_lens/HookedTransformer.py @@ -1313,6 +1313,17 @@ def from_pretrained( 3. Global default ("right") first_n_layers: If specified, only load the first n layers of the model. """ + import warnings + + warnings.warn( + "HookedTransformer.from_pretrained is deprecated and will be removed in a " + "future major release. Use TransformerBridge.boot_transformers(...) instead, " + "then call enable_compatibility_mode() for HookedTransformer-equivalent " + "numerics. See docs/source/content/migrating_to_v3.md.", + DeprecationWarning, + stacklevel=2, + ) + if checkpoint_value is not None and checkpoint_label is not None: raise ValueError( "Specify checkpoint_value or checkpoint_label, not both — they are aliases." diff --git a/transformer_lens/conversion_utils/hook_conversion_utils.py b/transformer_lens/conversion_utils/hook_conversion_utils.py index 5c1dd37bbc..d0469f061d 100644 --- a/transformer_lens/conversion_utils/hook_conversion_utils.py +++ b/transformer_lens/conversion_utils/hook_conversion_utils.py @@ -2,8 +2,6 @@ import torch -from transformer_lens.loading_from_pretrained import get_pretrained_model_config - def get_weight_conversion_field_set(weights: dict) -> str: """Creates a formatted string showing how weights are mapped between frameworks. @@ -72,5 +70,7 @@ def model_info(model_name): model_name (str): Name of the pretrained model to analyze (e.g., 'gpt2', 'bert-base-uncased', etc.) """ + from transformer_lens.loading_from_pretrained import get_pretrained_model_config + cfg = get_pretrained_model_config(model_name) model_info_cfg(cfg) diff --git a/transformer_lens/evals.py b/transformer_lens/evals.py index 69bbe6e2ed..4200ea8b94 100644 --- a/transformer_lens/evals.py +++ b/transformer_lens/evals.py @@ -446,7 +446,7 @@ def ioi_eval(model, dataset=None, batch_size=8, num_samples=1000, tokenizer=None """Evaluate the Model on the Indirect Object Identification Task. Args: - model: HookedTransformer model. + model: A HookedTransformer or TransformerBridge model. dataset: PyTorch Dataset that returns a dict with keys "prompt", "IO", and "S". batch_size: Batch size to use. num_samples: Number of samples to use. @@ -533,7 +533,7 @@ def mmlu_eval( Paper: https://arxiv.org/abs/2009.03300 Args: - model: HookedTransformer model to evaluate. + model: A HookedTransformer or TransformerBridge model to evaluate. tokenizer: Tokenizer to use. If None, uses model.tokenizer. subjects: Subject(s) to evaluate on. Can be None (all 57 subjects), a single subject string, or a list of subjects. See :const:`MMLU_SUBJECTS` for valid names. diff --git a/transformer_lens/factories/architecture_adapter_factory.py b/transformer_lens/factories/architecture_adapter_factory.py index 1514e9f870..1e1f96bd0e 100644 --- a/transformer_lens/factories/architecture_adapter_factory.py +++ b/transformer_lens/factories/architecture_adapter_factory.py @@ -74,6 +74,7 @@ Phi3ArchitectureAdapter, PhiArchitectureAdapter, PhiMoEArchitectureAdapter, + PretrainArchitectureAdapter, Qwen2ArchitectureAdapter, Qwen2MoeArchitectureAdapter, Qwen3_5ArchitectureAdapter, @@ -82,7 +83,9 @@ Qwen3MoeArchitectureAdapter, Qwen3NextArchitectureAdapter, QwenArchitectureAdapter, + RavenArchitectureAdapter, RecurrentGemmaArchitectureAdapter, + RWKV7ArchitectureAdapter, SmolLM3ArchitectureAdapter, StableLmArchitectureAdapter, T5ArchitectureAdapter, @@ -170,7 +173,9 @@ "Qwen3NextForCausalLM": Qwen3NextArchitectureAdapter, "Qwen3_5ForCausalLM": Qwen3_5ArchitectureAdapter, "Qwen3_5ForConditionalGeneration": Qwen3_5MultimodalArchitectureAdapter, + "RavenForCausalLM": RavenArchitectureAdapter, "RecurrentGemmaForCausalLM": RecurrentGemmaArchitectureAdapter, + "RWKV7ForCausalLM": RWKV7ArchitectureAdapter, "SmolLM3ForCausalLM": SmolLM3ArchitectureAdapter, "StableLmForCausalLM": StableLmArchitectureAdapter, "T5ForConditionalGeneration": T5ArchitectureAdapter, @@ -181,6 +186,7 @@ "Zamba2ForCausalLM": Zamba2ArchitectureAdapter, "NanoGPTForCausalLM": NanogptArchitectureAdapter, "TransformerLensNative": NativeArchitectureAdapter, + "TransformerLensPretrain": PretrainArchitectureAdapter, "MinGPTForCausalLM": MingptArchitectureAdapter, "GPTNeoForCausalLM": NeoArchitectureAdapter, "GPTNeoXForCausalLM": NeoxArchitectureAdapter, diff --git a/transformer_lens/head_detector.py b/transformer_lens/head_detector.py index e05ad64122..95d4fdc7df 100644 --- a/transformer_lens/head_detector.py +++ b/transformer_lens/head_detector.py @@ -12,7 +12,7 @@ from typing_extensions import Literal, get_args from transformer_lens.ActivationCache import ActivationCache -from transformer_lens.HookedTransformer import HookedTransformer +from transformer_lens.model_protocol import TransformerLensModel from transformer_lens.utilities import is_lower_triangular, is_square HeadName = Literal["previous_token_head", "duplicate_token_head", "induction_head"] @@ -32,7 +32,7 @@ def detect_head( - model: HookedTransformer, + model: TransformerLensModel, seq: Union[str, List[str]], detection_pattern: Union[torch.Tensor, HeadName], heads: Optional[Union[List[LayerHeadTuple], LayerToHead]] = None, diff --git a/transformer_lens/hook_points.py b/transformer_lens/hook_points.py index bbaf10f4d3..1d0c041f9d 100644 --- a/transformer_lens/hook_points.py +++ b/transformer_lens/hook_points.py @@ -440,7 +440,7 @@ def __getattr__(name: str): warnings.warn( "Importing HookedRootModule from transformer_lens.hook_points is " - "deprecated and will be removed in a future release. Import it from " + "deprecated and will be removed in TransformerLens 4.0. Import it from " "transformer_lens (preferred) or transformer_lens.HookedRootModule instead.", DeprecationWarning, stacklevel=2, diff --git a/transformer_lens/loading_from_pretrained.py b/transformer_lens/loading_from_pretrained.py index 0c5ee3ca63..606bd1ebaf 100644 --- a/transformer_lens/loading_from_pretrained.py +++ b/transformer_lens/loading_from_pretrained.py @@ -56,6 +56,11 @@ convert_t5_weights, ) from transformer_lens.supported_models import MODEL_ALIASES, OFFICIAL_MODEL_NAMES +from transformer_lens.tools.model_registry.checkpoints import ( + PYTHIA_CHECKPOINTS, + PYTHIA_V0_CHECKPOINTS, + STANFORD_CRFM_CHECKPOINTS, +) from transformer_lens.utilities.hf_utils import get_rotary_pct_from_config NON_HF_HOSTED_MODEL_NAMES = [ @@ -1756,21 +1761,9 @@ def get_num_params_of_pretrained(model_name: str) -> int: # %% Load checkpointed model state dicts -# The steps for which there are checkpoints in the stanford crfm models -STANFORD_CRFM_CHECKPOINTS = ( - list(range(0, 100, 10)) - + list(range(100, 2000, 50)) - + list(range(2000, 20000, 100)) - + list(range(20000, 400000 + 1, 1000)) -) - -# Linearly spaced checkpoints for Pythia models, taken every 1000 steps. -# Batch size 2,097,152 tokens, so checkpoints every 2.1B tokens -PYTHIA_CHECKPOINTS = [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + list( - range(1000, 143000 + 1, 1000) -) -# Pythia V1 has log-spaced early checkpoints (see line above), but V0 doesn't -PYTHIA_V0_CHECKPOINTS = list(range(1000, 143000 + 1, 1000)) +# Checkpoint schedules (STANFORD_CRFM_CHECKPOINTS, PYTHIA_CHECKPOINTS, +# PYTHIA_V0_CHECKPOINTS) are imported from tools/model_registry/checkpoints.py, +# their canonical home. def get_checkpoint_labels(model_name: str, **kwargs: Any) -> tuple[list[int], str]: diff --git a/transformer_lens/model_bridge/bridge_core.py b/transformer_lens/model_bridge/bridge_core.py index 9637cdadce..ee35b0a0a4 100644 --- a/transformer_lens/model_bridge/bridge_core.py +++ b/transformer_lens/model_bridge/bridge_core.py @@ -3,12 +3,13 @@ import re import warnings -from contextlib import contextmanager -from functools import lru_cache +from contextlib import contextmanager, nullcontext +from functools import lru_cache, partial from typing import ( Any, Callable, Dict, + Iterable, Iterator, List, Literal, @@ -16,6 +17,7 @@ Optional, Tuple, Union, + cast, overload, ) @@ -28,6 +30,7 @@ from transformer_lens.model_bridge.exceptions import StopAtLayerException from transformer_lens.utilities.aliases import resolve_alias from transformer_lens.utilities.lm_utils import lm_cross_entropy_loss +from transformer_lens.utilities.slice import Slice, SliceInput _BLOCK_PATTERN = re.compile("blocks\\.(\\d+)") @@ -121,6 +124,24 @@ def hook_dict(self) -> dict[str, HookPoint]: self._add_aliases_to_hooks(hooks) return hooks + @property + def mod_dict(self) -> Dict[str, Any]: + """Module/hook name -> object, HookedRootModule-compatible. + + Union of the named-module tree and the aliased hook view, so both + canonical (``blocks.0.mlp.hook_out``) and HT-style + (``blocks.0.hook_mlp_out``) names resolve to the same HookPoint. + """ + # BridgeCore is a mixin; concrete bridges (TransformerBridge/RemoteBridge) + # are nn.Modules, so named_modules() is always present at runtime. + mods: Dict[str, Any] = { + name: module + for name, module in cast(torch.nn.Module, self).named_modules() + if name != "" + } + mods.update(self.hook_dict) + return mods + # ---- alias registry ---- def _register_aliases(self) -> None: @@ -365,6 +386,19 @@ def _check_loss_supported(self, return_type: Optional[str]) -> None: # ---- hook lookup / mutation ---- + @staticmethod + def _is_embedding_stage_hook(name: str) -> bool: + """Hooks belonging to the pre-block token/positional embedding stage. + + Excluded from ``start_at_layer`` output: the caller's residual already + carries the embedding, so the embedding stage is logically skipped even + though HF still computes (and discards) it. ``unembed``/``hook_unembed`` + are the output stage and deliberately not matched. + """ + return name in ("hook_embed", "hook_pos_embed", "hook_tokens") or name.startswith( + ("embed.", "pos_embed.") + ) + def _check_hook_fireable(self, *names: str) -> None: """Fail loud when the driver declares it can't fire a requested hook — attaching anyway would yield a silently-unhooked forward / empty cache.""" @@ -404,6 +438,33 @@ def get_hook_point(self, hook_name: str) -> Optional[HookPoint]: pass return None + def check_hooks_to_add( + self, + hook_point: HookPoint, + hook_point_name: str, + hook: Callable, + dir: Literal["fwd", "bwd"] = "fwd", + is_permanent: bool = False, + prepend: bool = False, + ) -> None: + """Validate a hook before it is added; override to add checks. + + No-op by default (mirrors ``HookedRootModule.check_hooks_to_add``). + Driver-fireability is enforced separately in ``_check_hook_fireable``. + """ + + def _add_fn_to_hook_point( + self, + hook_point: HookPoint, + name: str, + hook_fn: Callable, + dir: Literal["fwd", "bwd"], + is_permanent: bool, + ) -> None: + """Run the extension-point check, then attach the hook function.""" + self.check_hooks_to_add(hook_point, name, hook_fn, dir=dir, is_permanent=is_permanent) + hook_point.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) + def add_hook( self, name: Union[str, Callable[[str], bool]], @@ -431,7 +492,7 @@ def add_hook( if hook_id in seen_hooks: continue seen_hooks.add(hook_id) - hook_point.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) + self._add_fn_to_hook_point(hook_point, hook_name, hook_fn, dir, is_permanent) return # Fast path: canonical registry names skip the alias-map build (hook_dict + @@ -439,7 +500,7 @@ def add_hook( registry_hp = self._hook_registry.get(name) if registry_hp is not None: self._check_hook_fireable(name) - registry_hp.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) + self._add_fn_to_hook_point(registry_hp, name, hook_fn, dir, is_permanent) return # Same alias resolution run_with_hooks uses, so HT-style names work here too. canonical = build_alias_to_canonical_map(self.hook_dict).get(name, name) @@ -449,7 +510,7 @@ def add_hook( self._check_hook_fireable(name) registry_hp = self._hook_registry.get(canonical) if registry_hp is not None: - registry_hp.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) + self._add_fn_to_hook_point(registry_hp, canonical, hook_fn, dir, is_permanent) return component: Any = self @@ -463,7 +524,7 @@ def add_hook( if hasattr(component, hook_name): hook_point = getattr(component, hook_name) if isinstance(hook_point, HookPoint): - hook_point.add_hook(hook_fn, dir=dir, is_permanent=is_permanent) + self._add_fn_to_hook_point(hook_point, name, hook_fn, dir, is_permanent) else: raise AttributeError( f"'{hook_name}' is not a hook point. Found object of type: {type(hook_point)} with value: {hook_point}" @@ -485,23 +546,177 @@ def add_perma_hook( """ self.add_hook(name, hook_fn, dir=dir, is_permanent=True) - def reset_hooks(self, clear_contexts: bool = True) -> None: - """Remove all hooks. Registry is canonical; nn.Module children walked additively.""" + def hook_points(self) -> Iterable[HookPoint]: + """All :class:`HookPoint` instances (registry is canonical and complete).""" + return self._hook_registry.values() + + def clear_contexts(self) -> None: + """Clear the stored ``ctx`` on every hook point.""" for hp in self._hook_registry.values(): - hp.remove_hooks() - if hasattr(self, "children"): - from transformer_lens.model_bridge.generalized_components.base import ( - GeneralizedComponent, - ) + hp.clear_context() + + def remove_all_hook_fns( + self, + direction: Literal["fwd", "bwd", "both"] = "both", + including_permanent: bool = False, + level: Optional[int] = None, + ) -> None: + """Remove hook functions from every hook point.""" + for hp in self._hook_registry.values(): + hp.remove_hooks(direction, including_permanent=including_permanent, level=level) + + def reset_hooks( + self, + clear_contexts: bool = True, + direction: Literal["fwd", "bwd", "both"] = "both", + including_permanent: bool = False, + level: Optional[int] = None, + ) -> None: + """Remove hooks from every hook point; mirrors ``HookedRootModule.reset_hooks``. + + The hook registry is canonical and complete (every component's HookPoint + is registered), so a single pass covers the whole model. + + Args: + clear_contexts: Also clear each hook point's stored ``ctx``. + direction: Which direction(s) to remove — ``"fwd"``, ``"bwd"``, or ``"both"``. + including_permanent: If True, also remove hooks added via ``add_perma_hook``. + level: If set, only remove hooks registered at this context level. + """ + if clear_contexts: + self.clear_contexts() + self.remove_all_hook_fns(direction, including_permanent=including_permanent, level=level) + + @staticmethod + def _normalize_names_filter( + names_filter: Optional[Union[str, List[str], Callable[[str], bool]]], + ) -> Callable[[str], bool]: + """Turn None / str / list / callable into a name predicate.""" + if names_filter is None: + return lambda name: True + if isinstance(names_filter, str): + return lambda name: name == names_filter + if isinstance(names_filter, list): + return lambda name: name in names_filter + if callable(names_filter): + return names_filter + raise ValueError("names_filter must be None, a string, a list of strings, or a callable") + + @staticmethod + def _pos_slice_dim(name: str) -> int: + """Position dimension for a hook's activation (see ``run_with_cache``).""" + return -2 if name.endswith(("hook_pattern", "hook_attn_scores")) else 1 + + def get_caching_hooks( + self, + names_filter: Optional[Union[str, List[str], Callable[[str], bool]]] = None, + incl_bwd: bool = False, + device: Any = None, + remove_batch_dim: bool = False, + cache: Optional[dict] = None, + pos_slice: Optional[Union[Slice, SliceInput]] = None, + ) -> Tuple[dict, list, list]: + """Build caching hooks without adding them. Mirrors ``HookedRootModule.get_caching_hooks``. + + Returns ``(cache, fwd_hooks, bwd_hooks)`` where each hook is a + ``(name, hook_fn)`` pair suitable for ``hooks()`` / ``run_with_hooks``. + Activations are keyed by the HookPoint's canonical name; backward hooks + append ``"_grad"``. ``bwd_hooks`` is empty unless ``incl_bwd``. + """ + if cache is None: + cache = {} + pos_slice_obj = Slice.unwrap(pos_slice) + filter_fn = self._normalize_names_filter(names_filter) + + def save_hook(tensor: Any, hook: Any, is_backward: bool = False) -> None: + assert hook.name is not None + key = hook.name + "_grad" if is_backward else hook.name + stored = tensor.detach().to(device) + if remove_batch_dim: + stored = stored[0] + if pos_slice_obj is not None and stored.dim() >= 2: + stored = pos_slice_obj.apply(stored, dim=self._pos_slice_dim(hook.name)) + cache[key] = stored + + fwd_hooks: list = [] + bwd_hooks: list = [] + seen: set = set() + for name, hook_point in self.hook_dict.items(): + if filter_fn(name) and id(hook_point) not in seen: + seen.add(id(hook_point)) + fwd_hooks.append((name, partial(save_hook, is_backward=False))) + if incl_bwd: + bwd_hooks.append((name, partial(save_hook, is_backward=True))) + return cache, fwd_hooks, bwd_hooks + + def add_caching_hooks( + self, + names_filter: Optional[Union[str, List[str], Callable[[str], bool]]] = None, + incl_bwd: bool = False, + device: Any = None, + remove_batch_dim: bool = False, + cache: Optional[dict] = None, + ) -> dict: + """Attach caching hooks to the model (does not run it). Returns the cache dict. + + Mirrors ``HookedRootModule.add_caching_hooks``. The hooks persist until + ``reset_hooks()``. + """ + cache, fwd_hooks, bwd_hooks = self.get_caching_hooks( + names_filter, + incl_bwd=incl_bwd, + device=device, + remove_batch_dim=remove_batch_dim, + cache=cache, + ) + for name, hook_fn in fwd_hooks: + self.add_hook(name, hook_fn, dir="fwd") + for name, hook_fn in bwd_hooks: + self.add_hook(name, hook_fn, dir="bwd") + return cache - def remove_hooks_recursive(module: Any) -> None: - if isinstance(module, GeneralizedComponent): - module.remove_hooks() - if hasattr(module, "children"): - for child in module.children(): - remove_hooks_recursive(child) + def cache_all( + self, + cache: Optional[dict], + incl_bwd: bool = False, + device: Any = None, + remove_batch_dim: bool = False, + ) -> None: + """Deprecated: cache every activation. Use ``run_with_cache`` / ``add_caching_hooks``.""" + warnings.warn( + "cache_all is deprecated; use run_with_cache or add_caching_hooks.", + DeprecationWarning, + stacklevel=2, + ) + self.add_caching_hooks( + names_filter=None, + cache=cache, + incl_bwd=incl_bwd, + device=device, + remove_batch_dim=remove_batch_dim, + ) - remove_hooks_recursive(self) + def cache_some( + self, + cache: Optional[dict], + names: Callable[[str], bool], + incl_bwd: bool = False, + device: Any = None, + remove_batch_dim: bool = False, + ) -> None: + """Deprecated: cache activations matching ``names``. Use ``run_with_cache``.""" + warnings.warn( + "cache_some is deprecated; use run_with_cache or add_caching_hooks.", + DeprecationWarning, + stacklevel=2, + ) + self.add_caching_hooks( + names_filter=names, + cache=cache, + incl_bwd=incl_bwd, + device=device, + remove_batch_dim=remove_batch_dim, + ) def hooks( self, @@ -527,6 +742,7 @@ def add_hook_to_point( name: str, dir: Literal["fwd", "bwd"] = "fwd", ) -> None: + self.check_hooks_to_add(hook_point, name, hook_fn, dir=dir) if self.compatibility_mode and name != hook_point.name: alias_names_list: list = [] if hook_point.name is not None: @@ -580,14 +796,17 @@ def run_with_hooks( clear_contexts: bool = False, return_type: Optional[str] = "logits", stop_at_layer: Optional[int] = None, + start_at_layer: Optional[int] = None, remove_batch_dim: bool = False, **kwargs: Any, ) -> Any: """Run the model with specified forward and backward hooks. ``stop_at_layer`` raises :class:`StopAtLayerException` to stop early - (KV cache cleaned up on stop). ``remove_batch_dim`` squeezes/unsqueezes - the batch dim around hook callbacks (batch_size==1 only). + (KV cache cleaned up on stop). ``start_at_layer`` treats ``input`` as the + residual entering block ``k`` (see :meth:`forward`); hooks on blocks below + ``k`` are skipped to match HookedTransformer. ``remove_batch_dim`` + squeezes/unsqueezes the batch dim around hook callbacks (batch_size==1 only). """ if "names_filter" in kwargs: # **kwargs would silently absorb it; fail loud. @@ -602,6 +821,11 @@ def run_with_hooks( effective_stop_layer = len(self.blocks) + stop_at_layer else: effective_stop_layer = stop_at_layer + effective_start_layer = None + if start_at_layer is not None and hasattr(self, "blocks"): + effective_start_layer = ( + len(self.blocks) + start_at_layer if start_at_layer < 0 else start_at_layer + ) def add_hook_to_point( hook_point: HookPoint, @@ -609,13 +833,19 @@ def add_hook_to_point( name: str, dir: Literal["fwd", "bwd"] = "fwd", ) -> None: - if effective_stop_layer is not None and name.startswith("blocks."): + if effective_start_layer is not None and self._is_embedding_stage_hook(name): + return + if name.startswith("blocks."): try: - layer_num = int(name.split(".")[1]) - if layer_num >= effective_stop_layer: - return + layer_num: Optional[int] = int(name.split(".")[1]) except (IndexError, ValueError): - pass + layer_num = None + if layer_num is not None: + if effective_stop_layer is not None and layer_num >= effective_stop_layer: + return + if effective_start_layer is not None and layer_num < effective_start_layer: + return + self.check_hooks_to_add(hook_point, name, hook_fn, dir=dir) if self.compatibility_mode and name != hook_point.name: alias_names_list: list = [] if hook_point.name is not None: @@ -681,6 +911,8 @@ def wrapped_hook_fn(tensor, hook, _orig_fn=original_hook_fn): try: apply_hooks(fwd_hooks, True) apply_hooks(bwd_hooks, False) + if start_at_layer is not None: + kwargs["start_at_layer"] = start_at_layer try: output = self.forward( input, return_type=return_type, stop_at_layer=stop_at_layer, **kwargs @@ -724,14 +956,26 @@ def run_with_cache( remove_batch_dim: bool = False, names_filter: Optional[Union[str, List[str], Callable[[str], bool]]] = None, stop_at_layer: Optional[int] = None, + start_at_layer: Optional[int] = None, + pos_slice: Optional[Union[Slice, SliceInput]] = None, + incl_bwd: bool = False, **kwargs, ) -> Tuple[Any, Union[ActivationCache, Dict[str, torch.Tensor]]]: """Run the model and cache activations. Returns ``(output, cache)``. ``stop_at_layer`` raises :class:`StopAtLayerException` to stop early. - ``device`` offloads cached activations (matches ``ActivationCache.to``); the - model and inputs stay where the caller put them. + ``start_at_layer`` treats ``input`` as the residual entering block ``k`` + (see :meth:`forward`); blocks below ``k`` are excluded from the cache to + match HookedTransformer. ``pos_slice`` slices each cached activation along + its position dimension (dim 1 for resid/per-head/token-id activations; the + query position ``-2`` for attention patterns/scores). ``incl_bwd`` also + caches gradients under ``"_grad"`` by running ``output.backward()``; + the caller must request a scalar output (``return_type="loss"``) and the + model must be on the gradients-capable transformers driver. ``device`` + offloads cached activations (matches ``ActivationCache.to``); the model and + inputs stay where the caller put them. """ + pos_slice_obj = Slice.unwrap(pos_slice) aliases = build_alias_to_canonical_map(self.hook_dict) def create_names_filter_fn(filter_input): @@ -768,21 +1012,30 @@ def create_names_filter_fn(filter_input): # None → no-op .to(None), tensors stay on their current device. cache_device = kwargs.pop("device", None) + def _store(name: str, value: torch.Tensor) -> None: + stored = value.detach().to(cache_device) + if pos_slice_obj is not None and stored.dim() >= 2: + # Position is dim 1 for every bridge activation (resid [b,p,d], per-head + # [b,p,h,d], token ids [b,p]) except attention patterns/scores, which + # slice the query position at -2 — see _pos_slice_dim. + stored = pos_slice_obj.apply(stored, dim=self._pos_slice_dim(name)) + cache[name] = stored + def make_cache_hook(name: str): def cache_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: if tensor is None: cache[name] = None elif isinstance(tensor, torch.Tensor): - cache[name] = tensor.detach().to(cache_device) + _store(name, tensor) elif isinstance(tensor, tuple): if len(tensor) > 0 and isinstance(tensor[0], torch.Tensor): - cache[name] = tensor[0].detach().to(cache_device) + _store(name, tensor[0]) else: pass else: try: if hasattr(tensor, "detach"): - cache[name] = tensor.detach().to(cache_device) + _store(name, tensor) except: pass return tensor @@ -796,18 +1049,30 @@ def cache_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: effective_stop_layer = len(self.blocks) + stop_at_layer else: effective_stop_layer = stop_at_layer + effective_start_layer = None + if start_at_layer is not None and hasattr(self, "blocks"): + effective_start_layer = ( + len(self.blocks) + start_at_layer if start_at_layer < 0 else start_at_layer + ) matched_any = False for hook_name, hook in hook_dict.items(): if names_filter_fn(hook_name): matched_any = True - if effective_stop_layer is not None: - if hook_name.startswith("blocks."): - try: - layer_num = int(hook_name.split(".")[1]) - if layer_num >= effective_stop_layer: - continue - except (IndexError, ValueError): - pass + if effective_start_layer is not None and self._is_embedding_stage_hook(hook_name): + continue + if hook_name.startswith("blocks."): + try: + layer_num = int(hook_name.split(".")[1]) + except (IndexError, ValueError): + layer_num = None + if layer_num is not None: + # stop/start bound the executed range; blocks outside it + # either don't run (stop) or run on discarded input (start), + # so their activations must not enter the cache. + if effective_stop_layer is not None and layer_num >= effective_stop_layer: + continue + if effective_start_layer is not None and layer_num < effective_start_layer: + continue hooks.append((hook, hook_name)) # Explicit string/list filters matching nothing must not return (logits, {}) silently. if not matched_any and names_filter and isinstance(names_filter, (str, list)): @@ -815,8 +1080,19 @@ def cache_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: f"names_filter {names_filter!r} matched no hook points on this model; " "check the name against model.hook_dict (this backend may not serve it)." ) + + def make_grad_cache_hook(name: str): + def grad_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: + if isinstance(tensor, torch.Tensor): + _store(name + "_grad", tensor) + return tensor + + return grad_hook + for hp, name in hooks: hp.add_hook(make_cache_hook(name)) + if incl_bwd: + hp.add_hook(make_grad_cache_hook(name), dir="bwd") processed_args = [input] # Driver-aware input placement: torch drivers move input_ids to the model's # device; remote drivers (no local parameters) leave them as-is. @@ -864,30 +1140,37 @@ def stop_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: "will remain on their per-layer devices.", stacklevel=2, ) + if start_at_layer is not None: + filtered_kwargs["start_at_layer"] = start_at_layer try: if ( "output_attentions" not in filtered_kwargs and self.adapter.supports_hf_output_attentions ): filtered_kwargs["output_attentions"] = True - if processed_args: - output = self.forward(processed_args[0], **filtered_kwargs) - elif "input_ids" in filtered_kwargs: - output = self.forward( - filtered_kwargs["input_ids"], - **{k: v for k, v in filtered_kwargs.items() if k != "input_ids"}, - ) - else: - output = self.forward(**filtered_kwargs) - if hasattr(output, "logits"): - output = output.logits + # incl_bwd needs grad to build the graph and run backward while the + # bwd cache hooks are still attached (i.e. before the finally below). + with torch.enable_grad() if incl_bwd else nullcontext(): + if processed_args: + output = self.forward(processed_args[0], **filtered_kwargs) + elif "input_ids" in filtered_kwargs: + output = self.forward( + filtered_kwargs["input_ids"], + **{k: v for k, v in filtered_kwargs.items() if k != "input_ids"}, + ) + else: + output = self.forward(**filtered_kwargs) + if hasattr(output, "logits"): + output = output.logits + if incl_bwd: + output.backward() except StopAtLayerException as e: output = e.layer_output except Exception as e: raise e finally: for hp, _ in hooks: - hp.remove_hooks(dir="fwd") + hp.remove_hooks(dir="both" if incl_bwd else "fwd") if self.compatibility_mode == True: reverse_aliases = {} for old_name, new_name in aliases.items(): diff --git a/transformer_lens/model_bridge/generalized_components/__init__.py b/transformer_lens/model_bridge/generalized_components/__init__.py index 1ab4685eb5..5ed5826e2b 100644 --- a/transformer_lens/model_bridge/generalized_components/__init__.py +++ b/transformer_lens/model_bridge/generalized_components/__init__.py @@ -1,4 +1,11 @@ """Bridge components for transformer architectures.""" + +from transformer_lens.model_bridge.generalized_components.alibi_joint_qkv_attention import ( + ALiBiJointQKVAttentionBridge, +) +from transformer_lens.model_bridge.generalized_components.altup_block import ( + AltUpBlockBridge, +) from transformer_lens.model_bridge.generalized_components.attention import ( AttentionBridge, ) @@ -14,9 +21,6 @@ from transformer_lens.model_bridge.generalized_components.bloom_attention import ( BloomAttentionBridge, ) -from transformer_lens.model_bridge.generalized_components.codegen_attention import ( - CodeGenAttentionBridge, -) from transformer_lens.model_bridge.generalized_components.bloom_block import ( BloomBlockBridge, ) @@ -27,6 +31,9 @@ CLIPVisionEncoderBridge, CLIPVisionEncoderLayerBridge, ) +from transformer_lens.model_bridge.generalized_components.codegen_attention import ( + CodeGenAttentionBridge, +) from transformer_lens.model_bridge.generalized_components.conv1d import Conv1DBridge from transformer_lens.model_bridge.generalized_components.conv_pos_embed import ( ConvPosEmbedBridge, @@ -37,24 +44,18 @@ from transformer_lens.model_bridge.generalized_components.embedding import ( EmbeddingBridge, ) -from transformer_lens.model_bridge.generalized_components.alibi_joint_qkv_attention import ( - ALiBiJointQKVAttentionBridge, -) from transformer_lens.model_bridge.generalized_components.gated_delta_net import ( GatedDeltaNetBridge, ) -from transformer_lens.model_bridge.generalized_components.altup_block import ( - AltUpBlockBridge, -) from transformer_lens.model_bridge.generalized_components.gated_mlp import ( GatedMLPBridge, ) -from transformer_lens.model_bridge.generalized_components.glm_moe_dsa_attention import ( - GlmMoeDsaAttentionBridge, -) from transformer_lens.model_bridge.generalized_components.gated_rms_norm import ( GatedRMSNormBridge, ) +from transformer_lens.model_bridge.generalized_components.glm_moe_dsa_attention import ( + GlmMoeDsaAttentionBridge, +) from transformer_lens.model_bridge.generalized_components.joint_gate_up_mlp import ( JointGateUpMLPBridge, ) @@ -72,23 +73,26 @@ MLAAttentionBridge, ) from transformer_lens.model_bridge.generalized_components.mlp import MLPBridge +from transformer_lens.model_bridge.generalized_components.moe import MoEBridge from transformer_lens.model_bridge.generalized_components.mpt_alibi_attention import ( MPTALiBiAttentionBridge, ) -from transformer_lens.model_bridge.generalized_components.moe import MoEBridge from transformer_lens.model_bridge.generalized_components.normalization import ( NormalizationBridge, ) +from transformer_lens.model_bridge.generalized_components.opaque_block import ( + OpaqueBlockBridge, +) from transformer_lens.model_bridge.generalized_components.pos_embed import ( PosEmbedBridge, ) +from transformer_lens.model_bridge.generalized_components.position_embeddings_attention import ( + PositionEmbeddingsAttentionBridge, +) from transformer_lens.model_bridge.generalized_components.qwen3_5_vision_encoder import ( Qwen3_5VisionBlockBridge, Qwen3_5VisionEncoderBridge, ) -from transformer_lens.model_bridge.generalized_components.position_embeddings_attention import ( - PositionEmbeddingsAttentionBridge, -) from transformer_lens.model_bridge.generalized_components.rms_normalization import ( RMSNormalizationBridge, ) @@ -99,9 +103,15 @@ SiglipVisionEncoderBridge, SiglipVisionEncoderLayerBridge, ) -from transformer_lens.model_bridge.generalized_components.ssm2_mixer import SSM2MixerBridge -from transformer_lens.model_bridge.generalized_components.ssm_block import SSMBlockBridge -from transformer_lens.model_bridge.generalized_components.ssm_mixer import SSMMixerBridge +from transformer_lens.model_bridge.generalized_components.ssm2_mixer import ( + SSM2MixerBridge, +) +from transformer_lens.model_bridge.generalized_components.ssm_block import ( + SSMBlockBridge, +) +from transformer_lens.model_bridge.generalized_components.ssm_mixer import ( + SSMMixerBridge, +) from transformer_lens.model_bridge.generalized_components.ssm_protocol import ( SSMMixerProtocol, SSMStateHookMixin, @@ -109,15 +119,15 @@ ) from transformer_lens.model_bridge.generalized_components.symbolic import SymbolicBridge from transformer_lens.model_bridge.generalized_components.t5_block import T5BlockBridge -from transformer_lens.model_bridge.generalized_components.t5gemma_decoder_block import ( - T5GemmaDecoderBlockBridge, -) from transformer_lens.model_bridge.generalized_components.t5gemma2_decoder_block import ( T5Gemma2DecoderBlockBridge, ) from transformer_lens.model_bridge.generalized_components.t5gemma2_merged_attention import ( T5Gemma2MergedAttentionBridge, ) +from transformer_lens.model_bridge.generalized_components.t5gemma_decoder_block import ( + T5GemmaDecoderBlockBridge, +) from transformer_lens.model_bridge.generalized_components.unembedding import ( UnembeddingBridge, ) @@ -175,6 +185,7 @@ "SSMMixerProtocol", "SSMStateHookMixin", "find_ssm_mixer", + "OpaqueBlockBridge", "SSMBlockBridge", "SSMMixerBridge", "VisionProjectionBridge", diff --git a/transformer_lens/model_bridge/generalized_components/block.py b/transformer_lens/model_bridge/generalized_components/block.py index 00151fc0b3..b998c15836 100644 --- a/transformer_lens/model_bridge/generalized_components/block.py +++ b/transformer_lens/model_bridge/generalized_components/block.py @@ -194,6 +194,7 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: ) self._maybe_wire_pre_ln_capture() + args, kwargs = self._maybe_inject_start_residual(args, kwargs) self._check_stop_at_layer(*args, **kwargs) args, kwargs = self._hook_input_hidden_states(args, kwargs) @@ -257,35 +258,57 @@ def _is_standalone_hidden_state_call(args: tuple, kwargs: dict) -> bool: and isinstance(kwargs["hidden_states"], torch.Tensor) ) + def _extract_layer_idx(self) -> Optional[int]: + """Parse this block's layer index from its name (TL/GPT-2/LLaMA patterns).""" + if self.name is None: + return None + match = ( + re.search(r"blocks\.(\d+)", self.name) + or re.search(r"\.h\.(\d+)", self.name) + or re.search(r"\.layers\.(\d+)", self.name) + ) + return int(match.group(1)) if match else None + def _check_stop_at_layer(self, *args: Any, **kwargs: Any) -> None: """Check if execution should stop before this block. Raises StopAtLayerException. The _stop_at_layer_idx attribute is set by the bridge's forward method. Supports TL/GPT-2/LLaMA naming patterns for layer index extraction. """ - if not (hasattr(self, "_stop_at_layer_idx") and self._stop_at_layer_idx is not None): + if getattr(self, "_stop_at_layer_idx", None) is None: return - if self.name is not None: - match = ( - re.search(r"blocks\.(\d+)", self.name) - or re.search(r"\.h\.(\d+)", self.name) - or re.search(r"\.layers\.(\d+)", self.name) - ) - else: - match = None - if match: - layer_idx = int(match.group(1)) - if layer_idx == self._stop_at_layer_idx: - if len(args) > 0 and isinstance(args[0], torch.Tensor): - input_tensor = args[0] - elif "hidden_states" in kwargs and isinstance( - kwargs["hidden_states"], torch.Tensor - ): - input_tensor = kwargs["hidden_states"] - else: - raise ValueError(f"Cannot find input tensor to stop at layer {layer_idx}") - input_tensor = self.hook_in(input_tensor) - raise StopAtLayerException(input_tensor) + layer_idx = self._extract_layer_idx() + if layer_idx is not None and layer_idx == self._stop_at_layer_idx: + if len(args) > 0 and isinstance(args[0], torch.Tensor): + input_tensor = args[0] + elif "hidden_states" in kwargs and isinstance(kwargs["hidden_states"], torch.Tensor): + input_tensor = kwargs["hidden_states"] + else: + raise ValueError(f"Cannot find input tensor to stop at layer {layer_idx}") + input_tensor = self.hook_in(input_tensor) + raise StopAtLayerException(input_tensor) + + def _maybe_inject_start_residual(self, args: tuple, kwargs: dict) -> tuple[tuple, dict]: + """If this is the start_at_layer block, swap in the caller's residual. + + Mirror of ``_check_stop_at_layer``: the bridge's forward stashes the + residual-stream input on the block via ``_start_residual`` and sets + ``_start_at_layer_idx``. This block replaces its incoming hidden states + with that residual; ``_hook_input_hidden_states`` then fires ``hook_in`` + on it, so ``hook_resid_pre`` reflects the injected value. + """ + if getattr(self, "_start_at_layer_idx", None) is None: + return args, kwargs + if self._extract_layer_idx() != self._start_at_layer_idx: + return args, kwargs + residual = getattr(self, "_start_residual", None) + if residual is None: + return args, kwargs + if len(args) > 0 and isinstance(args[0], torch.Tensor): + args = (residual,) + args[1:] + elif "hidden_states" in kwargs: + kwargs = {**kwargs, "hidden_states": residual} + return args, kwargs def _hook_input_hidden_states(self, args: tuple, kwargs: dict) -> tuple[tuple, dict]: """Apply hook_in to the hidden_states input, whether in args or kwargs.""" diff --git a/transformer_lens/model_bridge/generalized_components/moe.py b/transformer_lens/model_bridge/generalized_components/moe.py index 18bbdac7ad..67a677ffc6 100644 --- a/transformer_lens/model_bridge/generalized_components/moe.py +++ b/transformer_lens/model_bridge/generalized_components/moe.py @@ -2,6 +2,7 @@ This module contains the bridge component for Mixture of Experts layers. """ + from __future__ import annotations from typing import Any, Dict, Optional @@ -110,10 +111,24 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: kwargs = {**kwargs, "hidden_states": hooked} output = self.original_component(*args, **kwargs) if isinstance(output, tuple): + if not output: + raise TypeError( + f"{self.name}: expected a non-empty tuple whose first " + "element is a torch.Tensor from the wrapped MoE component, " + "got an empty tuple." + ) + hidden_states = output[0] + if not isinstance(hidden_states, torch.Tensor): + raise TypeError( + f"{self.name}: expected the first tuple element from the " + f"wrapped MoE component to be a torch.Tensor, got " + f"{type(hidden_states).__name__}." + ) if len(output) > 1: router_scores = output[1] - self.hook_router_scores(router_scores) + if isinstance(router_scores, torch.Tensor): + self.hook_router_scores(router_scores) hidden_states = self.hook_out(hidden_states) return (hidden_states,) + output[1:] else: diff --git a/transformer_lens/model_bridge/generalized_components/opaque_block.py b/transformer_lens/model_bridge/generalized_components/opaque_block.py new file mode 100644 index 0000000000..2bfb990fc6 --- /dev/null +++ b/transformer_lens/model_bridge/generalized_components/opaque_block.py @@ -0,0 +1,112 @@ +"""Generic opaque block bridge for non-SSM, non-standard-transformer architectures.""" + +from __future__ import annotations + +import re +from typing import Any, Dict, Optional + +import torch + +from transformer_lens.model_bridge.exceptions import StopAtLayerException +from transformer_lens.model_bridge.generalized_components.base import ( + GeneralizedComponent, +) + + +class OpaqueBlockBridge(GeneralizedComponent): + """Generic block bridge that delegates a full block forward unchanged. + + Exposes ``hook_resid_pre`` / ``hook_resid_post`` (aliases of ``hook_in`` + / ``hook_out``) on the residual stream only — no assumptions are made + about the block's internal structure. + + Use this for architectures whose block internals do not follow a standard + SSM or transformer pre-norm flow, e.g.: + + - Post-residual (sandwich-norm) blocks like Raven / Huginn. + - Attention-free recurrent blocks like RWKV-7. + - Any custom block where ``hook_mixer_in`` / ``hook_mixer_out`` would be + semantically wrong or structurally absent. + + For SSM architectures (Mamba, Falcon-H1) use :class:`SSMBlockBridge`, + which extends this class and adds the ``hook_mixer_in`` / + ``hook_mixer_out`` aliases pointing at ``mixer.hook_in`` / + ``mixer.hook_out``. + """ + + is_list_item: bool = True + hook_out_is_single_residual_stream: bool = True + hook_aliases = { + "hook_resid_pre": "hook_in", + "hook_resid_post": "hook_out", + } + + def __init__( + self, + name: str, + config: Optional[Any] = None, + submodules: Optional[Dict[str, GeneralizedComponent]] = None, + hook_alias_overrides: Optional[Dict[str, str]] = None, + ): + super().__init__( + name, + config, + submodules=submodules if submodules is not None else {}, + hook_alias_overrides=hook_alias_overrides, + ) + + def forward(self, *args: Any, **kwargs: Any) -> Any: + """Delegate to the HF block with hook_in/hook_out wrapped around it.""" + if self.original_component is None: + raise RuntimeError( + f"Original component not set for {self.name}. " + "Call set_original_component() first." + ) + + self._check_stop_at_layer(*args, **kwargs) + args, kwargs = self._hook_input_hidden_states(args, kwargs) + output = self.original_component(*args, **kwargs) + return self._apply_output_hook(output) + + def _apply_output_hook(self, output: Any) -> Any: + """Hook the primary output tensor, preserving tuple structure if present.""" + if isinstance(output, tuple) and len(output) > 0: + first = output[0] + if isinstance(first, torch.Tensor): + first = self.hook_out(first) + return (first,) + output[1:] + return output + if isinstance(output, torch.Tensor): + return self.hook_out(output) + return output + + def _hook_input_hidden_states(self, args: tuple, kwargs: dict) -> tuple[tuple, dict]: + """Hook the hidden_states input whether it arrives positionally or by name.""" + if len(args) > 0 and isinstance(args[0], torch.Tensor): + hooked = self.hook_in(args[0]) + args = (hooked,) + args[1:] + elif "hidden_states" in kwargs and isinstance(kwargs["hidden_states"], torch.Tensor): + kwargs["hidden_states"] = self.hook_in(kwargs["hidden_states"]) + return args, kwargs + + def _check_stop_at_layer(self, *args: Any, **kwargs: Any) -> None: + """Raise StopAtLayerException when the configured stop index matches this block.""" + if not (hasattr(self, "_stop_at_layer_idx") and self._stop_at_layer_idx is not None): + return + if self.name is None: + return + # Mamba uses `.layers.{i}`; `blocks.{i}` is the fallback TL convention. + match = re.search(r"\.layers\.(\d+)", self.name) or re.search(r"blocks\.(\d+)", self.name) + if not match: + return + layer_idx = int(match.group(1)) + if layer_idx != self._stop_at_layer_idx: + return + if len(args) > 0 and isinstance(args[0], torch.Tensor): + input_tensor = args[0] + elif "hidden_states" in kwargs and isinstance(kwargs["hidden_states"], torch.Tensor): + input_tensor = kwargs["hidden_states"] + else: + raise ValueError(f"Cannot find input tensor to stop at layer {layer_idx}") + input_tensor = self.hook_in(input_tensor) + raise StopAtLayerException(input_tensor) diff --git a/transformer_lens/model_bridge/generalized_components/ssm_block.py b/transformer_lens/model_bridge/generalized_components/ssm_block.py index ad9d8c14f9..28d0720fd9 100644 --- a/transformer_lens/model_bridge/generalized_components/ssm_block.py +++ b/transformer_lens/model_bridge/generalized_components/ssm_block.py @@ -1,99 +1,27 @@ """Block container for State Space Model (Mamba) layers: norm → mixer → residual.""" + from __future__ import annotations -import re -from typing import Any, Dict, Optional +from transformer_lens.model_bridge.generalized_components.opaque_block import ( + OpaqueBlockBridge, +) -import torch -from transformer_lens.model_bridge.exceptions import StopAtLayerException -from transformer_lens.model_bridge.generalized_components.base import ( - GeneralizedComponent, -) +class SSMBlockBridge(OpaqueBlockBridge): + """Block bridge for SSM layers (Mamba, Falcon-H1, etc.). + Extends :class:`OpaqueBlockBridge` with SSM-specific hook aliases: -class SSMBlockBridge(GeneralizedComponent): - """Block bridge for SSM layers — direct GeneralizedComponent subclass. + - ``hook_mixer_in`` → ``mixer.hook_in`` + - ``hook_mixer_out`` → ``mixer.hook_out`` - Does not inherit from BlockBridge because BlockBridge's hook_aliases hardcode - transformer-specific names (hook_attn_*, hook_mlp_*, hook_resid_mid). + These aliases are only meaningful for architectures whose blocks contain a + ``mixer`` submodule (SSM or hybrid SSM/attention). For non-SSM architectures + use :class:`OpaqueBlockBridge` directly. """ - is_list_item: bool = True - hook_out_is_single_residual_stream: bool = True hook_aliases = { - "hook_resid_pre": "hook_in", - "hook_resid_post": "hook_out", + **OpaqueBlockBridge.hook_aliases, "hook_mixer_in": "mixer.hook_in", "hook_mixer_out": "mixer.hook_out", } - - def __init__( - self, - name: str, - config: Optional[Any] = None, - submodules: Optional[Dict[str, GeneralizedComponent]] = None, - hook_alias_overrides: Optional[Dict[str, str]] = None, - ): - super().__init__( - name, - config, - submodules=submodules if submodules is not None else {}, - hook_alias_overrides=hook_alias_overrides, - ) - - def forward(self, *args: Any, **kwargs: Any) -> Any: - """Delegate to the HF block with hook_in/hook_out wrapped around it.""" - if self.original_component is None: - raise RuntimeError( - f"Original component not set for {self.name}. " - "Call set_original_component() first." - ) - - self._check_stop_at_layer(*args, **kwargs) - args, kwargs = self._hook_input_hidden_states(args, kwargs) - output = self.original_component(*args, **kwargs) - return self._apply_output_hook(output) - - def _apply_output_hook(self, output: Any) -> Any: - """Hook the primary output tensor, preserving tuple structure if present.""" - if isinstance(output, tuple) and len(output) > 0: - first = output[0] - if isinstance(first, torch.Tensor): - first = self.hook_out(first) - return (first,) + output[1:] - return output - if isinstance(output, torch.Tensor): - return self.hook_out(output) - return output - - def _hook_input_hidden_states(self, args: tuple, kwargs: dict) -> tuple[tuple, dict]: - """Hook the hidden_states input whether it arrives positionally or by name.""" - if len(args) > 0 and isinstance(args[0], torch.Tensor): - hooked = self.hook_in(args[0]) - args = (hooked,) + args[1:] - elif "hidden_states" in kwargs and isinstance(kwargs["hidden_states"], torch.Tensor): - kwargs["hidden_states"] = self.hook_in(kwargs["hidden_states"]) - return args, kwargs - - def _check_stop_at_layer(self, *args: Any, **kwargs: Any) -> None: - """Raise StopAtLayerException when the configured stop index matches this block.""" - if not (hasattr(self, "_stop_at_layer_idx") and self._stop_at_layer_idx is not None): - return - if self.name is None: - return - # Mamba uses `.layers.{i}`; `blocks.{i}` is the fallback TL convention. - match = re.search(r"\.layers\.(\d+)", self.name) or re.search(r"blocks\.(\d+)", self.name) - if not match: - return - layer_idx = int(match.group(1)) - if layer_idx != self._stop_at_layer_idx: - return - if len(args) > 0 and isinstance(args[0], torch.Tensor): - input_tensor = args[0] - elif "hidden_states" in kwargs and isinstance(kwargs["hidden_states"], torch.Tensor): - input_tensor = kwargs["hidden_states"] - else: - raise ValueError(f"Cannot find input tensor to stop at layer {layer_idx}") - input_tensor = self.hook_in(input_tensor) - raise StopAtLayerException(input_tensor) diff --git a/transformer_lens/model_bridge/remote_bridge.py b/transformer_lens/model_bridge/remote_bridge.py index 6b9c5e8b83..50f56d3b5c 100644 --- a/transformer_lens/model_bridge/remote_bridge.py +++ b/transformer_lens/model_bridge/remote_bridge.py @@ -127,6 +127,7 @@ def run_with_hooks( clear_contexts: bool = False, return_type: Optional[str] = "logits", stop_at_layer: Optional[int] = None, + start_at_layer: Optional[int] = None, remove_batch_dim: bool = False, **kwargs: Any, ) -> Any: @@ -138,6 +139,7 @@ def run_with_hooks( "RemoteBridge has no backward pass; bwd_hooks are unsupported." ) self._reject_stop_at_layer(stop_at_layer) + self._reject_start_at_layer(start_at_layer) if fwd_hooks: warnings.warn( "RemoteBridge fwd_hooks fire on already-captured activations (read-only): " @@ -169,9 +171,20 @@ def _reject_stop_at_layer(stop_at_layer: Any) -> None: "always runs the full forward pass." ) + @staticmethod + def _reject_start_at_layer(start_at_layer: Any) -> None: + # Residual-stream injection needs a local `blocks` tree; the remote engine + # only runs a full forward from tokens. + if start_at_layer is not None: + raise NotImplementedError( + "RemoteBridge does not support start_at_layer: the remote engine " + "always runs the full forward pass from tokens." + ) + def run_with_cache(self, *args: Any, **kwargs: Any) -> Any: - """Cache via driver captures; stop_at_layer rejected (full remote forward).""" + """Cache via driver captures; stop/start_at_layer rejected (full remote forward).""" self._reject_stop_at_layer(kwargs.get("stop_at_layer")) + self._reject_start_at_layer(kwargs.get("start_at_layer")) return super().run_with_cache(*args, **kwargs) def to_tokens(self, input: Any, prepend_bos: bool | None = None, truncate: bool = True) -> Any: diff --git a/transformer_lens/model_bridge/sources/_bridge_builder.py b/transformer_lens/model_bridge/sources/_bridge_builder.py index d389bcd8b9..006d3cf07f 100644 --- a/transformer_lens/model_bridge/sources/_bridge_builder.py +++ b/transformer_lens/model_bridge/sources/_bridge_builder.py @@ -127,6 +127,28 @@ "index_head_dim", "index_topk", "q_lora_rank", + # Raven / Huginn (depth-recurrent) + "mean_recurrence", + "mean_backprop_depth", + "n_layers_in_prelude", + "n_layers_in_recurrent_block", + "n_layers_in_coda", + "injection_type", + "qk_bias", + # RWKV-7 (attention-free recurrent, generalized delta-rule time-mixing). + # head_dim is intentionally omitted: it is a read-only alias of d_head on + # TransformerBridgeConfig, so a passthrough setattr would raise. + "num_heads", + "value_dim", + "decay_low_rank_dim", + "gate_low_rank_dim", + "a_low_rank_dim", + "v_low_rank_dim", + "norm_first", + "norm_bias", + "fuse_norm", + "attn_mode", + "hidden_act", ] diff --git a/transformer_lens/model_bridge/sources/transformers/helpers.py b/transformer_lens/model_bridge/sources/transformers/helpers.py index 922c30f5f8..eb88f7d2b8 100644 --- a/transformer_lens/model_bridge/sources/transformers/helpers.py +++ b/transformer_lens/model_bridge/sources/transformers/helpers.py @@ -7,6 +7,8 @@ AutoModelForSeq2SeqLM, ) +from transformer_lens.tools.model_registry.checkpoints import get_checkpoint_labels + def get_hf_model_class_for_architecture(architecture: str): """Pick the correct HuggingFace ``AutoModel*`` class for the architecture.""" @@ -65,8 +67,6 @@ def _resolve_checkpoint_to_revision( f"families: {list(_CHECKPOINT_REVISION_FORMATS.keys())}." ) - from transformer_lens.loading_from_pretrained import get_checkpoint_labels - labels, _ = get_checkpoint_labels(model_name) if checkpoint_value is not None: if checkpoint_value not in labels: diff --git a/transformer_lens/model_bridge/sources/transformers/source.py b/transformer_lens/model_bridge/sources/transformers/source.py index bcfa47f1c8..19caeb45eb 100644 --- a/transformer_lens/model_bridge/sources/transformers/source.py +++ b/transformer_lens/model_bridge/sources/transformers/source.py @@ -23,7 +23,7 @@ determine_architecture_from_hf_config, setup_tokenizer, ) -from transformer_lens.supported_models import MODEL_ALIASES +from transformer_lens.tools.model_registry.registry_io import resolve_model_alias from transformer_lens.utilities import get_device from .helpers import ( @@ -108,13 +108,12 @@ def boot( Returns: The bridge to the loaded model. """ - for official_name, aliases in MODEL_ALIASES.items(): - if model_name in aliases: - logging.warning( - f"DEPRECATED: You are using a deprecated, model_name alias '{model_name}'. TransformerLens will now load the official transformers model name, '{official_name}' instead.\n Please update your code to use the official name by changing model_name from '{model_name}' to '{official_name}'.\nSince TransformerLens v3, all model names should be the official transformers model names.\nThe aliases will be removed in the next version of TransformerLens, so please do the update now." - ) - model_name = official_name - break + official_name = resolve_model_alias(model_name) + if official_name is not None: + logging.warning( + f"DEPRECATED: You are using a deprecated, model_name alias '{model_name}'. TransformerLens will now load the official transformers model name, '{official_name}' instead.\n Please update your code to use the official name by changing model_name from '{model_name}' to '{official_name}'.\nSince TransformerLens v3, all model names should be the official transformers model names.\nThe aliases will be removed in the next version of TransformerLens, so please do the update now." + ) + model_name = official_name if checkpoint_index is not None or checkpoint_value is not None: if revision is not None: raise ValueError( diff --git a/transformer_lens/model_bridge/supported_architectures/__init__.py b/transformer_lens/model_bridge/supported_architectures/__init__.py index aee5182f01..dcec4fdc68 100644 --- a/transformer_lens/model_bridge/supported_architectures/__init__.py +++ b/transformer_lens/model_bridge/supported_architectures/__init__.py @@ -193,6 +193,9 @@ from transformer_lens.model_bridge.supported_architectures.phimoe import ( PhiMoEArchitectureAdapter, ) +from transformer_lens.model_bridge.supported_architectures.pretrain import ( + PretrainArchitectureAdapter, +) from transformer_lens.model_bridge.supported_architectures.qwen import ( QwenArchitectureAdapter, ) @@ -217,9 +220,15 @@ from transformer_lens.model_bridge.supported_architectures.qwen3_next import ( Qwen3NextArchitectureAdapter, ) +from transformer_lens.model_bridge.supported_architectures.raven import ( + RavenArchitectureAdapter, +) from transformer_lens.model_bridge.supported_architectures.recurrent_gemma import ( RecurrentGemmaArchitectureAdapter, ) +from transformer_lens.model_bridge.supported_architectures.rwkv7 import ( + RWKV7ArchitectureAdapter, +) from transformer_lens.model_bridge.supported_architectures.smollm3 import ( SmolLM3ArchitectureAdapter, ) @@ -307,6 +316,7 @@ "PhiArchitectureAdapter", "Phi3ArchitectureAdapter", "PhiMoEArchitectureAdapter", + "PretrainArchitectureAdapter", "QwenArchitectureAdapter", "Qwen2ArchitectureAdapter", "Qwen2MoeArchitectureAdapter", @@ -315,7 +325,9 @@ "Qwen3NextArchitectureAdapter", "Qwen3_5ArchitectureAdapter", "Qwen3_5MultimodalArchitectureAdapter", + "RavenArchitectureAdapter", "RecurrentGemmaArchitectureAdapter", + "RWKV7ArchitectureAdapter", "SmolLM3ArchitectureAdapter", "StableLmArchitectureAdapter", "T5ArchitectureAdapter", diff --git a/transformer_lens/model_bridge/supported_architectures/pretrain.py b/transformer_lens/model_bridge/supported_architectures/pretrain.py new file mode 100644 index 0000000000..20091b65ff --- /dev/null +++ b/transformer_lens/model_bridge/supported_architectures/pretrain.py @@ -0,0 +1,442 @@ +"""Architecture adapter for a lightweight decoder-only pretraining model. + +Maps a decoder-only transformer using RoPE, RMSNorm, gated SwiGLU MLPs, and +optional sparse mixture-of-experts feed-forward layers into +TransformerBridge, by wrapping the source module and delegating to its own +`forward` rather than translating parameters into a second implementation. + +Usage: `build_pretrain_bridge(model, cfg)` -- the public entry point. +`PretrainModelContainer` and direct `build_bridge_from_module` use are +internal/advanced details (see `PretrainModelContainer`'s docstring). + +Scope: maps a live module into TransformerBridge. Does not load +checkpoints, merge tensor-parallel shards, or depend on a training +framework. + +Required module protocol -- "lightweight decoder-only pretraining models" +describes intent, not a generality guarantee. The wrapped model must +expose: + + model.embed (embedding lookup) + model.blocks[i].norm1 (pre-attention norm) + model.blocks[i].attn (called as attn(x, ...)) + model.blocks[i].norm2 (pre-MLP norm) + model.blocks[i].mlp (gate/up/down, or router/experts) + model.norm_f (final norm) + model.lm_head (unembedding) + +`gate`/`up`/`down` and `router`/`experts` name the supported protocol. +`DenseOrMoEFeedForwardBridge` checks these structurally -- attribute +presence plus basic type (each is a module, `experts` is a registered +module collection) -- and raises clearly on a mismatch, but that is +structural validation only: it does not and cannot validate that a +module satisfying the shape actually implements matching forward +semantics. Blocks must take more than the bare hidden state (this target +passes `cos`/`sin`) -- see `PretrainModelContainer`. +""" +from __future__ import annotations + +from typing import Any + +import torch + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.bridge import TransformerBridge +from transformer_lens.model_bridge.generalized_components import ( + AttentionBridge, + DelegatedAttentionBlockBridge, + EmbeddingBridge, + GatedMLPBridge, + LinearBridge, + MoEBridge, + RMSNormalizationBridge, + UnembeddingBridge, +) +from transformer_lens.model_bridge.generalized_components.base import ( + GeneralizedComponent, +) + +ARCHITECTURE_NAME = "TransformerLensPretrain" + +# Reserved bridge kwargs removed before forwarding to the wrapped model. +# Only known bridge-compatibility kwargs are stripped so genuine caller +# mistakes (e.g. `target=` vs `targets=`) still raise naturally, and no +# signature introspection is needed to support an arbitrary forward. +_BRIDGE_COMPAT_KWARGS = frozenset({"output_attentions"}) + + +class DenseOrMoEFeedForwardBridge(GeneralizedComponent): + """Wraps either a dense SwiGLU MLP or a sparse MoE layer behind a + common interface. Dispatch is determined by structural inspection + (`router`/`experts` vs `gate`/`up`/`down`) rather than configuration, + so dense, MoE, and mixed architectures all use the same component + mapping. This identifies the supported protocol -- it does not + validate that a module merely sharing those attribute names actually + implements the matching forward behavior. + + `hasattr` alone would let a module with, say, both `router`/`experts` + and `gate`/`up`/`down` (or `router`/`experts` of the wrong types) win + the MoE branch by attribute-name coincidence and fail later with a + confusing error from deep inside `MoEBridge`, or not fail until + forward time. `set_original_component` therefore also checks the + basic shape of whichever protocol wins: `router`/`gate`/`up`/`down` + must themselves be modules, and `experts` must be a *registered* + module collection (`nn.ModuleList`/`nn.ModuleDict`) -- a plain Python + list/tuple of `nn.Module` experts is rejected even though every + element is itself a valid module, because modules held in an + ordinary list aren't registered as children and would silently drop + out of `parameters()`/`state_dict()`/`.to(...)`/`train()`/`eval()`, + contradicting this adapter's lifecycle guarantees. + """ + + def __init__(self, name: str, config: Any): + super().__init__(name, config=config, submodules={}) + self._delegate: GeneralizedComponent | None = None + + def set_original_component(self, component: torch.nn.Module) -> None: + super().set_original_component(component) + # This subclass's own __init__ takes `name: str` (non-optional), so + # self.name is always a str here -- but the base GeneralizedComponent + # attribute is typed `str | None`, which is all mypy sees without this + # narrowing. MoEBridge/GatedMLPBridge both require a plain `str` name. + assert self.name is not None + if hasattr(component, "router") and hasattr(component, "experts"): + if not isinstance(component.router, torch.nn.Module): + raise TypeError( + f"{type(component).__name__}.router must be an nn.Module; " + f"got {type(component.router).__name__}." + ) + if not isinstance(component.experts, (torch.nn.ModuleList, torch.nn.ModuleDict)): + raise TypeError( + f"{type(component).__name__}.experts must be a registered " + "module collection (nn.ModuleList or nn.ModuleDict); got " + f"{type(component.experts).__name__}." + ) + delegate: GeneralizedComponent = MoEBridge( + name=self.name, + config=self.config, + submodules={"gate": LinearBridge(name="router")}, + ) + elif hasattr(component, "gate") and hasattr(component, "up") and hasattr(component, "down"): + for field in ("gate", "up", "down"): + value = getattr(component, field) + if not isinstance(value, torch.nn.Module): + raise TypeError( + f"{type(component).__name__}.{field} must be an " + f"nn.Module; got {type(value).__name__}." + ) + delegate = GatedMLPBridge( + name=self.name, + config=self.config, + submodules={ + "gate": LinearBridge(name="gate"), + "in": LinearBridge(name="up"), + "out": LinearBridge(name="down"), + }, + ) + else: + raise ValueError( + f"Block.mlp is a {type(component).__name__} with neither " + "an MoE layer's (router, experts) nor a gated MLP's " + "(gate, up, down) attributes -- this adapter doesn't know " + "how to wrap it." + ) + delegate.set_original_component(component) + # Not `self._delegate = delegate`: normal registration would + # duplicate hook_in/hook_out under a nested `._delegate.` path, + # risking a broad hook selector firing twice. + # `_delegate` is an execution helper, absent from named_modules() + # -- safe since parameters/state_dict/dtype are read from the raw + # wrapped model (see PretrainModelContainer), not this tree. + object.__setattr__(self, "_delegate", delegate) + + def forward(self, *args: Any, **kwargs: Any) -> Any: + assert self._delegate is not None, f"{self.name}: original component not set" + if args: + args = (self.hook_in(args[0]),) + args[1:] + elif "hidden_states" in kwargs: + kwargs = {**kwargs, "hidden_states": self.hook_in(kwargs["hidden_states"])} + output = self._delegate(*args, **kwargs) + + if isinstance(output, tuple): + if len(output) == 0: + raise TypeError( + "DenseOrMoEFeedForwardBridge expected a non-empty tuple " + "whose first element is a torch.Tensor" + ) + + first = output[0] + + if not isinstance(first, torch.Tensor): + raise TypeError( + "DenseOrMoEFeedForwardBridge expected the first tuple element " + f"to be a torch.Tensor, got {type(first).__name__}" + ) + + hooked_first = self.hook_out(first) + + # Preserve every auxiliary element without sending it through HookPoint. + return (hooked_first, *output[1:]) + + if not isinstance(output, torch.Tensor): + raise TypeError( + "DenseOrMoEFeedForwardBridge expected a torch.Tensor or a tuple " + f"whose first element is a torch.Tensor, got {type(output).__name__}" + ) + + return self.hook_out(output) + + +class _LogitsAttrDict(dict): + """Makes a plain dict's keys accessible as attributes (`d.logits` reads + `d["logits"]`), so a source model's plain-dict forward output satisfies + the `hasattr(output, "logits")` contract `TransformerBridge` expects. + Behaves as a plain dict everywhere else (indexing, `.get`, `in`, ...). + """ + + def __getattr__(self, key: str) -> Any: + try: + return self[key] + except KeyError as e: + raise AttributeError(key) from e + + +class PretrainModelContainer(torch.nn.Module): + """Internal detail -- `build_pretrain_bridge` applies this + automatically. Three responsibilities, all in this container's own + `forward`: + + 1. Avoids a `TransformerBridge.__getattr__` collision: a source + model's own `self.embed`/`self.blocks` clashes with identically + named component_mapping keys. Wrapping one level deeper + (`container.inner.embed`) fixes this without touching the source + model. + 2. Normalizes the return value to the `.logits` contract + `TransformerBridge` expects: a plain `"logits"` dict (the target + architecture's actual shape) is wrapped in `_LogitsAttrDict`; a bare + tensor, tensor-first tuple, or object already exposing `.logits` + passes through after validating the tensor is present and is a + tensor; anything else raises immediately with a clear message. + 3. Strips `_BRIDGE_COMPAT_KWARGS` from kwargs before calling the + wrapped model (see that constant's comment). + + Also sidesteps a `BlockBridge` convention where a bare-tensor block + output gets wrapped in a 1-tuple for "standalone hidden_states calls": + since this target's blocks take `cos`/`sin` too, that path never + triggers, so the source forward loop needs no changes to be bridged. + + `self.inner` is a regular registered submodule, so + `container.train()`/`.eval()` already recurse into it via the normal + `nn.Module` traversal -- no override needed here. The propagation gap + lives one level up, at `TransformerBridge` itself (see + `build_pretrain_bridge`), whose `.train()`/`.eval()` do not walk down + to `original_model`. + """ + + def __init__(self, model: torch.nn.Module) -> None: + super().__init__() + self.inner = model + + def forward(self, *args: Any, **kwargs: Any) -> Any: + filtered = {k: v for k, v in kwargs.items() if k not in _BRIDGE_COMPAT_KWARGS} + output = self.inner(*args, **filtered) + + # Already-normalized or already-HF-style outputs pass through + # unchanged, but only after checking .logits is actually a tensor + # -- an object merely exposing the attribute isn't enough. + if isinstance(output, _LogitsAttrDict): + if "logits" not in output: + raise ValueError( + f"{type(self.inner).__name__}.forward returned a " + "_LogitsAttrDict without a 'logits' key." + ) + if not isinstance(output["logits"], torch.Tensor): + raise TypeError( + f"{type(self.inner).__name__}.forward returned a " + f"_LogitsAttrDict with a non-tensor 'logits' value: " + f"{type(output['logits']).__name__}." + ) + return output + + # try/except, not hasattr(): hasattr() would evaluate a + # property-backed .logits once, then a separate read would + # evaluate it again. This reads it exactly once. + try: + logits = output.logits + except AttributeError: + pass + else: + if not isinstance(logits, torch.Tensor): + raise TypeError( + f"{type(self.inner).__name__}.forward returned a " + f"{type(output).__name__} with a non-tensor .logits value: " + f"{type(logits).__name__}." + ) + return output + + if isinstance(output, torch.Tensor): + return output + + # TransformerBridge extracts output[0] as logits for tuple returns. + if isinstance(output, tuple): + if output and isinstance(output[0], torch.Tensor): + return output + raise TypeError( + f"{type(self.inner).__name__}.forward returned a tuple whose " + f"first element is a {type(output[0]).__name__ if output else 'empty tuple'}, " + "not a torch.Tensor -- TransformerBridge extracts output[0] as " + "logits for tuple returns, so it must be a tensor." + ) + + # The primary case: a plain dict, normalized into _LogitsAttrDict. + if isinstance(output, dict): + if "logits" not in output: + raise ValueError( + f"{type(self.inner).__name__}.forward returned a dict with keys " + # list(), not sorted(): sorted() raises TypeError on + # heterogeneous keys, which would mask this error. + f"{list(output.keys())}, but PretrainModelContainer requires a " + "'logits' key -- without it, TransformerBridge's own " + "hasattr(output, 'logits') check would silently fail the same " + "way this container exists to prevent." + ) + if not isinstance(output["logits"], torch.Tensor): + raise TypeError( + f"{type(self.inner).__name__}.forward returned a dict whose " + f"'logits' value is a {type(output['logits']).__name__}, not a " + "torch.Tensor." + ) + return _LogitsAttrDict(output) + + raise TypeError( + f"{type(self.inner).__name__}.forward must return a torch.Tensor, a " + "tuple whose first element is a tensor, a dict containing a " + "tensor-valued 'logits' key, or an object with a tensor-valued " + f".logits attribute; got {type(output).__name__}." + ) + + +class NativeForwardAttentionBridge(AttentionBridge): + """Opaque attention bridge that delegates to the source attention. + + This adapter intentionally exposes only input/output attention hooks. + It has no mapped Q/K/V/O projection components, so the standard + per-head aliases and weight aliases do not apply. + """ + + hook_aliases = {} + property_aliases = {} + supports_split_qkv_fork = False + + +class PretrainArchitectureAdapter(ArchitectureAdapter): + """Adapter for a decoder-only transformer using RoPE, RMSNorm, gated + SwiGLU MLPs, and optional sparse MoE feed-forward layers. + + Uses an opaque `NativeForwardAttentionBridge` with no attention + projection submodules, not `JointQKVAttentionBridge`/ + `PositionEmbeddingsAttentionBridge`: those reimplement RoPE via HF's + rotate-half convention, wrong for a source model using the + adjacent-pair convention. The opaque bridge delegates unchanged to + `Attention.forward`, so RoPE runs as written -- at the cost of no + per-head hooks, only block-level + `resid_pre`/`resid_mid`/`resid_post`. + + Blocks use `DelegatedAttentionBlockBridge` rather than plain + `BlockBridge`: that existing abstraction already exists for + architectures where attention is delegated wholesale and the + split-qkv-fork block-level aliases (`hook_attn_in`/`hook_q_input`/ + `hook_k_input`/`hook_v_input`) don't apply. It complements + `NativeForwardAttentionBridge.supports_split_qkv_fork = False` (which + prevents the split-QKV-fork machinery and its associated HookPoints + from being exposed for this attention component) by also removing the + now-dangling block-level aliases that would otherwise point at them. + `hook_attn_out` is untouched by either change, since the attention + component still fires its own `hook_out` normally. + + `self.cfg` is mutated in place, not copied (matches `nanogpt.py`'s + convention) -- callers holding another reference to the same config + will see these fields change. + + Bridges built through `build_pretrain_bridge` are given a + mode-propagating subclass so `.train()`/`.eval()` reach the wrapped + source model (see that function's docstring) -- this adapter class + itself has no lifecycle behavior of its own. + """ + + def __init__(self, cfg: Any) -> None: + super().__init__(cfg) + + self.cfg.normalization_type = "RMS" + self.cfg.positional_embedding_type = "rotary" + self.cfg.final_rms = True + self.cfg.gated_mlp = True + self.cfg.attn_only = False + + self.component_mapping = { + # "inner." because this adapter expects the source model to + # arrive wrapped in `PretrainModelContainer` -- see that + # class's docstring for why. + "embed": EmbeddingBridge(name="inner.embed"), + "blocks": DelegatedAttentionBlockBridge( + name="inner.blocks", + config=self.cfg, + submodules={ + "ln1": RMSNormalizationBridge(name="norm1", config=self.cfg), + "attn": NativeForwardAttentionBridge( + name="attn", + config=self.cfg, + submodules={}, # opaque wrap -- see class docstring + ), + "ln2": RMSNormalizationBridge(name="norm2", config=self.cfg), + "mlp": DenseOrMoEFeedForwardBridge(name="mlp", config=self.cfg), + }, + ), + "ln_final": RMSNormalizationBridge(name="inner.norm_f", config=self.cfg), + "unembed": UnembeddingBridge(name="inner.lm_head"), + } + + +def build_pretrain_bridge( + model: torch.nn.Module, + cfg: TransformerBridgeConfig, + *, + device: Any = None, + dtype: torch.dtype | None = None, + model_name: str | None = None, +) -> TransformerBridge: + """Public entry point: wraps `model` in `PretrainModelContainer` and + builds a `TransformerBridge` around it. Prefer this over calling + `build_bridge_from_module` directly -- the container is easy to forget. + + `device`/`dtype`/`model_name` forward to `build_bridge_from_module` + only when explicitly given. + + `bridge.train()`/`.eval()` propagate to `model` via + `TransformerBridge.train()` itself, which sets mode on + `original_model` in addition to the registered module tree + (`original_model` is deliberately not a registered submodule, so + `nn.Module.train()`'s own recursion never reaches it). This adapter + needs nothing extra for mode propagation. + + Setting mode on `model` directly still works too and stays in sync. + """ + from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_from_module, + ) + + kwargs: dict[str, Any] = {} + if device is not None: + kwargs["device"] = device + if dtype is not None: + kwargs["dtype"] = dtype + if model_name is not None: + kwargs["model_name"] = model_name + + return build_bridge_from_module( + PretrainModelContainer(model), + architecture=ARCHITECTURE_NAME, + tl_config=cfg, + **kwargs, + ) diff --git a/transformer_lens/model_bridge/supported_architectures/raven.py b/transformer_lens/model_bridge/supported_architectures/raven.py new file mode 100644 index 0000000000..28fc3836f1 --- /dev/null +++ b/transformer_lens/model_bridge/supported_architectures/raven.py @@ -0,0 +1,303 @@ +"""Raven / Huginn architecture adapter (RavenForCausalLM). + +Model family: tomg-group-umd/huginn-0125 ("Huginn"), a depth-recurrent +("latent reasoning") decoder from Geiping et al. Loaded via remote code +(``auto_map`` → ``raven_modeling_minimal.RavenForCausalLM``), so +``trust_remote_code=True`` is required. + +Architecture overview +--------------------- +Huginn is NOT a flat stack of transformer layers. Its forward has three +phases, all operating on the same residual width ``n_embd`` (5280): + + wte → prelude (P physical blocks) + → [ recurrent core: R physical blocks, applied N times ] + → coda (C physical blocks) → ln_f → lm_head + +with P = ``n_layers_in_prelude`` (2), R = ``n_layers_in_recurrent_block`` +(4), C = ``n_layers_in_coda`` (2). ``num_hidden_layers`` (8) counts the +*physical* blocks (2 + 4 + 2), stored as three separate ``ModuleList``s under +``model.transformer`` (``prelude`` / ``core_block`` / ``coda``), NOT one +``model.layers``. + +The recurrence is the defining feature. ``RavenForCausalLM.forward`` calls +``iterate_forward`` → ``core_block_forward``, which runs the SAME four +``core_block`` modules N times. Each step re-injects the prelude output: + + x = adapter(cat([latent_state, prelude_output], dim=-1)) # injection + for block in core_block: x = block(x) # 4 blocks + +``N`` (``num_steps``) is a RUNTIME argument to ``forward``, not a fixed +config value. At eval it defaults to ``config.mean_recurrence`` (32) via +``randomized_iteration_sampler``; a caller may pass any ``num_steps``. + +Each block is a ``SandwichBlock``: four RMSNorms with post-residual +normalisation (``x = norm_2(attn(norm_1(x)) + x)``; ``x = norm_4(mlp(norm_3 +(x)) + x)``) — the residual stream itself is renormalised after each add, +unlike a standard pre-norm transformer. Attention is MHA (55 heads == 55 kv +heads) with a COMBINED ``Wqkv`` projection plus a learned additive ``qk_bias`` +parameter and RoPE (base 50000). The MLP is a gated SiLU MLP with a combined +gate+up ``fc`` projection. Embeddings are scaled by ``√n_embd`` (≈72.66) in +the HF forward, and ``lm_head`` is tied to ``wte``. + +Key adapter decisions +--------------------- +1. Full delegation, like Ouro. The recurrence, the prelude re-injection, the + emb-scale, the sandwich norms and RoPE all live inside the remote-code + ``forward`` that the bridge delegates to, so a single forward pass is + numerically correct with no loop handling here. + +2. ``OpaqueBlockBridge`` for all three block lists (``prelude`` / ``core_block`` + / ``coda``): ``BlockBridge``'s hook aliases hardcode a standard pre-norm flow + (hook_resid_mid, ln1→attn→ln2→mlp) that the post-residual ``SandwichBlock`` + does not follow. ``OpaqueBlockBridge`` delegates the whole block and exposes + only ``hook_in`` / ``hook_out`` on the residual stream, which is correct + regardless of the internal norm placement. Each block's inner attn / mlp / + norms are still declared as submodules so they wrap the live HF modules and + their hooks fire. + +3. Recurrent core hooks. Because the core loop lives inside the HF forward, + the four ``core_block`` blocks' ``hook_in`` / ``hook_out`` fire once PER + recurrence step (N times per forward), and ``run_with_cache`` keeps the + final step. This is the load-bearing behavioural difference from a flat + decoder and mirrors Ouro's looped-depth semantics. Separately addressing + an individual recurrence step (e.g. logit-lens across steps) is NOT + expressible through the static ``core_block.{i}.hook_out`` names — it + requires the model's native ``iterate_one_step`` / ``predict_from_latents`` + interface. Deliberately not mapped: ``transformer.adapter`` (the injection + Linear) and ``HuginnDynamicCache``'s block-indexed slot layout. + +4. ``applicable_phases = []``. Huginn diverges too far from the transformer- + shaped ``verify_models`` phases to score meaningfully: post-residual + sandwich norms, a runtime recurrence count, combined QKV + ``qk_bias``, + and — decisively — a RANDOM initial latent state (``initialize_state`` + uses ``torch.randn_like``), which makes the forward non-deterministic + across calls unless the RNG is seeded or ``input_states`` is supplied. + Correctness is covered in the integration tests, where the seed is pinned + before both the bridge and HF calls (same decision spirit as nemotron_h / + mamba). ``ln_f`` is applied mid-network (after the recurrence, feeding the + coda) as well as at the end, so ``supports_fold_ln = False`` — folding it + into ``W_U`` would corrupt the coda input. + +Config propagation +------------------ +The recurrence-shape attributes (``mean_recurrence``, ``n_layers_in_prelude`` +/ ``_recurrent_block`` / ``_coda``, ``mean_backprop_depth``, ``injection_type``, +``qk_bias``) are surfaced on ``self.cfg`` here AND added to the two +``_HF_PASSTHROUGH_ATTRS`` lists so analysis tooling can read them off a booted +bridge. + +Remote-code loading (transformers v5) +------------------------------------ +Huginn's remote code targets transformers 4.44 and breaks under v5 (5.8.1) in +two ways that ``prepare_loading`` patches: (1) ``_tied_weights_keys`` is a list, +but v5's ``tie_weights`` expects a dict, so the model does not even construct; +(2) v5's meta-device load re-invokes ``PreTrainedModel._init_weights`` on +already-materialised modules and would re-randomise the checkpoint. See +``prepare_loading`` for both. (Huginn does not use ``ROPE_INIT_FUNCTIONS``; it +precomputes its own ``freqs_cis``, so no RoPE patch is needed.) + +Optional parameters (may be absent from a state_dict) +---------------------------------------------------- +Huginn has ``bias=False`` everywhere except the attention ``qk_bias`` +parameter; RMSNorm has no bias. Weight processing must tolerate missing +biases via ``ProcessWeights._safe_get_tensor()``. +""" + +import sys +from typing import Any + +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.generalized_components import ( + EmbeddingBridge, + LinearBridge, + MLPBridge, + OpaqueBlockBridge, + RMSNormalizationBridge, + UnembeddingBridge, +) +from transformer_lens.model_bridge.generalized_components.attention import ( + AttentionBridge, +) + + +class RavenArchitectureAdapter(ArchitectureAdapter): + """Architecture adapter for RavenForCausalLM (Huginn depth-recurrent decoder). + + Prelude / weight-tied recurrent core / coda phases over a shared residual + width. The recurrence and prelude re-injection live inside the remote-code + HF forward, which the bridge delegates to; see the module docstring for the + full set of adapter decisions. + """ + + # Huginn is off the transformer-shaped verify_models path: post-residual + # sandwich norms, runtime recurrence count, and a random initial latent + # state make the phases non-meaningful. Correctness lives in the + # integration tests (seed pinned before bridge and HF calls). + applicable_phases: list[int] = [] + + def __init__(self, cfg: Any) -> None: + """Initialize the Raven / Huginn architecture adapter.""" + super().__init__(cfg) + + # Standard weight-processing / norm flags. + self.cfg.normalization_type = "RMS" + self.cfg.uses_rms_norm = True + self.cfg.positional_embedding_type = "rotary" + self.cfg.final_rms = True + self.cfg.gated_mlp = True + self.cfg.attn_only = False + + # ln_f (transformer.ln_f) is applied after the recurrence (feeding the + # coda) AND at the very end, so it is not a final-only norm. Folding it + # into W_U would corrupt the coda's input. + self.supports_fold_ln = False + + # Surface the recurrence-shape attributes on cfg so they are present on + # both the HF-boot path (also via _HF_PASSTHROUGH_ATTRS) and the + # synthetic-config path used by the unit tests. + setattr(self.cfg, "mean_recurrence", getattr(cfg, "mean_recurrence", 32)) + setattr(self.cfg, "mean_backprop_depth", getattr(cfg, "mean_backprop_depth", 8)) + setattr(self.cfg, "n_layers_in_prelude", getattr(cfg, "n_layers_in_prelude", 2)) + setattr( + self.cfg, "n_layers_in_recurrent_block", getattr(cfg, "n_layers_in_recurrent_block", 4) + ) + setattr(self.cfg, "n_layers_in_coda", getattr(cfg, "n_layers_in_coda", 2)) + setattr(self.cfg, "injection_type", getattr(cfg, "injection_type", "linear")) + setattr(self.cfg, "qk_bias", getattr(cfg, "qk_bias", True)) + + # Full delegation to the HF forward — no HT-format weight reshaping. + self.weight_processing_conversions = {} + + self.component_mapping = { + "embed": EmbeddingBridge(name="transformer.wte"), + # Three separate physical block lists. Each uses OpaqueBlockBridge so + # the delegated SandwichBlock forward keeps its post-residual norm + # placement while hook_in / hook_out wrap the residual stream. Fresh + # submodule instances per list (they bind to distinct HF modules). + "prelude": OpaqueBlockBridge( + name="transformer.prelude", + submodules=self._sandwich_submodules(), + ), + "core_block": OpaqueBlockBridge( + name="transformer.core_block", + submodules=self._sandwich_submodules(), + ), + "coda": OpaqueBlockBridge( + name="transformer.coda", + submodules=self._sandwich_submodules(), + ), + "ln_final": RMSNormalizationBridge(name="transformer.ln_f", config=self.cfg), + "unembed": UnembeddingBridge(name="lm_head"), + } + + def _sandwich_submodules(self) -> dict[str, Any]: + """Build a fresh set of SandwichBlock submodule bridges. + + Returns new instances on every call so each of the three block lists + wraps its own live HF modules rather than sharing bridge objects. + + Submodule keys mirror the HF attribute names (``norm_1``..``norm_4``, + ``attn``, ``mlp``) so weight-key translation is identity. Attention is + native (combined ``Wqkv`` + ``qk_bias``, RoPE, custom SDPA path), so it + is delegated via ``maintain_native_attention``; only the combined + ``qkv`` and output ``o`` projections are exposed. The gated MLP is + likewise delegated with its combined gate+up ``fc`` and output ``proj``. + """ + attn = AttentionBridge( + name="attn", + config=self.cfg, + submodules={ + "qkv": LinearBridge(name="Wqkv"), + "o": LinearBridge(name="proj"), + }, + maintain_native_attention=True, + requires_attention_mask=True, + ) + # Raven uses a combined Wqkv projection; no separate q/k/v submodules exist. + # Strip the default hook_q/hook_k/hook_v aliases so they don't appear as + # dead (unresolvable) aliases in the hook-alias resolution audit. + attn.hook_aliases = { + k: v + for k, v in AttentionBridge.hook_aliases.items() + if k not in {"hook_q", "hook_k", "hook_v"} + } + return { + "norm_1": RMSNormalizationBridge(name="norm_1", config=self.cfg), + "attn": attn, + "norm_2": RMSNormalizationBridge(name="norm_2", config=self.cfg), + "norm_3": RMSNormalizationBridge(name="norm_3", config=self.cfg), + "mlp": MLPBridge( + name="mlp", + config=self.cfg, + submodules={ + "in": LinearBridge(name="fc"), + "out": LinearBridge(name="proj"), + }, + ), + "norm_4": RMSNormalizationBridge(name="norm_4", config=self.cfg), + } + + def prepare_loading(self, model_name: str, model_kwargs: dict) -> None: + """Patch Huginn's remote code for transformers v5 compatibility. + + Huginn's modeling code targets transformers 4.44; two things break under + v5 (5.8.1), so two patches: + + 1. Tied-weights format. ``RavenForCausalLM._tied_weights_keys`` is a list + (``["lm_head.weight"]``, the 4.x format), but v5's ``tie_weights`` -> + ``get_expanded_tied_weights_keys`` calls ``.keys()`` on it and raises + ``AttributeError``. The model does not even construct. Rewrite it to + the v5 dict form ``{"lm_head.weight": "transformer.wte.weight"}`` + (Huginn ties ``lm_head`` to ``transformer.wte``). + + 2. Weight re-init. Under v5's meta-device load-then-materialise flow, + ``PreTrainedModel._init_weights`` is invoked on modules that already + hold checkpoint weights, re-randomising them. Guard it to skip modules + whose parameters are already on a real (non-meta) device — the same + defensive patch openelm.py applies. + + Args: + model_name: The HuggingFace model name/path. + model_kwargs: The kwargs dict for from_pretrained(). + """ + # Force-import the modeling module so it appears in sys.modules to patch. + try: + from transformers.dynamic_module_utils import get_class_from_dynamic_module + + get_class_from_dynamic_module( + "raven_modeling_minimal.RavenForCausalLM", + model_name, + ) + except Exception: + return + + # Each checkpoint revision gets its own module in sys.modules; patch all. + for key in list(sys.modules.keys()): + if "raven" not in key.lower() or "modeling" not in key.lower(): + continue + module = sys.modules[key] + + # Patch 1: tied-weights keys list -> v5 dict form. + causal_lm_class = getattr(module, "RavenForCausalLM", None) + if causal_lm_class is not None and isinstance( + getattr(causal_lm_class, "_tied_weights_keys", None), list + ): + causal_lm_class._tied_weights_keys = {"lm_head.weight": "transformer.wte.weight"} + + # Patch 2: don't re-randomise already-loaded weights. + pretrained_class = getattr(module, "RavenPreTrainedModel", None) + if pretrained_class is None or getattr(pretrained_class, "_tl_patched", False): + continue + original_init_weights = pretrained_class._init_weights + + def safe_init_weights(self, mod, _original=original_init_weights): + # Only initialise modules still on meta device (pre-loading); + # never re-randomise weights already read from the checkpoint. + first_param = next(mod.parameters(), None) + if first_param is not None and first_param.device.type != "meta": + return + _original(self, mod) + + pretrained_class._init_weights = safe_init_weights + pretrained_class._tl_patched = True diff --git a/transformer_lens/model_bridge/supported_architectures/rwkv7.py b/transformer_lens/model_bridge/supported_architectures/rwkv7.py new file mode 100644 index 0000000000..f4186330a1 --- /dev/null +++ b/transformer_lens/model_bridge/supported_architectures/rwkv7.py @@ -0,0 +1,296 @@ +"""RWKV-7 ("Goose") architecture adapter (RWKV7ForCausalLM). + +Model family: ``fla-hub/rwkv7-*`` (e.g. ``fla-hub/rwkv7-0.1B-g1``), an +attention-free recurrent language model from the flash-linear-attention (``fla``) +library. Loaded via remote code (``trust_remote_code=True``); the modeling +classes live in ``fla.models.rwkv7``. + +Architecture overview +--------------------- +RWKV-7 is a flat stack of recurrent blocks over a shared residual width, wrapped +by standard (biased) LayerNorm rather than RMSNorm and with no positional +embeddings:: + + embeddings -> [ RWKV7Block x N ] -> norm -> lm_head + +Each ``RWKV7Block`` is a pre-norm pair of a time-mixing and a channel-mixing +sublayer:: + + x = x + attn(attn_norm(x)) # time-mixing (RWKV7Attention) + x = x + ffn(ffn_norm(x)) # channel-mixing (RWKV7FeedForward) + +with an extra ``pre_norm`` LayerNorm on layer 0 only (``config.norm_first``). +The time-mixing sublayer is the RWKV-7 "generalized delta rule": token-shifted +lerp coefficients (``x_r``..``x_g``), receptance / key / value / output +projections (``r_proj`` / ``k_proj`` / ``v_proj`` / ``o_proj``), low-rank LoRAs +for the log-space decay / value blend / a-coefficient / gate, and a GroupNorm. +The channel-mixing sublayer is a token-shifted squared-ReLU MLP (``key`` up, +``value`` down). + +Cross-block ``v_first`` threading: ``RWKV7Model.forward`` initialises +``v_first = torch.zeros_like(hidden_states)`` and threads it through every block +(layer 0 fills it; later layers blend their value with it). This is managed +entirely by the HF model-level forward, so the bridge — which delegates each +block's forward to HF — gets it for free. + +Key adapter decisions +--------------------- +1. Full delegation. The recurrence, the token shift, the LoRAs, the GroupNorm, + and the ``v_first`` threading all live inside the ``fla`` remote-code forward + that the bridge delegates to, so a single forward pass is numerically correct + with no scan handling here. ``weight_processing_conversions = {}``. + +2. ``OpaqueBlockBridge`` for the block list, for the same reason nemotron_h / raven + use it: ``BlockBridge``'s hook aliases hardcode a standard pre-norm attention + flow (``hook_resid_mid``, ``ln1 -> attn -> ln2 -> mlp``) that a recurrent + RWKV-7 block does not follow. ``OpaqueBlockBridge`` delegates the whole block and + exposes only ``hook_in`` / ``hook_out`` on the residual stream, which is + correct regardless of the internal mixing. The block's inner norms / mixers + are still declared as submodules so they wrap the live HF modules and fire. + +3. ``GeneralizedComponent`` for both the time-mixing (``attn``) and channel-mixing + (``ffn``) sublayers. Its forward is a generic I/O-hooked passthrough that + delegates to the live HF module and hooks the primary output while preserving + the rest of the tuple. ``attn`` returns a 4-tuple ``(hidden_states, attentions, + past_key_values, v_first)`` and ``ffn`` returns a 2-tuple + ``(hidden_states, state)``; the generic passthrough handles both. The + ``r_proj`` / ``k_proj`` / ``v_proj`` / ``o_proj`` (time-mixing) and ``key`` / + ``value`` (channel-mixing) projections are exposed as ``LinearBridge`` + submodules so their hooks fire; the LoRAs, GroupNorm, and per-channel lerp + parameters are deliberately left inside the delegated forward. + +4. Norms. RWKV-7 uses standard biased LayerNorm, so ``normalization_type = "LN"`` + and the norms are ``NormalizationBridge`` (not ``RMSNormalizationBridge``). + ``attn_norm`` and the final ``model.norm`` are ordinary single-input norms. + ``ffn_norm`` is the exception: under ``config.fuse_norm`` (default ``True``) + the block calls it as ``ffn_norm(hidden_states, residual, True)`` — a fused + add-and-norm returning ``(normed, new_residual)`` — which the reimplementing + ``NormalizationBridge`` cannot express. It is therefore wrapped with a plain + delegating ``GeneralizedComponent`` (generic I/O-hooked passthrough) that + forwards any signature to the live HF module and hooks the primary output. + +5. ``applicable_phases = []``. RWKV-7 is attention-free and recurrent, off the + transformer-shaped ``verify_models`` phases; correctness is covered by the + integration tests (bridge-vs-HF logit parity). ``ln_final`` (``model.norm``) + feeds only ``lm_head``, so folding is unnecessary and ``supports_fold_ln`` is + left at its default. + +6. ``is_stateful = False``. ``fla`` manages recurrent decode state through its + own ``Cache`` object, not the ``cache_params`` convention the stateful-cache + path expects, so the adapter does not advertise a stateful cache. + +Config propagation +------------------ +The RWKV-7 shape attributes (``head_dim``, ``num_heads``, ``value_dim``, the four +LoRA low-rank dims, ``norm_first`` / ``norm_bias`` / ``fuse_norm``, ``attn_mode``, +``hidden_act``) are surfaced on ``self.cfg`` here AND added to the +``_HF_PASSTHROUGH_ATTRS`` list so analysis tooling can read them off a booted +bridge; the synthetic-config unit tests read the same names. + +Remote-code loading (transformers v5) +------------------------------------ +``fla``'s modeling code predates transformers v5; ``prepare_loading`` applies the +same two defensive patches raven does: (1) rewrite ``_tied_weights_keys`` from the +4.x list form to the v5 dict form so ``get_expanded_tied_weights_keys`` does not +call ``.keys()`` on a list (only reached on tied checkpoints — RWKV-7 defaults +``tie_word_embeddings=False`` — but harmless when untied); (2) guard +``_init_weights`` so v5's meta-device materialise pass does not re-randomise +weights already read from the checkpoint. +""" + +import sys +from typing import Any + +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.generalized_components import ( + EmbeddingBridge, + LinearBridge, + NormalizationBridge, + OpaqueBlockBridge, + UnembeddingBridge, +) +from transformer_lens.model_bridge.generalized_components.base import ( + GeneralizedComponent, +) + + +class RWKV7ArchitectureAdapter(ArchitectureAdapter): + """Architecture adapter for RWKV7ForCausalLM (RWKV-7 "Goose"). + + Attention-free recurrent decoder: a flat stack of pre-norm blocks, each a + generalized-delta-rule time-mixing sublayer plus a token-shifted squared-ReLU + channel-mixing sublayer, wrapped by standard biased LayerNorm. The recurrence + and the cross-block ``v_first`` threading live inside the ``fla`` remote-code + forward, which the bridge delegates to; see the module docstring for the full + set of adapter decisions. + """ + + # Attention-free and recurrent — off the transformer-shaped verify_models + # path; correctness lives in the integration tests (bridge-vs-HF parity). + applicable_phases: list[int] = [] + + def __init__(self, cfg: Any) -> None: + """Initialize the RWKV-7 architecture adapter.""" + super().__init__(cfg) + + # Standard biased LayerNorm, no positional embeddings, ungated FFN. + self.cfg.normalization_type = "LN" + self.cfg.uses_rms_norm = False + self.cfg.positional_embedding_type = "none" + self.cfg.final_rms = False + self.cfg.attn_only = False + self.cfg.gated_mlp = False + + # fla drives recurrent decode state via its own Cache, not the + # cache_params convention, so the adapter does not advertise a cache. + setattr(self.cfg, "is_stateful", False) + + # Surface the RWKV-7 shape attributes on cfg so they are present on both + # the HF-boot path (also via _HF_PASSTHROUGH_ATTRS) and the synthetic + # config path used by the unit tests. getattr-with-default keeps a bare + # TransformerBridgeConfig from raising. + # head_dim is a read-only property on TransformerBridgeConfig (aliases + # d_head), so it is read but never assigned. num_heads defaults to + # d_model // head_dim. + head_dim = getattr(cfg, "head_dim", 64) or 64 + num_heads = getattr(cfg, "num_heads", None) or max(1, self.cfg.d_model // head_dim) + value_dim = getattr(cfg, "value_dim", None) or [self.cfg.d_model] * self.cfg.n_layers + # setattr (not direct assignment) for the RWKV-7-specific names so mypy + # does not flag them as undeclared on TransformerBridgeConfig. + setattr(self.cfg, "num_heads", num_heads) + setattr(self.cfg, "value_dim", value_dim) + setattr(self.cfg, "decay_low_rank_dim", getattr(cfg, "decay_low_rank_dim", 64)) + setattr(self.cfg, "gate_low_rank_dim", getattr(cfg, "gate_low_rank_dim", 128)) + setattr(self.cfg, "a_low_rank_dim", getattr(cfg, "a_low_rank_dim", 64)) + setattr(self.cfg, "v_low_rank_dim", getattr(cfg, "v_low_rank_dim", 16)) + setattr(self.cfg, "norm_first", getattr(cfg, "norm_first", True)) + setattr(self.cfg, "norm_bias", getattr(cfg, "norm_bias", True)) + setattr(self.cfg, "fuse_norm", getattr(cfg, "fuse_norm", True)) + setattr(self.cfg, "attn_mode", getattr(cfg, "attn_mode", "chunk")) + setattr(self.cfg, "hidden_act", getattr(cfg, "hidden_act", "sqrelu")) + # LayerNorm epsilon for the reimplementing NormalizationBridge path (eps + # is a real config field, so direct assignment is fine). + self.cfg.eps = getattr(cfg, "norm_eps", getattr(cfg, "eps", 1e-5)) + + # Full delegation to the fla forward — no HT-format weight reshaping. + self.weight_processing_conversions = {} + + self.component_mapping = { + "embed": EmbeddingBridge(name="model.embeddings"), + "blocks": OpaqueBlockBridge( + name="model.layers", + submodules={ + # Pre-norm before time-mixing (standard single-input LayerNorm). + "attn_norm": NormalizationBridge( + name="attn_norm", config=self.cfg, uses_rms_norm=False + ), + # Time-mixing: generalized delta rule. Delegated passthrough; + # only the four projections are exposed (LoRAs / GroupNorm / + # lerp params stay inside the fla forward). + "attn": GeneralizedComponent( + name="attn", + config=self.cfg, + submodules={ + "r_proj": LinearBridge(name="r_proj"), + "k_proj": LinearBridge(name="k_proj"), + "v_proj": LinearBridge(name="v_proj"), + "o_proj": LinearBridge(name="o_proj"), + }, + ), + # Pre-norm before channel-mixing. Under config.fuse_norm the + # block calls this as ffn_norm(x, residual, True) -> (normed, + # residual), a fused signature the reimplementing + # NormalizationBridge can't express, so delegate any-signature + # to the live HF module with I/O hooks. + "ffn_norm": GeneralizedComponent(name="ffn_norm", config=self.cfg), + # Channel-mixing: token-shifted squared-ReLU MLP. Delegated + # passthrough exposing the up ("key") and down ("value") + # projections. HF confusingly names the output proj "value". + "ffn": GeneralizedComponent( + name="ffn", + config=self.cfg, + submodules={ + "key": LinearBridge(name="key"), + "value": LinearBridge(name="value"), + }, + ), + }, + ), + "ln_final": NormalizationBridge( + name="model.norm", config=self.cfg, uses_rms_norm=False + ), + "unembed": UnembeddingBridge(name="lm_head"), + } + + def prepare_loading(self, model_name: str, model_kwargs: dict) -> None: + """Patch fla's RWKV-7 remote code for transformers v5 compatibility. + + Two defensive patches, mirroring raven: + + 1. Tied-weights format. ``RWKV7ForCausalLM._tied_weights_keys`` is a list + (``["lm_head.weight"]``, the 4.x form). v5's ``tie_weights`` -> + ``get_expanded_tied_weights_keys`` calls ``.keys()`` on the mapping, + which raises ``AttributeError`` on a list. Rewrite it to the v5 dict + form ``{"lm_head.weight": "model.embeddings.weight"}``. RWKV-7 defaults + ``tie_word_embeddings=False`` (so the list path is usually short- + circuited before ``.keys()``), but the rewrite is harmless when untied + and prevents the crash on any tied checkpoint. + + 2. Weight re-init. Under v5's meta-device load-then-materialise flow, + ``PreTrainedModel._init_weights`` is invoked on modules that already + hold checkpoint weights, re-randomising them. Guard it to skip modules + whose parameters are already on a real (non-meta) device — the same + defensive patch openelm.py / raven.py apply. + + Args: + model_name: The HuggingFace model name/path. + model_kwargs: The kwargs dict for from_pretrained(). + """ + # Force-import the fla RWKV-7 modeling module so its classes appear in + # sys.modules to patch. fla is normally a pip package; fall back to the + # dynamic-module route for genuinely bundled remote code. + try: + import fla.models.rwkv7.modeling_rwkv7 # noqa: F401 + except Exception: + try: + from transformers.dynamic_module_utils import ( + get_class_from_dynamic_module, + ) + + get_class_from_dynamic_module( + "modeling_rwkv7.RWKV7ForCausalLM", + model_name, + ) + except Exception: + return + + # Patch every loaded RWKV-7 modeling module (each remote revision gets its + # own module object in sys.modules). + for key in list(sys.modules.keys()): + if "rwkv7" not in key.lower() or "modeling" not in key.lower(): + continue + module = sys.modules[key] + + # Patch 1: tied-weights keys list -> v5 dict form. + causal_lm_class = getattr(module, "RWKV7ForCausalLM", None) + if causal_lm_class is not None and isinstance( + getattr(causal_lm_class, "_tied_weights_keys", None), list + ): + causal_lm_class._tied_weights_keys = {"lm_head.weight": "model.embeddings.weight"} + + # Patch 2: don't re-randomise already-loaded weights. + pretrained_class = getattr(module, "RWKV7PreTrainedModel", None) + if pretrained_class is None or getattr(pretrained_class, "_tl_patched", False): + continue + original_init_weights = pretrained_class._init_weights + + def safe_init_weights(self, mod, _original=original_init_weights): + # Only initialise modules still on meta device (pre-loading); + # never re-randomise weights already read from the checkpoint. + first_param = next(mod.parameters(), None) + if first_param is not None and first_param.device.type != "meta": + return + _original(self, mod) + + pretrained_class._init_weights = safe_init_weights + pretrained_class._tl_patched = True diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index cd874f9f93..aff4229f4b 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -282,8 +282,12 @@ def __setattr__(self, name: str, value: Any) -> None: def _scan_existing_hooks(self, module: nn.Module, prefix: str = "") -> None: """Scan existing modules for hooks and add them to registry.""" visited = set() - # Protect canonical HookPoint names from alias overwrites - named_hook_ids: set = set() + # Protect canonical HookPoint names from alias overwrites. Seeded with + # already-registered hooks so the post-alias-registration re-scan adds + # alias registry entries without renaming shared HookPoint instances + # (dir() sorts alphabetically, so block-level alias attributes like + # hook_mlp_out are visited before the mlp child they point into). + named_hook_ids: set = {id(hp) for hp in self._hook_registry.values() if hp.name is not None} def scan_module(mod: nn.Module, path: str = "") -> None: obj_id = id(mod) @@ -1295,6 +1299,40 @@ def tl_named_parameters(self) -> Iterator[tuple[str, torch.Tensor]]: """ return iter(self.get_params().items()) + def input_to_embed( + self, + input: Union[str, List[str], torch.Tensor], + prepend_bos: Optional[bool] = None, + padding_side: Optional[str] = None, + attention_mask: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor, None, Optional[torch.Tensor]]: + """Convert input to the residual stream entering block 0 (``resid_pre[0]``). + + Bridge analog of :meth:`HookedTransformer.input_to_embed`. Returns + ``(residual, tokens, shortformer_pos_embed, attention_mask)``; feed the + residual to ``forward(..., start_at_layer=0)`` to resume the pass. + + ``shortformer_pos_embed`` is always ``None``: for the models the bridge + supports the residual already carries positional information, so there is + no separate positional stream to return. + """ + if isinstance(input, (str, list)): + assert self.tokenizer is not None, "Must provide a tokenizer for string input." + tokens = self.to_tokens(input, prepend_bos=prepend_bos, padding_side=padding_side) + else: + tokens = input + if isinstance(tokens, torch.Tensor) and tokens.ndim == 1: + tokens = tokens.unsqueeze(0) + if ( + attention_mask is None + and self.tokenizer is not None + and self.tokenizer.padding_side == "left" + ): + _prepend = self.cfg.default_prepend_bos if prepend_bos is None else prepend_bos + attention_mask = utils.get_attention_mask(self.tokenizer, tokens, _prepend) + residual = self.forward(tokens, stop_at_layer=0, attention_mask=attention_mask) + return residual, tokens, None, attention_mask + def forward( self, input: Union[str, List[str], torch.Tensor], @@ -1307,21 +1345,32 @@ def forward( stop_at_layer: Optional[int] = None, pixel_values: Optional[torch.Tensor] = None, input_values: Optional[torch.Tensor] = None, + past_key_values: Optional[Any] = None, **kwargs, ) -> Any: """Forward pass through the model. Args: input: Input to the model - return_type: Type of output to return ('logits', 'loss', 'both', 'predictions', None) + return_type: Type of output to return ('logits', 'loss', 'both', 'predictions', + 'logits_and_cache', None). 'logits_and_cache' returns + ``(logits, past_key_values)`` — the HuggingFace cache after this step, to + feed back on the next call for incremental decoding. loss_per_token: Whether to return loss per token prepend_bos: Whether to prepend BOS token padding_side: Which side to pad on - start_at_layer: Not implemented in TransformerBridge. The bridge delegates - to HuggingFace's model.forward() which owns the layer iteration loop, - making start_at_layer infeasible without monkey-patching HF internals - (fragile across HF versions) or exception-based layer skipping (corrupts - model state). Raises NotImplementedError if a non-None value is passed. + past_key_values: HuggingFace KV cache from a prior ``use_cache=True`` step + (e.g. the second element of a ``return_type='logits_and_cache'`` return). + When provided, KV caching is enabled automatically and only the new + tokens' keys/values are computed; HF derives the position offset from the + cache length. This is the bridge's manual KV-cache entry point — it uses + HF's native cache object rather than a TransformerLens cache. + start_at_layer: Resume the forward from block ``k``, treating ``input`` as + the residual-stream tensor ``[batch, pos, d_model]`` entering that block + (mirrors HookedTransformer). Blocks 0..k-1 still execute internally (their + output is discarded when block k swaps in the residual) but are excluded + from ``run_with_cache`` output. Requires an HF model that accepts + ``inputs_embeds``; only supported on the standard ``blocks`` stack. stop_at_layer: Layer to stop forward pass at pixel_values: Optional image tensor for multimodal models (e.g., LLaVA, Gemma3). The tensor is passed directly to the underlying HuggingFace model. @@ -1344,12 +1393,7 @@ def forward( ) if start_at_layer is not None: - raise NotImplementedError( - "start_at_layer is not supported in TransformerBridge. " - "The bridge delegates to HuggingFace's model.forward() which controls " - "the layer iteration loop. See the TransformerBridge review plan for a " - "detailed analysis of implementation approaches and their tradeoffs." - ) + input = self._setup_start_at_layer(input, start_at_layer) # Set stop_at_layer flag on all blocks if requested if stop_at_layer is not None: @@ -1452,6 +1496,11 @@ def forward( if attention_mask is not None: kwargs["attention_mask"] = attention_mask + if past_key_values is not None: + # Manual KV-cache injection: hand HF its own cache back and let it + # extend it, computing only the new tokens' keys/values. + kwargs["past_key_values"] = past_key_values + kwargs["use_cache"] = True if kwargs.pop("use_past_kv_cache", False) or kwargs.get("use_cache", False): kwargs["use_cache"] = True # Auto-generate decoder_input_ids for encoder-decoder models @@ -1563,6 +1612,53 @@ def forward( if hasattr(self, "_last_hf_cache"): del self._last_hf_cache + if start_at_layer is not None: + self._teardown_start_at_layer() + + def _setup_start_at_layer(self, input: Any, start_at_layer: int) -> torch.Tensor: + """Arm residual-stream injection at block ``start_at_layer``. + + ``input`` is the residual entering that block. It is returned (batch-promoted) + to drive the HF forward as ``inputs_embeds`` so position ids / attention mask + are computed for the right sequence length; block ``start_at_layer`` then swaps + it back in for its own input, discarding whatever blocks 0..k-1 produced. + """ + for alt in ("encoder_blocks", "decoder_blocks", "L_blocks", "H_blocks"): + if hasattr(self, alt): + raise NotImplementedError( + "start_at_layer is only supported on the standard 'blocks' stack, " + f"not {alt!r}." + ) + if not hasattr(self, "blocks"): + raise NotImplementedError("start_at_layer requires a 'blocks' stack.") + if not (isinstance(input, torch.Tensor) and input.is_floating_point()): + raise ValueError( + "start_at_layer requires a residual-stream tensor [batch, pos, d_model]; " + f"got {type(input).__name__}. Capture it from a prior run_with_cache " + "(e.g. cache['blocks.k.hook_in'])." + ) + residual = input if input.ndim == 3 else input.unsqueeze(0) + n_blocks = len(self.blocks) + if start_at_layer < 0: + start_at_layer += n_blocks + if not 0 <= start_at_layer < n_blocks: + raise ValueError(f"start_at_layer={start_at_layer} out of range [0, {n_blocks}).") + # The returned tensor is fed to HF as inputs_embeds; stash a clone so an + # architecture that mutates inputs_embeds in place can't corrupt the value + # block ``start_at_layer`` swaps back in. + injected = residual.clone() + for block in self.blocks: + block._start_at_layer_idx = start_at_layer + block._start_residual = injected + return residual + + def _teardown_start_at_layer(self) -> None: + """Clear the residual-injection state set by ``_setup_start_at_layer``.""" + if hasattr(self, "blocks"): + for block in self.blocks: + block._start_at_layer_idx = None + block._start_residual = None + # loss_fn inherited from BridgeCore def _resolve_stopping_criteria( @@ -2879,6 +2975,19 @@ def mps(self) -> "TransformerBridge": """ return self.to(torch.device("mps")) + def train(self, mode: bool = True) -> "TransformerBridge": + """Set training mode, propagating to the wrapped source model. + + ``original_model`` lives in ``__dict__`` rather than the registered + module tree, so the inherited ``nn.Module.train()`` recursion does not + reach it. + """ + super().train(mode) + original = getattr(self, "original_model", None) + if isinstance(original, torch.nn.Module): + original.train(mode) + return self + def set_use_attn_result(self, use_attn_result: bool): """Toggle whether to explicitly calculate and expose the result for each attention head. diff --git a/transformer_lens/model_protocol.py b/transformer_lens/model_protocol.py new file mode 100644 index 0000000000..1ebb7b8292 --- /dev/null +++ b/transformer_lens/model_protocol.py @@ -0,0 +1,65 @@ +"""Structural type shared by :class:`HookedTransformer` and :class:`TransformerBridge`. + +The interpretability utilities (``patching``, ``head_detector``, ``ActivationCache``) +are model-agnostic at runtime — they only need ``cfg`` plus the ``run_with_*`` / +tokenization surface and a few weight-processing helpers. Historically their +signatures said ``HookedTransformer``, which made a ``TransformerBridge`` fail to +type-check even though it works. Typing the model parameter as this Protocol accepts +either (and any future structural match, e.g. ``RemoteBridge``). + +Members are typed loosely on purpose: this is a compatibility shim over two concrete +classes whose method signatures differ in detail but agree in use. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable + +if TYPE_CHECKING: + from transformer_lens.config import TransformerLensConfig + + +# runtime_checkable so the interp utilities' beartype-decorated signatures can +# isinstance-check the parameter at runtime (presence-only; both models qualify). +@runtime_checkable +class TransformerLensModel(Protocol): + """Minimal structural interface common to HookedTransformer and TransformerBridge.""" + + # Read-only property (not a bare attribute) so it is covariant: a concrete model + # whose cfg is a TransformerLensConfig *subclass* (HookedTransformerConfig / + # TransformerBridgeConfig) still conforms. A mutable attribute would be invariant. + @property + def cfg(self) -> "TransformerLensConfig": + ... + + def run_with_cache(self, *args: Any, **kwargs: Any) -> Any: + ... + + def run_with_hooks(self, *args: Any, **kwargs: Any) -> Any: + ... + + def to_tokens(self, *args: Any, **kwargs: Any) -> Any: + ... + + +@runtime_checkable +class TransformerLensModelWithWeights(TransformerLensModel, Protocol): + """Adds the weight-processing surface that ``ActivationCache``'s advanced helpers + (LayerNorm folding, residual-direction projection) reach for. Both concrete models + expose these; the bridge builds them from its adapter.""" + + @property + def blocks(self) -> Any: + ... + + @property + def ln_final(self) -> Any: + ... + + def accumulated_bias(self, *args: Any, **kwargs: Any) -> Any: + ... + + def to_single_token(self, *args: Any, **kwargs: Any) -> Any: + ... + + def tokens_to_residual_directions(self, *args: Any, **kwargs: Any) -> Any: + ... diff --git a/transformer_lens/patching.py b/transformer_lens/patching.py index 868b1f2156..24a50ed126 100644 --- a/transformer_lens/patching.py +++ b/transformer_lens/patching.py @@ -62,7 +62,7 @@ import transformer_lens.utilities as utils from transformer_lens.ActivationCache import ActivationCache -from transformer_lens.HookedTransformer import HookedTransformer +from transformer_lens.model_protocol import TransformerLensModel # %% Logits = torch.Tensor @@ -90,7 +90,7 @@ def make_df_from_ranges( @overload def generic_activation_patch( - model: HookedTransformer, + model: TransformerLensModel, corrupted_tokens: Int[torch.Tensor, "batch pos"], clean_cache: ActivationCache, patching_metric: Callable[[Float[torch.Tensor, "batch pos d_vocab"]], Float[torch.Tensor, ""]], @@ -107,7 +107,7 @@ def generic_activation_patch( @overload def generic_activation_patch( - model: HookedTransformer, + model: TransformerLensModel, corrupted_tokens: Int[torch.Tensor, "batch pos"], clean_cache: ActivationCache, patching_metric: Callable[[Float[torch.Tensor, "batch pos d_vocab"]], Float[torch.Tensor, ""]], @@ -123,7 +123,7 @@ def generic_activation_patch( def generic_activation_patch( - model: HookedTransformer, + model: TransformerLensModel, corrupted_tokens: Int[torch.Tensor, "batch pos"], clean_cache: ActivationCache, patching_metric: Callable[[Float[torch.Tensor, "batch pos d_vocab"]], Float[torch.Tensor, ""]], diff --git a/transformer_lens/supported_models.py b/transformer_lens/supported_models.py index ab6b2f2d8a..bd670c20f8 100644 --- a/transformer_lens/supported_models.py +++ b/transformer_lens/supported_models.py @@ -1,6 +1,8 @@ # HookedTransformers model registry. If you are looking for TransformerBridge's supported models # see transformer_lens/tools/model_registry/data/supported_models.json or the docs page at # https://transformerlensorg.github.io/TransformerLens/generated/transformer_bridge_models.html +from transformer_lens.tools.model_registry.registry_io import load_model_aliases + OFFICIAL_MODEL_NAMES: list[str] = [ "01-ai/Yi-34B", "01-ai/Yi-34B-Chat", @@ -255,400 +257,10 @@ """Official model names for models on HuggingFace.""" # Model Aliases: -MODEL_ALIASES: dict[str, list[str]] = { - "01-ai/Yi-34B": ["yi-34b", "Yi-34B"], - "01-ai/Yi-34B-Chat": ["yi-34b-chat", "Yi-34B-Chat"], - "01-ai/Yi-6B": ["yi-6b", "Yi-6B"], - "01-ai/Yi-6B-Chat": ["yi-6b-chat", "Yi-6B-Chat"], - "ai-forever/mGPT": ["mGPT"], - "allenai/OLMo-1B-hf": ["olmo-1b"], - "allenai/OLMo-2-0425-1B": ["olmo-2-1b"], - "allenai/OLMo-2-1124-7B": ["olmo-2-7b"], - "allenai/Olmo-3-1025-7B": ["olmo-3-1025-7b"], - "allenai/Olmo-3-1125-32B": ["olmo-3-1125-32b"], - "allenai/Olmo-3-32B-Think": ["olmo-3-32b-think"], - "allenai/Olmo-3-7B-Instruct": ["olmo-3-7b-instruct"], - "allenai/Olmo-3-7B-Think": ["olmo-3-7b-think"], - "allenai/Olmo-3.1-32B-Instruct": ["olmo-3.1-32b-instruct"], - "allenai/Olmo-3.1-32B-Think": ["olmo-3.1-32b-think"], - "allenai/OLMo-7B-hf": ["olmo-7b"], - "allenai/OLMoE-1B-7B-0924": ["olmoe"], - "ArthurConmy/redwood_attn_2l": ["redwood_attn_2l"], - "Baidicoot/Othello-GPT-Transformer-Lens": ["othello-gpt"], - "bigcode/santacoder": ["santacoder"], - "bigscience/bloom-1b1": ["bloom-1b1"], - "bigscience/bloom-1b7": ["bloom-1b7"], - "bigscience/bloom-3b": ["bloom-3b"], - "bigscience/bloom-560m": ["bloom-560m"], - "bigscience/bloom-7b1": ["bloom-7b1"], - "codellama/CodeLlama-7b-hf": ["CodeLlamallama-2-7b"], - "codellama/CodeLlama-7b-Instruct-hf": ["CodeLlama-7b-instruct"], - "codellama/CodeLlama-7b-Python-hf": ["CodeLlama-7b-python"], - "distilgpt2": ["distillgpt2", "distill-gpt2", "distil-gpt2", "gpt2-xs"], - "EleutherAI/gpt-j-6B": ["gpt-j-6B", "gpt-j", "gptj"], - "EleutherAI/gpt-neo-1.3B": ["gpt-neo-1.3B", "gpt-neo-medium", "neo-medium"], - "EleutherAI/gpt-neo-125M": ["gpt-neo-125M", "gpt-neo-small", "neo-small", "neo"], - "EleutherAI/gpt-neo-2.7B": ["gpt-neo-2.7B", "gpt-neo-large", "neo-large"], - "EleutherAI/gpt-neox-20b": ["gpt-neox-20b", "gpt-neox", "neox"], - "EleutherAI/pythia-1.4b": ["pythia-1.4b", "EleutherAI/pythia-1.3b", "pythia-1.3b"], - "EleutherAI/pythia-1.4b-deduped": [ - "pythia-1.4b-deduped", - "EleutherAI/pythia-1.3b-deduped", - "pythia-1.3b-deduped", - ], - "EleutherAI/pythia-1.4b-deduped-v0": [ - "pythia-1.4b-deduped-v0", - "EleutherAI/pythia-1.3b-deduped-v0", - "pythia-1.3b-deduped-v0", - ], - "EleutherAI/pythia-1.4b-v0": ["pythia-1.4b-v0", "EleutherAI/pythia-1.3b-v0", "pythia-1.3b-v0"], - "EleutherAI/pythia-12b": ["pythia-12b", "EleutherAI/pythia-13b", "pythia-13b"], - "EleutherAI/pythia-12b-deduped": [ - "pythia-12b-deduped", - "EleutherAI/pythia-13b-deduped", - "pythia-13b-deduped", - ], - "EleutherAI/pythia-12b-deduped-v0": [ - "pythia-12b-deduped-v0", - "EleutherAI/pythia-13b-deduped-v0", - "pythia-13b-deduped-v0", - ], - "EleutherAI/pythia-12b-v0": ["pythia-12b-v0", "EleutherAI/pythia-13b-v0", "pythia-13b-v0"], - "EleutherAI/pythia-14m": ["pythia-14m"], - "EleutherAI/pythia-160m": ["pythia-160m", "EleutherAI/pythia-125m", "pythia-125m"], - "EleutherAI/pythia-160m-deduped": [ - "pythia-160m-deduped", - "EleutherAI/pythia-125m-deduped", - "pythia-125m-deduped", - ], - "EleutherAI/pythia-160m-deduped-v0": [ - "pythia-160m-deduped-v0", - "EleutherAI/pythia-125m-deduped-v0", - "pythia-125m-deduped-v0", - ], - "EleutherAI/pythia-160m-seed1": [ - "pythia-160m-seed1", - "EleutherAI/pythia-125m-seed1", - "pythia-125m-seed1", - ], - "EleutherAI/pythia-160m-seed2": [ - "pythia-160m-seed2", - "EleutherAI/pythia-125m-seed2", - "pythia-125m-seed2", - ], - "EleutherAI/pythia-160m-seed3": [ - "pythia-160m-seed3", - "EleutherAI/pythia-125m-seed3", - "pythia-125m-seed3", - ], - "EleutherAI/pythia-160m-v0": ["pythia-160m-v0", "EleutherAI/pythia-125m-v0", "pythia-125m-v0"], - "EleutherAI/pythia-1b": ["pythia-1b", "EleutherAI/pythia-800m", "pythia-800m"], - "EleutherAI/pythia-1b-deduped": [ - "pythia-1b-deduped", - "EleutherAI/pythia-800m-deduped", - "pythia-800m-deduped", - ], - "EleutherAI/pythia-1b-deduped-v0": [ - "pythia-1b-deduped-v0", - "EleutherAI/pythia-800m-deduped-v0", - "pythia-800m-deduped-v0", - ], - "EleutherAI/pythia-1b-v0": ["pythia-1b-v0", "EleutherAI/pythia-800m-v0", "pythia-800m-v0"], - "EleutherAI/pythia-2.8b": ["pythia-2.8b", "EleutherAI/pythia-2.7b", "pythia-2.7b"], - "EleutherAI/pythia-2.8b-deduped": [ - "pythia-2.8b-deduped", - "EleutherAI/pythia-2.7b-deduped", - "pythia-2.7b-deduped", - ], - "EleutherAI/pythia-2.8b-deduped-v0": [ - "pythia-2.8b-deduped-v0", - "EleutherAI/pythia-2.7b-deduped-v0", - "pythia-2.7b-deduped-v0", - ], - "EleutherAI/pythia-2.8b-v0": ["pythia-2.8b-v0", "EleutherAI/pythia-2.7b-v0", "pythia-2.7b-v0"], - "EleutherAI/pythia-31m": ["pythia-31m"], - "EleutherAI/pythia-410m": ["pythia-410m", "EleutherAI/pythia-350m", "pythia-350m"], - "EleutherAI/pythia-410m-deduped": [ - "pythia-410m-deduped", - "EleutherAI/pythia-350m-deduped", - "pythia-350m-deduped", - ], - "EleutherAI/pythia-410m-deduped-v0": [ - "pythia-410m-deduped-v0", - "EleutherAI/pythia-350m-deduped-v0", - "pythia-350m-deduped-v0", - ], - "EleutherAI/pythia-410m-v0": ["pythia-410m-v0", "EleutherAI/pythia-350m-v0", "pythia-350m-v0"], - "EleutherAI/pythia-6.9b": ["pythia-6.9b", "EleutherAI/pythia-6.7b", "pythia-6.7b"], - "EleutherAI/pythia-6.9b-deduped": [ - "pythia-6.9b-deduped", - "EleutherAI/pythia-6.7b-deduped", - "pythia-6.7b-deduped", - ], - "EleutherAI/pythia-6.9b-deduped-v0": [ - "pythia-6.9b-deduped-v0", - "EleutherAI/pythia-6.7b-deduped-v0", - "pythia-6.7b-deduped-v0", - ], - "EleutherAI/pythia-6.9b-v0": ["pythia-6.9b-v0", "EleutherAI/pythia-6.7b-v0", "pythia-6.7b-v0"], - "EleutherAI/pythia-70m": ["pythia-70m", "pythia", "EleutherAI/pythia-19m", "pythia-19m"], - "EleutherAI/pythia-70m-deduped": [ - "pythia-70m-deduped", - "EleutherAI/pythia-19m-deduped", - "pythia-19m-deduped", - ], - "EleutherAI/pythia-70m-deduped-v0": [ - "pythia-70m-deduped-v0", - "EleutherAI/pythia-19m-deduped-v0", - "pythia-19m-deduped-v0", - ], - "EleutherAI/pythia-70m-v0": [ - "pythia-70m-v0", - "pythia-v0", - "EleutherAI/pythia-19m-v0", - "pythia-19m-v0", - ], - "facebook/hubert-base-ls960": ["hubert-base-ls960"], - "facebook/opt-1.3b": ["opt-1.3b", "opt-medium"], - "facebook/opt-125m": ["opt-125m", "opt-small", "opt"], - "facebook/opt-13b": ["opt-13b", "opt-xxl"], - "facebook/opt-2.7b": ["opt-2.7b", "opt-large"], - "facebook/opt-30b": ["opt-30b", "opt-xxxl"], - "facebook/opt-6.7b": ["opt-6.7b", "opt-xl"], - "facebook/opt-66b": ["opt-66b", "opt-xxxxl"], - "facebook/wav2vec2-base": ["wav2vec2-base", "w2v2-base"], - "facebook/wav2vec2-large": ["wav2vec2-large", "w2v2-large"], - "google-bert/bert-base-cased": ["bert-base-cased"], - "google-bert/bert-base-uncased": ["bert-base-uncased"], - "google-bert/bert-large-cased": ["bert-large-cased"], - "google-bert/bert-large-uncased": ["bert-large-uncased"], - "google-t5/t5-base": ["t5-base"], - "google-t5/t5-large": ["t5-large"], - "google-t5/t5-small": ["t5-small"], - "google/gemma-2-27b": ["gemma-2-27b"], - "google/gemma-2-27b-it": ["gemma-2-27b-it"], - "google/gemma-2-2b": ["gemma-2-2b"], - "google/gemma-2-2b-it": ["gemma-2-2b-it"], - "google/gemma-2-9b": ["gemma-2-9b"], - "google/gemma-2-9b-it": ["gemma-2-9b-it"], - "google/gemma-2b": ["gemma-2b"], - "google/gemma-2b-it": ["gemma-2b-it"], - "google/gemma-3-12b-it": ["gemma-3-12b-it"], - "google/gemma-3-12b-pt": ["gemma-3-12b-pt"], - "google/gemma-3-1b-it": ["gemma-3-1b-it"], - "google/gemma-3-1b-pt": ["gemma-3-1b-pt"], - "google/gemma-3-270m": ["gemma-3-270m"], - "google/gemma-3-270m-it": ["gemma-3-270m-it"], - "google/gemma-3-27b-it": ["gemma-3-27b-it"], - "google/gemma-3-27b-pt": ["gemma-3-27b-pt"], - "google/gemma-3-4b-it": ["gemma-3-4b-it"], - "google/gemma-3-4b-pt": ["gemma-3-4b-pt"], - "google/gemma-7b": ["gemma-7b"], - "google/gemma-7b-it": ["gemma-7b-it"], - "google/medgemma-27b-it": ["medgemma-27b-it"], - "google/medgemma-27b-text-it": ["medgemma-27b-text-it"], - "google/medgemma-4b-it": ["medgemma-4b-it"], - "google/medgemma-4b-pt": ["medgemma-4b-pt"], - "gpt2": ["gpt2-small"], - "llama-13b-hf": ["llama-13b"], - "llama-30b-hf": ["llama-30b"], - "llama-65b-hf": ["llama-65b"], - "llama-7b-hf": ["llama-7b"], - "meta-llama/Llama-2-13b-chat-hf": ["Llama-2-13b-chat"], - "meta-llama/Llama-2-13b-hf": ["Llama-2-13b"], - "meta-llama/Llama-2-70b-chat-hf": ["Llama-2-70b-chat", "meta-llama-2-70b-chat-hf"], - "meta-llama/Llama-2-7b-chat-hf": ["Llama-2-7b-chat"], - "meta-llama/Llama-2-7b-hf": ["Llama-2-7b"], - "microsoft/phi-1": ["phi-1"], - "microsoft/phi-1_5": ["phi-1_5"], - "microsoft/phi-2": ["phi-2"], - "microsoft/Phi-3-mini-4k-instruct": ["phi-3"], - "microsoft/phi-4": ["phi-4"], - "mistralai/Mistral-7B-Instruct-v0.1": ["mistral-7b-instruct"], - "mistralai/Mistral-7B-v0.1": ["mistral-7b"], - "mistralai/Mistral-Nemo-Base-2407": ["mistral-nemo-base-2407"], - "mistralai/Mixtral-8x7B-Instruct-v0.1": ["mixtral-instruct", "mixtral-8x7b-instruct"], - "mistralai/Mixtral-8x7B-v0.1": ["mixtral", "mixtral-8x7b"], - "NeelNanda/Attn-Only-2L512W-Shortformer-6B-big-lr": [ - "attn-only-2l-demo", - "attn-only-2l-shortformer-6b-big-lr", - "attn-only-2l-induction-demo", - "attn-only-demo", - ], - "NeelNanda/Attn_Only_1L512W_C4_Code": [ - "attn-only-1l", - "attn-only-1l-new", - "attn-only-1l-c4-code", - ], - "NeelNanda/Attn_Only_2L512W_C4_Code": [ - "attn-only-2l", - "attn-only-2l-new", - "attn-only-2l-c4-code", - ], - "NeelNanda/Attn_Only_3L512W_C4_Code": [ - "attn-only-3l", - "attn-only-3l-new", - "attn-only-3l-c4-code", - ], - "NeelNanda/Attn_Only_4L512W_C4_Code": [ - "attn-only-4l", - "attn-only-4l-new", - "attn-only-4l-c4-code", - ], - "NeelNanda/GELU_1L512W_C4_Code": ["gelu-1l", "gelu-1l-new", "gelu-1l-c4-code"], - "NeelNanda/GELU_2L512W_C4_Code": ["gelu-2l", "gelu-2l-new", "gelu-2l-c4-code"], - "NeelNanda/GELU_3L512W_C4_Code": ["gelu-3l", "gelu-3l-new", "gelu-3l-c4-code"], - "NeelNanda/GELU_4L512W_C4_Code": ["gelu-4l", "gelu-4l-new", "gelu-4l-c4-code"], - "NeelNanda/SoLU_10L1280W_C4_Code": ["solu-10l", "solu-10l-new", "solu-10l-c4-code"], - "NeelNanda/SoLU_10L_v22_old": ["solu-10l-pile", "solu-10l-old"], - "NeelNanda/SoLU_12L1536W_C4_Code": ["solu-12l", "solu-12l-new", "solu-12l-c4-code"], - "NeelNanda/SoLU_12L_v23_old": ["solu-12l-pile", "solu-12l-old"], - "NeelNanda/SoLU_1L512W_C4_Code": ["solu-1l", "solu-1l-new", "solu-1l-c4-code"], - "NeelNanda/SoLU_1L512W_Wiki_Finetune": [ - "solu-1l-wiki", - "solu-1l-wiki-finetune", - "solu-1l-finetune", - ], - "NeelNanda/SoLU_1L_v9_old": ["solu-1l-pile", "solu-1l-old"], - "NeelNanda/SoLU_2L512W_C4_Code": ["solu-2l", "solu-2l-new", "solu-2l-c4-code"], - "NeelNanda/SoLU_2L_v10_old": ["solu-2l-pile", "solu-2l-old"], - "NeelNanda/SoLU_3L512W_C4_Code": ["solu-3l", "solu-3l-new", "solu-3l-c4-code"], - "NeelNanda/SoLU_4L512W_C4_Code": ["solu-4l", "solu-4l-new", "solu-4l-c4-code"], - "NeelNanda/SoLU_4L512W_Wiki_Finetune": [ - "solu-4l-wiki", - "solu-4l-wiki-finetune", - "solu-4l-finetune", - ], - "NeelNanda/SoLU_4L_v11_old": ["solu-4l-pile", "solu-4l-old"], - "NeelNanda/SoLU_6L768W_C4_Code": ["solu-6l", "solu-6l-new", "solu-6l-c4-code"], - "NeelNanda/SoLU_6L_v13_old": ["solu-6l-pile", "solu-6l-old"], - "NeelNanda/SoLU_8L1024W_C4_Code": ["solu-8l", "solu-8l-new", "solu-8l-c4-code"], - "NeelNanda/SoLU_8L_v21_old": ["solu-8l-pile", "solu-8l-old"], - "openai/gpt-oss-20b": ["gpt-oss-20b", "gpt-oss"], - "Qwen/Qwen-14B": ["qwen-14b"], - "Qwen/Qwen-14B-Chat": ["qwen-14b-chat"], - "Qwen/Qwen-1_8B": ["qwen-1.8b"], - "Qwen/Qwen-1_8B-Chat": ["qwen-1.8b-chat"], - "Qwen/Qwen-7B": ["qwen-7b"], - "Qwen/Qwen-7B-Chat": ["qwen-7b-chat"], - "Qwen/Qwen1.5-0.5B": ["qwen1.5-0.5b"], - "Qwen/Qwen1.5-0.5B-Chat": ["qwen1.5-0.5b-chat"], - "Qwen/Qwen1.5-1.8B": ["qwen1.5-1.8b"], - "Qwen/Qwen1.5-1.8B-Chat": ["qwen1.5-1.8b-chat"], - "Qwen/Qwen1.5-14B": ["qwen1.5-14b"], - "Qwen/Qwen1.5-14B-Chat": ["qwen1.5-14b-chat"], - "Qwen/Qwen1.5-4B": ["qwen1.5-4b"], - "Qwen/Qwen1.5-4B-Chat": ["qwen1.5-4b-chat"], - "Qwen/Qwen1.5-7B": ["qwen1.5-7b"], - "Qwen/Qwen1.5-7B-Chat": ["qwen1.5-7b-chat"], - "Qwen/Qwen2-0.5B": ["qwen2-0.5b"], - "Qwen/Qwen2-0.5B-Instruct": ["qwen2-0.5b-instruct"], - "Qwen/Qwen2-1.5B": ["qwen2-1.5b"], - "Qwen/Qwen2-1.5B-Instruct": ["qwen2-1.5b-instruct"], - "Qwen/Qwen2-7B": ["qwen2-7b"], - "Qwen/Qwen2-7B-Instruct": ["qwen2-7b-instruct"], - "Qwen/Qwen2.5-0.5B": ["qwen2.5-0.5b"], - "Qwen/Qwen2.5-0.5B-Instruct": ["qwen2.5-0.5b-instruct"], - "Qwen/Qwen2.5-1.5B": ["qwen2.5-1.5b"], - "Qwen/Qwen2.5-1.5B-Instruct": ["qwen2.5-1.5b-instruct"], - "Qwen/Qwen2.5-14B": ["qwen2.5-14b"], - "Qwen/Qwen2.5-14B-Instruct": ["qwen2.5-14b-instruct"], - "Qwen/Qwen2.5-32B": ["qwen2.5-32b"], - "Qwen/Qwen2.5-32B-Instruct": ["qwen2.5-32b-instruct"], - "Qwen/Qwen2.5-3B": ["qwen2.5-3b"], - "Qwen/Qwen2.5-3B-Instruct": ["qwen2.5-3b-instruct"], - "Qwen/Qwen2.5-72B": ["qwen2.5-72b"], - "Qwen/Qwen2.5-72B-Instruct": ["qwen2.5-72b-instruct"], - "Qwen/Qwen2.5-7B": ["qwen2.5-7b"], - "Qwen/Qwen2.5-7B-Instruct": ["qwen2.5-7b-instruct"], - "Qwen/Qwen3-0.6B": ["qwen3-0.6b"], - "Qwen/Qwen3-0.6B-Base": ["qwen3-0.6b-base"], - "Qwen/Qwen3-1.7B": ["qwen3-1.7b"], - "Qwen/Qwen3-14B": ["qwen3-14b"], - "Qwen/Qwen3-4B": ["qwen3-4b"], - "Qwen/Qwen3-8B": ["qwen3-8b"], - "Qwen/QwQ-32B-Preview": ["qwen-32b-preview"], - "roneneldan/TinyStories-1Layer-21M": ["tiny-stories-1L-21M"], - "roneneldan/TinyStories-1M": ["tiny-stories-1M"], - "roneneldan/TinyStories-28M": ["tiny-stories-28M"], - "roneneldan/TinyStories-2Layers-33M": ["tiny-stories-2L-33M"], - "roneneldan/TinyStories-33M": ["tiny-stories-33M"], - "roneneldan/TinyStories-3M": ["tiny-stories-3M"], - "roneneldan/TinyStories-8M": ["tiny-stories-8M"], - "roneneldan/TinyStories-Instruct-1M": ["tiny-stories-instruct-1M"], - "roneneldan/TinyStories-Instruct-28M": ["tiny-stories-instruct-28M"], - "roneneldan/TinyStories-Instruct-2Layers-33M": ["tiny-stories-instruct-2L-33M"], - "roneneldan/TinyStories-Instruct-33M": ["tiny-stories-instruct-33M"], - "roneneldan/TinyStories-Instruct-3M": ["tiny-stories-instruct-3M"], - "roneneldan/TinyStories-Instruct-8M": ["tiny-stories-instruct-8M"], - "roneneldan/TinyStories-Instuct-1Layer-21M": ["tiny-stories-instruct-1L-21M"], - "stabilityai/stablelm-base-alpha-3b": ["stablelm-base-alpha-3b", "stablelm-base-3b"], - "stabilityai/stablelm-base-alpha-7b": ["stablelm-base-alpha-7b", "stablelm-base-7b"], - "stabilityai/stablelm-tuned-alpha-3b": ["stablelm-tuned-alpha-3b", "stablelm-tuned-3b"], - "stabilityai/stablelm-tuned-alpha-7b": ["stablelm-tuned-alpha-7b", "stablelm-tuned-7b"], - "stanford-crfm/alias-gpt2-small-x21": [ - "stanford-gpt2-small-a", - "alias-gpt2-small-x21", - "gpt2-mistral-small-a", - "gpt2-stanford-small-a", - ], - "stanford-crfm/arwen-gpt2-medium-x21": [ - "stanford-gpt2-medium-a", - "arwen-gpt2-medium-x21", - "gpt2-medium-small-a", - "gpt2-stanford-medium-a", - ], - "stanford-crfm/battlestar-gpt2-small-x49": [ - "stanford-gpt2-small-b", - "battlestar-gpt2-small-x49", - "gpt2-mistral-small-b", - "gpt2-mistral-small-b", - ], - "stanford-crfm/beren-gpt2-medium-x49": [ - "stanford-gpt2-medium-b", - "beren-gpt2-medium-x49", - "gpt2-medium-small-b", - "gpt2-stanford-medium-b", - ], - "stanford-crfm/caprica-gpt2-small-x81": [ - "stanford-gpt2-small-c", - "caprica-gpt2-small-x81", - "gpt2-mistral-small-c", - "gpt2-stanford-small-c", - ], - "stanford-crfm/celebrimbor-gpt2-medium-x81": [ - "stanford-gpt2-medium-c", - "celebrimbor-gpt2-medium-x81", - "gpt2-medium-small-c", - "gpt2-medium-small-c", - ], - "stanford-crfm/darkmatter-gpt2-small-x343": [ - "stanford-gpt2-small-d", - "darkmatter-gpt2-small-x343", - "gpt2-mistral-small-d", - "gpt2-mistral-small-d", - ], - "stanford-crfm/durin-gpt2-medium-x343": [ - "stanford-gpt2-medium-d", - "durin-gpt2-medium-x343", - "gpt2-medium-small-d", - "gpt2-stanford-medium-d", - ], - "stanford-crfm/eowyn-gpt2-medium-x777": [ - "stanford-gpt2-medium-e", - "eowyn-gpt2-medium-x777", - "gpt2-medium-small-e", - "gpt2-stanford-medium-e", - ], - "stanford-crfm/expanse-gpt2-small-x777": [ - "stanford-gpt2-small-e", - "expanse-gpt2-small-x777", - "gpt2-mistral-small-e", - "gpt2-mistral-small-e", - ], - "swiss-ai/Apertus-8B-2509": ["apertus-8b", "apertus"], - "swiss-ai/Apertus-8B-Instruct-2509": ["apertus-8b-instruct", "apertus-instruct"], -} +# Canonical data lives in the bridge model registry +# (tools/model_registry/data/model_aliases.json); this re-export serves the +# legacy HookedTransformer loaders until their removal at 4.0. +MODEL_ALIASES: dict[str, list[str]] = load_model_aliases() """Model aliases for models on HuggingFace.""" diff --git a/transformer_lens/tools/__init__.py b/transformer_lens/tools/__init__.py index 2b78fa84ec..98af80819d 100644 --- a/transformer_lens/tools/__init__.py +++ b/transformer_lens/tools/__init__.py @@ -6,8 +6,25 @@ Subpackages: - analysis: High-level interpretability analyses (e.g. Direct Logit Attribution) - model_registry: Tools for discovering and documenting supported models + +Subpackages load lazily (PEP 562): ``analysis`` imports HookedTransformer, so an +eager import here would create a cycle for core modules that consume +``model_registry`` data at import time. """ -from . import analysis, model_registry +import importlib +from typing import Any + +_SUBMODULES = ("analysis", "model_registry") __all__ = ["analysis", "model_registry"] + + +def __getattr__(name: str) -> Any: + if name in _SUBMODULES: + return importlib.import_module(f".{name}", __name__) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(_SUBMODULES)) diff --git a/transformer_lens/tools/model_registry/__init__.py b/transformer_lens/tools/model_registry/__init__.py index b04c5bb7ab..30a3839267 100644 --- a/transformer_lens/tools/model_registry/__init__.py +++ b/transformer_lens/tools/model_registry/__init__.py @@ -115,7 +115,9 @@ "Qwen3NextForCausalLM", "Qwen3_5ForCausalLM", "Qwen3_5ForConditionalGeneration", + "RavenForCausalLM", "RecurrentGemmaForCausalLM", + "RWKV7ForCausalLM", "SmolLM3ForCausalLM", "StableLmForCausalLM", "T5ForConditionalGeneration", @@ -200,7 +202,9 @@ "Qwen3_5ForCausalLM": ["Qwen"], "Qwen3_5ForConditionalGeneration": ["Qwen"], "QwenForCausalLM": ["Qwen"], + "RavenForCausalLM": ["tomg-group-umd"], "RecurrentGemmaForCausalLM": ["google"], + "RWKV7ForCausalLM": ["fla-hub"], "SmolLM3ForCausalLM": ["HuggingFaceTB"], "StableLmForCausalLM": ["stabilityai"], "T5ForConditionalGeneration": ["google-t5", "google", "Salesforce", "MBZUAI"], @@ -210,10 +214,29 @@ "Zamba2ForCausalLM": ["Zyphra"], } +# Model-name prefixes that require trust_remote_code=True to load (custom +# modeling code on the HF Hub). +REMOTE_CODE_MODEL_PREFIXES: tuple[str, ...] = ( + "bigcode/santacoder", + "Qwen/Qwen-", + "Qwen/Qwen3-", + "microsoft/phi-2", + "microsoft/phi-4", + "apple/OpenELM", + "openai/gpt-oss-", + "swiss-ai/Apertus-", + "baichuan-inc/", # BaichuanForCausalLM — ships own modeling_baichuan.py + "ByteDance/Ouro-", # OuroForCausalLM — ships own modeling_ouro.py + "internlm/", # InternLM2ForCausalLM — ships own modeling_internlm2.py + "GSAI-ML/LLaDA", # LLaDAModelLM — ships configuration_llada.py/modeling_llada.py + "kuleshov-group/", # BD3LM — ships own custom modeling_d_dit.py +) + __all__ = [ # Constants "HF_SUPPORTED_ARCHITECTURES", "CANONICAL_AUTHORS_BY_ARCH", + "REMOTE_CODE_MODEL_PREFIXES", # Exceptions "ModelRegistryError", "ModelNotFoundError", diff --git a/transformer_lens/tools/model_registry/checkpoints.py b/transformer_lens/tools/model_registry/checkpoints.py new file mode 100644 index 0000000000..fbcab44d9f --- /dev/null +++ b/transformer_lens/tools/model_registry/checkpoints.py @@ -0,0 +1,45 @@ +"""Training-checkpoint label data for checkpointed model families. + +Canonical home for the checkpoint schedules previously defined in +``transformer_lens/loading_from_pretrained.py``. The schedules are frozen +historical artifacts of the published training runs. +""" + +import logging + +from .registry_io import resolve_model_alias + +# The steps for which there are checkpoints in the stanford crfm models +STANFORD_CRFM_CHECKPOINTS: list[int] = ( + list(range(0, 100, 10)) + + list(range(100, 2000, 50)) + + list(range(2000, 20000, 100)) + + list(range(20000, 400000 + 1, 1000)) +) + +# Linearly spaced checkpoints for Pythia models, taken every 1000 steps. +# Batch size 2,097,152 tokens, so checkpoints every 2.1B tokens +PYTHIA_CHECKPOINTS: list[int] = [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + list( + range(1000, 143000 + 1, 1000) +) +# Pythia V1 has log-spaced early checkpoints (see line above), but V0 doesn't +PYTHIA_V0_CHECKPOINTS: list[int] = list(range(1000, 143000 + 1, 1000)) + + +def get_checkpoint_labels(model_name: str) -> tuple[list[int], str]: + """Return (checkpoint labels, label type) for a checkpointed model family. + + Covers the HF-revision-checkpointed families (Pythia, stanford-crfm). + Raises ValueError for models without published checkpoint schedules. + """ + official_name = resolve_model_alias(model_name) or model_name + if official_name.startswith("stanford-crfm/"): + return STANFORD_CRFM_CHECKPOINTS, "step" + if official_name.startswith("EleutherAI/pythia"): + if "v0" in official_name: + return PYTHIA_V0_CHECKPOINTS, "step" + logging.warning( + "Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models." + ) + return PYTHIA_CHECKPOINTS, "step" + raise ValueError(f"Model {official_name} is not checkpointed.") diff --git a/transformer_lens/tools/model_registry/data/model_aliases.json b/transformer_lens/tools/model_registry/data/model_aliases.json new file mode 100644 index 0000000000..60124aab0c --- /dev/null +++ b/transformer_lens/tools/model_registry/data/model_aliases.json @@ -0,0 +1,883 @@ +{ + "description": "Canonical model-name alias table. Maps official HuggingFace model names to deprecated TransformerLens short aliases. Aliases are frozen: they warn on use and will be removed in TransformerLens 4.0.", + "aliases": { + "01-ai/Yi-34B": [ + "yi-34b", + "Yi-34B" + ], + "01-ai/Yi-34B-Chat": [ + "yi-34b-chat", + "Yi-34B-Chat" + ], + "01-ai/Yi-6B": [ + "yi-6b", + "Yi-6B" + ], + "01-ai/Yi-6B-Chat": [ + "yi-6b-chat", + "Yi-6B-Chat" + ], + "ai-forever/mGPT": [ + "mGPT" + ], + "allenai/OLMo-1B-hf": [ + "olmo-1b" + ], + "allenai/OLMo-2-0425-1B": [ + "olmo-2-1b" + ], + "allenai/OLMo-2-1124-7B": [ + "olmo-2-7b" + ], + "allenai/Olmo-3-1025-7B": [ + "olmo-3-1025-7b" + ], + "allenai/Olmo-3-1125-32B": [ + "olmo-3-1125-32b" + ], + "allenai/Olmo-3-32B-Think": [ + "olmo-3-32b-think" + ], + "allenai/Olmo-3-7B-Instruct": [ + "olmo-3-7b-instruct" + ], + "allenai/Olmo-3-7B-Think": [ + "olmo-3-7b-think" + ], + "allenai/Olmo-3.1-32B-Instruct": [ + "olmo-3.1-32b-instruct" + ], + "allenai/Olmo-3.1-32B-Think": [ + "olmo-3.1-32b-think" + ], + "allenai/OLMo-7B-hf": [ + "olmo-7b" + ], + "allenai/OLMoE-1B-7B-0924": [ + "olmoe" + ], + "ArthurConmy/redwood_attn_2l": [ + "redwood_attn_2l" + ], + "Baidicoot/Othello-GPT-Transformer-Lens": [ + "othello-gpt" + ], + "bigcode/santacoder": [ + "santacoder" + ], + "bigscience/bloom-1b1": [ + "bloom-1b1" + ], + "bigscience/bloom-1b7": [ + "bloom-1b7" + ], + "bigscience/bloom-3b": [ + "bloom-3b" + ], + "bigscience/bloom-560m": [ + "bloom-560m" + ], + "bigscience/bloom-7b1": [ + "bloom-7b1" + ], + "codellama/CodeLlama-7b-hf": [ + "CodeLlamallama-2-7b" + ], + "codellama/CodeLlama-7b-Instruct-hf": [ + "CodeLlama-7b-instruct" + ], + "codellama/CodeLlama-7b-Python-hf": [ + "CodeLlama-7b-python" + ], + "distilgpt2": [ + "distillgpt2", + "distill-gpt2", + "distil-gpt2", + "gpt2-xs" + ], + "EleutherAI/gpt-j-6B": [ + "gpt-j-6B", + "gpt-j", + "gptj" + ], + "EleutherAI/gpt-neo-1.3B": [ + "gpt-neo-1.3B", + "gpt-neo-medium", + "neo-medium" + ], + "EleutherAI/gpt-neo-125M": [ + "gpt-neo-125M", + "gpt-neo-small", + "neo-small", + "neo" + ], + "EleutherAI/gpt-neo-2.7B": [ + "gpt-neo-2.7B", + "gpt-neo-large", + "neo-large" + ], + "EleutherAI/gpt-neox-20b": [ + "gpt-neox-20b", + "gpt-neox", + "neox" + ], + "EleutherAI/pythia-1.4b": [ + "pythia-1.4b", + "EleutherAI/pythia-1.3b", + "pythia-1.3b" + ], + "EleutherAI/pythia-1.4b-deduped": [ + "pythia-1.4b-deduped", + "EleutherAI/pythia-1.3b-deduped", + "pythia-1.3b-deduped" + ], + "EleutherAI/pythia-1.4b-deduped-v0": [ + "pythia-1.4b-deduped-v0", + "EleutherAI/pythia-1.3b-deduped-v0", + "pythia-1.3b-deduped-v0" + ], + "EleutherAI/pythia-1.4b-v0": [ + "pythia-1.4b-v0", + "EleutherAI/pythia-1.3b-v0", + "pythia-1.3b-v0" + ], + "EleutherAI/pythia-12b": [ + "pythia-12b", + "EleutherAI/pythia-13b", + "pythia-13b" + ], + "EleutherAI/pythia-12b-deduped": [ + "pythia-12b-deduped", + "EleutherAI/pythia-13b-deduped", + "pythia-13b-deduped" + ], + "EleutherAI/pythia-12b-deduped-v0": [ + "pythia-12b-deduped-v0", + "EleutherAI/pythia-13b-deduped-v0", + "pythia-13b-deduped-v0" + ], + "EleutherAI/pythia-12b-v0": [ + "pythia-12b-v0", + "EleutherAI/pythia-13b-v0", + "pythia-13b-v0" + ], + "EleutherAI/pythia-14m": [ + "pythia-14m" + ], + "EleutherAI/pythia-160m": [ + "pythia-160m", + "EleutherAI/pythia-125m", + "pythia-125m" + ], + "EleutherAI/pythia-160m-deduped": [ + "pythia-160m-deduped", + "EleutherAI/pythia-125m-deduped", + "pythia-125m-deduped" + ], + "EleutherAI/pythia-160m-deduped-v0": [ + "pythia-160m-deduped-v0", + "EleutherAI/pythia-125m-deduped-v0", + "pythia-125m-deduped-v0" + ], + "EleutherAI/pythia-160m-seed1": [ + "pythia-160m-seed1", + "EleutherAI/pythia-125m-seed1", + "pythia-125m-seed1" + ], + "EleutherAI/pythia-160m-seed2": [ + "pythia-160m-seed2", + "EleutherAI/pythia-125m-seed2", + "pythia-125m-seed2" + ], + "EleutherAI/pythia-160m-seed3": [ + "pythia-160m-seed3", + "EleutherAI/pythia-125m-seed3", + "pythia-125m-seed3" + ], + "EleutherAI/pythia-160m-v0": [ + "pythia-160m-v0", + "EleutherAI/pythia-125m-v0", + "pythia-125m-v0" + ], + "EleutherAI/pythia-1b": [ + "pythia-1b", + "EleutherAI/pythia-800m", + "pythia-800m" + ], + "EleutherAI/pythia-1b-deduped": [ + "pythia-1b-deduped", + "EleutherAI/pythia-800m-deduped", + "pythia-800m-deduped" + ], + "EleutherAI/pythia-1b-deduped-v0": [ + "pythia-1b-deduped-v0", + "EleutherAI/pythia-800m-deduped-v0", + "pythia-800m-deduped-v0" + ], + "EleutherAI/pythia-1b-v0": [ + "pythia-1b-v0", + "EleutherAI/pythia-800m-v0", + "pythia-800m-v0" + ], + "EleutherAI/pythia-2.8b": [ + "pythia-2.8b", + "EleutherAI/pythia-2.7b", + "pythia-2.7b" + ], + "EleutherAI/pythia-2.8b-deduped": [ + "pythia-2.8b-deduped", + "EleutherAI/pythia-2.7b-deduped", + "pythia-2.7b-deduped" + ], + "EleutherAI/pythia-2.8b-deduped-v0": [ + "pythia-2.8b-deduped-v0", + "EleutherAI/pythia-2.7b-deduped-v0", + "pythia-2.7b-deduped-v0" + ], + "EleutherAI/pythia-2.8b-v0": [ + "pythia-2.8b-v0", + "EleutherAI/pythia-2.7b-v0", + "pythia-2.7b-v0" + ], + "EleutherAI/pythia-31m": [ + "pythia-31m" + ], + "EleutherAI/pythia-410m": [ + "pythia-410m", + "EleutherAI/pythia-350m", + "pythia-350m" + ], + "EleutherAI/pythia-410m-deduped": [ + "pythia-410m-deduped", + "EleutherAI/pythia-350m-deduped", + "pythia-350m-deduped" + ], + "EleutherAI/pythia-410m-deduped-v0": [ + "pythia-410m-deduped-v0", + "EleutherAI/pythia-350m-deduped-v0", + "pythia-350m-deduped-v0" + ], + "EleutherAI/pythia-410m-v0": [ + "pythia-410m-v0", + "EleutherAI/pythia-350m-v0", + "pythia-350m-v0" + ], + "EleutherAI/pythia-6.9b": [ + "pythia-6.9b", + "EleutherAI/pythia-6.7b", + "pythia-6.7b" + ], + "EleutherAI/pythia-6.9b-deduped": [ + "pythia-6.9b-deduped", + "EleutherAI/pythia-6.7b-deduped", + "pythia-6.7b-deduped" + ], + "EleutherAI/pythia-6.9b-deduped-v0": [ + "pythia-6.9b-deduped-v0", + "EleutherAI/pythia-6.7b-deduped-v0", + "pythia-6.7b-deduped-v0" + ], + "EleutherAI/pythia-6.9b-v0": [ + "pythia-6.9b-v0", + "EleutherAI/pythia-6.7b-v0", + "pythia-6.7b-v0" + ], + "EleutherAI/pythia-70m": [ + "pythia-70m", + "pythia", + "EleutherAI/pythia-19m", + "pythia-19m" + ], + "EleutherAI/pythia-70m-deduped": [ + "pythia-70m-deduped", + "EleutherAI/pythia-19m-deduped", + "pythia-19m-deduped" + ], + "EleutherAI/pythia-70m-deduped-v0": [ + "pythia-70m-deduped-v0", + "EleutherAI/pythia-19m-deduped-v0", + "pythia-19m-deduped-v0" + ], + "EleutherAI/pythia-70m-v0": [ + "pythia-70m-v0", + "pythia-v0", + "EleutherAI/pythia-19m-v0", + "pythia-19m-v0" + ], + "facebook/hubert-base-ls960": [ + "hubert-base-ls960" + ], + "facebook/opt-1.3b": [ + "opt-1.3b", + "opt-medium" + ], + "facebook/opt-125m": [ + "opt-125m", + "opt-small", + "opt" + ], + "facebook/opt-13b": [ + "opt-13b", + "opt-xxl" + ], + "facebook/opt-2.7b": [ + "opt-2.7b", + "opt-large" + ], + "facebook/opt-30b": [ + "opt-30b", + "opt-xxxl" + ], + "facebook/opt-6.7b": [ + "opt-6.7b", + "opt-xl" + ], + "facebook/opt-66b": [ + "opt-66b", + "opt-xxxxl" + ], + "facebook/wav2vec2-base": [ + "wav2vec2-base", + "w2v2-base" + ], + "facebook/wav2vec2-large": [ + "wav2vec2-large", + "w2v2-large" + ], + "google-bert/bert-base-cased": [ + "bert-base-cased" + ], + "google-bert/bert-base-uncased": [ + "bert-base-uncased" + ], + "google-bert/bert-large-cased": [ + "bert-large-cased" + ], + "google-bert/bert-large-uncased": [ + "bert-large-uncased" + ], + "google-t5/t5-base": [ + "t5-base" + ], + "google-t5/t5-large": [ + "t5-large" + ], + "google-t5/t5-small": [ + "t5-small" + ], + "google/gemma-2-27b": [ + "gemma-2-27b" + ], + "google/gemma-2-27b-it": [ + "gemma-2-27b-it" + ], + "google/gemma-2-2b": [ + "gemma-2-2b" + ], + "google/gemma-2-2b-it": [ + "gemma-2-2b-it" + ], + "google/gemma-2-9b": [ + "gemma-2-9b" + ], + "google/gemma-2-9b-it": [ + "gemma-2-9b-it" + ], + "google/gemma-2b": [ + "gemma-2b" + ], + "google/gemma-2b-it": [ + "gemma-2b-it" + ], + "google/gemma-3-12b-it": [ + "gemma-3-12b-it" + ], + "google/gemma-3-12b-pt": [ + "gemma-3-12b-pt" + ], + "google/gemma-3-1b-it": [ + "gemma-3-1b-it" + ], + "google/gemma-3-1b-pt": [ + "gemma-3-1b-pt" + ], + "google/gemma-3-270m": [ + "gemma-3-270m" + ], + "google/gemma-3-270m-it": [ + "gemma-3-270m-it" + ], + "google/gemma-3-27b-it": [ + "gemma-3-27b-it" + ], + "google/gemma-3-27b-pt": [ + "gemma-3-27b-pt" + ], + "google/gemma-3-4b-it": [ + "gemma-3-4b-it" + ], + "google/gemma-3-4b-pt": [ + "gemma-3-4b-pt" + ], + "google/gemma-7b": [ + "gemma-7b" + ], + "google/gemma-7b-it": [ + "gemma-7b-it" + ], + "google/medgemma-27b-it": [ + "medgemma-27b-it" + ], + "google/medgemma-27b-text-it": [ + "medgemma-27b-text-it" + ], + "google/medgemma-4b-it": [ + "medgemma-4b-it" + ], + "google/medgemma-4b-pt": [ + "medgemma-4b-pt" + ], + "gpt2": [ + "gpt2-small" + ], + "llama-13b-hf": [ + "llama-13b" + ], + "llama-30b-hf": [ + "llama-30b" + ], + "llama-65b-hf": [ + "llama-65b" + ], + "llama-7b-hf": [ + "llama-7b" + ], + "meta-llama/Llama-2-13b-chat-hf": [ + "Llama-2-13b-chat" + ], + "meta-llama/Llama-2-13b-hf": [ + "Llama-2-13b" + ], + "meta-llama/Llama-2-70b-chat-hf": [ + "Llama-2-70b-chat", + "meta-llama-2-70b-chat-hf" + ], + "meta-llama/Llama-2-7b-chat-hf": [ + "Llama-2-7b-chat" + ], + "meta-llama/Llama-2-7b-hf": [ + "Llama-2-7b" + ], + "microsoft/phi-1": [ + "phi-1" + ], + "microsoft/phi-1_5": [ + "phi-1_5" + ], + "microsoft/phi-2": [ + "phi-2" + ], + "microsoft/Phi-3-mini-4k-instruct": [ + "phi-3" + ], + "microsoft/phi-4": [ + "phi-4" + ], + "mistralai/Mistral-7B-Instruct-v0.1": [ + "mistral-7b-instruct" + ], + "mistralai/Mistral-7B-v0.1": [ + "mistral-7b" + ], + "mistralai/Mistral-Nemo-Base-2407": [ + "mistral-nemo-base-2407" + ], + "mistralai/Mixtral-8x7B-Instruct-v0.1": [ + "mixtral-instruct", + "mixtral-8x7b-instruct" + ], + "mistralai/Mixtral-8x7B-v0.1": [ + "mixtral", + "mixtral-8x7b" + ], + "NeelNanda/Attn-Only-2L512W-Shortformer-6B-big-lr": [ + "attn-only-2l-demo", + "attn-only-2l-shortformer-6b-big-lr", + "attn-only-2l-induction-demo", + "attn-only-demo" + ], + "NeelNanda/Attn_Only_1L512W_C4_Code": [ + "attn-only-1l", + "attn-only-1l-new", + "attn-only-1l-c4-code" + ], + "NeelNanda/Attn_Only_2L512W_C4_Code": [ + "attn-only-2l", + "attn-only-2l-new", + "attn-only-2l-c4-code" + ], + "NeelNanda/Attn_Only_3L512W_C4_Code": [ + "attn-only-3l", + "attn-only-3l-new", + "attn-only-3l-c4-code" + ], + "NeelNanda/Attn_Only_4L512W_C4_Code": [ + "attn-only-4l", + "attn-only-4l-new", + "attn-only-4l-c4-code" + ], + "NeelNanda/GELU_1L512W_C4_Code": [ + "gelu-1l", + "gelu-1l-new", + "gelu-1l-c4-code" + ], + "NeelNanda/GELU_2L512W_C4_Code": [ + "gelu-2l", + "gelu-2l-new", + "gelu-2l-c4-code" + ], + "NeelNanda/GELU_3L512W_C4_Code": [ + "gelu-3l", + "gelu-3l-new", + "gelu-3l-c4-code" + ], + "NeelNanda/GELU_4L512W_C4_Code": [ + "gelu-4l", + "gelu-4l-new", + "gelu-4l-c4-code" + ], + "NeelNanda/SoLU_10L1280W_C4_Code": [ + "solu-10l", + "solu-10l-new", + "solu-10l-c4-code" + ], + "NeelNanda/SoLU_10L_v22_old": [ + "solu-10l-pile", + "solu-10l-old" + ], + "NeelNanda/SoLU_12L1536W_C4_Code": [ + "solu-12l", + "solu-12l-new", + "solu-12l-c4-code" + ], + "NeelNanda/SoLU_12L_v23_old": [ + "solu-12l-pile", + "solu-12l-old" + ], + "NeelNanda/SoLU_1L512W_C4_Code": [ + "solu-1l", + "solu-1l-new", + "solu-1l-c4-code" + ], + "NeelNanda/SoLU_1L512W_Wiki_Finetune": [ + "solu-1l-wiki", + "solu-1l-wiki-finetune", + "solu-1l-finetune" + ], + "NeelNanda/SoLU_1L_v9_old": [ + "solu-1l-pile", + "solu-1l-old" + ], + "NeelNanda/SoLU_2L512W_C4_Code": [ + "solu-2l", + "solu-2l-new", + "solu-2l-c4-code" + ], + "NeelNanda/SoLU_2L_v10_old": [ + "solu-2l-pile", + "solu-2l-old" + ], + "NeelNanda/SoLU_3L512W_C4_Code": [ + "solu-3l", + "solu-3l-new", + "solu-3l-c4-code" + ], + "NeelNanda/SoLU_4L512W_C4_Code": [ + "solu-4l", + "solu-4l-new", + "solu-4l-c4-code" + ], + "NeelNanda/SoLU_4L512W_Wiki_Finetune": [ + "solu-4l-wiki", + "solu-4l-wiki-finetune", + "solu-4l-finetune" + ], + "NeelNanda/SoLU_4L_v11_old": [ + "solu-4l-pile", + "solu-4l-old" + ], + "NeelNanda/SoLU_6L768W_C4_Code": [ + "solu-6l", + "solu-6l-new", + "solu-6l-c4-code" + ], + "NeelNanda/SoLU_6L_v13_old": [ + "solu-6l-pile", + "solu-6l-old" + ], + "NeelNanda/SoLU_8L1024W_C4_Code": [ + "solu-8l", + "solu-8l-new", + "solu-8l-c4-code" + ], + "NeelNanda/SoLU_8L_v21_old": [ + "solu-8l-pile", + "solu-8l-old" + ], + "openai/gpt-oss-20b": [ + "gpt-oss-20b", + "gpt-oss" + ], + "Qwen/Qwen-14B": [ + "qwen-14b" + ], + "Qwen/Qwen-14B-Chat": [ + "qwen-14b-chat" + ], + "Qwen/Qwen-1_8B": [ + "qwen-1.8b" + ], + "Qwen/Qwen-1_8B-Chat": [ + "qwen-1.8b-chat" + ], + "Qwen/Qwen-7B": [ + "qwen-7b" + ], + "Qwen/Qwen-7B-Chat": [ + "qwen-7b-chat" + ], + "Qwen/Qwen1.5-0.5B": [ + "qwen1.5-0.5b" + ], + "Qwen/Qwen1.5-0.5B-Chat": [ + "qwen1.5-0.5b-chat" + ], + "Qwen/Qwen1.5-1.8B": [ + "qwen1.5-1.8b" + ], + "Qwen/Qwen1.5-1.8B-Chat": [ + "qwen1.5-1.8b-chat" + ], + "Qwen/Qwen1.5-14B": [ + "qwen1.5-14b" + ], + "Qwen/Qwen1.5-14B-Chat": [ + "qwen1.5-14b-chat" + ], + "Qwen/Qwen1.5-4B": [ + "qwen1.5-4b" + ], + "Qwen/Qwen1.5-4B-Chat": [ + "qwen1.5-4b-chat" + ], + "Qwen/Qwen1.5-7B": [ + "qwen1.5-7b" + ], + "Qwen/Qwen1.5-7B-Chat": [ + "qwen1.5-7b-chat" + ], + "Qwen/Qwen2-0.5B": [ + "qwen2-0.5b" + ], + "Qwen/Qwen2-0.5B-Instruct": [ + "qwen2-0.5b-instruct" + ], + "Qwen/Qwen2-1.5B": [ + "qwen2-1.5b" + ], + "Qwen/Qwen2-1.5B-Instruct": [ + "qwen2-1.5b-instruct" + ], + "Qwen/Qwen2-7B": [ + "qwen2-7b" + ], + "Qwen/Qwen2-7B-Instruct": [ + "qwen2-7b-instruct" + ], + "Qwen/Qwen2.5-0.5B": [ + "qwen2.5-0.5b" + ], + "Qwen/Qwen2.5-0.5B-Instruct": [ + "qwen2.5-0.5b-instruct" + ], + "Qwen/Qwen2.5-1.5B": [ + "qwen2.5-1.5b" + ], + "Qwen/Qwen2.5-1.5B-Instruct": [ + "qwen2.5-1.5b-instruct" + ], + "Qwen/Qwen2.5-14B": [ + "qwen2.5-14b" + ], + "Qwen/Qwen2.5-14B-Instruct": [ + "qwen2.5-14b-instruct" + ], + "Qwen/Qwen2.5-32B": [ + "qwen2.5-32b" + ], + "Qwen/Qwen2.5-32B-Instruct": [ + "qwen2.5-32b-instruct" + ], + "Qwen/Qwen2.5-3B": [ + "qwen2.5-3b" + ], + "Qwen/Qwen2.5-3B-Instruct": [ + "qwen2.5-3b-instruct" + ], + "Qwen/Qwen2.5-72B": [ + "qwen2.5-72b" + ], + "Qwen/Qwen2.5-72B-Instruct": [ + "qwen2.5-72b-instruct" + ], + "Qwen/Qwen2.5-7B": [ + "qwen2.5-7b" + ], + "Qwen/Qwen2.5-7B-Instruct": [ + "qwen2.5-7b-instruct" + ], + "Qwen/Qwen3-0.6B": [ + "qwen3-0.6b" + ], + "Qwen/Qwen3-0.6B-Base": [ + "qwen3-0.6b-base" + ], + "Qwen/Qwen3-1.7B": [ + "qwen3-1.7b" + ], + "Qwen/Qwen3-14B": [ + "qwen3-14b" + ], + "Qwen/Qwen3-4B": [ + "qwen3-4b" + ], + "Qwen/Qwen3-8B": [ + "qwen3-8b" + ], + "Qwen/QwQ-32B-Preview": [ + "qwen-32b-preview" + ], + "roneneldan/TinyStories-1Layer-21M": [ + "tiny-stories-1L-21M" + ], + "roneneldan/TinyStories-1M": [ + "tiny-stories-1M" + ], + "roneneldan/TinyStories-28M": [ + "tiny-stories-28M" + ], + "roneneldan/TinyStories-2Layers-33M": [ + "tiny-stories-2L-33M" + ], + "roneneldan/TinyStories-33M": [ + "tiny-stories-33M" + ], + "roneneldan/TinyStories-3M": [ + "tiny-stories-3M" + ], + "roneneldan/TinyStories-8M": [ + "tiny-stories-8M" + ], + "roneneldan/TinyStories-Instruct-1M": [ + "tiny-stories-instruct-1M" + ], + "roneneldan/TinyStories-Instruct-28M": [ + "tiny-stories-instruct-28M" + ], + "roneneldan/TinyStories-Instruct-2Layers-33M": [ + "tiny-stories-instruct-2L-33M" + ], + "roneneldan/TinyStories-Instruct-33M": [ + "tiny-stories-instruct-33M" + ], + "roneneldan/TinyStories-Instruct-3M": [ + "tiny-stories-instruct-3M" + ], + "roneneldan/TinyStories-Instruct-8M": [ + "tiny-stories-instruct-8M" + ], + "roneneldan/TinyStories-Instuct-1Layer-21M": [ + "tiny-stories-instruct-1L-21M" + ], + "stabilityai/stablelm-base-alpha-3b": [ + "stablelm-base-alpha-3b", + "stablelm-base-3b" + ], + "stabilityai/stablelm-base-alpha-7b": [ + "stablelm-base-alpha-7b", + "stablelm-base-7b" + ], + "stabilityai/stablelm-tuned-alpha-3b": [ + "stablelm-tuned-alpha-3b", + "stablelm-tuned-3b" + ], + "stabilityai/stablelm-tuned-alpha-7b": [ + "stablelm-tuned-alpha-7b", + "stablelm-tuned-7b" + ], + "stanford-crfm/alias-gpt2-small-x21": [ + "stanford-gpt2-small-a", + "alias-gpt2-small-x21", + "gpt2-mistral-small-a", + "gpt2-stanford-small-a" + ], + "stanford-crfm/arwen-gpt2-medium-x21": [ + "stanford-gpt2-medium-a", + "arwen-gpt2-medium-x21", + "gpt2-medium-small-a", + "gpt2-stanford-medium-a" + ], + "stanford-crfm/battlestar-gpt2-small-x49": [ + "stanford-gpt2-small-b", + "battlestar-gpt2-small-x49", + "gpt2-mistral-small-b" + ], + "stanford-crfm/beren-gpt2-medium-x49": [ + "stanford-gpt2-medium-b", + "beren-gpt2-medium-x49", + "gpt2-medium-small-b", + "gpt2-stanford-medium-b" + ], + "stanford-crfm/caprica-gpt2-small-x81": [ + "stanford-gpt2-small-c", + "caprica-gpt2-small-x81", + "gpt2-mistral-small-c", + "gpt2-stanford-small-c" + ], + "stanford-crfm/celebrimbor-gpt2-medium-x81": [ + "stanford-gpt2-medium-c", + "celebrimbor-gpt2-medium-x81", + "gpt2-medium-small-c" + ], + "stanford-crfm/darkmatter-gpt2-small-x343": [ + "stanford-gpt2-small-d", + "darkmatter-gpt2-small-x343", + "gpt2-mistral-small-d" + ], + "stanford-crfm/durin-gpt2-medium-x343": [ + "stanford-gpt2-medium-d", + "durin-gpt2-medium-x343", + "gpt2-medium-small-d", + "gpt2-stanford-medium-d" + ], + "stanford-crfm/eowyn-gpt2-medium-x777": [ + "stanford-gpt2-medium-e", + "eowyn-gpt2-medium-x777", + "gpt2-medium-small-e", + "gpt2-stanford-medium-e" + ], + "stanford-crfm/expanse-gpt2-small-x777": [ + "stanford-gpt2-small-e", + "expanse-gpt2-small-x777", + "gpt2-mistral-small-e" + ], + "swiss-ai/Apertus-8B-2509": [ + "apertus-8b", + "apertus" + ], + "swiss-ai/Apertus-8B-Instruct-2509": [ + "apertus-8b-instruct", + "apertus-instruct" + ] + } +} diff --git a/transformer_lens/tools/model_registry/data/supported_models.json b/transformer_lens/tools/model_registry/data/supported_models.json index 0a377103fe..713cb2da50 100644 --- a/transformer_lens/tools/model_registry/data/supported_models.json +++ b/transformer_lens/tools/model_registry/data/supported_models.json @@ -182721,6 +182721,20 @@ "phase7_score": null, "phase8_score": null }, + { + "architecture_id": "RavenForCausalLM", + "model_id": "tomg-group-umd/huginn-0125", + "status": 0, + "verified_date": null, + "metadata": null, + "note": null, + "phase1_score": null, + "phase2_score": null, + "phase3_score": null, + "phase4_score": null, + "phase7_score": null, + "phase8_score": null + }, { "architecture_id": "HrmTextForCausalLM", "model_id": "sapientinc/HRM-Text-1B", diff --git a/transformer_lens/tools/model_registry/generate_report.py b/transformer_lens/tools/model_registry/generate_report.py index 2478116e07..0a4f8cb05e 100644 --- a/transformer_lens/tools/model_registry/generate_report.py +++ b/transformer_lens/tools/model_registry/generate_report.py @@ -72,6 +72,8 @@ "HunYuanDenseV1ForCausalLM": "Tencent's open source decoder models", "Cohere2ForCausalLM": "Cohere's Command-A architecture with interleaved sliding-window RoPE and full-attention NoPE layers", "OuroForCausalLM": "ByteDance's Ouro looped language model (LoopLM) with weight-shared iterated depth", + "RavenForCausalLM": "tomg-group-umd's Huginn depth-recurrent decoder (prelude / weight-tied recurrent core / coda) with runtime recurrence count", + "RWKV7ForCausalLM": 'fla-hub\'s RWKV-7 ("Goose") attention-free recurrent decoder with generalized-delta-rule time-mixing and squared-ReLU channel-mixing', # Unsupported architectures "BertModel": "Google's BERT bidirectional encoder for understanding tasks", "BertForMaskedLM": "BERT with masked language modeling head", diff --git a/transformer_lens/tools/model_registry/registry_io.py b/transformer_lens/tools/model_registry/registry_io.py index 29d31ad988..a29079c245 100644 --- a/transformer_lens/tools/model_registry/registry_io.py +++ b/transformer_lens/tools/model_registry/registry_io.py @@ -8,6 +8,7 @@ import json import logging from datetime import date +from functools import lru_cache from pathlib import Path from typing import Callable, Optional @@ -18,6 +19,7 @@ _DATA_DIR = Path(__file__).parent / "data" _SUPPORTED_MODELS_PATH = _DATA_DIR / "supported_models.json" _VERIFICATION_HISTORY_PATH = _DATA_DIR / "verification_history.json" +_MODEL_ALIASES_PATH = _DATA_DIR / "model_aliases.json" # Status codes STATUS_UNVERIFIED = 0 @@ -136,6 +138,21 @@ def is_quantized_model(model_id: str) -> bool: return is_incompatible_quantized(model_id) +@lru_cache(maxsize=1) +def load_model_aliases() -> dict[str, list[str]]: + """Load the canonical alias table: official HF model name -> deprecated short aliases.""" + with open(_MODEL_ALIASES_PATH) as f: + return json.load(f)["aliases"] + + +def resolve_model_alias(model_name: str) -> Optional[str]: + """Return the official HF name if ``model_name`` is a deprecated alias, else None.""" + for official_name, aliases in load_model_aliases().items(): + if model_name in aliases: + return official_name + return None + + def load_supported_models_raw() -> dict: """Load supported_models.json as a raw dict.""" with open(_SUPPORTED_MODELS_PATH) as f: diff --git a/transformer_lens/tools/model_registry/verify_models.py b/transformer_lens/tools/model_registry/verify_models.py index ed7ad23075..fcfb713dcb 100644 --- a/transformer_lens/tools/model_registry/verify_models.py +++ b/transformer_lens/tools/model_registry/verify_models.py @@ -43,6 +43,7 @@ # between models without corrupting state. _interrupt_requested = False +from . import REMOTE_CODE_MODEL_PREFIXES from .registry_io import ( QUANTIZED_NOTE, STATUS_FAILED, @@ -58,16 +59,6 @@ logger = logging.getLogger(__name__) -# Architectures added via the TransformerBridge system that need trust_remote_code=True. -# These are not in the legacy NEED_REMOTE_CODE_MODELS tuple (loading_from_pretrained.py). -_BRIDGE_REMOTE_CODE_PREFIXES: tuple[str, ...] = ( - "baichuan-inc/", # BaichuanForCausalLM — ships own modeling_baichuan.py - "ByteDance/Ouro-", # OuroForCausalLM — ships own modeling_ouro.py - "internlm/", # InternLM2ForCausalLM — ships own modeling_internlm2.py - "GSAI-ML/LLaDA", # LLaDAModelLM — ships configuration_llada.py/modeling_llada.py - "kuleshov-group/", # BD3LM — ships own custom modeling_d_dit.py -) - # Data directory for registry files _DATA_DIR = Path(__file__).parent / "data" _CHECKPOINT_PATH = _DATA_DIR / "verification_checkpoint.json" @@ -199,10 +190,7 @@ def estimate_model_params(model_id: str) -> int: """ from transformers import AutoConfig - from transformer_lens.loading_from_pretrained import NEED_REMOTE_CODE_MODELS - - _all_remote_prefixes = NEED_REMOTE_CODE_MODELS + _BRIDGE_REMOTE_CODE_PREFIXES - trust_remote_code = any(model_id.startswith(prefix) for prefix in _all_remote_prefixes) + trust_remote_code = any(model_id.startswith(prefix) for prefix in REMOTE_CODE_MODEL_PREFIXES) from transformer_lens.utilities.hf_utils import get_hf_token config = AutoConfig.from_pretrained( @@ -889,10 +877,9 @@ def verify_models( all_results: list = [] error_msg: Optional[str] = None - from transformer_lens.loading_from_pretrained import NEED_REMOTE_CODE_MODELS - - _all_remote_prefixes = NEED_REMOTE_CODE_MODELS + _BRIDGE_REMOTE_CODE_PREFIXES - needs_remote_code = any(model_id.startswith(prefix) for prefix in _all_remote_prefixes) + needs_remote_code = any( + model_id.startswith(prefix) for prefix in REMOTE_CODE_MODEL_PREFIXES + ) # Convert string dtype to torch.dtype for benchmark suite import torch diff --git a/transformer_lens/utilities/tracr.py b/transformer_lens/utilities/tracr.py index a8db6b579c..fdfa74c9dd 100644 --- a/transformer_lens/utilities/tracr.py +++ b/transformer_lens/utilities/tracr.py @@ -19,7 +19,7 @@ def infer_tracr_output_label(model: Any) -> str: """Infer the RASP output label used in ``model.residual_labels``. - Tracr stores residual basis labels as strings like ``"reverse:3"`` while + Tracr stores residual basis labels as strings like ``"reverse_1:3"`` while its categorical output encoder stores only values and output-column ids. The output label is the unique residual-label prefix whose value set exactly matches the output encoder's categorical value set. From 596e584846982c940b1418bc165d61d7c58e9eca Mon Sep 17 00:00:00 2001 From: Jonah Larson Date: Fri, 24 Jul 2026 18:04:54 -0500 Subject: [PATCH 15/87] Comment cleanup jul 26 (#1541) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add dedicated GraniteMoe adapter tests (#1302) (#1524) granite_moe had substantive test coverage, but embedded inside test_granite_adapter.py rather than in its own file — inconsistent with every other architecture listed in #1302 (including granite_moe_hybrid), which each get a dedicated test__adapter.py. Moves that coverage into test_granite_moe_adapter.py and adds the one piece that was missing: an explicit config-flag test class (GraniteMoe doesn't override config setup, so it inherits dense Granite's flags, but nothing asserted that directly before). test_granite_adapter.py goes back to covering only GraniteArchitectureAdapter, matching its own docstring. * Updating documentation for future deprecation (#1523) * feat: add RWKV-7 (Goose) TransformerBridge adapter (RWKV7ForCausalLM) (#1521) * feat: add RWKV-7 (Goose) TransformerBridge adapter (RWKV7ForCausalLM) Signed-off-by: Mukund Pandey * refactor: use OpaqueBlockBridge for RWKV-7; introduce OpaqueBlockBridge base class Signed-off-by: Mukund Pandey * added hook_out_is_single_residual_stream default to OpaqueBlockBridge * style: apply black formatting to opaque_block, ssm_block, __init__ Signed-off-by: Mukund Pandey * style: fix import order in generalized_components/__init__.py (isort) Signed-off-by: Mukund Pandey * style: fix import order in generalized_components/__init__.py (isort) Signed-off-by: Mukund Pandey * style: fix import order in generalized_components/__init__.py (isort) Signed-off-by: Mukund Pandey * refactor: replace SSM2MixerBridge with GeneralizedComponent for RWKV-7 attn/ffn sublayers Signed-off-by: Mukund Pandey * style: fix black formatting in generate_report.py Signed-off-by: Mukund Pandey --------- Signed-off-by: Mukund Pandey Co-authored-by: jlarson4 * feat: add RavenForCausalLM (Huginn) depth-recurrent adapter (#1520) * Add RavenForCausalLM (Huginn) TransformerBridge adapter Adds a TransformerBridge architecture adapter for RavenForCausalLM (tomg-group-umd/huginn-0125), a depth-recurrent decoder with a prelude / weight-tied recurrent core / coda structure, resolving #1469. - raven.py delegates the recurrence to the remote-code HF forward and maps the three physical block lists (prelude / core_block / coda) via SSMBlockBridge, with combined-QKV native attention and a gated MLP. Sets applicable_phases=[] (a random initial latent state and post-residual sandwich norms diverge from the verify_models phases) and supports_fold_ln=False (ln_f is reused mid-network). prepare_loading patches Huginn's remote code for transformers v5 (tied-weights-keys dict form + a weight re-init guard). - Registers in the adapter factory, the model registry (canonical author and description) and supported_models.json; surfaces the recurrence-shape config via both _HF_PASSTHROUGH_ATTRS lists. - Adds synthetic-config unit tests and CI-gated integration tests. * fix: pass num_steps as int to iterate_forward (0-d tensor has no len) * fix: remove unused torch import and use setattr for dynamic cfg attrs * style: apply black formatting to raven.py Signed-off-by: Mukund Pandey * refactor: introduce OpaqueBlockBridge; use it for Raven instead of SSMBlockBridge Signed-off-by: Mukund Pandey * style: apply black formatting to opaque_block, ssm_block, __init__ Signed-off-by: Mukund Pandey * style: fix import order in generalized_components/__init__.py (isort) Signed-off-by: Mukund Pandey * fix: strip dead hook_q/k/v aliases from Raven AttentionBridge Raven uses a combined Wqkv projection (no separate q/k/v submodules), so AttentionBridge's default hook_q/hook_k/hook_v aliases (which target q.hook_out / k.hook_out / v.hook_out) are unresolvable. The upstream test_every_hook_alias_resolves_to_hookpoint audit catches these as 9 dead aliases across prelude / core_block / coda. Strip them by setting an instance-level hook_aliases that omits those three keys, leaving only the aliases that have real HookPoint targets. Signed-off-by: Mukund Pandey * chore: merge upstream/dev — add RWKV-7 passthrough attrs and registry entry Brings in the RWKV-7 additions from #1521 that landed in dev: - _bridge_builder.py: added RWKV-7 passthrough attrs (num_heads, value_dim, decay/gate/a/v_low_rank_dim, norm_first, norm_bias, fuse_norm, attn_mode, hidden_act) - generate_report.py: added RWKV7ForCausalLM description entry Signed-off-by: Mukund Pandey --------- Signed-off-by: Mukund Pandey * Fix Tracr demo output_label: compiled labels are auto-numbered (#1525) The Tracr demo passes output_label="reverse", but Tracr builds labels as f"{name}_{unique_id}", so the compiled expression's residual labels are reverse_1:* and the demo raises ValueError at the state-dict cell. Pass reverse.label instead of hardcoding the name, and correct the same wrong label shape in the unit-test fixture (real Tracr never emits "reverse:1") and in the infer_tracr_output_label docstring example. Co-authored-by: Claude Fable 5 * Add TransformerBridge adapter for lightweight decoder-only pretraining models (#1519) * Add TransformerLens pretrain bridge adapter * Add TransformerLens pretrain bridge adapter * Setup registry for hookedtransformer deprecation * bug fixes and rebase to 4.x * Hook management improvements * implemented `stop_at_layer` on TransformerBridge * input_to_embed and pos_slice added to bridge * prevent corruption, check_hooks_to_add placed in all locations that need it * Adjusted head detector to be HookedTransformer agnostic, added key value injection to bridge * Add specific migration differences section * Attempting to let pytest run in parallel * Attempting to fix the OOM on the multi-device Full Coverage run * Big comment sweep * Comment fix --------- Signed-off-by: Mukund Pandey Co-authored-by: Delaida Muminovic Co-authored-by: Mukund Pandey Co-authored-by: Joseph Quevedo <41499530+dewstend@users.noreply.github.com> Co-authored-by: Claude Fable 5 Co-authored-by: Cacapice --- docs/make_docs.py | 2 +- docs/source/conf.py | 1 - .../compatibility/test_hook_completeness.py | 6 +- ...test_legacy_hooked_transformer_coverage.py | 2 +- .../model_bridge/test_n_ctx_override.py | 18 ++-- tests/acceptance/test_activation_cache.py | 5 - .../acceptance/test_hooked_encoder_decoder.py | 1 - tests/acceptance/test_hooked_transformer.py | 11 --- .../test_bridge_generate_return_cache.py | 2 +- .../model_bridge/test_bridge_stop_at_layer.py | 3 - .../model_bridge/test_falcon_h1_adapter.py | 2 +- .../test_granite_moe_hybrid_adapter.py | 8 +- ..._joint_qkv_attention_hook_compatibility.py | 1 - .../model_bridge/test_mamba2_adapter.py | 14 +-- .../model_bridge/test_mamba_adapter.py | 19 ++-- .../model_bridge/test_pretrain_adapter.py | 9 +- .../model_bridge/test_qwen3_moe_bridge.py | 2 +- .../test_ssm_effective_attention_dispatch.py | 6 +- .../test_centralized_weight_processing.py | 2 +- .../test_fold_layer_integration.py | 1 - .../test_grouped_query_attention.py | 7 +- .../test_main_demo_pattern_hooks.py | 1 - tests/integration/test_start_at_layer.py | 3 +- .../test_next_sentence_prediction.py | 4 - .../test_codegen_attention_bridge.py | 4 +- .../test_joint_qkv_attention.py | 12 --- .../test_bd3lm_adapter.py | 1 - .../test_gemma3n_adapter.py | 2 +- .../test_hunyuan_v1_dense_adapter.py | 2 +- .../test_lfm2_adapter.py | 2 +- .../test_mpt_adapter.py | 4 +- .../test_pretrain_model_container.py | 30 +----- .../test_stablelm_adapter.py | 4 +- .../model_bridge/test_component_hooks_fire.py | 6 +- .../test_hook_alias_resolution.py | 2 +- .../test_inspect_vllm_provider.py | 2 +- .../unit/model_bridge/test_native_features.py | 3 +- .../test_optimizer_compatibility.py | 2 - tests/unit/test_hook_points.py | 4 +- tests/unit/test_hooked_transformer_config.py | 2 - tests/unit/test_key_value_cache_entry.py | 35 +------ tests/unit/test_supported_models.py | 3 - tests/unit/test_svd_interpreter.py | 2 +- tests/unit/test_utils.py | 4 +- tests/unit/test_weight_processing.py | 2 - tests/unit/utilities/test_devices.py | 1 - .../test_base_tensor_conversion.py | 4 - .../test_callable_tensor_conversion.py | 3 - .../test_rearrange_tensor_conversion.py | 6 +- .../test_repeat_tensor_conversion.py | 2 - .../test_terenary_tensor_conversion.py | 10 +- transformer_lens/HookedAudioEncoder.py | 1 - transformer_lens/HookedEncoder.py | 1 - transformer_lens/HookedEncoderDecoder.py | 3 - transformer_lens/HookedTransformer.py | 22 ----- transformer_lens/benchmarks/forward_pass.py | 3 +- transformer_lens/benchmarks/main_benchmark.py | 2 +- .../components/abstract_attention.py | 6 -- transformer_lens/components/attention.py | 1 - transformer_lens/components/t5_block.py | 4 +- .../components/transformer_block.py | 1 - .../config/hooked_transformer_config.py | 1 - .../config/transformer_lens_config.py | 8 -- .../conversion_steps/tensor_conversion_set.py | 1 - .../helpers/merge_quantiziation_fields.py | 2 - .../conversion_utils/hook_conversion_utils.py | 6 -- transformer_lens/evals.py | 2 - transformer_lens/head_detector.py | 2 +- transformer_lens/loading_from_pretrained.py | 4 - .../model_bridge/architecture_adapter.py | 3 +- .../generalized_components/base.py | 2 - .../generalized_components/block.py | 7 +- .../generalized_components/bloom_block.py | 5 - .../generalized_components/bloom_mlp.py | 1 - .../clip_vision_encoder.py | 1 - .../generalized_components/linear.py | 1 - .../position_embeddings_attention.py | 3 - .../rotary_embedding.py | 1 - .../siglip_vision_encoder.py | 1 - .../vision_projection.py | 3 - .../sources/transformers/source.py | 3 +- .../supported_architectures/gemma1.py | 1 - .../supported_architectures/gemma2.py | 2 - .../gemma3_multimodal.py | 1 - .../gpt2_lm_head_custom.py | 2 +- .../hunyuan_v1_dense.py | 2 - .../supported_architectures/llama.py | 1 - .../supported_architectures/llava.py | 1 - .../supported_architectures/olmo2.py | 1 - .../supported_architectures/olmoe.py | 1 - .../supported_architectures/pretrain.py | 96 ++++--------------- .../supported_architectures/qwen2.py | 2 - .../supported_architectures/qwen3.py | 3 +- .../supported_architectures/qwen3_moe.py | 1 - .../model_bridge/transformer_bridge.py | 3 - transformer_lens/patching.py | 3 - .../pretrained/weight_conversions/hubert.py | 1 - .../pretrained/weight_conversions/olmoe.py | 1 - .../pretrained/weight_conversions/qwen3.py | 1 - transformer_lens/utilities/aliases.py | 1 - transformer_lens/utilities/attribute_utils.py | 1 - .../utilities/bridge_components.py | 3 - transformer_lens/utilities/hf_utils.py | 3 - transformer_lens/utilities/matrix.py | 1 - transformer_lens/utilities/tokenize_utils.py | 2 +- transformer_lens/weight_processing.py | 1 - 106 files changed, 97 insertions(+), 423 deletions(-) diff --git a/docs/make_docs.py b/docs/make_docs.py index 242ce043e0..5f96d36405 100644 --- a/docs/make_docs.py +++ b/docs/make_docs.py @@ -217,7 +217,7 @@ def tokenizer_vocab_hash(tokenizer: PreTrainedTokenizer) -> str: # convert to base64 return base64.b64encode( hash_obj.digest(), - altchars=b"-_", # - and _ as altchars + altchars=b"-_", ).decode("UTF-8") diff --git a/docs/source/conf.py b/docs/source/conf.py index 1960b0c4ac..eea59e26a8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -184,4 +184,3 @@ def setup(app): """Sphinx setup overrides.""" # Connect functions to run when watch detects a file change app.connect("builder-inited", run_apidoc) - # app.connect("builder-inited", copy_demos) # Don't run as too slow diff --git a/tests/acceptance/model_bridge/compatibility/test_hook_completeness.py b/tests/acceptance/model_bridge/compatibility/test_hook_completeness.py index 84c6a2d223..fdcfcdca28 100644 --- a/tests/acceptance/model_bridge/compatibility/test_hook_completeness.py +++ b/tests/acceptance/model_bridge/compatibility/test_hook_completeness.py @@ -19,13 +19,13 @@ # Diverse architectures for hook completeness testing. # Constraint: these tests compare bridge vs legacy HookedTransformer, so each -# entry must be in HookedTransformer's OFFICIAL_MODEL_NAMES. Tiny C1-affected -# families (Llama/Qwen/Gemma under ~150M) aren't registered with HT; for those, +# entry must be in HookedTransformer's OFFICIAL_MODEL_NAMES. Tiny Llama/Qwen/Gemma +# families (under ~150M) aren't registered with HT; for those, # tests/unit/model_bridge/test_component_hooks_fire.py (Tier 2) provides # direct per-adapter hook-firing coverage without needing an HT counterpart. MODELS_TO_TEST = [ "gpt2", # JointQKVAttentionBridge (standard decoder-only) - "EleutherAI/pythia-14m", # ParallelBlockBridge (C15 regression guard) + "EleutherAI/pythia-14m", # ParallelBlockBridge regression guard ] # Gemma2: local only (too large for CI) diff --git a/tests/acceptance/model_bridge/compatibility/test_legacy_hooked_transformer_coverage.py b/tests/acceptance/model_bridge/compatibility/test_legacy_hooked_transformer_coverage.py index ad2b53874a..e8d82544d2 100644 --- a/tests/acceptance/model_bridge/compatibility/test_legacy_hooked_transformer_coverage.py +++ b/tests/acceptance/model_bridge/compatibility/test_legacy_hooked_transformer_coverage.py @@ -229,7 +229,7 @@ def test_memory_efficiency(self, bridge_model): # Run multiple forward passes for _ in range(5): output = bridge_model(prompt) - del output # Explicitly delete + del output # release tensor so the leak probe below is meaningful # Clean up gc.collect() diff --git a/tests/acceptance/model_bridge/test_n_ctx_override.py b/tests/acceptance/model_bridge/test_n_ctx_override.py index be35474e68..54b2a49be1 100644 --- a/tests/acceptance/model_bridge/test_n_ctx_override.py +++ b/tests/acceptance/model_bridge/test_n_ctx_override.py @@ -51,24 +51,20 @@ def test_n_ctx_combined_with_hf_config_overrides(): assert bridge.cfg.n_ctx == 256 -# --- Coverage for code-review items #2, #4, #5, #7 --- - - def test_n_ctx_zero_raises_value_error(): - """#2: n_ctx must be positive; zero should raise ValueError.""" + """n_ctx must be positive; zero should raise ValueError.""" with pytest.raises(ValueError, match="positive integer"): TransformerBridge.boot_transformers("gpt2", device="cpu", n_ctx=0, load_weights=False) def test_n_ctx_negative_raises_value_error(): - """#2: n_ctx must be positive; negative should raise ValueError.""" + """n_ctx must be positive; negative should raise ValueError.""" with pytest.raises(ValueError, match="positive integer"): TransformerBridge.boot_transformers("gpt2", device="cpu", n_ctx=-1, load_weights=False) def test_n_ctx_conflict_with_hf_config_overrides_warns(caplog): - """#4: When both n_ctx and the same hf_config_overrides field are set with different values, - a warning should be emitted explaining that n_ctx wins.""" + """When both n_ctx and the same hf_config_overrides field are set with different values, a warning should be emitted explaining that n_ctx wins.""" with caplog.at_level(logging.WARNING): TransformerBridge.boot_transformers( "gpt2", @@ -85,7 +81,7 @@ def test_n_ctx_conflict_with_hf_config_overrides_warns(caplog): def test_n_ctx_no_conflict_when_values_match(caplog): - """#4: If n_ctx and hf_config_overrides agree on the value, no conflict warning is emitted.""" + """If n_ctx and hf_config_overrides agree on the value, no conflict warning is emitted.""" with caplog.at_level(logging.WARNING): TransformerBridge.boot_transformers( "gpt2", @@ -98,8 +94,7 @@ def test_n_ctx_no_conflict_when_values_match(caplog): def test_n_ctx_shrink_with_load_weights_gives_clear_error(): - """#5: Shrinking a learned-pos-embed model's n_ctx at weight-load time should raise - with a message explaining the cause and suggesting alternatives.""" + """Shrinking a learned-pos-embed model's n_ctx at weight-load time should raise with a message explaining the cause and suggesting alternatives.""" with pytest.raises(RuntimeError) as exc_info: TransformerBridge.boot_transformers("gpt2", device="cpu", n_ctx=256, load_weights=True) err = str(exc_info.value) @@ -108,8 +103,7 @@ def test_n_ctx_shrink_with_load_weights_gives_clear_error(): def test_n_ctx_override_verified_on_loaded_model(): - """#7: After load, the override should be visible on hf_model.config so users - can trust that the longer/shorter context is actually in effect.""" + """After load, the override should be visible on hf_model.config so users can trust that the longer/shorter context is actually in effect.""" bridge = TransformerBridge.boot_transformers( "gpt2", device="cpu", n_ctx=2048, load_weights=False ) diff --git a/tests/acceptance/test_activation_cache.py b/tests/acceptance/test_activation_cache.py index 68f6fd0b39..60875620f7 100644 --- a/tests/acceptance/test_activation_cache.py +++ b/tests/acceptance/test_activation_cache.py @@ -23,7 +23,6 @@ def get_ioi_tokens_and_answer_tokens(model): - # List of prompts prompts = [] # List of answers, in the format (correct, incorrect) answers = [] @@ -62,15 +61,12 @@ def load_model(name): @torch.no_grad def test_logit_attrs_matches_reference_code(): - # Load solu-2l model = load_model("solu-2l") tokens, answer_tokens = get_ioi_tokens_and_answer_tokens(model) - # Run the model and cache all activations _, cache = model.run_with_cache(tokens) - # Get accumulated resid accumulated_residual = cache.accumulated_resid(layer=-1, incl_mid=True, pos_slice=-1) # Get ref ave logit diffs (cribbed notebook code) @@ -745,7 +741,6 @@ def test_compute_test_head_results_does_not_compute_results_twice(): cache.compute_head_results() assert "blocks.0.attn.hook_result" in cache.cache_dict - # set infinity to the first element of the head results assert cache.cache_dict["blocks.0.attn.hook_result"][0, 0, 0, 0] != float("inf") cache.cache_dict["blocks.0.attn.hook_result"][0, 0, 0, 0] = float("inf") cache.compute_head_results() diff --git a/tests/acceptance/test_hooked_encoder_decoder.py b/tests/acceptance/test_hooked_encoder_decoder.py index d3628e7afb..2be657c479 100644 --- a/tests/acceptance/test_hooked_encoder_decoder.py +++ b/tests/acceptance/test_hooked_encoder_decoder.py @@ -229,7 +229,6 @@ def test_cross_attention_layer(our_model, huggingface_model, hello_world_tokens, hf_layer = huggingface_model.decoder.block[0].layer[1] our_layer = our_model.decoder[0] - # assert ln weights are the same assert_close(hf_layer.layer_norm.weight, our_layer.ln2.w) our_cross_attn_out = ( diff --git a/tests/acceptance/test_hooked_transformer.py b/tests/acceptance/test_hooked_transformer.py index 8020c19237..a2d472e497 100644 --- a/tests/acceptance/test_hooked_transformer.py +++ b/tests/acceptance/test_hooked_transformer.py @@ -73,16 +73,6 @@ text = "Hello world!" -""" -# Code to regenerate loss store -store = {} -for name in model_names: - model = HookedTransformer.from_pretrained(name, device='cuda') - loss = model(text,return_type="loss") - store[name] = loss.item() -print(store) -""" - # Loss values for minimal testing SMALL_LOSS_STORE = { "gpt2-small": 5.331855773925781, @@ -385,7 +375,6 @@ def benchmark_model_options( device ) - # hf_model = hf_model.to(device) hf_logits = hf_model(tokens).logits.detach() hf_logits = hf_logits.to("cpu") diff --git a/tests/integration/model_bridge/test_bridge_generate_return_cache.py b/tests/integration/model_bridge/test_bridge_generate_return_cache.py index c6323818be..3b74f2226b 100644 --- a/tests/integration/model_bridge/test_bridge_generate_return_cache.py +++ b/tests/integration/model_bridge/test_bridge_generate_return_cache.py @@ -78,7 +78,7 @@ def test_cache_matches_run_with_cache_over_full_sequence(self, bridge): assert torch.allclose(g, r, atol=1e-5, rtol=1e-4), f"{name} differs from run_with_cache" def test_includes_attention_patterns(self, bridge): - """The cache includes full [batch, heads, q, k] attention patterns (an Option B benefit).""" + """The cache includes full [batch, heads, q, k] attention patterns.""" tokens = bridge.to_tokens("The quick brown") with torch.no_grad(): out, cache = bridge.generate( diff --git a/tests/integration/model_bridge/test_bridge_stop_at_layer.py b/tests/integration/model_bridge/test_bridge_stop_at_layer.py index 20e6e2298e..8e2c7e5ba2 100644 --- a/tests/integration/model_bridge/test_bridge_stop_at_layer.py +++ b/tests/integration/model_bridge/test_bridge_stop_at_layer.py @@ -200,7 +200,6 @@ def test_stop_at_embed_processed(bridge_with_processed_weights): """Test stop_at_layer=0 with processed weights.""" rand_input = torch.randint(0, 100, (2, 10)) - # Run with stop_at_layer=0 output, cache = bridge_with_processed_weights.run_with_cache(rand_input, stop_at_layer=0) # Verify output shape @@ -245,7 +244,6 @@ def test_no_stop_processed(bridge_with_processed_weights): """Test stop_at_layer=None with processed weights (full forward pass).""" rand_input = torch.randint(0, 100, (2, 10)) - # Run with stop_at_layer=None output, cache = bridge_with_processed_weights.run_with_cache(rand_input, stop_at_layer=None) # Verify output shape is correct (logits) @@ -328,7 +326,6 @@ def test_stop_at_final_compat_no_processing(bridge_with_compat_no_processing): """Test stop_at_layer=-1 with compatibility mode (no processing).""" rand_input = torch.randint(0, 100, (2, 10)) - # Run with stop_at_layer=-1 output, cache = bridge_with_compat_no_processing.run_with_cache(rand_input, stop_at_layer=-1) # Verify output shape diff --git a/tests/integration/model_bridge/test_falcon_h1_adapter.py b/tests/integration/model_bridge/test_falcon_h1_adapter.py index cd38eecc03..51c82af218 100644 --- a/tests/integration/model_bridge/test_falcon_h1_adapter.py +++ b/tests/integration/model_bridge/test_falcon_h1_adapter.py @@ -2,7 +2,7 @@ Verifies wrap-don't-reimplement behaviour against Falcon-H1 checkpoints: -- ``tiiuae/Falcon-H1-Tiny-90M-Instruct`` — smallest variant (issue #1403 verify-first). +- ``tiiuae/Falcon-H1-Tiny-90M-Instruct`` — smallest variant. - ``tiiuae/Falcon-H1-0.5B-Base`` — primary parity suite. - Forward-pass logits match HF **exactly** (the bridge delegates the whole block to HF, which applies Falcon-H1's ~12 scalar multipliers natively). diff --git a/tests/integration/model_bridge/test_granite_moe_hybrid_adapter.py b/tests/integration/model_bridge/test_granite_moe_hybrid_adapter.py index f363e545b1..105677d34a 100644 --- a/tests/integration/model_bridge/test_granite_moe_hybrid_adapter.py +++ b/tests/integration/model_bridge/test_granite_moe_hybrid_adapter.py @@ -10,7 +10,7 @@ access or weight downloads (mirrors tests/unit/model_bridge/test_gpt_oss_moe.py and test_qwen3_moe_bridge.py for the from_config + direct-constructor pattern). -Coverage focus (Phase 0, SSM mixer access normalization): +Coverage focus (SSM mixer access normalization): - The Mamba-2 mixer is reachable at the canonical ``.mixer`` slot on SSM layers. - compute_effective_attention runs on a Granite SSM layer and reconstructs the SSM output (proves dims are sourced from the wrapped HF mixer, not the shared @@ -168,7 +168,7 @@ def test_shared_mlp_and_moe_hooks_fire(self, cache) -> None: # --------------------------------------------------------------------------- -# Effective attention: the Phase 0 acceptance surface +# Effective attention # --------------------------------------------------------------------------- @@ -351,7 +351,7 @@ def _available_devices(): class TestGraniteMoeHybridProcessedParity: - """Phase-3 parity guard: processed (compat-mode) bridge vs raw HF via log_softmax. + """Parity guard: processed (compat-mode) bridge vs raw HF via log_softmax. The forward test only checks UNPROCESSED delegation (==0.0); this pins the PROCESSED path so a compat-mode regression is caught in CI without the full-size checkpoint. @@ -376,7 +376,7 @@ def test_compat_mode_logits_match_raw_hf(self): class TestGraniteMoeHybridEagerScanIntervention: - """Phase 4 eager-scan intervention on Granite's Mamba-2 mixer layers (hybrid): + """Eager-scan intervention on Granite's Mamba-2 mixer layers (hybrid): hooks fire on the Mamba layers only, interventions propagate to logits, and the default path is untouched. Eager scan needs use_cache=False (prefill).""" diff --git a/tests/integration/model_bridge/test_joint_qkv_attention_hook_compatibility.py b/tests/integration/model_bridge/test_joint_qkv_attention_hook_compatibility.py index b7b0461556..d602978ecc 100644 --- a/tests/integration/model_bridge/test_joint_qkv_attention_hook_compatibility.py +++ b/tests/integration/model_bridge/test_joint_qkv_attention_hook_compatibility.py @@ -13,7 +13,6 @@ def test_v_hook_out_equals_blocks_attn_hook_v(self): # Load DistilGPT-2 in TransformerBridge (faster for testing) bridge = TransformerBridge.boot_transformers("distilgpt2", device="cpu") - # Turn on compatibility mode bridge.enable_compatibility_mode(disable_warnings=True) # Create test input diff --git a/tests/integration/model_bridge/test_mamba2_adapter.py b/tests/integration/model_bridge/test_mamba2_adapter.py index 511baea59c..b503921b9e 100644 --- a/tests/integration/model_bridge/test_mamba2_adapter.py +++ b/tests/integration/model_bridge/test_mamba2_adapter.py @@ -10,10 +10,10 @@ - Generation via is_stateful fallback delegates to hf_generate - d_mlp == 0 assertion: in_proj output features match the 3-way split formula -Cache clone safety: Same guarantee as Phase 1 — the wrap-don't-reimplement -design keeps Mamba2Mixer opaque, so `ssm_states` is never hooked. The only -hooked tensors are projection inputs/outputs, which are per-step and not -mutated by the cache machinery. +Cache clone safety: the wrap-don't-reimplement design keeps Mamba2Mixer +opaque, so `ssm_states` is never hooked. The only hooked tensors are +projection inputs/outputs, which are per-step and not mutated by the cache +machinery. """ import contextlib @@ -81,7 +81,7 @@ def test_no_x_proj_or_dt_proj(self, mamba2_bridge): assert not hasattr(hf_mixer, "dt_proj") def test_d_mlp_is_zero(self, mamba2_bridge): - """Plan's assertion: projection_size = intermediate + conv_dim + num_heads. + """projection_size = intermediate + conv_dim + num_heads. If a future HF release introduces non-zero d_mlp, this test will fail, signaling that the 3-way in_proj split assumption no longer holds. @@ -462,7 +462,7 @@ def test_d_shape(self, mamba2_bridge): class TestMamba2StatefulGeneration: - """Phase 3: bridge.generate() runs a dedicated stateful loop with + """bridge.generate() runs a dedicated stateful loop with Mamba2Cache. Tokens match HF's native generate() exactly, and projection hooks fire on every step so interventions are possible. """ @@ -683,7 +683,7 @@ def _eager_scan(bridge): class TestMamba2EagerScanIntervention: - """Phase 4: opt-in eager-scan path exposes hook_ssm_write / hook_ssm_state for + """opt-in eager-scan path exposes hook_ssm_write / hook_ssm_state for interventions (write-knockout, state-patch) that propagate to logits, while the default run_with_cache path is untouched. Eager scan requires use_cache=False (so cache_params is None — prefill only).""" diff --git a/tests/integration/model_bridge/test_mamba_adapter.py b/tests/integration/model_bridge/test_mamba_adapter.py index 32cae35dd3..e3710438e6 100644 --- a/tests/integration/model_bridge/test_mamba_adapter.py +++ b/tests/integration/model_bridge/test_mamba_adapter.py @@ -7,15 +7,8 @@ - SSM blocks correctly exclude transformer-specific hook_resid_mid - Parameter access via __getattr__ fallback (A_log, D) -Note on cache clone safety: The Mamba adapter plan flagged in-place MambaCache -mutation as a risk — hooks that capture `ssm_states` would see corrupted values -on subsequent decode steps because HF mutates the cache in place. Phase 1 avoids -this risk entirely by design: the wrap-don't-reimplement approach keeps the -MambaMixer opaque, so `ssm_states` is never exposed through a hook. Phase 1 -hooks only observe projection inputs/outputs (in_proj, conv1d, x_proj, dt_proj, -out_proj), which are per-step tensors and are never mutated by the cache -machinery. If a future phase adds per-step SSM state hooks (compatibility mode), -those hooks MUST `.clone()` captured state tensors. +Cache-clone safety: hooks never expose HF's in-place-mutated `ssm_states`; any +future per-step SSM-state hook MUST `.clone()` captured state. """ import contextlib @@ -109,7 +102,7 @@ def test_mixer_submodule_hooks_fire(self, cache): assert f"blocks.{i}.mixer.{submod}.hook_out" in cache def test_projection_shapes(self, cache, mamba_bridge): - """Hook tensor shapes match the plan's Step 1.5 shape summary.""" + """Hook tensor shapes match the expected projection shapes.""" d_model = mamba_bridge.cfg.d_model # 768 intermediate = mamba_bridge.cfg.intermediate_size # 1536 conv_kernel = mamba_bridge.cfg.conv_kernel # 4 @@ -219,7 +212,7 @@ def test_full_forward_works_after_stop(self, mamba_bridge): class TestMambaStatefulGeneration: - """Phase 3: bridge.generate() runs a proper stateful loop instead of + """bridge.generate() runs a proper stateful loop instead of delegating to hf_generate(). This gives hook integration during generation. """ @@ -449,7 +442,7 @@ def test_raises_on_empty_cache(self, mamba_bridge): class TestMamba1SSMState: - """Mamba-1 recurrent-state reconstruction (Phase 4.5): read-parity with Mamba-2. + """Mamba-1 recurrent-state reconstruction: read-parity with Mamba-2. The vectorized state must match a naive fp64 step-by-step S6 scan, and ``y = C·S + D·x`` reconstructed through gate + out_proj must match HF's cached @@ -566,7 +559,7 @@ def _eager_scan(bridge): class TestMamba1EagerScanIntervention: - """Phase 4 (Mamba-1): opt-in eager S6 scan exposes hook_ssm_write / hook_ssm_state + """Opt-in eager S6 scan exposes hook_ssm_write / hook_ssm_state for interventions that propagate to logits, while the default path is untouched. Eager scan needs use_cache=False (prefill; cache_params is None).""" diff --git a/tests/integration/model_bridge/test_pretrain_adapter.py b/tests/integration/model_bridge/test_pretrain_adapter.py index 5878532e38..18f7e8b937 100644 --- a/tests/integration/model_bridge/test_pretrain_adapter.py +++ b/tests/integration/model_bridge/test_pretrain_adapter.py @@ -15,13 +15,8 @@ cross-checked against an independent way of expressing the same edit; and float64 construction-time preservation. -Note on scope: since the adapter delegates the whole forward to the -source model rather than reimplementing attention/RoPE, exact logit -parity mainly demonstrates that wrapping and output normalization don't -alter an unhooked forward, not that the RoPE convention itself is -correct (the same implementation runs on both sides of the comparison). -`TestResidualStreamDecomposition` and `TestHookIntervention` are the -stronger evidence for correct intervention points. +Scope: parity only checks that wrapping doesn't alter an unhooked forward; +`TestResidualStreamDecomposition`/`TestHookIntervention` verify intervention points. """ from __future__ import annotations diff --git a/tests/integration/model_bridge/test_qwen3_moe_bridge.py b/tests/integration/model_bridge/test_qwen3_moe_bridge.py index 4cb9604faf..62aeac3f8b 100644 --- a/tests/integration/model_bridge/test_qwen3_moe_bridge.py +++ b/tests/integration/model_bridge/test_qwen3_moe_bridge.py @@ -136,7 +136,7 @@ def test_block_attn_has_q_norm_k_norm(self, tiny_qwen3moe_bridge) -> None: # Forward-pass tests require real weights — meta-device tensor ops raise -# NotImplementedError. Run these manually during Step 3 verification. +# NotImplementedError. Run these manually during model verification. @pytest.mark.skip(reason="Requires real weights — run manually during verification") diff --git a/tests/integration/model_bridge/test_ssm_effective_attention_dispatch.py b/tests/integration/model_bridge/test_ssm_effective_attention_dispatch.py index 11ed879001..43d308e2a1 100644 --- a/tests/integration/model_bridge/test_ssm_effective_attention_dispatch.py +++ b/tests/integration/model_bridge/test_ssm_effective_attention_dispatch.py @@ -1,4 +1,4 @@ -"""Phase 3: family-agnostic SSM effective-attention dispatch + canonical hook vocabulary. +"""Family-agnostic SSM effective-attention dispatch + canonical hook vocabulary. Builds tiny synthetic models (no Hub access) for each SSM family and verifies: - SSMMixerProtocol conformance (Mamba-1, Mamba-2, gated-delta-net; not attention), @@ -330,7 +330,7 @@ def test_qwen35_dict_and_include_dt_scaling_unsupported(self, qwen35_bridge): # --------------------------------------------------------------------------- -# Canonical hook vocabulary (3b) — survives the hybrid alias resolution +# Canonical hook vocabulary — survives the hybrid alias resolution # --------------------------------------------------------------------------- @@ -364,7 +364,7 @@ def test_mamba1_hook_ssm_dt_alias(self, mamba1_bridge): # --------------------------------------------------------------------------- -# cache.compute_ssm_state (Phase 4.5) — family-agnostic over Mamba-1 / Mamba-2 +# cache.compute_ssm_state — family-agnostic over Mamba-1 / Mamba-2 # and gated-delta-net (each reconstructs its own recurrent state). # --------------------------------------------------------------------------- diff --git a/tests/integration/test_centralized_weight_processing.py b/tests/integration/test_centralized_weight_processing.py index b4636f262b..ee278b21ad 100644 --- a/tests/integration/test_centralized_weight_processing.py +++ b/tests/integration/test_centralized_weight_processing.py @@ -53,7 +53,7 @@ def test_processing_without_architecture_adapter( processed_without_adapter = ProcessWeights.process_weights( state_dict=raw_state_dict, cfg=cfg, - adapter=None, # No adapter + adapter=None, fold_ln=False, center_writing_weights=False, center_unembed=False, diff --git a/tests/integration/test_fold_layer_integration.py b/tests/integration/test_fold_layer_integration.py index c45067bc5a..6871ae2b72 100644 --- a/tests/integration/test_fold_layer_integration.py +++ b/tests/integration/test_fold_layer_integration.py @@ -39,7 +39,6 @@ def gpt2_model_and_config(self): tl_model = HookedTransformer.from_pretrained(model_name, device=device) # Create architecture adapter - # Convert HookedTransformerConfig to TransformerBridgeConfig bridge_config = TransformerBridgeConfig.from_dict(tl_model.cfg.__dict__) bridge_config.architecture = "gpt2" adapter = GPT2ArchitectureAdapter(bridge_config) diff --git a/tests/integration/test_grouped_query_attention.py b/tests/integration/test_grouped_query_attention.py index a8cf6f3972..7d560adc3c 100644 --- a/tests/integration/test_grouped_query_attention.py +++ b/tests/integration/test_grouped_query_attention.py @@ -92,12 +92,7 @@ def test_grouped_query_attention_output_is_correct(): split_query_input, split_key_input, split_value_input ) - # Use both relative and absolute tolerances for numerical stability - # Different code paths (split vs non-split) can have tiny floating point differences - # rtol=5e-5 allows 0.005% relative error, atol=0.5 handles absolute differences - # CI shows max absolute diff: ~0.39, max relative diff: ~1.3e-5 (0.0013%) - # Local shows max absolute diff: ~0.008, max relative diff: ~2e-7 - # Variation due to different hardware/compiler optimizations - relative error is what matters + # split vs non-split paths differ in fp; loose atol/rtol needed across hardware # Calculate differences for debugging abs_diff = torch.abs(regular_attn_output - split_grouped_query_attn_output) diff --git a/tests/integration/test_main_demo_pattern_hooks.py b/tests/integration/test_main_demo_pattern_hooks.py index 4539cb42a1..97ad674ffb 100644 --- a/tests/integration/test_main_demo_pattern_hooks.py +++ b/tests/integration/test_main_demo_pattern_hooks.py @@ -50,7 +50,6 @@ def induction_score_hook(pattern, hook): induction_stripe, "batch head_index position -> head_index", "mean" ) - # Store the result induction_score_store[hook.layer(), :] = induction_score return pattern diff --git a/tests/integration/test_start_at_layer.py b/tests/integration/test_start_at_layer.py index f1d007829d..a849887d47 100644 --- a/tests/integration/test_start_at_layer.py +++ b/tests/integration/test_start_at_layer.py @@ -59,8 +59,7 @@ def count_hook(activation, hook): ("blocks.0.attn.hook_k", count_hook), ("blocks.1.mlp.hook_pre", count_hook), ("blocks.2.attn.hook_k", count_hook), - ("blocks.2.mlp.hook_pre", count_hook) - # ("blocks.2.mlp.hook_mid", count_hook), + ("blocks.2.mlp.hook_pre", count_hook), ], ) diff --git a/tests/unit/model_bridge/compatibility/test_next_sentence_prediction.py b/tests/unit/model_bridge/compatibility/test_next_sentence_prediction.py index 59fa5c3722..b7999127fa 100644 --- a/tests/unit/model_bridge/compatibility/test_next_sentence_prediction.py +++ b/tests/unit/model_bridge/compatibility/test_next_sentence_prediction.py @@ -67,7 +67,6 @@ def test_tokenizer_integration(bert_nsp, mock_transformer_bridge): } mock_transformer_bridge.tokenizer.return_value = mock_encodings - # Call to_tokens tokens, type_ids, mask = bert_nsp.to_tokens(input_sentences) # Verify tokenizer was called correctly @@ -101,14 +100,12 @@ def test_device_handling_to_tokens(bert_nsp, mock_transformer_bridge): } mock_transformer_bridge.tokenizer.return_value = mock_encodings - # Call to_tokens with move_to_device=True tokens, type_ids, mask = bert_nsp.to_tokens(input_data, move_to_device=True) # Verify each tensor was moved to the correct device for tensor in [tokens, type_ids, mask]: assert tensor.device.type == mock_transformer_bridge.cfg.device - # Call with move_to_device=False tokens, type_ids, mask = bert_nsp.to_tokens(input_data, move_to_device=False) # Verify tensors remained on CPU @@ -177,7 +174,6 @@ def test_run_with_cache(bert_nsp, mock_transformer_bridge): input_data = ["First sentence.", "Second sentence."] - # Run with cache output, cache = bert_nsp.run_with_cache( input_data, return_type="logits", return_cache_object=True ) diff --git a/tests/unit/model_bridge/generalized_components/test_codegen_attention_bridge.py b/tests/unit/model_bridge/generalized_components/test_codegen_attention_bridge.py index df2c2b004e..9e8418d609 100644 --- a/tests/unit/model_bridge/generalized_components/test_codegen_attention_bridge.py +++ b/tests/unit/model_bridge/generalized_components/test_codegen_attention_bridge.py @@ -467,9 +467,7 @@ def capture_q_after_rope(tensor, hook): q_after_rope.append(tensor.clone()) return tensor - # We patch _reconstruct_attention to intercept Q after RoPE. - # Simpler: capture attn_scores and back-compute is complex. - # Instead, we patch the module-level function with a wrapper. + # Patch the module-level _apply_rotary_pos_emb to capture Q/K post-RoPE. import transformer_lens.model_bridge.generalized_components.codegen_attention as codegen_attn_mod original_fn = codegen_attn_mod._apply_rotary_pos_emb diff --git a/tests/unit/model_bridge/generalized_components/test_joint_qkv_attention.py b/tests/unit/model_bridge/generalized_components/test_joint_qkv_attention.py index 31064a5d17..d7191fc106 100644 --- a/tests/unit/model_bridge/generalized_components/test_joint_qkv_attention.py +++ b/tests/unit/model_bridge/generalized_components/test_joint_qkv_attention.py @@ -143,13 +143,11 @@ def forward(self, input): def q_hook_id_fn(q_output, hook): return q_output - # Add the hook to q.hook_out qkv_bridge.q.hook_out.add_hook(q_hook_id_fn) # Run forward pass with identity hook to get baseline baseline_output, _ = qkv_bridge(test_input) - # Remove the identity hook qkv_bridge.q.hook_out.remove_hooks() # Add a hook to q.hook_out that modifies the output @@ -159,10 +157,8 @@ def q_hook_id_fn(q_output, hook): def q_hook_fn(q_output, hook): nonlocal q_mutation_applied q_mutation_applied = True - # Modify the q output by adding a distinct value return q_output + q_mutated_value - # Add the hook to q.hook_out qkv_bridge.q.hook_out.add_hook(q_hook_fn) # Run forward pass with hook @@ -230,13 +226,11 @@ def forward(self, input): def k_hook_id_fn(k_output, hook): return k_output - # Add the hook to k.hook_out qkv_bridge.k.hook_out.add_hook(k_hook_id_fn) # Run forward pass with identity hook to get baseline baseline_output, _ = qkv_bridge(test_input) - # Remove the identity hook qkv_bridge.k.hook_out.remove_hooks() # Add a hook to k.hook_out that modifies the output @@ -246,10 +240,8 @@ def k_hook_id_fn(k_output, hook): def k_hook_fn(k_output, hook): nonlocal k_mutation_applied k_mutation_applied = True - # Modify the k output by adding a distinct value return k_output + k_mutated_value - # Add the hook to k_hook_out qkv_bridge.k.hook_out.add_hook(k_hook_fn) # Run forward pass with hook @@ -317,13 +309,11 @@ def forward(self, input): def v_hook_id_fn(v_output, hook): return v_output - # Add the hook to v.hook_out qkv_bridge.v.hook_out.add_hook(v_hook_id_fn) # Run forward pass with identity hook to get baseline baseline_output, _ = qkv_bridge(test_input) - # Remove the identity hook qkv_bridge.v.hook_out.remove_hooks() # Add a hook to v.hook_out that modifies the output @@ -333,10 +323,8 @@ def v_hook_id_fn(v_output, hook): def v_hook_fn(v_output, hook): nonlocal v_mutation_applied v_mutation_applied = True - # Modify the v output by adding a distinct value return v_output + v_mutated_value - # Add the hook to v.hook_out qkv_bridge.v.hook_out.add_hook(v_hook_fn) # Run forward pass with hook diff --git a/tests/unit/model_bridge/supported_architectures/test_bd3lm_adapter.py b/tests/unit/model_bridge/supported_architectures/test_bd3lm_adapter.py index 01873d72fc..3626f607e1 100644 --- a/tests/unit/model_bridge/supported_architectures/test_bd3lm_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_bd3lm_adapter.py @@ -92,7 +92,6 @@ def test_final_rms_false(self, adapter: BD3LMArchitectureAdapter) -> None: assert adapter.cfg.final_rms is False def test_applicable_phases(self, adapter: BD3LMArchitectureAdapter) -> None: - # BD3LM now supports all phases correctly assert adapter.applicable_phases == [1, 2, 3] def test_supports_generation(self, adapter: BD3LMArchitectureAdapter) -> None: diff --git a/tests/unit/model_bridge/supported_architectures/test_gemma3n_adapter.py b/tests/unit/model_bridge/supported_architectures/test_gemma3n_adapter.py index 458fa9e719..87c42b11d7 100644 --- a/tests/unit/model_bridge/supported_architectures/test_gemma3n_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_gemma3n_adapter.py @@ -75,7 +75,7 @@ def test_text_path_nested_under_language_model(): assert isinstance(m["embed"], EmbeddingBridge) assert isinstance(m["blocks"], AltUpBlockBridge) assert isinstance(m["unembed"], UnembeddingBridge) - # Vision/audio are referenced-but-unbridged (Route D groundwork). + # Vision/audio are referenced but not bridged (text-only adapter for now). assert "vision_encoder" not in m and "audio_encoder" not in m diff --git a/tests/unit/model_bridge/supported_architectures/test_hunyuan_v1_dense_adapter.py b/tests/unit/model_bridge/supported_architectures/test_hunyuan_v1_dense_adapter.py index d0511b0104..bb5f120874 100644 --- a/tests/unit/model_bridge/supported_architectures/test_hunyuan_v1_dense_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_hunyuan_v1_dense_adapter.py @@ -123,7 +123,7 @@ class TestHunYuanDenseV1AdapterConfig: """Adapter must set all required config flags to the values HunYuanDenseV1 expects.""" def test_attn_implementation_is_eager(self, adapter: HunYuanDenseV1ArchitectureAdapter) -> None: - """Set to eager.""" + """Adapter forces eager attention so component-level activations are hookable.""" assert adapter.cfg.attn_implementation == "eager" diff --git a/tests/unit/model_bridge/supported_architectures/test_lfm2_adapter.py b/tests/unit/model_bridge/supported_architectures/test_lfm2_adapter.py index b80a67cdd8..4acadc33ae 100644 --- a/tests/unit/model_bridge/supported_architectures/test_lfm2_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_lfm2_adapter.py @@ -136,7 +136,7 @@ class TestLfm2AdapterConfig: """Adapter must set all required config flags to the values Lfm2 expects.""" def test_attn_implementation_is_eager(self, adapter: Lfm2ArchitectureAdapter) -> None: - """Set to eager.""" + """Adapter forces eager attention so component-level activations are hookable.""" assert adapter.cfg.attn_implementation == "eager" diff --git a/tests/unit/model_bridge/supported_architectures/test_mpt_adapter.py b/tests/unit/model_bridge/supported_architectures/test_mpt_adapter.py index 36333ecac1..d5a1ba7d39 100644 --- a/tests/unit/model_bridge/supported_architectures/test_mpt_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_mpt_adapter.py @@ -66,7 +66,7 @@ def test_no_mlp_conversion_keys(self, adapter: MPTArchitectureAdapter) -> None: # --------------------------------------------------------------------------- -# Component mapping structure tests (Phase B-1) +# Component mapping structure tests # --------------------------------------------------------------------------- @@ -100,7 +100,7 @@ def test_mlp_submodule_keys(self, adapter: MPTArchitectureAdapter) -> None: # --------------------------------------------------------------------------- -# _split_mpt_qkv tests (Phase B-1) +# _split_mpt_qkv tests # --------------------------------------------------------------------------- diff --git a/tests/unit/model_bridge/supported_architectures/test_pretrain_model_container.py b/tests/unit/model_bridge/supported_architectures/test_pretrain_model_container.py index 43c40901ea..8ccb981b0d 100644 --- a/tests/unit/model_bridge/supported_architectures/test_pretrain_model_container.py +++ b/tests/unit/model_bridge/supported_architectures/test_pretrain_model_container.py @@ -197,33 +197,13 @@ def test_train_and_eval_return_the_bridge_itself(self) -> None: assert bridge.eval() is bridge def test_bridge_can_be_reconstructed_from_supported_state(self) -> None: - """The adapter's actual persistence contract is: rebuild the - bridge from the source model class + cfg + the source model's - *raw* state_dict, then confirm behavior (outputs, train/eval - propagation) survives the round trip. + """Rebuild the bridge from the source model's *raw* state_dict and + confirm behavior survives the round trip. The state_dict must be snapshotted BEFORE `build_pretrain_bridge` - is called. Building the bridge wraps the model's submodules in - place (each one gains an `_original_component` wrapper for - hooking), so `model.state_dict()` taken *after* wrapping has - keys like `embed._original_component.weight` instead of - `embed.weight` -- a fresh, unwrapped `TinyPretrainModel` can't - load that. Capturing the state_dict first, then wrapping a - second fresh model and loading the pre-wrap snapshot into it, - avoids that trap. - - Only the source model's state_dict is used -- not - `bridge.state_dict()` -- since the bridge doesn't own an - independent set of learned weights; it exposes the wrapped - source model's parameters under bridge-shaped names. - - This is deliberately weaker than whole-object `pickle.dumps`, - which would also require every nested `TransformerBridge` - implementation detail (e.g. `AttentionBridge`'s hook - conversions) to be picklable -- a guarantee unrelated to this - adapter and not part of what it needs to promise. If a stronger - whole-object-pickle guarantee is later required, it belongs in a - `TransformerBridge`-level test, not here. + wraps the model's submodules in place; taken after, keys become + `embed._original_component.weight` and a fresh unwrapped model + can't load them. """ cfg = _make_cfg(n_layers=1) model = TinyPretrainModel(d_model=16, n_heads=2, n_layers=1, d_ff=32, vocab_size=64) diff --git a/tests/unit/model_bridge/supported_architectures/test_stablelm_adapter.py b/tests/unit/model_bridge/supported_architectures/test_stablelm_adapter.py index 9d49b92cdf..6767848a6d 100644 --- a/tests/unit/model_bridge/supported_architectures/test_stablelm_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_stablelm_adapter.py @@ -387,9 +387,7 @@ def parallel_adapter(self) -> StableLmArchitectureAdapter: def test_parallel_block_submodule_keys( self, parallel_adapter: StableLmArchitectureAdapter ) -> None: - """Container is ParallelBlockBridge so the no-ln2 layout is the supported shape. - BlockBridge's C15 guard rejects `attn + mlp` without `ln2`, which is exactly the - regression #1386 was filed for.""" + """Container is ParallelBlockBridge so the no-ln2 layout (attn + mlp without ln2) is the supported shape that BlockBridge would otherwise reject.""" block = _mapping(parallel_adapter)["blocks"] assert isinstance(block, ParallelBlockBridge) assert set(block.submodules.keys()) == {"ln1", "attn", "mlp"} diff --git a/tests/unit/model_bridge/test_component_hooks_fire.py b/tests/unit/model_bridge/test_component_hooks_fire.py index 8c9ce751b5..3cd1e1e54b 100644 --- a/tests/unit/model_bridge/test_component_hooks_fire.py +++ b/tests/unit/model_bridge/test_component_hooks_fire.py @@ -1,7 +1,7 @@ -"""Regression test: attention hooks fire on forward (C1 guard). +"""Regression test: attention hooks fire on forward. Complements the alias-resolution test: aliases can resolve yet the HookPoint -may never fire if forward bypasses the LinearBridge (the original C1 bug). +may never fire if forward bypasses the LinearBridge (the original bug). Isolates blocks.0.attn per PositionEmbeddingsAttentionBridge adapter with a synthetic HF module and asserts hook_q/k/v/z all fire. """ @@ -121,7 +121,7 @@ def _wire_attention_submodules( @pytest.mark.parametrize("architecture", _position_embeddings_adapters()) def test_attention_critical_hooks_fire_on_forward(architecture: str) -> None: - """Assert hook_q/k/v/z fire during attention forward (C1 regression guard).""" + """Assert hook_q/k/v/z fire during attention forward (regression guard).""" adapter_cls = SUPPORTED_ARCHITECTURES[architecture] adapter = adapter_cls(_stub_cfg(architecture)) attn = adapter.component_mapping["blocks"].submodules["attn"] diff --git a/tests/unit/model_bridge/test_hook_alias_resolution.py b/tests/unit/model_bridge/test_hook_alias_resolution.py index fd5b8d85b0..f3ad05cd5c 100644 --- a/tests/unit/model_bridge/test_hook_alias_resolution.py +++ b/tests/unit/model_bridge/test_hook_alias_resolution.py @@ -66,7 +66,7 @@ def _resolve(component: GeneralizedComponent, target: str) -> Any: # xfail(strict=True) so future fixes XPASS and force the marker to be removed. -# Each entry maps to a specific audit finding deferred from the C1+C15 PR. +# Each entry marks an adapter with known-dead hook aliases; xfail until fixed. _KNOWN_DEAD_ALIASES = { "GPT2LMHeadCustomModel": "audit H27 — stale adapter, delete candidate", "NanoGPTForCausalLM": "audit H28 — broken weight conversion, delete candidate", diff --git a/tests/unit/model_bridge/test_inspect_vllm_provider.py b/tests/unit/model_bridge/test_inspect_vllm_provider.py index 2d9b1e81fd..e9a4478dda 100644 --- a/tests/unit/model_bridge/test_inspect_vllm_provider.py +++ b/tests/unit/model_bridge/test_inspect_vllm_provider.py @@ -183,7 +183,7 @@ def test_passes_locked_kwargs_to_llm(self, monkeypatch): assert kwargs["tensor_parallel_size"] == 1 assert kwargs["skip_tokenizer_init"] is True assert kwargs["disable_log_stats"] is True - # Inc 2: capture wiring requires worker_extension_cls + full-vocab logprobs. + # Capture wiring requires worker_extension_cls + full-vocab logprobs. assert kwargs["worker_extension_cls"].endswith("TLWorkerExtension") assert kwargs["max_logprobs"] == 128 # _fake_hf_config().vocab_size assert kwargs["max_num_batched_tokens"] == 2048 # default diff --git a/tests/unit/model_bridge/test_native_features.py b/tests/unit/model_bridge/test_native_features.py index 85f6b5ad46..b658f2c06d 100644 --- a/tests/unit/model_bridge/test_native_features.py +++ b/tests/unit/model_bridge/test_native_features.py @@ -173,8 +173,7 @@ def test_output_logits_soft_cap_bounds_logits(): """Logits must be bounded by ±cap when output_logits_soft_cap > 0.""" cap = 5.0 cfg = _cfg(output_logits_soft_cap=cap, seed=0) - # Force-large outputs by skipping the soft-cap → then re-enabling. Easier: - # just pick a cap and assert |logits| <= cap. tanh-cap math guarantees it. + # tanh soft-cap guarantees |logits| <= cap. bridge = TransformerBridge.boot_native(cfg) logits = _forward(bridge) assert logits.abs().max().item() <= cap + 1e-5 diff --git a/tests/unit/model_bridge/test_optimizer_compatibility.py b/tests/unit/model_bridge/test_optimizer_compatibility.py index 8342dbff1a..2d948d769a 100644 --- a/tests/unit/model_bridge/test_optimizer_compatibility.py +++ b/tests/unit/model_bridge/test_optimizer_compatibility.py @@ -44,7 +44,6 @@ def test_tl_parameters_provides_tl_style_names(self, small_bridge_model): For optimization, use parameters() which returns only leaf tensors. """ - # Get TL-style parameters tl_params = small_bridge_model.tl_parameters() # Check that we have TL-style names (blocks.X.attn.W_Y format) @@ -63,7 +62,6 @@ def test_tl_named_parameters_provides_iterator(self, small_bridge_model): This method provides the same content as tl_parameters() but as an iterator, maintaining consistency with PyTorch's named_parameters() API pattern. """ - # Get TL-style parameters as iterator tl_named_params = list(small_bridge_model.tl_named_parameters()) tl_params_dict = small_bridge_model.tl_parameters() diff --git a/tests/unit/test_hook_points.py b/tests/unit/test_hook_points.py index df85849f06..51947a9cc2 100644 --- a/tests/unit/test_hook_points.py +++ b/tests/unit/test_hook_points.py @@ -128,10 +128,10 @@ def test_reshape_functionality_integration(): # Create a test hook conversion class TestHookConversion(BaseTensorConversion): def handle_conversion(self, input_value, *full_context): - return input_value * 2 # Double the input + return input_value * 2 def revert(self, input_value, *full_context): - return input_value + 10 # Add 10 to the output + return input_value + 10 # Create a simple test module that uses HookPoint class TestModule(torch.nn.Module): diff --git a/tests/unit/test_hooked_transformer_config.py b/tests/unit/test_hooked_transformer_config.py index c9a2d13694..e44e7409f8 100644 --- a/tests/unit/test_hooked_transformer_config.py +++ b/tests/unit/test_hooked_transformer_config.py @@ -10,7 +10,6 @@ def test_hooked_transformer_config_object(): n_layers=2, d_vocab=100, d_model=6, n_ctx=5, d_head=2, attn_only=True ) result = HookedTransformerConfig.unwrap(hooked_transformer_config) - # Assert that the same object was returned assert result is hooked_transformer_config @@ -24,7 +23,6 @@ def test_hooked_transformer_config_dict(): "attn_only": True, } result = HookedTransformerConfig.unwrap(hooked_transformer_config_dict) - # Assert that the new returned value has been transformed into a config object assert isinstance(result, HookedTransformerConfig) diff --git a/tests/unit/test_key_value_cache_entry.py b/tests/unit/test_key_value_cache_entry.py index e4b300017e..e17b718719 100644 --- a/tests/unit/test_key_value_cache_entry.py +++ b/tests/unit/test_key_value_cache_entry.py @@ -1,19 +1,4 @@ -"""Tests for TransformerLensKeyValueCacheEntry.init_cache_entry dtype behaviour. - -The buggy pre-fix code used ``torch.get_default_dtype()`` to initialise -``past_keys`` and ``past_values``. PyTorch's default is ``torch.float32``, -so the bug silently produced the correct dtype for fp32 models but the -wrong dtype (fp32 instead of fp16/bf16) for reduced-precision ones. Of -the tests below, ``test_init_cache_entry_uses_cfg_dtype_float32`` is -therefore a baseline sanity check that passes against both the buggy -and fixed code — it verifies the common case works, not that the bug is -absent. The real regression guards are -``test_init_cache_entry_uses_cfg_dtype_float16``, -``..._bfloat16``, ``..._dtype_independent_of_global_default``, and -``test_append_preserves_cfg_dtype``, which all fail against the buggy -code (the fp16 cache was getting promoted to fp32 by the bug, breaking -the downstream attention-score matmul). -""" +"""Tests that KeyValueCacheEntry.init_cache_entry buffers follow cfg.dtype, not torch's global default.""" import torch @@ -36,15 +21,7 @@ def _make_cfg(dtype: torch.dtype, n_heads: int = 4, d_head: int = 8, n_key_value def test_init_cache_entry_uses_cfg_dtype_float32(): - """Baseline: cfg.dtype=float32 produces fp32 buffers. - - Note: this test passes against both the buggy and fixed implementations - because torch's default dtype is also float32. It is a sanity check - that the common case works, not a regression guard for the specific - bug this module was added to prevent. See module docstring and - ``test_init_cache_entry_dtype_independent_of_global_default`` for the - tests that discriminate fix vs bug. - """ + """Baseline: cfg.dtype=float32 produces fp32 buffers.""" cfg = _make_cfg(dtype=torch.float32) entry = TransformerLensKeyValueCacheEntry.init_cache_entry(cfg, device="cpu") assert entry.past_keys.dtype == torch.float32 @@ -66,13 +43,7 @@ def test_init_cache_entry_uses_cfg_dtype_bfloat16(): def test_init_cache_entry_dtype_independent_of_global_default(): - """Regression guard: cache dtype follows cfg.dtype, not the global default. - - Also covers the fp32 case indirectly: if someone reintroduces the old - ``torch.get_default_dtype()`` behaviour, this test plus the fp16 / - bfloat16 / append / GQA tests catch it; the fp32-only baseline above - would not, since fp32 happens to be torch's global default. - """ + """Regression guard: cache dtype follows cfg.dtype, not the global default.""" cfg = _make_cfg(dtype=torch.float16) original_default = torch.get_default_dtype() try: diff --git a/tests/unit/test_supported_models.py b/tests/unit/test_supported_models.py index 2571907c31..56a59d2054 100644 --- a/tests/unit/test_supported_models.py +++ b/tests/unit/test_supported_models.py @@ -14,13 +14,10 @@ def test_official_model_names_is_alphabetical(): def test_model_aliases_is_alphabetical(): - # Extract the keys as they appear in the dictionary actual_keys = list(MODEL_ALIASES.keys()) - # Create a sorted version, ignoring case expected_keys = sorted(actual_keys, key=str.casefold) - # Compare the actual insertion order to the expected alphabetical order assert actual_keys == expected_keys, "MODEL_ALIASES keys are not in alphabetical order. " diff --git a/tests/unit/test_svd_interpreter.py b/tests/unit/test_svd_interpreter.py index dfe22444cb..e08e98b4b4 100644 --- a/tests/unit/test_svd_interpreter.py +++ b/tests/unit/test_svd_interpreter.py @@ -12,7 +12,7 @@ MODEL = "solu-2l" VECTOR_TYPES = ["OV", "w_in", "w_out"] -ATOL = 2e-4 # Absolute tolerance - how far does a float have to be before we consider it no longer equal? +ATOL = 2e-4 # absolute tolerance for float comparison @pytest.fixture(scope="module") diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 78050e503b..3f73ae1421 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -263,9 +263,7 @@ def test_get_attention_mask(self, model, padding_side, prepend_bos, prompts_with def test_calc_fan_in_fan_out(): - """ - Test for the calc_fan_in_and_fan_out function in the utils module. - """ + """Verifies fan_in/fan_out for 1D/2D/3D tensors and raises ValueError for 0D/4D.""" # Test for the case when the tensor is 1D tensor_1d = torch.tensor([1, 2, 3, 4, 5]) fan_in, fan_out = utils.calc_fan_in_and_fan_out(tensor_1d) diff --git a/tests/unit/test_weight_processing.py b/tests/unit/test_weight_processing.py index 6735de1ea2..d5d7f59e2e 100644 --- a/tests/unit/test_weight_processing.py +++ b/tests/unit/test_weight_processing.py @@ -14,8 +14,6 @@ from transformer_lens.config.transformer_lens_config import TransformerLensConfig from transformer_lens.weight_processing import ProcessWeights -# from typing import Dict # Unused import - def deep_copy_state_dict(state_dict): """Create a deep copy of a state dict with cloned tensors. diff --git a/tests/unit/utilities/test_devices.py b/tests/unit/utilities/test_devices.py index 5834486a89..ca2bf7ae26 100644 --- a/tests/unit/utilities/test_devices.py +++ b/tests/unit/utilities/test_devices.py @@ -28,7 +28,6 @@ def state_dict(self): return self._parameters def to(self, device_or_dtype): - # Mock the to method if isinstance(device_or_dtype, torch.device): self.cfg.device = device_or_dtype.type elif isinstance(device_or_dtype, str): diff --git a/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_base_tensor_conversion.py b/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_base_tensor_conversion.py index adffa3704f..538f6dd1fc 100644 --- a/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_base_tensor_conversion.py +++ b/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_base_tensor_conversion.py @@ -26,12 +26,8 @@ def test_mock_weight_conversion_adds_five(): output_tensor = weight_conversion.convert(input_tensor) expected_tensor = torch.full((1, 4), 5.0, dtype=torch.float32) - # Option 1: simple equality check assert torch.equal(output_tensor, expected_tensor) - # Option 2: more robust approximate check - # torch.testing.assert_close(output_tensor, expected_tensor) - @pytest.mark.parametrize("shape", [(1, 4), (2, 2), (3,)]) def test_mock_weight_conversion_various_shapes(shape): diff --git a/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_callable_tensor_conversion.py b/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_callable_tensor_conversion.py index 977d02e68b..c86a2045f6 100644 --- a/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_callable_tensor_conversion.py +++ b/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_callable_tensor_conversion.py @@ -12,7 +12,6 @@ def test_callable_tensor_conversion_basic(): """ def my_callable(tensor_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - # Example transformation: add 1 to each tensor value new_dict = {} for k, v in tensor_dict.items(): new_dict[k] = v + 1 @@ -52,7 +51,6 @@ def my_callable(tensor_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor] return new_dict def my_input_filter(tensor_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - # Multiply each tensor by 2 filtered_dict = {} for k, v in tensor_dict.items(): filtered_dict[k] = v * 2 @@ -86,7 +84,6 @@ def my_callable(tensor_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor] return new_dict def my_output_filter(tensor_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - # Subtract 5 from each element filtered_dict = {} for k, v in tensor_dict.items(): filtered_dict[k] = v - 5 diff --git a/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_rearrange_tensor_conversion.py b/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_rearrange_tensor_conversion.py index 43f74014f0..92232b627e 100644 --- a/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_rearrange_tensor_conversion.py +++ b/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_rearrange_tensor_conversion.py @@ -41,7 +41,6 @@ def test_rearrange_tensor_conversion_input_filter(): """ def input_filter(tensor): - # E.g., multiply by 2 return tensor * 2 conversion = RearrangeTensorConversion("(n h) m->n m h", input_filter=input_filter, n=8) @@ -67,7 +66,6 @@ def test_rearrange_tensor_conversion_output_filter(): """ def output_filter(tensor): - # E.g., add 10 return tensor + 10 conversion = RearrangeTensorConversion("(n h) m->n m h", output_filter=output_filter, n=8) @@ -96,10 +94,10 @@ def test_rearrange_tensor_conversion_input_output_filters(): """ def input_filter(tensor): - return tensor * 2 # Double + return tensor * 2 def output_filter(tensor): - return tensor + 3 # Then add 3 + return tensor + 3 conversion = RearrangeTensorConversion( "(n h) m->n m h", input_filter=input_filter, output_filter=output_filter, n=8 diff --git a/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_repeat_tensor_conversion.py b/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_repeat_tensor_conversion.py index d99f402017..e210b9aae1 100644 --- a/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_repeat_tensor_conversion.py +++ b/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_repeat_tensor_conversion.py @@ -13,8 +13,6 @@ def test_repeat_tensor_conversion_basic(): Example pattern (h w -> h 2 w): We repeat along the second dimension. """ - # Pattern says: "h w -> h repeat1 w" - # let's do: pattern="h w -> h 2 w", so we define repeat2=2 for the new axis conversion = RepeatTensorConversion("h w -> h 2 w", h=3, w=4) # Start with a known shape: [3,4], fill with arange to track values diff --git a/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_terenary_tensor_conversion.py b/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_terenary_tensor_conversion.py index 16b2ef495d..76716f0764 100644 --- a/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_terenary_tensor_conversion.py +++ b/tests/unit/weight_conversion/conversion_utils/conversion_steps/test_terenary_tensor_conversion.py @@ -153,18 +153,12 @@ def handle_conversion(self, input_value: torch.Tensor, *unused) -> torch.Tensor: def test_ternary_tensor_conversion_find_context_field_failure(): """ - If fallback is a str, but the context doesn't contain that key, - find_context_field returns None, and so fallback is effectively None => - This might cause an error or we accept returning None. - We'll see how your code handles that scenario. + If fallback is a str missing from the context, find_context_field returns None and convert returns None. """ ternary = TernaryTensorConversion(fallback_conversion="missing_key", primary_conversion=None) # No context => won't find 'missing_key' output = ternary.convert(None, *{}) - # According to your code, if nothing is found, we return None from find_context_field. - # handle_fallback_conversion will return that None from the function. - # => This might break if the calling code expects a tensor. - # We'll just check it's None. + # missing key -> find_context_field returns None, so convert returns None assert output is None, "Expected None if the fallback str wasn't found in the provided context." diff --git a/transformer_lens/HookedAudioEncoder.py b/transformer_lens/HookedAudioEncoder.py index 5b58351588..5b5d6747a2 100644 --- a/transformer_lens/HookedAudioEncoder.py +++ b/transformer_lens/HookedAudioEncoder.py @@ -264,7 +264,6 @@ def forward( # ---------- 1) Normalize input: get (frames, frame_mask) ---------- frames = None frame_mask = None # one_zero_attention_mask: 1 = valid, 0 = padding - # print(type(inputs)) # If user passed (frames, mask) tuple if isinstance(inputs, tuple) and len(inputs) == 2 and isinstance(inputs[0], torch.Tensor): frames, frame_mask = inputs diff --git a/transformer_lens/HookedEncoder.py b/transformer_lens/HookedEncoder.py index f751fafbfe..64cb09b141 100644 --- a/transformer_lens/HookedEncoder.py +++ b/transformer_lens/HookedEncoder.py @@ -300,7 +300,6 @@ def forward( # If input was a list of strings, split predictions into a list if " " in predictions: - # Split along space predictions = predictions.split(" ") predictions = [f"Prediction {i}: {p}" for i, p in enumerate(predictions)] return predictions diff --git a/transformer_lens/HookedEncoderDecoder.py b/transformer_lens/HookedEncoderDecoder.py index aa08483914..2961190183 100644 --- a/transformer_lens/HookedEncoderDecoder.py +++ b/transformer_lens/HookedEncoderDecoder.py @@ -112,7 +112,6 @@ def __init__( ] ) self.decoder_final_ln = RMSNorm(self.cfg) - # self.lm_head = nn.Linear(self.cfg.d_model, self.cfg.d_vocab_out) self.unembed = Unembed(self.cfg) self.hook_embed = HookPoint() @@ -470,7 +469,6 @@ def generate( ) ) - # Append new token to the decoder input decoder_input = torch.cat([decoder_input, sampled_tokens.unsqueeze(-1)], dim=-1) if stop_at_eos and finished_sequences.all(): @@ -478,7 +476,6 @@ def generate( if return_type == "str": assert self.tokenizer is not None - # Convert tokens to string return cast(str, self.tokenizer.decode(decoder_input[0], skip_special_tokens=True)) else: diff --git a/transformer_lens/HookedTransformer.py b/transformer_lens/HookedTransformer.py index 133eb5dae9..fbaaec11a0 100644 --- a/transformer_lens/HookedTransformer.py +++ b/transformer_lens/HookedTransformer.py @@ -1508,27 +1508,6 @@ def init_weights(self): weights! Note that this function assumes that weight names being with `W_`. Set seed here to ensure determinism. - - This does NOT follow the PyTorch scheme, which as far as I can tell is super out of date but - no one has gotten round to updating it? https://github.com/pytorch/pytorch/issues/18182 - - The default PyTorch scheme is the following: all linear layers use uniform(-1/sqrt(fan_in), - 1/sqrt(fan_in)) for weights, and uniform(-1/sqrt(fan_in), 1/sqrt(fan_in)) for biases. For - biases, fan_in is computed using the fan_in for the weight matrix of the linear layer. Note - that it *does not actually* use Kaiming initialization, despite the fact that it calls the - function. - - However, for Transformer blocks, it instead initializes biases to zero and weights using Xavier uniform, that - is: uniform(-sqrt(6 / (fan_in + fan_out)), sqrt(6 / (fan_in + fan_out))) for weights. - - PyTorch Transformers are especially bad - TransformerEncoder initializes all layers to the - exact same weights?! https://github.com/pytorch/pytorch/issues/72253. - - The best paper I've found on transformer initialization is the muP paper, but haven't - integrated those ideas yet: https://arxiv.org/abs/2203.03466 - - We split off the initialization into separate functions because muP initialization handles - different parts of the model differently. """ if self.cfg.seed is not None: @@ -2257,7 +2236,6 @@ def generate( def _logits_to_tuple(logits_list: list[torch.Tensor]) -> tuple[torch.Tensor, ...]: assert logits_list is not None - # Convert to tuple of tensors return tuple(logits_list) try: diff --git a/transformer_lens/benchmarks/forward_pass.py b/transformer_lens/benchmarks/forward_pass.py index a4940b2a05..559224cfc9 100644 --- a/transformer_lens/benchmarks/forward_pass.py +++ b/transformer_lens/benchmarks/forward_pass.py @@ -131,8 +131,7 @@ def benchmark_forward_pass( else: reference_output = hf_output.last_hidden_state else: - # HuggingFace model (reference_model is guaranteed non-None here - # because we returned early at line 80 when both are None) + # reference_model is non-None here: the both-None case returns early above assert reference_model is not None assert isinstance(test_input, str), "Text model requires string input" tokens = bridge.to_tokens(test_input) diff --git a/transformer_lens/benchmarks/main_benchmark.py b/transformer_lens/benchmarks/main_benchmark.py index d2ccbfa707..d8894dc577 100644 --- a/transformer_lens/benchmarks/main_benchmark.py +++ b/transformer_lens/benchmarks/main_benchmark.py @@ -880,7 +880,7 @@ def cleanup_model(model, model_name_str: str): print(f"✓ Detected attn_implementation={attn_implementation}") # Clean up config-only bridge immediately to free memory del bridge_config_only - gc.collect() # Force garbage collection immediately + gc.collect() except Exception as e: if verbose: print(f"⚠ Could not detect config (will use defaults): {str(e)}") diff --git a/transformer_lens/components/abstract_attention.py b/transformer_lens/components/abstract_attention.py index 69b6667300..855ea35aa8 100644 --- a/transformer_lens/components/abstract_attention.py +++ b/transformer_lens/components/abstract_attention.py @@ -867,7 +867,6 @@ def create_alibi_slope( # Use broadcasting to create the desired lower triangular part of the matrix slope_matrix = rows - cols - # Use the clamp method to set all positive values (upper right triangle) to return slope_matrix.clamp(max=0).to(torch.float32) @staticmethod @@ -898,10 +897,8 @@ def create_alibi_multipliers( Returns: A tensor of shape (n_heads,) containing the scalar multiplier for each head. """ - # Calculate the starting value start = 2 ** (-8 / n_heads) - # Generate the indices [0, 1, ..., n_heads-1] indices = torch.arange(n_heads, device=device) # Compute the multipliers, with the starting value being the same as the ratio @@ -943,12 +940,10 @@ def create_alibi_bias( Returns: The ALiBi bias that should be added to the attention scores before the softmax. """ - # Create the slope matrix slope: Float[torch.Tensor, "query key"] = AbstractAttention.create_alibi_slope( n_ctx, device ) - # Create the scalar multiplier for each head. multipliers: Float[torch.Tensor, "head_idx"] = AbstractAttention.create_alibi_multipliers( n_heads, device ) @@ -957,7 +952,6 @@ def create_alibi_bias( slope = einops.rearrange(slope, "query key -> 1 query key") multipliers = einops.rearrange(multipliers, "head_idx -> head_idx 1 1") - # Element-wise multiplication of the slope and multipliers alibi_bias = multipliers * slope return alibi_bias diff --git a/transformer_lens/components/attention.py b/transformer_lens/components/attention.py index cfce395d4e..11d94cb4ac 100644 --- a/transformer_lens/components/attention.py +++ b/transformer_lens/components/attention.py @@ -16,7 +16,6 @@ from bitsandbytes.nn.modules import Params4bit -# Attention class Attention(AbstractAttention): def __init__( self, diff --git a/transformer_lens/components/t5_block.py b/transformer_lens/components/t5_block.py index e93b70e7f6..d3b54210e5 100644 --- a/transformer_lens/components/t5_block.py +++ b/transformer_lens/components/t5_block.py @@ -102,9 +102,7 @@ def forward( value_input = attn_in attn_out = self.hook_attn_out( - # hook the residual stream states that are used to calculate the - # queries, keys and values, independently. - # Then take the layer norm of these inputs, and pass these to the attention module. + # ln1 applied per Q/K/V input so split-QKV hooks stay independent self.attn( query_input=self.ln1(query_input), key_input=self.ln1(key_input), diff --git a/transformer_lens/components/transformer_block.py b/transformer_lens/components/transformer_block.py index 92ac7484ad..b80289ae6d 100644 --- a/transformer_lens/components/transformer_block.py +++ b/transformer_lens/components/transformer_block.py @@ -27,7 +27,6 @@ from transformer_lens.utilities import repeat_along_head_dimension -# Transformer Block class TransformerBlock(nn.Module): ln1: nn.Module ln2: nn.Module diff --git a/transformer_lens/config/hooked_transformer_config.py b/transformer_lens/config/hooked_transformer_config.py index 0e4a757ef6..b726b60662 100644 --- a/transformer_lens/config/hooked_transformer_config.py +++ b/transformer_lens/config/hooked_transformer_config.py @@ -299,7 +299,6 @@ class HookedTransformerConfig(TransformerLensConfig): norm_topk_prob: bool = False def __post_init__(self): - # Call parent's post_init first super().__post_init__() if self.seed is not None: diff --git a/transformer_lens/config/transformer_lens_config.py b/transformer_lens/config/transformer_lens_config.py index fb1f5f0459..dacae23a16 100644 --- a/transformer_lens/config/transformer_lens_config.py +++ b/transformer_lens/config/transformer_lens_config.py @@ -74,10 +74,8 @@ class TransformerLensConfig: # GQA configuration n_key_value_heads: Optional[int] = None - # Attention only model attn_only: bool = False - # Gated MLP gated_mlp: bool = False # Normalization configuration @@ -86,22 +84,16 @@ class TransformerLensConfig: # Epsilon for normalization eps: float = 1e-5 - # Layer norm folding activated layer_norm_folding: bool = False - # Activation function act_fn: str = "relu" - # Normalization type normalization_type: Optional[str] = "LN" - # Number of experts num_experts: Optional[int] = None - # Number of experts per token experts_per_token: Optional[int] = None - # Final RMS norm final_rms: bool = False # Model dtype for LayerNormPre compatibility diff --git a/transformer_lens/conversion_utils/conversion_steps/tensor_conversion_set.py b/transformer_lens/conversion_utils/conversion_steps/tensor_conversion_set.py index c5bd804c6e..ec486d3877 100644 --- a/transformer_lens/conversion_utils/conversion_steps/tensor_conversion_set.py +++ b/transformer_lens/conversion_utils/conversion_steps/tensor_conversion_set.py @@ -41,7 +41,6 @@ def get_component(self, model: Any, name: str) -> Any: else: field_name, conversion_step = field_info - # Get the component from the model component = find_property(field_name, model) # Apply conversion step if specified diff --git a/transformer_lens/conversion_utils/helpers/merge_quantiziation_fields.py b/transformer_lens/conversion_utils/helpers/merge_quantiziation_fields.py index e32ed8e6c7..0a69ff7e76 100644 --- a/transformer_lens/conversion_utils/helpers/merge_quantiziation_fields.py +++ b/transformer_lens/conversion_utils/helpers/merge_quantiziation_fields.py @@ -16,7 +16,6 @@ def merge_quantization_fields(field_set: Any, quantization_fields: dict[str, Any Returns: The merged field set (same object, modified in-place). """ - # Merge the quantization fields into the existing field_set for field_name, new_field_value in quantization_fields.items(): existing_field = field_set.fields.get(field_name) @@ -52,7 +51,6 @@ def merge_quantization_fields(field_set: Any, quantization_fields: dict[str, Any ) else: # new_field_value is a simple value (like torch.Tensor) - # Simply overwrite the existing field field_set.fields[field_name] = new_field_value return field_set diff --git a/transformer_lens/conversion_utils/hook_conversion_utils.py b/transformer_lens/conversion_utils/hook_conversion_utils.py index d0469f061d..2712c7f242 100644 --- a/transformer_lens/conversion_utils/hook_conversion_utils.py +++ b/transformer_lens/conversion_utils/hook_conversion_utils.py @@ -52,13 +52,7 @@ def model_info_cfg(cfg): """ # TODO: WeightConversionFactory import needs to be updated or removed - # from transformer_lens.factories.weight_conversion_factory import ( - # WeightConversionFactory, - # ) - - # weight_conversion = WeightConversionFactory.select_weight_conversion_config(cfg) print(f"Hook conversion details for architecture {cfg.original_architecture}:") - # print(weight_conversion.__repr__()) print("Hook conversion factory not yet implemented") diff --git a/transformer_lens/evals.py b/transformer_lens/evals.py index 4200ea8b94..8423f301e4 100644 --- a/transformer_lens/evals.py +++ b/transformer_lens/evals.py @@ -609,7 +609,6 @@ def mmlu_eval( # Tokenize the prompt tokens = tokenizer.encode(prompt, return_tensors="pt").to(model.cfg.device) - # Get logits logits = model(tokens, return_type="logits") # Get log probabilities at the last position (predicting the answer letter) @@ -624,7 +623,6 @@ def mmlu_eval( # Select the choice with highest log probability predicted_answer = choice_log_probs.index(max(choice_log_probs)) - # Check if correct is_correct = predicted_answer == correct_answer num_correct += int(is_correct) num_total += 1 diff --git a/transformer_lens/head_detector.py b/transformer_lens/head_detector.py index 95d4fdc7df..5e32c56183 100644 --- a/transformer_lens/head_detector.py +++ b/transformer_lens/head_detector.py @@ -220,7 +220,7 @@ def get_induction_head_detection_pattern( def get_supported_heads() -> None: - """Returns a list of supported heads.""" + """Print the supported head names.""" print(f"Supported heads: {HEAD_NAMES}") diff --git a/transformer_lens/loading_from_pretrained.py b/transformer_lens/loading_from_pretrained.py index 606bd1ebaf..492078fa04 100644 --- a/transformer_lens/loading_from_pretrained.py +++ b/transformer_lens/loading_from_pretrained.py @@ -2037,11 +2037,8 @@ def fill_missing_keys( Returns: dict: State dict with missing keys filled in """ - # Get the default state dict default_state_dict = model.state_dict() - # Get the keys that are missing from the pretrained model missing_keys = set(default_state_dict.keys()) - set(state_dict.keys()) - # Fill in the missing keys with the default initialization for key in missing_keys: if "hf_model" in key: # Skip keys that are from the HuggingFace model, if loading from HF. @@ -2070,7 +2067,6 @@ class Config: n_layers: int = 12 -# Returns the configuration parameters of the model as a basic Config dataclass def get_basic_config(model_name: str, **kwargs: Any) -> Config: """Returns the configuration parameters of the model as a basic Config dataclass.""" return Config( diff --git a/transformer_lens/model_bridge/architecture_adapter.py b/transformer_lens/model_bridge/architecture_adapter.py index d69524543f..95e9bd55f2 100644 --- a/transformer_lens/model_bridge/architecture_adapter.py +++ b/transformer_lens/model_bridge/architecture_adapter.py @@ -41,8 +41,7 @@ class ArchitectureAdapter: # in specific phases (e.g. SSMs don't have the transformer-shaped hooks/ # weights the benchmark phases assume) should override. An empty list # means "skip verify_models entirely; verification lives in integration - # tests." The full refactor that would make SSM phases meaningful is - # documented in ~/.claude/plans/ssm-verification-compatibility.md. + # tests." applicable_phases: list[int] = [1, 2, 3, 4] # Whether this architecture supports text generation via generate(). diff --git a/transformer_lens/model_bridge/generalized_components/base.py b/transformer_lens/model_bridge/generalized_components/base.py index 65768857e3..2dcc13c95d 100644 --- a/transformer_lens/model_bridge/generalized_components/base.py +++ b/transformer_lens/model_bridge/generalized_components/base.py @@ -70,7 +70,6 @@ def __init__( # Copy class-level hook_aliases and apply any overrides if hook_alias_overrides is not None: - # Make a copy of class-level aliases and update with overrides self.hook_aliases = self.__class__.hook_aliases.copy() self.hook_aliases.update(hook_alias_overrides) @@ -234,7 +233,6 @@ def set_processed_weights( if hasattr(self.original_component, key): param = getattr(self.original_component, key) if param is not None and isinstance(param, torch.nn.Parameter): - # Check that shapes match if param.shape != weight_tensor.shape: raise ValueError( f"Shape mismatch when setting weight '{key}' in {type(self.original_component).__name__}: " diff --git a/transformer_lens/model_bridge/generalized_components/block.py b/transformer_lens/model_bridge/generalized_components/block.py index b998c15836..89d0347a59 100644 --- a/transformer_lens/model_bridge/generalized_components/block.py +++ b/transformer_lens/model_bridge/generalized_components/block.py @@ -79,9 +79,9 @@ def __init__( auto_overrides["hook_mlp_out"] = "ln2_post.hook_out" merged_overrides = {**auto_overrides, **(hook_alias_overrides or {})} - # Guard against the C15 bug class: sequential transformer block (attn + - # mlp) with no ln2 would silently point hook_resid_mid at the wrong - # tensor. Use ParallelBlockBridge for parallel-residual architectures. + # Guard against the bug where a sequential block (attn + mlp) with no ln2 + # silently points hook_resid_mid at the wrong tensor. Use + # ParallelBlockBridge for parallel-residual architectures. # Skip the check on generic-container / attn-only uses (no mlp). has_attn_like = submodules is not None and any( k in submodules for k in _VARIANT_SUBMODULE_SET @@ -95,7 +95,6 @@ def __init__( f"parallel-residual architecture." ) - # Call parent with merged overrides super().__init__( name, config, diff --git a/transformer_lens/model_bridge/generalized_components/bloom_block.py b/transformer_lens/model_bridge/generalized_components/bloom_block.py index e233880348..d3a2f9a020 100644 --- a/transformer_lens/model_bridge/generalized_components/bloom_block.py +++ b/transformer_lens/model_bridge/generalized_components/bloom_block.py @@ -74,9 +74,6 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: Returns: Output from the original BLOOM block """ - # Debug: Check if alibi is being passed - # print(f"BloomBlockBridge.forward() called with kwargs keys: {list(kwargs.keys())}") - if self.original_component is None: raise RuntimeError( f"Original component not set for {self.name}. Call set_original_component() first." @@ -131,10 +128,8 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: alibi = self.build_alibi_tensor(attention_mask_2d, num_heads, dtype) alibi = alibi.reshape(batch_size * num_heads, 1, seq_length) - # Add alibi to kwargs kwargs["alibi"] = alibi # else: alibi is already present from HF, don't overwrite it! - # Call original component output = self.original_component(*args, **kwargs) return self._apply_output_hook(output, wrap_single_element=False) diff --git a/transformer_lens/model_bridge/generalized_components/bloom_mlp.py b/transformer_lens/model_bridge/generalized_components/bloom_mlp.py index eb45dada1f..54791b4e41 100644 --- a/transformer_lens/model_bridge/generalized_components/bloom_mlp.py +++ b/transformer_lens/model_bridge/generalized_components/bloom_mlp.py @@ -68,7 +68,6 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: # The original BLOOM block passes it, so we just pass everything through # No need to validate since the original component will handle it - # Call the original BLOOM MLP component with all arguments output = self.original_component(*args, **kwargs) # Apply hook_out diff --git a/transformer_lens/model_bridge/generalized_components/clip_vision_encoder.py b/transformer_lens/model_bridge/generalized_components/clip_vision_encoder.py index e0debb530e..d7d81481b8 100644 --- a/transformer_lens/model_bridge/generalized_components/clip_vision_encoder.py +++ b/transformer_lens/model_bridge/generalized_components/clip_vision_encoder.py @@ -159,7 +159,6 @@ def forward( f"Original component not set for {self.name}. Call set_original_component() first." ) - # Apply input hook to pixel values pixel_values = self.hook_in(pixel_values) # Forward through the vision tower diff --git a/transformer_lens/model_bridge/generalized_components/linear.py b/transformer_lens/model_bridge/generalized_components/linear.py index 391f5a7640..9356b8031e 100644 --- a/transformer_lens/model_bridge/generalized_components/linear.py +++ b/transformer_lens/model_bridge/generalized_components/linear.py @@ -39,7 +39,6 @@ def forward(self, input: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tenso return output def __repr__(self) -> str: - """String representation of the LinearBridge.""" if self.original_component is not None: try: in_features = self.original_component.in_features diff --git a/transformer_lens/model_bridge/generalized_components/position_embeddings_attention.py b/transformer_lens/model_bridge/generalized_components/position_embeddings_attention.py index 5e43abf4e2..5af6599404 100644 --- a/transformer_lens/model_bridge/generalized_components/position_embeddings_attention.py +++ b/transformer_lens/model_bridge/generalized_components/position_embeddings_attention.py @@ -48,7 +48,6 @@ def _setup_eager_attention_hook_wrapper() -> None: if _EAGER_ATTENTION_WRAPPED: return - # Store the original function _ORIGINAL_EAGER_ATTENTION_FORWARD = gemma2_module.eager_attention_forward def hooked_eager_attention_forward( @@ -82,7 +81,6 @@ def hooked_eager_attention_forward( if hasattr(bridge, "hook_rot_k"): key = bridge.hook_rot_k(key) - # Call the original function assert _ORIGINAL_EAGER_ATTENTION_FORWARD is not None return _ORIGINAL_EAGER_ATTENTION_FORWARD( module, query, key, value, attention_mask, **kwargs @@ -271,7 +269,6 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: position_embeddings = kwargs.pop("position_embeddings", None) attention_mask = kwargs.pop("attention_mask", None) - # Apply input hook hidden_states = self.hook_in(hidden_states) # Match dtype of HF module diff --git a/transformer_lens/model_bridge/generalized_components/rotary_embedding.py b/transformer_lens/model_bridge/generalized_components/rotary_embedding.py index c560fff960..d1cfd22b28 100644 --- a/transformer_lens/model_bridge/generalized_components/rotary_embedding.py +++ b/transformer_lens/model_bridge/generalized_components/rotary_embedding.py @@ -133,7 +133,6 @@ def forward( # Apply hooks to match HookedTransformer's rotary_cos/rotary_sin pattern cos = self.hook_cos(cos) sin = self.hook_sin(sin) - # Return the hooked cos and sin as a tuple # Note: Don't pass tuple through hook_out as it expects a tensor return (cos, sin) else: diff --git a/transformer_lens/model_bridge/generalized_components/siglip_vision_encoder.py b/transformer_lens/model_bridge/generalized_components/siglip_vision_encoder.py index f81f8c8f67..8da7c748dd 100644 --- a/transformer_lens/model_bridge/generalized_components/siglip_vision_encoder.py +++ b/transformer_lens/model_bridge/generalized_components/siglip_vision_encoder.py @@ -155,7 +155,6 @@ def forward( f"Original component not set for {self.name}. Call set_original_component() first." ) - # Apply input hook to pixel values pixel_values = self.hook_in(pixel_values) # Forward through the vision tower diff --git a/transformer_lens/model_bridge/generalized_components/vision_projection.py b/transformer_lens/model_bridge/generalized_components/vision_projection.py index e79652228b..b33489bdbd 100644 --- a/transformer_lens/model_bridge/generalized_components/vision_projection.py +++ b/transformer_lens/model_bridge/generalized_components/vision_projection.py @@ -66,13 +66,10 @@ def forward( f"Original component not set for {self.name}. Call set_original_component() first." ) - # Apply input hook vision_features = self.hook_in(vision_features) - # Forward through the projection layer output = self.original_component(vision_features, **kwargs) - # Apply output hook if isinstance(output, tuple): output = (self.hook_out(output[0]),) + output[1:] else: diff --git a/transformer_lens/model_bridge/sources/transformers/source.py b/transformer_lens/model_bridge/sources/transformers/source.py index 19caeb45eb..3c4fc3c3c8 100644 --- a/transformer_lens/model_bridge/sources/transformers/source.py +++ b/transformer_lens/model_bridge/sources/transformers/source.py @@ -51,8 +51,7 @@ def boot( revision: str | None = None, checkpoint_index: int | None = None, checkpoint_value: int | None = None, - # Multi-device placement (accelerate-dispatched). GPU-validated 2026-07-16: - # tests/acceptance/model_bridge/test_bridge_multigpu*.py + scripts/bridge_multi_device_parity.py. + # Multi-device placement (accelerate-dispatched). Mutually exclusive with device. device_map: str | dict[str, str | int] | None = None, n_devices: int | None = None, max_memory: dict[str | int, str | int] | None = None, diff --git a/transformer_lens/model_bridge/supported_architectures/gemma1.py b/transformer_lens/model_bridge/supported_architectures/gemma1.py index 63bdab124d..a85220c6fd 100644 --- a/transformer_lens/model_bridge/supported_architectures/gemma1.py +++ b/transformer_lens/model_bridge/supported_architectures/gemma1.py @@ -129,7 +129,6 @@ def setup_component_testing(self, hf_model: Any, bridge_model: Any = None) -> No hf_model: The HuggingFace Gemma1 model instance bridge_model: The TransformerBridge model (if available, set rotary_emb on actual instances) """ - # Get rotary embedding instance from the model rotary_emb = hf_model.model.rotary_emb # Force HF model to use "eager" attention to match bridge implementation diff --git a/transformer_lens/model_bridge/supported_architectures/gemma2.py b/transformer_lens/model_bridge/supported_architectures/gemma2.py index 719ead0e5f..674732848e 100644 --- a/transformer_lens/model_bridge/supported_architectures/gemma2.py +++ b/transformer_lens/model_bridge/supported_architectures/gemma2.py @@ -40,8 +40,6 @@ def __init__(self, cfg: Any) -> None: self.cfg.gated_mlp = True self.cfg.attn_only = False - # Gemma models were not trained with BOS tokens - # self.cfg.default_prepend_bos = False self.cfg.uses_rms_norm = True # Gemma models use (1.0 + weight) in RMSNorm instead of just weight # See: https://github.com/huggingface/transformers/pull/29402 diff --git a/transformer_lens/model_bridge/supported_architectures/gemma3_multimodal.py b/transformer_lens/model_bridge/supported_architectures/gemma3_multimodal.py index a0c6d09ee3..d30061ef7a 100644 --- a/transformer_lens/model_bridge/supported_architectures/gemma3_multimodal.py +++ b/transformer_lens/model_bridge/supported_architectures/gemma3_multimodal.py @@ -51,7 +51,6 @@ def __init__(self, cfg: Any) -> None: """Initialize the Gemma3 multimodal architecture adapter.""" super().__init__(cfg) - # Mark this as a multimodal model self.cfg.is_multimodal = True # Language model configuration (same as text-only Gemma 3) diff --git a/transformer_lens/model_bridge/supported_architectures/gpt2_lm_head_custom.py b/transformer_lens/model_bridge/supported_architectures/gpt2_lm_head_custom.py index 4d5f53386c..f0557e09fa 100644 --- a/transformer_lens/model_bridge/supported_architectures/gpt2_lm_head_custom.py +++ b/transformer_lens/model_bridge/supported_architectures/gpt2_lm_head_custom.py @@ -62,7 +62,7 @@ def __init__(self, cfg: Any) -> None: ), source_key="transformer.h.{i}.attn.c_proj.weight", ), - # "unembed.b_U": "lm_head.bias", # gpt2 has no unembed bias + # no unembed.b_U mapping: gpt2 has no unembed bias } # Set up component mapping diff --git a/transformer_lens/model_bridge/supported_architectures/hunyuan_v1_dense.py b/transformer_lens/model_bridge/supported_architectures/hunyuan_v1_dense.py index 8629bfb2c6..a5ab20a1b6 100644 --- a/transformer_lens/model_bridge/supported_architectures/hunyuan_v1_dense.py +++ b/transformer_lens/model_bridge/supported_architectures/hunyuan_v1_dense.py @@ -19,7 +19,6 @@ class HunYuanDenseV1ArchitectureAdapter(ArchitectureAdapter): """Architecture adapter for HunYuanDenseV1 models.""" def __init__(self, cfg: Any) -> None: - """Initialize the HunYuanDenseV1 architecture adapter.""" super().__init__(cfg) self.cfg.normalization_type = "RMS" @@ -85,7 +84,6 @@ def __init__(self, cfg: Any) -> None: def setup_component_testing(self, hf_model: Any, bridge_model: Any = None) -> None: """Set up model-specific references for component testing.""" - # Get rotary embedding instance from the HF model rotary_emb = hf_model.model.rotary_emb # Set attention implementation on HF model to eager (vs sdpa default) diff --git a/transformer_lens/model_bridge/supported_architectures/llama.py b/transformer_lens/model_bridge/supported_architectures/llama.py index b7e731a393..98eb195e19 100644 --- a/transformer_lens/model_bridge/supported_architectures/llama.py +++ b/transformer_lens/model_bridge/supported_architectures/llama.py @@ -113,7 +113,6 @@ def setup_component_testing(self, hf_model: Any, bridge_model: Any = None) -> No hf_model: The HuggingFace Llama model instance bridge_model: The TransformerBridge model (if available, set rotary_emb on actual instances) """ - # Get rotary embedding instance from the model rotary_emb = hf_model.model.rotary_emb # Set rotary_emb on actual bridge instances in bridge_model if available diff --git a/transformer_lens/model_bridge/supported_architectures/llava.py b/transformer_lens/model_bridge/supported_architectures/llava.py index 4993ffdb55..b77a5a7a9b 100644 --- a/transformer_lens/model_bridge/supported_architectures/llava.py +++ b/transformer_lens/model_bridge/supported_architectures/llava.py @@ -51,7 +51,6 @@ def __init__(self, cfg: Any) -> None: """Initialize the LLava architecture adapter.""" super().__init__(cfg) - # Mark this as a multimodal model self.cfg.is_multimodal = True # Language model configuration (same as LLaMA) diff --git a/transformer_lens/model_bridge/supported_architectures/olmo2.py b/transformer_lens/model_bridge/supported_architectures/olmo2.py index 80b51a13e0..6bc5f1fc75 100644 --- a/transformer_lens/model_bridge/supported_architectures/olmo2.py +++ b/transformer_lens/model_bridge/supported_architectures/olmo2.py @@ -139,7 +139,6 @@ def setup_component_testing(self, hf_model: Any, bridge_model: Any = None) -> No hf_model: The HuggingFace OLMo 2 model instance bridge_model: The TransformerBridge model (if available) """ - # Get rotary embedding instance from the model rotary_emb = hf_model.model.rotary_emb # Force HF model to use "eager" attention to match bridge implementation diff --git a/transformer_lens/model_bridge/supported_architectures/olmoe.py b/transformer_lens/model_bridge/supported_architectures/olmoe.py index f8fc8e3440..912680a9b0 100644 --- a/transformer_lens/model_bridge/supported_architectures/olmoe.py +++ b/transformer_lens/model_bridge/supported_architectures/olmoe.py @@ -156,7 +156,6 @@ def setup_component_testing(self, hf_model: Any, bridge_model: Any = None) -> No hf_model: The HuggingFace OLMoE model instance bridge_model: The TransformerBridge model (if available) """ - # Get rotary embedding instance from the model rotary_emb = hf_model.model.rotary_emb # Force HF model to use "eager" attention to match bridge implementation diff --git a/transformer_lens/model_bridge/supported_architectures/pretrain.py b/transformer_lens/model_bridge/supported_architectures/pretrain.py index 20091b65ff..ee468ec2f1 100644 --- a/transformer_lens/model_bridge/supported_architectures/pretrain.py +++ b/transformer_lens/model_bridge/supported_architectures/pretrain.py @@ -67,27 +67,10 @@ class DenseOrMoEFeedForwardBridge(GeneralizedComponent): - """Wraps either a dense SwiGLU MLP or a sparse MoE layer behind a - common interface. Dispatch is determined by structural inspection - (`router`/`experts` vs `gate`/`up`/`down`) rather than configuration, - so dense, MoE, and mixed architectures all use the same component - mapping. This identifies the supported protocol -- it does not - validate that a module merely sharing those attribute names actually - implements the matching forward behavior. - - `hasattr` alone would let a module with, say, both `router`/`experts` - and `gate`/`up`/`down` (or `router`/`experts` of the wrong types) win - the MoE branch by attribute-name coincidence and fail later with a - confusing error from deep inside `MoEBridge`, or not fail until - forward time. `set_original_component` therefore also checks the - basic shape of whichever protocol wins: `router`/`gate`/`up`/`down` - must themselves be modules, and `experts` must be a *registered* - module collection (`nn.ModuleList`/`nn.ModuleDict`) -- a plain Python - list/tuple of `nn.Module` experts is rejected even though every - element is itself a valid module, because modules held in an - ordinary list aren't registered as children and would silently drop - out of `parameters()`/`state_dict()`/`.to(...)`/`train()`/`eval()`, - contradicting this adapter's lifecycle guarantees. + """Wraps a dense SwiGLU MLP or sparse MoE layer behind one interface. + Dispatch is by structural inspection (`router`/`experts` vs + `gate`/`up`/`down`), not config, so dense/MoE/mixed architectures all + share the same component mapping. """ def __init__(self, name: str, config: Any): @@ -101,6 +84,10 @@ def set_original_component(self, component: torch.nn.Module) -> None: # attribute is typed `str | None`, which is all mypy sees without this # narrowing. MoEBridge/GatedMLPBridge both require a plain `str` name. assert self.name is not None + # hasattr can match by attribute-name coincidence, so verify the + # protocol's shape too: router/gate/up/down must be modules, and + # experts a *registered* ModuleList/ModuleDict -- a plain list of + # experts drops out of parameters()/state_dict()/.to()/train()/eval(). if hasattr(component, "router") and hasattr(component, "experts"): if not isinstance(component.router, torch.nn.Module): raise TypeError( @@ -203,35 +190,11 @@ def __getattr__(self, key: str) -> Any: class PretrainModelContainer(torch.nn.Module): - """Internal detail -- `build_pretrain_bridge` applies this - automatically. Three responsibilities, all in this container's own - `forward`: - - 1. Avoids a `TransformerBridge.__getattr__` collision: a source - model's own `self.embed`/`self.blocks` clashes with identically - named component_mapping keys. Wrapping one level deeper - (`container.inner.embed`) fixes this without touching the source - model. - 2. Normalizes the return value to the `.logits` contract - `TransformerBridge` expects: a plain `"logits"` dict (the target - architecture's actual shape) is wrapped in `_LogitsAttrDict`; a bare - tensor, tensor-first tuple, or object already exposing `.logits` - passes through after validating the tensor is present and is a - tensor; anything else raises immediately with a clear message. - 3. Strips `_BRIDGE_COMPAT_KWARGS` from kwargs before calling the - wrapped model (see that constant's comment). - - Also sidesteps a `BlockBridge` convention where a bare-tensor block - output gets wrapped in a 1-tuple for "standalone hidden_states calls": - since this target's blocks take `cos`/`sin` too, that path never - triggers, so the source forward loop needs no changes to be bridged. - - `self.inner` is a regular registered submodule, so - `container.train()`/`.eval()` already recurse into it via the normal - `nn.Module` traversal -- no override needed here. The propagation gap - lives one level up, at `TransformerBridge` itself (see - `build_pretrain_bridge`), whose `.train()`/`.eval()` do not walk down - to `original_model`. + """Wraps the source model one level deeper (`container.inner`) so its + own `embed`/`blocks` attrs don't collide with `TransformerBridge` + component_mapping keys, normalizes the forward return to the `.logits` + contract, and strips `_BRIDGE_COMPAT_KWARGS`. Applied automatically by + `build_pretrain_bridge`. """ def __init__(self, model: torch.nn.Module) -> None: @@ -334,35 +297,10 @@ class PretrainArchitectureAdapter(ArchitectureAdapter): """Adapter for a decoder-only transformer using RoPE, RMSNorm, gated SwiGLU MLPs, and optional sparse MoE feed-forward layers. - Uses an opaque `NativeForwardAttentionBridge` with no attention - projection submodules, not `JointQKVAttentionBridge`/ - `PositionEmbeddingsAttentionBridge`: those reimplement RoPE via HF's - rotate-half convention, wrong for a source model using the - adjacent-pair convention. The opaque bridge delegates unchanged to - `Attention.forward`, so RoPE runs as written -- at the cost of no - per-head hooks, only block-level - `resid_pre`/`resid_mid`/`resid_post`. - - Blocks use `DelegatedAttentionBlockBridge` rather than plain - `BlockBridge`: that existing abstraction already exists for - architectures where attention is delegated wholesale and the - split-qkv-fork block-level aliases (`hook_attn_in`/`hook_q_input`/ - `hook_k_input`/`hook_v_input`) don't apply. It complements - `NativeForwardAttentionBridge.supports_split_qkv_fork = False` (which - prevents the split-QKV-fork machinery and its associated HookPoints - from being exposed for this attention component) by also removing the - now-dangling block-level aliases that would otherwise point at them. - `hook_attn_out` is untouched by either change, since the attention - component still fires its own `hook_out` normally. - - `self.cfg` is mutated in place, not copied (matches `nanogpt.py`'s - convention) -- callers holding another reference to the same config - will see these fields change. - - Bridges built through `build_pretrain_bridge` are given a - mode-propagating subclass so `.train()`/`.eval()` reach the wrapped - source model (see that function's docstring) -- this adapter class - itself has no lifecycle behavior of its own. + Uses an opaque `NativeForwardAttentionBridge` (delegates to + `Attention.forward`) so RoPE runs under the source's adjacent-pair + convention rather than HF's rotate-half -- at the cost of only + block-level hooks, no per-head hooks. """ def __init__(self, cfg: Any) -> None: diff --git a/transformer_lens/model_bridge/supported_architectures/qwen2.py b/transformer_lens/model_bridge/supported_architectures/qwen2.py index 7326201edf..ee400a11a5 100644 --- a/transformer_lens/model_bridge/supported_architectures/qwen2.py +++ b/transformer_lens/model_bridge/supported_architectures/qwen2.py @@ -100,12 +100,10 @@ def setup_component_testing(self, hf_model: Any, bridge_model: Any = None) -> No hf_model: The HuggingFace Qwen2 model instance bridge_model: The TransformerBridge model (if available, set rotary_emb on actual instances) """ - # Get rotary embedding instance from the model rotary_emb = hf_model.model.rotary_emb # Set rotary_emb on actual bridge instances in bridge_model if available if bridge_model is not None and hasattr(bridge_model, "blocks"): - # Set on each layer's actual attention bridge instance for block in bridge_model.blocks: if hasattr(block, "attn"): block.attn.set_rotary_emb(rotary_emb) diff --git a/transformer_lens/model_bridge/supported_architectures/qwen3.py b/transformer_lens/model_bridge/supported_architectures/qwen3.py index d8117b7530..d2a8356f41 100644 --- a/transformer_lens/model_bridge/supported_architectures/qwen3.py +++ b/transformer_lens/model_bridge/supported_architectures/qwen3.py @@ -132,8 +132,7 @@ def setup_component_testing(self, hf_model: Any, bridge_model: Any = None) -> No if "attn" in block._modules: block.attn.set_rotary_emb(rotary_emb) - # Set on template for get_generalized_component() calls - # Set on template — may not exist in hybrid adapters + # Set on template for get_generalized_component(); may be absent in hybrid adapters mapping = self.component_mapping or {} blocks_template = mapping.get("blocks") if isinstance(mapping, dict) else None if blocks_template and "attn" in getattr(blocks_template, "submodules", {}): diff --git a/transformer_lens/model_bridge/supported_architectures/qwen3_moe.py b/transformer_lens/model_bridge/supported_architectures/qwen3_moe.py index 643892cee1..58bb51db07 100644 --- a/transformer_lens/model_bridge/supported_architectures/qwen3_moe.py +++ b/transformer_lens/model_bridge/supported_architectures/qwen3_moe.py @@ -121,7 +121,6 @@ def setup_component_testing(self, hf_model: Any, bridge_model: Any = None) -> No hf_model: The HuggingFace Qwen3MoE model instance bridge_model: The TransformerBridge model (if available) """ - # Get rotary embedding instance from the model rotary_emb = hf_model.model.rotary_emb # Force HF model to use "eager" attention to match bridge implementation diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index aff4229f4b..aee061f39d 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -2372,7 +2372,6 @@ def generate( if hasattr(self, "_last_hf_cache"): del self._last_hf_cache - # Concatenate all sampled tokens sampled_tokens = torch.cat(sampled_tokens_list, dim=1) if is_encoder_decoder: # Reconstruct full decoder sequence: start token + generated tokens @@ -2796,7 +2795,6 @@ def hf_generate( if any_flag_set: generation_kwargs.setdefault("return_dict_in_generate", True) - # Generate using the original HuggingFace model with torch.no_grad(): outputs = self.original_model.generate(input_ids, **generation_kwargs) # type: ignore[operator] @@ -3149,7 +3147,6 @@ def _is_valid_bridge_path(self, hf_path: str) -> bool: Returns: True if the path is valid, False if it contains nested HF components """ - # Split the path into parts parts = hf_path.split(".") # Get the component mapping for validation diff --git a/transformer_lens/patching.py b/transformer_lens/patching.py index 24a50ed126..1e802d6ddf 100644 --- a/transformer_lens/patching.py +++ b/transformer_lens/patching.py @@ -206,7 +206,6 @@ def patching_hook(corrupted_activation, hook, index, clean_activation): corrupted_activation = corrupted_activation.clone() return patch_setter(corrupted_activation, index, clean_activation) - # Iterate over every list of indices, and make the appropriate patch! for c, index_row in enumerate(tqdm((list(index_df.iterrows())))): index = index_row[1].to_list() @@ -220,12 +219,10 @@ def patching_hook(corrupted_activation, hook, index, clean_activation): clean_activation=clean_cache[current_activation_name], ) - # Run the model with the patching hook and get the logits! patched_logits = model.run_with_hooks( corrupted_tokens, fwd_hooks=[(current_activation_name, current_hook)] ) - # Calculate the patching metric and store if flattened_output: patched_metric_output[c] = patching_metric(patched_logits).item() else: diff --git a/transformer_lens/pretrained/weight_conversions/hubert.py b/transformer_lens/pretrained/weight_conversions/hubert.py index 54e0e8567d..64f4940aa0 100644 --- a/transformer_lens/pretrained/weight_conversions/hubert.py +++ b/transformer_lens/pretrained/weight_conversions/hubert.py @@ -26,7 +26,6 @@ def convert_hubert_weights(hf_model, cfg: HookedTransformerConfig): # Use cfg dims for reshaping d_model = cfg.d_model n_heads = cfg.n_heads - # d_head = d_model // n_heads # implicit if needed for l, layer in enumerate(encoder_layers): # --- Attention module --- diff --git a/transformer_lens/pretrained/weight_conversions/olmoe.py b/transformer_lens/pretrained/weight_conversions/olmoe.py index a38ea758f5..7528a1a4a5 100644 --- a/transformer_lens/pretrained/weight_conversions/olmoe.py +++ b/transformer_lens/pretrained/weight_conversions/olmoe.py @@ -56,7 +56,6 @@ def convert_olmoe_weights(olmoe, cfg: HookedTransformerConfig): down = experts.down_proj # [num_experts, d_model, d_mlp] for e in range(cfg.num_experts): - # Split fused gate_up into gate and up projections state_dict[f"blocks.{l}.mlp.experts.{e}.W_gate.weight"] = gate_up[e, : cfg.d_mlp, :] state_dict[f"blocks.{l}.mlp.experts.{e}.W_in.weight"] = gate_up[e, cfg.d_mlp :, :] state_dict[f"blocks.{l}.mlp.experts.{e}.W_out.weight"] = down[e] diff --git a/transformer_lens/pretrained/weight_conversions/qwen3.py b/transformer_lens/pretrained/weight_conversions/qwen3.py index c2848213c2..ec5a8fd522 100644 --- a/transformer_lens/pretrained/weight_conversions/qwen3.py +++ b/transformer_lens/pretrained/weight_conversions/qwen3.py @@ -35,7 +35,6 @@ def convert_qwen3_weights(qwen: Any, cfg: HookedTransformerConfig): state_dict[f"blocks.{l}.attn.{gqa_uscore}W_K"] = W_K state_dict[f"blocks.{l}.attn.{gqa_uscore}W_V"] = W_V - # Load weights into RMSNorm modules state_dict[f"blocks.{l}.attn.q_norm.w"] = qwen.model.layers[l].self_attn.q_norm.weight state_dict[f"blocks.{l}.attn.k_norm.w"] = qwen.model.layers[l].self_attn.k_norm.weight diff --git a/transformer_lens/utilities/aliases.py b/transformer_lens/utilities/aliases.py index 5e618c4cbc..2d8980e8c2 100644 --- a/transformer_lens/utilities/aliases.py +++ b/transformer_lens/utilities/aliases.py @@ -57,7 +57,6 @@ def _resolve_single_target(target_name: str) -> Any: raise AttributeError( f"'{type(target_object).__name__}' object has no attribute '{target_name}'" ) - # Return the target hook return getattr(target_object, target_name) # if the target_name is a list, we check all elements diff --git a/transformer_lens/utilities/attribute_utils.py b/transformer_lens/utilities/attribute_utils.py index 2ecbc9bdc3..d133b12966 100644 --- a/transformer_lens/utilities/attribute_utils.py +++ b/transformer_lens/utilities/attribute_utils.py @@ -42,5 +42,4 @@ def set_nested_attr(obj, attr_str, value): for attr in attrs[:-1]: obj = getattr(obj, attr) - # Set the nested attribute's value setattr(obj, attrs[-1], value) diff --git a/transformer_lens/utilities/bridge_components.py b/transformer_lens/utilities/bridge_components.py index 8c0418fb6d..a9d525aa3d 100644 --- a/transformer_lens/utilities/bridge_components.py +++ b/transformer_lens/utilities/bridge_components.py @@ -30,11 +30,9 @@ def collect_all_submodules_of_component( if component_submodule.name is not None: submodules[block_prefix + component_submodule.name] = component_submodule - # If the component is a list item, we need to collect all submodules of the block bridge if component_submodule.is_list_item: submodules = collect_components_of_block_bridge(model, component_submodule, submodules) - # If the component has submodules, we need to collect them recursively if component_submodule.submodules: submodules = collect_all_submodules_of_component( model, component_submodule, submodules, block_prefix @@ -93,7 +91,6 @@ def collect_all_components(model: TransformerBridge, components: dict) -> dict: Dictionary mapping component names to their respective components """ - # Iterate through all components in component mapping for component in model.adapter.get_component_mapping().values(): components[component.name] = component components = collect_all_submodules_of_component(model, component, components) diff --git a/transformer_lens/utilities/hf_utils.py b/transformer_lens/utilities/hf_utils.py index 4b405770dc..cfaaf4ee68 100644 --- a/transformer_lens/utilities/hf_utils.py +++ b/transformer_lens/utilities/hf_utils.py @@ -190,8 +190,6 @@ def clear_huggingface_cache(): """ Deletes the Hugging Face cache directory and all its contents. - This function deletes the Hugging Face cache directory, which is used to store downloaded models and their associated files. Deleting the cache directory will remove all the downloaded models and their files, so you will need to download them again if you want to use them in your code. - This function is safe to call in parallel test execution - it will handle race conditions where multiple workers might try to delete the same directory. @@ -204,7 +202,6 @@ def clear_huggingface_cache(): print("Deleting Hugging Face cache directory and all its contents.") - # Check if cache directory exists if not os.path.exists(CACHE_DIR): return diff --git a/transformer_lens/utilities/matrix.py b/transformer_lens/utilities/matrix.py index 601d1d964e..2d768fdaef 100644 --- a/transformer_lens/utilities/matrix.py +++ b/transformer_lens/utilities/matrix.py @@ -42,7 +42,6 @@ def composition_scores( def get_matrix_corner(matrix: FactoredMatrix, n=3): - # Prints the top left corner of the tensor result = get_corner(matrix[tuple(slice(n) for _ in range(matrix.ndim))]) return result.AB diff --git a/transformer_lens/utilities/tokenize_utils.py b/transformer_lens/utilities/tokenize_utils.py index 0ba674febc..2b6c729e11 100644 --- a/transformer_lens/utilities/tokenize_utils.py +++ b/transformer_lens/utilities/tokenize_utils.py @@ -245,7 +245,7 @@ def get_attention_mask( torch.Tensor: The attention mask for the input. """ - # Initialize the attention mask with ones (indicating all tokens should be attended to) + # Default to attending every token; pad positions are zeroed below. attention_mask = torch.ones_like(tokens) if tokenizer is None: return attention_mask diff --git a/transformer_lens/weight_processing.py b/transformer_lens/weight_processing.py index 2db0e2b883..7d1ea647b6 100644 --- a/transformer_lens/weight_processing.py +++ b/transformer_lens/weight_processing.py @@ -1778,7 +1778,6 @@ def refactor_factored_attn_matrices( # Add singleton dimension for broadcasting b_V_expanded = einops.rearrange(b_V, "head_index d_head -> head_index d_head 1") - # Element-wise multiplication of b_V and W_O b_V_times_W_O = b_V_expanded * W_O # Sum over d_head and head_index dimensions From 1b2eaa416ad0862714b783175f6640900798c059 Mon Sep 17 00:00:00 2001 From: Jonah Larson Date: Wed, 29 Jul 2026 16:38:40 -0500 Subject: [PATCH 16/87] Release cleanup 3.6.0 Release port cleanup for 4.x (#1550) * Fixing tests that were broken by the migration from dev to 4.x * Updating docs for latest information, adding support for vision benchmarking * Clean up dead code in architecture adapters, properly documenting all our slow tests, fixing some test code configs * Test and architecture DRY cleanup * more DRY tests * Refactoring repeated architecture functions into helpers * Reduced duplication across adapters in component mapping and other setup components --- .github/workflows/checks.yml | 2 - .github/workflows/oracle-parity.yml | 16 +- AGENTS.md | 6 +- README.md | 4 +- docs/make_docs.py | 34 +- .../content/adapter_development/hf-scraper.md | 5 +- docs/source/content/compatibility_mode.md | 2 +- docs/source/content/contributing.md | 2 +- docs/source/content/drivers.md | 12 +- docs/source/content/getting_started.md | 2 +- docs/source/content/jacobian_lens_fitting.md | 6 +- docs/source/content/migrating_to_v3.md | 2 +- makefile | 2 +- scripts/audit_quarantines.py | 89 ++ tests/AGENTS.md | 6 +- tests/QUARANTINES.md | 49 +- tests/acceptance/model_bridge/conftest.py | 56 - tests/conftest.py | 38 + tests/integration/model_bridge/conftest.py | 56 +- tests/integration/model_bridge/helpers.py | 36 +- .../model_bridge/test_afmoe_adapter.py | 39 +- .../test_attention_weight_accessors.py | 6 +- .../model_bridge/test_bamba_adapter.py | 50 +- .../model_bridge/test_bitnet_adapter.py | 24 +- .../test_bridge_creation_modes.py | 34 +- .../model_bridge/test_bridge_integration.py | 64 +- .../test_bridge_start_at_layer.py | 18 + .../model_bridge/test_ernie4_5_adapter.py | 28 +- .../model_bridge/test_ernie4_5_moe_adapter.py | 24 +- .../model_bridge/test_exaone4_adapter.py | 44 +- .../model_bridge/test_exaone_adapter.py | 44 +- .../model_bridge/test_falcon_mamba_adapter.py | 44 +- .../model_bridge/test_gemma4_text_adapter.py | 19 +- .../model_bridge/test_glm4_adapter.py | 31 +- .../test_glm4_moe_lite_adapter.py | 44 +- .../model_bridge/test_glm4v_adapter.py | 31 +- .../model_bridge/test_glm_adapter.py | 26 +- .../model_bridge/test_glm_asr_adapter.py | 27 +- .../model_bridge/test_hrm_text_adapter.py | 23 +- .../model_bridge/test_idefics3_adapter.py | 38 +- .../model_bridge/test_jais2_adapter.py | 21 +- .../model_bridge/test_laguna_adapter.py | 21 +- .../model_bridge/test_llada2_moe_adapter.py | 21 +- .../model_bridge/test_llama4_adapter.py | 45 +- .../test_llama4_multimodal_adapter.py | 27 +- .../model_bridge/test_marian_adapter.py | 25 +- .../model_bridge/test_minimax_m2_adapter.py | 52 +- .../model_bridge/test_mistral3_adapter.py | 55 +- .../model_bridge/test_mixtral_adapter.py | 21 +- .../model_bridge/test_mpt_adapter.py | 31 +- .../model_bridge/test_nemotron_adapter.py | 30 +- .../model_bridge/test_olmo_hybrid_adapter.py | 21 +- .../model_bridge/test_openai_gpt_adapter.py | 30 +- .../model_bridge/test_qwen2_5_vl_adapter.py | 31 +- .../model_bridge/test_qwen2_audio_adapter.py | 36 +- .../model_bridge/test_qwen3_5_moe_adapter.py | 50 +- .../model_bridge/test_qwen3_moe_bridge.py | 29 +- .../model_bridge/test_qwen3_vl_adapter.py | 35 +- .../model_bridge/test_qwen3_vl_moe_adapter.py | 31 +- .../model_bridge/test_seed_oss_adapter.py | 42 +- .../model_bridge/test_stablelm_adapter.py | 21 +- .../model_bridge/test_starcoder2_adapter.py | 30 +- .../model_bridge/test_vaultgemma_adapter.py | 21 +- tests/mocks/mock_logits_bridge.py | 7 +- tests/mocks/vllm_boot.py | 122 ++ .../unit/benchmarks/test_vision_benchmarks.py | 285 +++++ tests/unit/model_bridge/sources/__init__.py | 0 .../test_boot_architecture_resolution.py | 213 ++++ .../test_inspect_provider_model_class.py | 104 ++ .../sources/test_modality_helpers.py | 248 ++++ .../supported_architectures/helpers.py | 39 + .../test_dream_adapter.py | 18 +- .../test_exaone_adapter.py | 4 +- .../test_gidd_adapter.py | 11 +- .../test_llada2_moe_adapter.py | 12 +- .../test_neox_adapter.py | 21 + .../test_pretrain_adapter.py | 37 + .../test_qwen3_5_adapter.py | 6 +- .../test_thin_subclass_adapters.py | 60 +- .../model_bridge/test_bridge_generate_bos.py | 56 +- .../test_bridge_generate_kv_cache.py | 28 +- .../model_bridge/test_checkpoint_revision.py | 6 +- .../model_bridge/test_diffusion_generate.py | 91 +- .../test_hook_alias_resolution.py | 18 +- .../model_bridge/test_remote_code_compat.py | 257 ++++ tests/unit/model_bridge/test_vllm_boot.py | 111 +- tests/unit/test_make_docs.py | 23 + .../unit/test_verify_models_phase_defaults.py | 21 +- .../test_registry_io_helpers.py | 124 ++ .../test_update_model_registry.py | 148 +++ .../tools/model_registry/test_verify_gates.py | 91 +- tests/unit/tools/test_model_registry.py | 86 +- .../utilities/test_attn_implementation.py | 126 ++ transformer_lens/benchmarks/audio.py | 9 +- transformer_lens/benchmarks/main_benchmark.py | 168 ++- transformer_lens/benchmarks/vision.py | 189 +++ transformer_lens/config/AGENTS.md | 4 +- .../model_bridge/architecture_adapter.py | 74 +- .../model_bridge/remote_bridge.py | 1 + .../model_bridge/sources/_bridge_builder.py | 6 + .../model_bridge/sources/_hf_format.py | 3 + .../model_bridge/sources/inspect/source.py | 15 +- .../sources/inspect/transformers_provider.py | 27 +- .../sources/inspect/vllm_provider.py | 15 +- .../sources/transformers/helpers.py | 76 +- .../sources/transformers/source.py | 111 +- .../model_bridge/sources/vllm/source.py | 19 +- .../supported_architectures/AGENTS.md | 2 +- .../supported_architectures/__init__.py | 1 + .../_remote_code_compat.py | 159 +++ .../supported_architectures/apertus.py | 17 +- .../supported_architectures/arcee.py | 17 +- .../supported_architectures/baichuan.py | 47 +- .../supported_architectures/bd3lm.py | 16 +- .../supported_architectures/cohere.py | 3 + .../supported_architectures/deepseek_v2.py | 148 ++- .../supported_architectures/deepseek_v3.py | 87 +- .../supported_architectures/deepseek_v4.py | 9 +- .../supported_architectures/dream.py | 52 +- .../supported_architectures/gemma3.py | 27 +- .../gemma3_multimodal.py | 13 +- .../supported_architectures/gidd.py | 50 +- .../supported_architectures/glm4_moe.py | 6 +- .../supported_architectures/glm4_moe_lite.py | 91 +- .../supported_architectures/glm_moe_dsa.py | 74 +- .../supported_architectures/gpt_oss.py | 6 - .../supported_architectures/granite.py | 21 +- .../supported_architectures/hrm_text.py | 30 +- .../supported_architectures/internlm2.py | 53 +- .../supported_architectures/jamba.py | 10 +- .../supported_architectures/jetmoe.py | 6 +- .../supported_architectures/laguna.py | 6 +- .../supported_architectures/lfm2.py | 27 +- .../supported_architectures/llada2_moe.py | 8 +- .../supported_architectures/llama.py | 19 - .../supported_architectures/mamba2.py | 2 +- .../supported_architectures/minimax_m2.py | 3 + .../supported_architectures/mixtral.py | 6 - .../modernbert_decoder.py | 6 +- .../supported_architectures/nanochat.py | 7 +- .../supported_architectures/neox.py | 26 +- .../supported_architectures/olmo.py | 8 +- .../supported_architectures/olmoe.py | 11 +- .../supported_architectures/openelm.py | 99 +- .../supported_architectures/ouro.py | 126 +- .../supported_architectures/pretrain.py | 11 +- .../supported_architectures/qwen2.py | 19 - .../supported_architectures/qwen3.py | 20 +- .../supported_architectures/qwen3_5.py | 30 +- .../supported_architectures/qwen3_5_moe.py | 44 +- .../supported_architectures/qwen3_moe.py | 3 + .../supported_architectures/raven.py | 203 +--- .../supported_architectures/rwkv7.py | 104 +- .../supported_architectures/t5gemma.py | 4 +- .../supported_architectures/t5gemma2.py | 4 +- .../model_bridge/transformer_bridge.py | 5 + .../tools/model_registry/__init__.py | 13 +- transformer_lens/tools/model_registry/api.py | 4 + .../model_registry/data/supported_models.json | 1062 ++++++++++++++++- .../tools/model_registry/generate_report.py | 1 - .../tools/model_registry/hf_scraper.py | 62 +- .../tools/model_registry/registry_io.py | 97 +- .../tools/model_registry/schemas.py | 4 + .../tools/model_registry/validate.py | 10 +- .../tools/model_registry/verify_models.py | 137 ++- transformer_lens/utilities/architectures.py | 12 +- .../utilities/attn_implementation.py | 59 + 167 files changed, 5482 insertions(+), 2624 deletions(-) create mode 100644 scripts/audit_quarantines.py delete mode 100644 tests/acceptance/model_bridge/conftest.py create mode 100644 tests/mocks/vllm_boot.py create mode 100644 tests/unit/benchmarks/test_vision_benchmarks.py create mode 100644 tests/unit/model_bridge/sources/__init__.py create mode 100644 tests/unit/model_bridge/sources/test_boot_architecture_resolution.py create mode 100644 tests/unit/model_bridge/sources/test_inspect_provider_model_class.py create mode 100644 tests/unit/model_bridge/sources/test_modality_helpers.py create mode 100644 tests/unit/model_bridge/test_remote_code_compat.py create mode 100644 tests/unit/tools/model_registry/test_registry_io_helpers.py create mode 100644 tests/unit/tools/model_registry/test_update_model_registry.py create mode 100644 tests/unit/utilities/test_attn_implementation.py create mode 100644 transformer_lens/benchmarks/vision.py create mode 100644 transformer_lens/model_bridge/supported_architectures/_remote_code_compat.py create mode 100644 transformer_lens/utilities/attn_implementation.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 4318eb942e..2802597bb2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -159,9 +159,7 @@ jobs: --ignore=tests/unit/model_bridge/test_optimizer_compatibility.py --ignore=tests/unit/model_bridge/test_gpt_oss_moe.py --ignore=tests/unit/model_bridge/test_component_inspection.py - --ignore=tests/unit/model_bridge/test_key_analysis.py --ignore=tests/unit/model_bridge/test_benchmark_gated_hooks_fire.py - --ignore=tests/unit/model_bridge/test_weight_processing_adapter_paths.py --ignore=tests/unit/model_bridge/test_bridge_generate_kv_cache.py --ignore=tests/unit/model_bridge/test_bridge_vs_hooked_transformer_patching.py --ignore=tests/unit/model_bridge/compatibility/ diff --git a/.github/workflows/oracle-parity.yml b/.github/workflows/oracle-parity.yml index 540729e327..388a48ded0 100644 --- a/.github/workflows/oracle-parity.yml +++ b/.github/workflows/oracle-parity.yml @@ -1,17 +1,11 @@ name: Oracle Parity # Runs the @pytest.mark.slow oracle parity suite for JacobianLens. -# Not part of the standard PR gate — requires ~4 GB VRAM and network access. -# Triggered manually or automatically when the test file or this workflow changes. +# Not part of the standard PR gate — requires ~4 GB VRAM (or a long CPU run) +# and network access, so it is manual-trigger only. Run it after changing +# tests/integration/test_jacobian_lens_oracle_parity.py or this workflow. on: workflow_dispatch: - push: - branches: - - main - - dev* - paths: - - "tests/integration/test_jacobian_lens_oracle_parity.py" - - ".github/workflows/oracle-parity.yml" permissions: contents: read @@ -21,6 +15,8 @@ jobs: name: Oracle Parity Checks runs-on: ubuntu-latest timeout-minutes: 60 + env: + HAS_HF_TOKEN: ${{ secrets.HF_TOKEN != '' }} steps: - uses: actions/checkout@v4 @@ -46,7 +42,7 @@ jobs: uv pip install "git+https://github.com/anthropics/jacobian-lens.git@${ORACLE_COMMIT}" - name: Authenticate HuggingFace - if: env.HF_TOKEN != '' + if: ${{ env.HAS_HF_TOKEN == 'true' }} run: uv run python -c "import os; from huggingface_hub import login; login(token=os.environ['HF_TOKEN'])" env: HF_TOKEN: ${{ secrets.HF_TOKEN }} diff --git a/AGENTS.md b/AGENTS.md index 12f81bdb8b..3d053dbbce 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,7 +22,7 @@ Sub-folder rules: [tests/AGENTS.md](tests/AGENTS.md) · [supported_architectures ## 1. What this repo is -**TransformerLens** — mechanistic-interpretability library. Loads 9,000+ models across 50+ architecture families (see [supported_models.json](transformer_lens/tools/model_registry/data/supported_models.json)) and exposes internal activations through a hook system for caching, editing, and ablating intermediate state. Built on HuggingFace `transformers`. +**TransformerLens** — mechanistic-interpretability library. Loads 15,000+ models across 140+ architecture families (see [supported_models.json](transformer_lens/tools/model_registry/data/supported_models.json)) and exposes internal activations through a hook system for caching, editing, and ablating intermediate state. Built on HuggingFace `transformers`. ## 2. Two systems live in this repo @@ -33,7 +33,7 @@ Sub-folder rules: [tests/AGENTS.md](tests/AGENTS.md) · [supported_architectures > ⚠ The **HookedTransformer acceptance suite is quarantined** ([test_hooked_transformer.py](tests/acceptance/test_hooked_transformer.py), [test_hooked_encoder.py](tests/acceptance/test_hooked_encoder.py), [test_hooked_encoder_decoder.py](tests/acceptance/test_hooked_encoder_decoder.py); see [QUARANTINES.md](tests/QUARANTINES.md)). HT changes land untested at the acceptance level — extra manual care required. -Bridge architecture-adapter pattern: each HF architecture has one file in [supported_architectures/](transformer_lens/model_bridge/supported_architectures/) mapping HF module paths to canonical names. Bridge hooks are architecture-native (e.g. `blocks.{i}.hook_out`); HT-style aliases live in [bridge.py](transformer_lens/model_bridge/bridge.py). +Bridge architecture-adapter pattern: each HF architecture has one file in [supported_architectures/](transformer_lens/model_bridge/supported_architectures/) mapping HF module paths to canonical names. Bridge hooks are architecture-native (e.g. `blocks.{i}.hook_out`); HT-style aliases live in [bridge_core.py](transformer_lens/model_bridge/bridge_core.py). **Mirroring rule:** if you change `HookedTransformer` behaviour that has a `TransformerBridge` counterpart, update both in the same PR. [supported_models.py](transformer_lens/supported_models.py) is HT-only — Bridge-only models go in the Bridge registry data file. @@ -107,7 +107,7 @@ Python: **>=3.10, <4.0**. CI tests 3.10, 3.11, 3.12. Format/type/docstring check ## 5. Hook naming — HT vs Bridge - **HT canonical**: uniform across architectures — `hook_embed`, `blocks.{i}.hook_resid_pre`, `blocks.{i}.attn.hook_q`, `blocks.{i}.hook_resid_post`. -- **Bridge-native**: architecture-shaped — `blocks.{i}.hook_out`, `blocks.{i}.attn.q.hook_out`. HT aliases registered via `build_alias_to_canonical_map()` in [bridge.py](transformer_lens/model_bridge/bridge.py). +- **Bridge-native**: architecture-shaped — `blocks.{i}.hook_out`, `blocks.{i}.attn.q.hook_out`. HT aliases registered via `build_alias_to_canonical_map()` in [bridge_core.py](transformer_lens/model_bridge/bridge_core.py). Prefer Bridge-native names in new code. Raw-HF-forward drivers comparing against `boot_transformers` must match its load configuration (fp32, eager attention) and probe for optional features like `resid_mid` rather than assume. diff --git a/README.md b/README.md index c76c8fe91e..2ad7365c1c 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ interpretability](https://distill.pub/2020/circuits/zoom-in/) of GPT-2 Style lan goal of mechanistic interpretability is to take a trained model and reverse engineer the algorithms the model learned during training from its weights. -TransformerLens lets you load in 9,000+ open source language models across 50+ architecture families, +TransformerLens lets you load in 15,000+ open source language models across 140+ architecture families, and exposes the internal activations of the model to you. You can cache any internal activation in the model, and add in functions to edit, remove or replace these activations as the model runs. @@ -52,7 +52,7 @@ logits, activations = bridge.run_with_cache("Hello World") > Gated models (Llama, Mistral, Gemma, ...) require `HF_TOKEN` in your environment. See [Environment Variables](https://TransformerLensOrg.github.io/TransformerLens/content/getting_started.html#environment-variables) for the full list. -`TransformerBridge` is the recommended 3.0 path and supports 9,000+ models across 50+ architecture families (see [`supported_models.json`](transformer_lens/tools/model_registry/data/supported_models.json) for the full inventory). By default it preserves raw HuggingFace weights – logits and activations match HF, *not* legacy `HookedTransformer` (which folds LayerNorm and centers weights by default). Call `bridge.enable_compatibility_mode()` after booting for HookedTransformer-equivalent numerics. The legacy `HookedTransformer.from_pretrained` API is still available but deprecated — see the [Migrating to TransformerLens 3](https://TransformerLensOrg.github.io/TransformerLens/content/migrating_to_v3.html) guide. +`TransformerBridge` is the recommended 3.0 path and supports 15,000+ models across 140+ architecture families (see [`supported_models.json`](transformer_lens/tools/model_registry/data/supported_models.json) for the full inventory). By default it preserves raw HuggingFace weights – logits and activations match HF, *not* legacy `HookedTransformer` (which folds LayerNorm and centers weights by default). Call `bridge.enable_compatibility_mode()` after booting for HookedTransformer-equivalent numerics. The legacy `HookedTransformer.from_pretrained` API is still available but deprecated — see the [Migrating to TransformerLens 3](https://TransformerLensOrg.github.io/TransformerLens/content/migrating_to_v3.html) guide. ## Key Tutorials diff --git a/docs/make_docs.py b/docs/make_docs.py index 9ad61ed16d..1810514c14 100644 --- a/docs/make_docs.py +++ b/docs/make_docs.py @@ -821,9 +821,10 @@ def copy_demos(_app: Optional[Any] = None): #bt-root tbody td a:hover { text-decoration: underline; } #bt-root .bt-rn { color: var(--color-foreground-muted, #999); font-size: 12px; width: 36px; text-align: right; } #bt-root .bt-badge { display: inline-block; padding: 2px 8px; border-radius: 10px; font-size: 11px; font-weight: 600; } +/* No .bt-s2: status-2 (skipped) rows deliberately render with the .bt-s0 + "Unverified" badge — see the status cell's class mapping. */ #bt-root .bt-s0 { background: #e8e8e8; color: #666; } #bt-root .bt-s1 { background: #d4edda; color: #155724; } -#bt-root .bt-s2 { background: #fff3cd; color: #856404; } #bt-root .bt-s3 { background: #f8d7da; color: #721c24; } #bt-root .bt-s4 { background: #d1ecf1; color: #0c5460; } #bt-root .bt-muted { color: var(--color-foreground-muted, #999); } @@ -902,7 +903,7 @@ def copy_demos(_app: Optional[Any] = None): - + @@ -928,7 +929,8 @@ def copy_demos(_app: Optional[Any] = None): \n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1122,9 +1158,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1188,9 +1225,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1231,9 +1269,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1304,9 +1343,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1377,9 +1417,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1410,9 +1451,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1455,9 +1497,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1488,9 +1531,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1550,9 +1594,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1594,9 +1639,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1648,9 +1694,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1681,9 +1728,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1725,9 +1773,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1776,9 +1825,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1819,9 +1869,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1863,15 +1914,17 @@ "name": "stdout", "output_type": "stream", "text": [ - "key_fourier_embed torch.Size([8, 128])\n" + "key_freqs [9, 10, 18, 30, 36]\n", + "key_fourier_embed torch.Size([10, 128])\n" ] }, { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1901,8 +1954,16 @@ } ], "source": [ - "key_freqs = [17, 25, 32, 47]\n", - "key_freq_indices = [33, 34, 49, 50, 63, 64, 93, 94]\n", + "# The key frequencies are a property of the trained run (which frequencies the\n", + "# network picks is seed-dependent), so detect them from the embedding's Fourier\n", + "# norms rather than hardcoding a past run's values. The spikes are ~10x the\n", + "# background, so a quarter-of-max threshold separates them cleanly.\n", + "fourier_norms = (fourier_basis @ W_E).norm(dim=-1)\n", + "key_freq_indices = [\n", + " i for i, norm in enumerate(fourier_norms) if i > 0 and norm > fourier_norms.max() / 4\n", + "]\n", + "key_freqs = sorted({(i + 1) // 2 for i in key_freq_indices})\n", + "print(\"key_freqs\", key_freqs)\n", "fourier_embed = fourier_basis @ W_E\n", "key_fourier_embed = fourier_embed[key_freq_indices]\n", "print(\"key_fourier_embed\", key_fourier_embed.shape)\n", @@ -1925,9 +1986,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -1968,9 +2030,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -2019,9 +2082,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -2064,9 +2128,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -2109,9 +2174,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -2152,9 +2218,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -2195,9 +2262,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -2238,9 +2306,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -2281,9 +2350,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -2352,9 +2422,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -2401,9 +2472,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -2476,9 +2548,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -2560,9 +2633,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -2600,7 +2674,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Study sin 17" + "Study the sin component of one key frequency" ] }, { @@ -2611,9 +2685,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -2643,10 +2718,10 @@ } ], "source": [ - "freq = 17\n", + "freq = key_freqs[0]\n", "W_logit_fourier = W_logit @ fourier_basis\n", - "neurons_sin_17 = W_logit_fourier[:, 2*freq-1]\n", - "line(neurons_sin_17)" + "neurons_sin_key = W_logit_fourier[:, 2*freq-1]\n", + "line(neurons_sin_key)" ] }, { @@ -2677,9 +2752,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -2709,8 +2785,8 @@ } ], "source": [ - "inputs_sin_17c = neuron_acts @ neurons_sin_17\n", - "imshow(fourier_basis @ inputs_sin_17c.reshape(p, p) @ fourier_basis.T, title=\"Fourier Heatmap over inputs for sin17c\", x=fourier_basis_names, y=fourier_basis_names)" + "inputs_sin_key = neuron_acts @ neurons_sin_key\n", + "imshow(fourier_basis @ inputs_sin_key.reshape(p, p) @ fourier_basis.T, title=f\"Fourier Heatmap over inputs for sin {freq} component\", x=fourier_basis_names, y=fourier_basis_names)" ] }, { @@ -2905,25 +2981,38 @@ "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "In IPython\n", - "In IPython\n", - "Set autoreload\n", - "Imported everything!\n" - ] - }, - { - "ename": "NameError", - "evalue": "name 'train_losses' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/tmp/ipykernel_1229617/2975677256.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mneel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnpx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mfig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnpx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_losses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_losses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_losses\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxaxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"Epoch\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myaxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"Loss\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlog_y\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtitle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"Training Curve for Modular Addition\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mline_labels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'test'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoggle_x\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoggle_y\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_fig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0madd_lines\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mNameError\u001b[0m: name 'train_losses' is not defined" - ] + "data": { + "text/html": [ + "
\n", + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ @@ -2941,7 +3030,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 97, "metadata": {}, "outputs": [ { @@ -2962,17 +3051,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 74, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Freq: 17\n", - "Freq: 25\n", - "Freq: 32\n", - "Freq: 47\n" + "Freq: 9\n", + "Freq: 10\n", + "Freq: 18\n", + "Freq: 30\n", + "Freq: 36\n" ] } ], @@ -2990,27 +3080,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 75, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Freq: 17\n", - "Coeff: tensor(-0.0254, device='cuda:0', grad_fn=)\n", - "Cosine Sim: tensor(-1.1664e-06, device='cuda:0', grad_fn=)\n", - "Freq: 25\n", - "Coeff: tensor(-0.0128, device='cuda:0', grad_fn=)\n", - "Cosine Sim: tensor(-5.9062e-07, device='cuda:0', grad_fn=)\n", - "Freq: 32\n", - "Coeff: tensor(0.0520, device='cuda:0', grad_fn=)\n", - "Cosine Sim: tensor(2.3932e-06, device='cuda:0', grad_fn=)\n", - "Freq: 47\n", - "Coeff: tensor(0.0008, device='cuda:0', grad_fn=)\n", - "Cosine Sim: tensor(3.7253e-08, device='cuda:0', grad_fn=)\n", - "Residual size: tensor(21740.6836, device='cuda:0', grad_fn=)\n", - "Residual fraction of norm: tensor(1., device='cuda:0', grad_fn=)\n" + "Freq: 9\n", + "Coeff: tensor(9648.4473, grad_fn=)\n", + "Cosine Sim: tensor(0.4165, grad_fn=)\n", + "Freq: 10\n", + "Coeff: tensor(6929.4863, grad_fn=)\n", + "Cosine Sim: tensor(0.2991, grad_fn=)\n", + "Freq: 18\n", + "Coeff: tensor(2.1624, grad_fn=)\n", + "Cosine Sim: tensor(9.3350e-05, grad_fn=)\n", + "Freq: 30\n", + "Coeff: tensor(18661.9688, grad_fn=)\n", + "Cosine Sim: tensor(0.8056, grad_fn=)\n", + "Freq: 36\n", + "Coeff: tensor(6209.9819, grad_fn=)\n", + "Cosine Sim: tensor(0.2681, grad_fn=)\n", + "Residual size: tensor(2955.6726, grad_fn=)\n", + "Residual fraction of norm: tensor(0.1276, grad_fn=)\n" ] } ], @@ -3030,14 +3123,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 76, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "tensor(0.0011, device='cuda:0', grad_fn=)\n" + "tensor(-0.0012, grad_fn=)\n" ] } ], @@ -3048,17 +3141,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 77, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor(1.7025e-07, device='cuda:0', dtype=torch.float64,\n", - " grad_fn=)" + "tensor(1.4906e-07, dtype=torch.float64, grad_fn=)" ] }, - "execution_count": 76, + "execution_count": 77, "metadata": {}, "output_type": "execute_result" } @@ -3069,16 +3161,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 78, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor(4.7274, device='cuda:0', dtype=torch.float64, grad_fn=)" + "tensor(6.2335e-08, dtype=torch.float64, grad_fn=)" ] }, - "execution_count": 77, + "execution_count": 78, "metadata": {}, "output_type": "execute_result" } @@ -3097,7 +3189,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 98, "metadata": {}, "outputs": [ { @@ -3123,13 +3215,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 80, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4d3444ac0c09485bb2297c35246614f4", + "model_id": "4b22bbb6b5804e3894d2d36005b18748", "version_major": 2, "version_minor": 0 }, @@ -3151,9 +3243,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:13: UserWarning:\n", + "/var/folders/kg/j3z_vbk52tl5d08sz1r87xq80000gn/T/ipykernel_4763/1361489951.py:13: UserWarning:\n", "\n", - "Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at /opt/conda/conda-bld/pytorch_1656352464346/work/torch/csrc/utils/tensor_new.cpp:201.)\n", + "Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/torch/csrc/utils/tensor_new.cpp:256.)\n", "\n" ] } @@ -3172,15 +3264,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 81, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -3216,13 +3309,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 82, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "64c4305b8e8f466f8dd9240817fbc5e2", + "model_id": "a3d353b9b43e4aa3b2a19b6a8c4e88b3", "version_major": 2, "version_minor": 0 }, @@ -3243,9 +3336,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -3290,13 +3384,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 83, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "93e129d45acf43d29dc422409440ad70", + "model_id": "47877757d7fe447facd0b4025e14a8ba", "version_major": 2, "version_minor": 0 }, @@ -3317,9 +3411,10 @@ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -3373,7 +3468,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 99, "metadata": {}, "outputs": [ { @@ -3382,7 +3477,7 @@ "torch.Size([12769, 512])" ] }, - "execution_count": 83, + "execution_count": 99, "metadata": {}, "output_type": "execute_result" } @@ -3393,15 +3488,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 85, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -3440,7 +3536,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 100, "metadata": {}, "outputs": [ { @@ -3459,14 +3555,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 87, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "tensor(4.8139, device='cuda:0', dtype=torch.float64, grad_fn=)\n" + "tensor(1.3544e-07, dtype=torch.float64, grad_fn=)\n" ] } ], @@ -3490,9 +3586,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 88, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor(1.4906e-07, dtype=torch.float64, grad_fn=)\n" + ] + } + ], "source": [ "print(loss_fn(original_logits, labels))" ] @@ -3507,17 +3611,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 89, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor(1.1483e-07, device='cuda:0', dtype=torch.float64,\n", - " grad_fn=)" + "tensor(1.1921e-07, dtype=torch.float64, grad_fn=)" ] }, - "execution_count": 177, + "execution_count": 89, "metadata": {}, "output_type": "execute_result" } @@ -3549,7 +3652,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 101, "metadata": {}, "outputs": [ { @@ -3581,15 +3684,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 91, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -3625,15 +3729,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 92, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, @@ -3678,14 +3783,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 93, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "tensor(7.5597, device='cuda:0', dtype=torch.float64, grad_fn=)\n" + "tensor(6.5147, dtype=torch.float64, grad_fn=)\n" ] } ], @@ -3710,24 +3815,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 94, "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([12769, 113])\n" - ] - }, { "data": { "text/plain": [ - "tensor(1.6301e-07, device='cuda:0', dtype=torch.float64,\n", - " grad_fn=)" + "tensor(5.0676, dtype=torch.float64, grad_fn=)" ] }, - "execution_count": 201, + "execution_count": 94, "metadata": {}, "output_type": "execute_result" } @@ -3759,7 +3856,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 102, "metadata": {}, "outputs": [ { @@ -3791,15 +3888,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 96, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
\n", + "
" + " }) }; " ] }, "metadata": {}, diff --git a/demos/No_Position_Experiment.ipynb b/demos/No_Position_Experiment.ipynb index 979c99e422..762c9feaef 100644 --- a/demos/No_Position_Experiment.ipynb +++ b/demos/No_Position_Experiment.ipynb @@ -64,14 +64,15 @@ "\n", "if IN_COLAB or IN_GITHUB:\n", " %pip install einops\n", - " %pip install transformer_lens@v1.15.0\n", + " %pip install transformer_lens\n", "\n", " # PySvelte is an unmaintained visualization library, use it as a backup if circuitsvis isn't working\n", " # # Install another version of node that makes PySvelte work way faster\n", " # !curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash -; sudo apt-get install -y nodejs\n", " # %pip install git+https://github.com/neelnanda-io/PySvelte.git\n", "\n", - "from transformer_lens import HookedTransformer, HookedTransformerConfig\n", + "from transformer_lens.config import TransformerBridgeConfig\n", + "from transformer_lens.model_bridge import TransformerBridge\n", "import torch\n", "import numpy as np\n", "import plotly.express as px\n", @@ -146,7 +147,7 @@ "metadata": {}, "outputs": [], "source": [ - "cfg = HookedTransformerConfig(\n", + "cfg = TransformerBridgeConfig(\n", " n_layers=2,\n", " d_model=64,\n", " d_head=64,\n", @@ -158,7 +159,9 @@ " normalization_type=\"LN\",\n", " device=device,\n", ")\n", - "model = HookedTransformer(cfg)" + "# boot_native builds a small randomly-initialized TL-native model (no\n", + "# HuggingFace download) and wraps it in a trainable TransformerBridge.\n", + "model = TransformerBridge.boot_native(cfg, device=device)" ] }, { @@ -184,57 +187,235 @@ "name": "stdout", "output_type": "stream", "text": [ - "HookedTransformer(\n", - " (embed): Embed()\n", - " (hook_embed): HookPoint()\n", - " (pos_embed): PosEmbed()\n", - " (hook_pos_embed): HookPoint()\n", + "TransformerBridge(\n", + " (embed): EmbeddingBridge(\n", + " (hook_in): HookPoint(name='embed.hook_in')\n", + " (hook_out): HookPoint(name='embed.hook_out')\n", + " (_original_component): Embedding(300, 64)\n", + " )\n", + " (pos_embed): PosEmbedBridge(\n", + " (hook_in): HookPoint(name='pos_embed.hook_in')\n", + " (hook_out): HookPoint(name='pos_embed.hook_out')\n", + " (_original_component): Embedding(50, 64)\n", + " )\n", " (blocks): ModuleList(\n", - " (0-1): 2 x TransformerBlock(\n", - " (ln1): LayerNorm(\n", - " (hook_scale): HookPoint()\n", - " (hook_normalized): HookPoint()\n", + " (0): BlockBridge(\n", + " (hook_in): HookPoint(name='blocks.0.hook_in')\n", + " (hook_out): HookPoint(name='blocks.0.hook_out')\n", + " (hook_mlp_in): HookPoint(name='blocks.0.hook_mlp_in')\n", + " (_original_component): NativeBlock(\n", + " (ln1): NormalizationBridge(\n", + " (hook_in): HookPoint(name='blocks.0.ln1.hook_in')\n", + " (hook_out): HookPoint(name='blocks.0.ln1.hook_out')\n", + " (hook_normalized): HookPoint(name='blocks.0.ln1.hook_normalized')\n", + " (hook_scale): HookPoint(name='blocks.0.ln1.hook_scale')\n", + " (_original_component): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (attn): AttentionBridge(\n", + " (hook_in): HookPoint(name='blocks.0.attn.hook_in')\n", + " (hook_out): HookPoint(name='blocks.0.attn.hook_out')\n", + " (hook_attn_scores): HookPoint(name='blocks.0.attn.hook_attn_scores')\n", + " (hook_pattern): HookPoint(name='blocks.0.attn.hook_pattern')\n", + " (hook_hidden_states): HookPoint(name='blocks.0.attn.hook_hidden_states')\n", + " (hook_result): HookPoint(name='blocks.0.attn.hook_result')\n", + " (hook_attn_in): HookPoint(name='blocks.0.attn.hook_attn_in')\n", + " (hook_q_input): HookPoint(name='blocks.0.attn.hook_q_input')\n", + " (hook_k_input): HookPoint(name='blocks.0.attn.hook_k_input')\n", + " (hook_v_input): HookPoint(name='blocks.0.attn.hook_v_input')\n", + " (_original_component): NativeAttention(\n", + " (q): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (k): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (v): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (o): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " )\n", + " (q): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (k): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (v): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (o): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " )\n", + " (ln2): NormalizationBridge(\n", + " (hook_in): HookPoint(name='blocks.0.ln2.hook_in')\n", + " (hook_out): HookPoint(name='blocks.0.ln2.hook_out')\n", + " (hook_normalized): HookPoint(name='blocks.0.ln2.hook_normalized')\n", + " (hook_scale): HookPoint(name='blocks.0.ln2.hook_scale')\n", + " (_original_component): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (mlp): MLPBridge(\n", + " (hook_in): HookPoint(name='blocks.0.mlp.hook_in')\n", + " (hook_out): HookPoint(name='blocks.0.mlp.hook_out')\n", + " (_original_component): NativeMLP(\n", + " (fc_in): LinearBridge(64 -> 256, bias=True, original_component=Linear)\n", + " (fc_out): LinearBridge(256 -> 64, bias=True, original_component=Linear)\n", + " )\n", + " (in): LinearBridge(64 -> 256, bias=True, original_component=Linear)\n", + " (out): LinearBridge(256 -> 64, bias=True, original_component=Linear)\n", + " )\n", + " )\n", + " (ln1): NormalizationBridge(\n", + " (hook_in): HookPoint(name='blocks.0.ln1.hook_in')\n", + " (hook_out): HookPoint(name='blocks.0.ln1.hook_out')\n", + " (hook_normalized): HookPoint(name='blocks.0.ln1.hook_normalized')\n", + " (hook_scale): HookPoint(name='blocks.0.ln1.hook_scale')\n", + " (_original_component): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (attn): AttentionBridge(\n", + " (hook_in): HookPoint(name='blocks.0.attn.hook_in')\n", + " (hook_out): HookPoint(name='blocks.0.attn.hook_out')\n", + " (hook_attn_scores): HookPoint(name='blocks.0.attn.hook_attn_scores')\n", + " (hook_pattern): HookPoint(name='blocks.0.attn.hook_pattern')\n", + " (hook_hidden_states): HookPoint(name='blocks.0.attn.hook_hidden_states')\n", + " (hook_result): HookPoint(name='blocks.0.attn.hook_result')\n", + " (hook_attn_in): HookPoint(name='blocks.0.attn.hook_attn_in')\n", + " (hook_q_input): HookPoint(name='blocks.0.attn.hook_q_input')\n", + " (hook_k_input): HookPoint(name='blocks.0.attn.hook_k_input')\n", + " (hook_v_input): HookPoint(name='blocks.0.attn.hook_v_input')\n", + " (_original_component): NativeAttention(\n", + " (q): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (k): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (v): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (o): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " )\n", + " (q): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (k): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (v): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (o): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " )\n", + " (ln2): NormalizationBridge(\n", + " (hook_in): HookPoint(name='blocks.0.ln2.hook_in')\n", + " (hook_out): HookPoint(name='blocks.0.ln2.hook_out')\n", + " (hook_normalized): HookPoint(name='blocks.0.ln2.hook_normalized')\n", + " (hook_scale): HookPoint(name='blocks.0.ln2.hook_scale')\n", + " (_original_component): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (mlp): MLPBridge(\n", + " (hook_in): HookPoint(name='blocks.0.mlp.hook_in')\n", + " (hook_out): HookPoint(name='blocks.0.mlp.hook_out')\n", + " (_original_component): NativeMLP(\n", + " (fc_in): LinearBridge(64 -> 256, bias=True, original_component=Linear)\n", + " (fc_out): LinearBridge(256 -> 64, bias=True, original_component=Linear)\n", + " )\n", + " (in): LinearBridge(64 -> 256, bias=True, original_component=Linear)\n", + " (out): LinearBridge(256 -> 64, bias=True, original_component=Linear)\n", + " )\n", + " )\n", + " (1): BlockBridge(\n", + " (hook_in): HookPoint(name='blocks.1.hook_in')\n", + " (hook_out): HookPoint(name='blocks.1.hook_out')\n", + " (hook_mlp_in): HookPoint(name='blocks.1.hook_mlp_in')\n", + " (_original_component): NativeBlock(\n", + " (ln1): NormalizationBridge(\n", + " (hook_in): HookPoint(name='blocks.1.ln1.hook_in')\n", + " (hook_out): HookPoint(name='blocks.1.ln1.hook_out')\n", + " (hook_normalized): HookPoint(name='blocks.1.ln1.hook_normalized')\n", + " (hook_scale): HookPoint(name='blocks.1.ln1.hook_scale')\n", + " (_original_component): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (attn): AttentionBridge(\n", + " (hook_in): HookPoint(name='blocks.1.attn.hook_in')\n", + " (hook_out): HookPoint(name='blocks.1.attn.hook_out')\n", + " (hook_attn_scores): HookPoint(name='blocks.1.attn.hook_attn_scores')\n", + " (hook_pattern): HookPoint(name='blocks.1.attn.hook_pattern')\n", + " (hook_hidden_states): HookPoint(name='blocks.1.attn.hook_hidden_states')\n", + " (hook_result): HookPoint(name='blocks.1.attn.hook_result')\n", + " (hook_attn_in): HookPoint(name='blocks.1.attn.hook_attn_in')\n", + " (hook_q_input): HookPoint(name='blocks.1.attn.hook_q_input')\n", + " (hook_k_input): HookPoint(name='blocks.1.attn.hook_k_input')\n", + " (hook_v_input): HookPoint(name='blocks.1.attn.hook_v_input')\n", + " (_original_component): NativeAttention(\n", + " (q): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (k): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (v): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (o): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " )\n", + " (q): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (k): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (v): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (o): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " )\n", + " (ln2): NormalizationBridge(\n", + " (hook_in): HookPoint(name='blocks.1.ln2.hook_in')\n", + " (hook_out): HookPoint(name='blocks.1.ln2.hook_out')\n", + " (hook_normalized): HookPoint(name='blocks.1.ln2.hook_normalized')\n", + " (hook_scale): HookPoint(name='blocks.1.ln2.hook_scale')\n", + " (_original_component): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (mlp): MLPBridge(\n", + " (hook_in): HookPoint(name='blocks.1.mlp.hook_in')\n", + " (hook_out): HookPoint(name='blocks.1.mlp.hook_out')\n", + " (_original_component): NativeMLP(\n", + " (fc_in): LinearBridge(64 -> 256, bias=True, original_component=Linear)\n", + " (fc_out): LinearBridge(256 -> 64, bias=True, original_component=Linear)\n", + " )\n", + " (in): LinearBridge(64 -> 256, bias=True, original_component=Linear)\n", + " (out): LinearBridge(256 -> 64, bias=True, original_component=Linear)\n", + " )\n", + " )\n", + " (ln1): NormalizationBridge(\n", + " (hook_in): HookPoint(name='blocks.1.ln1.hook_in')\n", + " (hook_out): HookPoint(name='blocks.1.ln1.hook_out')\n", + " (hook_normalized): HookPoint(name='blocks.1.ln1.hook_normalized')\n", + " (hook_scale): HookPoint(name='blocks.1.ln1.hook_scale')\n", + " (_original_component): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n", " )\n", - " (ln2): LayerNorm(\n", - " (hook_scale): HookPoint()\n", - " (hook_normalized): HookPoint()\n", + " (attn): AttentionBridge(\n", + " (hook_in): HookPoint(name='blocks.1.attn.hook_in')\n", + " (hook_out): HookPoint(name='blocks.1.attn.hook_out')\n", + " (hook_attn_scores): HookPoint(name='blocks.1.attn.hook_attn_scores')\n", + " (hook_pattern): HookPoint(name='blocks.1.attn.hook_pattern')\n", + " (hook_hidden_states): HookPoint(name='blocks.1.attn.hook_hidden_states')\n", + " (hook_result): HookPoint(name='blocks.1.attn.hook_result')\n", + " (hook_attn_in): HookPoint(name='blocks.1.attn.hook_attn_in')\n", + " (hook_q_input): HookPoint(name='blocks.1.attn.hook_q_input')\n", + " (hook_k_input): HookPoint(name='blocks.1.attn.hook_k_input')\n", + " (hook_v_input): HookPoint(name='blocks.1.attn.hook_v_input')\n", + " (_original_component): NativeAttention(\n", + " (q): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (k): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (v): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (o): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " )\n", + " (q): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (k): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (v): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", + " (o): LinearBridge(64 -> 64, bias=True, original_component=Linear)\n", " )\n", - " (attn): Attention(\n", - " (hook_k): HookPoint()\n", - " (hook_q): HookPoint()\n", - " (hook_v): HookPoint()\n", - " (hook_z): HookPoint()\n", - " (hook_attn_scores): HookPoint()\n", - " (hook_pattern): HookPoint()\n", - " (hook_result): HookPoint()\n", + " (ln2): NormalizationBridge(\n", + " (hook_in): HookPoint(name='blocks.1.ln2.hook_in')\n", + " (hook_out): HookPoint(name='blocks.1.ln2.hook_out')\n", + " (hook_normalized): HookPoint(name='blocks.1.ln2.hook_normalized')\n", + " (hook_scale): HookPoint(name='blocks.1.ln2.hook_scale')\n", + " (_original_component): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n", " )\n", - " (mlp): MLP(\n", - " (hook_pre): HookPoint()\n", - " (hook_post): HookPoint()\n", + " (mlp): MLPBridge(\n", + " (hook_in): HookPoint(name='blocks.1.mlp.hook_in')\n", + " (hook_out): HookPoint(name='blocks.1.mlp.hook_out')\n", + " (_original_component): NativeMLP(\n", + " (fc_in): LinearBridge(64 -> 256, bias=True, original_component=Linear)\n", + " (fc_out): LinearBridge(256 -> 64, bias=True, original_component=Linear)\n", + " )\n", + " (in): LinearBridge(64 -> 256, bias=True, original_component=Linear)\n", + " (out): LinearBridge(256 -> 64, bias=True, original_component=Linear)\n", " )\n", - " (hook_attn_in): HookPoint()\n", - " (hook_q_input): HookPoint()\n", - " (hook_k_input): HookPoint()\n", - " (hook_v_input): HookPoint()\n", - " (hook_mlp_in): HookPoint()\n", - " (hook_attn_out): HookPoint()\n", - " (hook_mlp_out): HookPoint()\n", - " (hook_resid_pre): HookPoint()\n", - " (hook_resid_mid): HookPoint()\n", - " (hook_resid_post): HookPoint()\n", " )\n", " )\n", - " (ln_final): LayerNorm(\n", - " (hook_scale): HookPoint()\n", - " (hook_normalized): HookPoint()\n", + " (ln_final): NormalizationBridge(\n", + " (hook_in): HookPoint(name='ln_final.hook_in')\n", + " (hook_out): HookPoint(name='ln_final.hook_out')\n", + " (hook_normalized): HookPoint(name='ln_final.hook_normalized')\n", + " (hook_scale): HookPoint(name='ln_final.hook_scale')\n", + " (_original_component): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (unembed): UnembeddingBridge(\n", + " (hook_in): HookPoint(name='unembed.hook_in')\n", + " (hook_out): HookPoint(name='unembed.hook_out')\n", + " (_original_component): Linear(in_features=64, out_features=300, bias=True)\n", " )\n", - " (unembed): Unembed()\n", ")\n" ] } ], "source": [ - "print(model)" + "print(repr(model))" ] }, { @@ -461,6 +642,8 @@ } ], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", + "# (unseeded training/eval prints vary per run)\n", "losses = []\n", "for epoch in tqdm.tqdm(range(num_epochs)):\n", " tokens = next(data_loader)\n", @@ -536,6 +719,8 @@ } ], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", + "# (unseeded training/eval prints vary per run)\n", "big_data_loader = make_data_generator(cfg, 4000)\n", "big_tokens = next(big_data_loader)\n", "big_tokens = big_tokens.to(device)\n", @@ -552,7 +737,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "ActivationCache with keys ['hook_embed', 'hook_pos_embed', 'blocks.0.hook_resid_pre', 'blocks.0.ln1.hook_scale', 'blocks.0.ln1.hook_normalized', 'blocks.0.attn.hook_q', 'blocks.0.attn.hook_k', 'blocks.0.attn.hook_v', 'blocks.0.attn.hook_attn_scores', 'blocks.0.attn.hook_pattern', 'blocks.0.attn.hook_z', 'blocks.0.hook_attn_out', 'blocks.0.hook_resid_mid', 'blocks.0.ln2.hook_scale', 'blocks.0.ln2.hook_normalized', 'blocks.0.mlp.hook_pre', 'blocks.0.mlp.hook_post', 'blocks.0.hook_mlp_out', 'blocks.0.hook_resid_post', 'blocks.1.hook_resid_pre', 'blocks.1.ln1.hook_scale', 'blocks.1.ln1.hook_normalized', 'blocks.1.attn.hook_q', 'blocks.1.attn.hook_k', 'blocks.1.attn.hook_v', 'blocks.1.attn.hook_attn_scores', 'blocks.1.attn.hook_pattern', 'blocks.1.attn.hook_z', 'blocks.1.hook_attn_out', 'blocks.1.hook_resid_mid', 'blocks.1.ln2.hook_scale', 'blocks.1.ln2.hook_normalized', 'blocks.1.mlp.hook_pre', 'blocks.1.mlp.hook_post', 'blocks.1.hook_mlp_out', 'blocks.1.hook_resid_post', 'ln_final.hook_scale', 'ln_final.hook_normalized']\n" + "ActivationCache with keys ['embed.hook_in', 'embed.hook_out', 'hook_embed', 'pos_embed.hook_in', 'pos_embed.hook_out', 'hook_pos_embed', 'blocks.0.hook_in', 'blocks.0.hook_resid_pre', 'blocks.0.ln1.hook_in', 'blocks.0.ln1.hook_scale', 'blocks.0.ln1.hook_normalized', 'blocks.0.ln1.hook_out', 'blocks.0.attn.hook_in', 'blocks.0.attn.q.hook_in', 'blocks.0.attn.q.hook_out', 'blocks.0.attn.hook_q', 'blocks.0.attn.k.hook_in', 'blocks.0.attn.k.hook_out', 'blocks.0.attn.hook_k', 'blocks.0.attn.v.hook_in', 'blocks.0.attn.v.hook_out', 'blocks.0.attn.hook_v', 'blocks.0.attn.o.hook_in', 'blocks.0.attn.hook_z', 'blocks.0.attn.o.hook_out', 'blocks.0.attn.hook_out', 'blocks.0.hook_attn_out', 'blocks.0.attn.hook_pattern', 'blocks.0.attn.hook_attn_scores', 'blocks.0.ln2.hook_in', 'blocks.0.hook_resid_mid', 'blocks.0.ln2.hook_scale', 'blocks.0.ln2.hook_normalized', 'blocks.0.ln2.hook_out', 'blocks.0.mlp.hook_in', 'blocks.0.mlp.in.hook_in', 'blocks.0.mlp.in.hook_out', 'blocks.0.mlp.hook_pre', 'blocks.0.mlp.out.hook_in', 'blocks.0.mlp.hook_post', 'blocks.0.mlp.out.hook_out', 'blocks.0.mlp.hook_out', 'blocks.0.hook_mlp_out', 'blocks.0.hook_out', 'blocks.0.hook_resid_post', 'blocks.1.hook_in', 'blocks.1.hook_resid_pre', 'blocks.1.ln1.hook_in', 'blocks.1.ln1.hook_scale', 'blocks.1.ln1.hook_normalized', 'blocks.1.ln1.hook_out', 'blocks.1.attn.hook_in', 'blocks.1.attn.q.hook_in', 'blocks.1.attn.q.hook_out', 'blocks.1.attn.hook_q', 'blocks.1.attn.k.hook_in', 'blocks.1.attn.k.hook_out', 'blocks.1.attn.hook_k', 'blocks.1.attn.v.hook_in', 'blocks.1.attn.v.hook_out', 'blocks.1.attn.hook_v', 'blocks.1.attn.o.hook_in', 'blocks.1.attn.hook_z', 'blocks.1.attn.o.hook_out', 'blocks.1.attn.hook_out', 'blocks.1.hook_attn_out', 'blocks.1.attn.hook_pattern', 'blocks.1.attn.hook_attn_scores', 'blocks.1.ln2.hook_in', 'blocks.1.hook_resid_mid', 'blocks.1.ln2.hook_scale', 'blocks.1.ln2.hook_normalized', 'blocks.1.ln2.hook_out', 'blocks.1.mlp.hook_in', 'blocks.1.mlp.in.hook_in', 'blocks.1.mlp.in.hook_out', 'blocks.1.mlp.hook_pre', 'blocks.1.mlp.out.hook_in', 'blocks.1.mlp.hook_post', 'blocks.1.mlp.out.hook_out', 'blocks.1.mlp.hook_out', 'blocks.1.hook_mlp_out', 'blocks.1.hook_out', 'blocks.1.hook_resid_post', 'ln_final.hook_in', 'ln_final.hook_scale', 'ln_final.hook_normalized', 'ln_final.hook_out', 'unembed.hook_in', 'unembed.hook_out', 'hook_unembed']\n" ] } ], @@ -592,9 +777,9 @@ "\n", "\n", "
\n", - "
\n", + " }) }; \n", "\n", "" ] @@ -631,9 +816,9 @@ "\n", "\n", "
\n", - "
\n", + " }) }; \n", "\n", "" ] @@ -746,9 +931,9 @@ "\n", "\n", "
\n", - "
\n", + " }) }; \n", "\n", "" ] @@ -800,7 +985,7 @@ "metadata": {}, "outputs": [], "source": [ - "analysis_cfg = HookedTransformerConfig(\n", + "analysis_cfg = TransformerBridgeConfig(\n", " n_layers=2,\n", " d_model=64,\n", " d_head=64,\n", @@ -809,13 +994,16 @@ " d_vocab=300,\n", " n_ctx=50,\n", " act_fn=\"relu\",\n", - " normalization_type=\"LNPre\",\n", + " normalization_type=\"LN\",\n", " init_weights=False,\n", + " device=device,\n", ")\n", - "analysis_model = HookedTransformer(analysis_cfg)\n", - "state_dict = model.state_dict()\n", - "analysis_model.load_and_process_state_dict(\n", - " state_dict, fold_ln=True, center_writing_weights=True, center_unembed=True\n", + "analysis_model = TransformerBridge.boot_native(analysis_cfg, device=device)\n", + "analysis_model.load_state_dict(model.state_dict())\n", + "# Fold LayerNorm into the downstream linear layers and center the\n", + "# residual-stream-writing / unembedding weights, in place.\n", + "analysis_model.process_weights(\n", + " fold_ln=True, center_writing_weights=True, center_unembed=True\n", ")\n", "deactivate_position(analysis_model)" ] @@ -848,9 +1036,9 @@ "\n", "\n", "
\n", - "
\n", + " }) }; \n", "\n", "" ] @@ -898,9 +1086,9 @@ "\n", "\n", "
\n", - "
\n", + " }) }; \n", "\n", "" ] @@ -948,9 +1136,9 @@ "\n", "\n", "
\n", - "
\n", + " }) }; \n", "\n", "" ] @@ -1004,9 +1192,9 @@ "\n", "\n", "
\n", - "
\n", + " }) }; \n", "\n", "" ] @@ -1043,9 +1231,9 @@ "\n", "\n", "
\n", - "
\n", + " }) }; \n", "\n", "" ] @@ -1082,9 +1270,9 @@ "\n", "\n", "
\n", - "
\n", + " }) }; \n", "\n", "" ] @@ -1121,9 +1309,9 @@ "\n", "\n", "
\n", - "
\n", + " }) }; \n", "\n", "" ] @@ -1203,6 +1391,8 @@ } ], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", + "# (unseeded training/eval prints vary per run)\n", "new_token_batch = next(big_data_loader).to(device)\n", "baseline_loss = loss_fn(model(new_token_batch), new_token_batch).item()\n", "print(\"Baseline loss:\", baseline_loss)" @@ -1258,14 +1448,18 @@ } ], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", + "# (unseeded training/eval prints vary per run)\n", "hook_list = list(model.hook_dict.keys())\n", "losses = []\n", "loss_labels = []\n", "for hook_name in hook_list:\n", " if (\n", " hook_name in cache\n", - " and hook_name != \"hook_pos_embed\"\n", + " and \"pos_embed\" not in hook_name\n", " and \"result\" not in hook_name\n", + " # embed.hook_in / pos_embed.hook_in cache the integer token ids\n", + " and cache[hook_name].is_floating_point()\n", " ):\n", " average_act = cache[hook_name].mean(0)\n", "\n", @@ -1296,9 +1490,9 @@ "\n", "\n", "
\n", - "
\n", + " }) }; \n", "\n", "" ] @@ -1341,10 +1535,10 @@ { "data": { "text/plain": [ - "dict_keys(['hook_embed', 'hook_pos_embed', 'blocks.0.hook_resid_pre', 'blocks.0.ln1.hook_scale', 'blocks.0.ln1.hook_normalized', 'blocks.0.attn.hook_q', 'blocks.0.attn.hook_k', 'blocks.0.attn.hook_v', 'blocks.0.attn.hook_attn_scores', 'blocks.0.attn.hook_pattern', 'blocks.0.attn.hook_z', 'blocks.0.hook_attn_out', 'blocks.0.hook_resid_mid', 'blocks.0.ln2.hook_scale', 'blocks.0.ln2.hook_normalized', 'blocks.0.mlp.hook_pre', 'blocks.0.mlp.hook_post', 'blocks.0.hook_mlp_out', 'blocks.0.hook_resid_post', 'blocks.1.hook_resid_pre', 'blocks.1.ln1.hook_scale', 'blocks.1.ln1.hook_normalized', 'blocks.1.attn.hook_q', 'blocks.1.attn.hook_k', 'blocks.1.attn.hook_v', 'blocks.1.attn.hook_attn_scores', 'blocks.1.attn.hook_pattern', 'blocks.1.attn.hook_z', 'blocks.1.hook_attn_out', 'blocks.1.hook_resid_mid', 'blocks.1.ln2.hook_scale', 'blocks.1.ln2.hook_normalized', 'blocks.1.mlp.hook_pre', 'blocks.1.mlp.hook_post', 'blocks.1.hook_mlp_out', 'blocks.1.hook_resid_post', 'ln_final.hook_scale', 'ln_final.hook_normalized'])" + "dict_keys(['embed.hook_in', 'embed.hook_out', 'hook_embed', 'pos_embed.hook_in', 'pos_embed.hook_out', 'hook_pos_embed', 'blocks.0.hook_in', 'blocks.0.hook_resid_pre', 'blocks.0.ln1.hook_in', 'blocks.0.ln1.hook_scale', 'blocks.0.ln1.hook_normalized', 'blocks.0.ln1.hook_out', 'blocks.0.attn.hook_in', 'blocks.0.attn.q.hook_in', 'blocks.0.attn.q.hook_out', 'blocks.0.attn.hook_q', 'blocks.0.attn.k.hook_in', 'blocks.0.attn.k.hook_out', 'blocks.0.attn.hook_k', 'blocks.0.attn.v.hook_in', 'blocks.0.attn.v.hook_out', 'blocks.0.attn.hook_v', 'blocks.0.attn.o.hook_in', 'blocks.0.attn.hook_z', 'blocks.0.attn.o.hook_out', 'blocks.0.attn.hook_out', 'blocks.0.hook_attn_out', 'blocks.0.attn.hook_pattern', 'blocks.0.attn.hook_attn_scores', 'blocks.0.ln2.hook_in', 'blocks.0.hook_resid_mid', 'blocks.0.ln2.hook_scale', 'blocks.0.ln2.hook_normalized', 'blocks.0.ln2.hook_out', 'blocks.0.mlp.hook_in', 'blocks.0.mlp.in.hook_in', 'blocks.0.mlp.in.hook_out', 'blocks.0.mlp.hook_pre', 'blocks.0.mlp.out.hook_in', 'blocks.0.mlp.hook_post', 'blocks.0.mlp.out.hook_out', 'blocks.0.mlp.hook_out', 'blocks.0.hook_mlp_out', 'blocks.0.hook_out', 'blocks.0.hook_resid_post', 'blocks.1.hook_in', 'blocks.1.hook_resid_pre', 'blocks.1.ln1.hook_in', 'blocks.1.ln1.hook_scale', 'blocks.1.ln1.hook_normalized', 'blocks.1.ln1.hook_out', 'blocks.1.attn.hook_in', 'blocks.1.attn.q.hook_in', 'blocks.1.attn.q.hook_out', 'blocks.1.attn.hook_q', 'blocks.1.attn.k.hook_in', 'blocks.1.attn.k.hook_out', 'blocks.1.attn.hook_k', 'blocks.1.attn.v.hook_in', 'blocks.1.attn.v.hook_out', 'blocks.1.attn.hook_v', 'blocks.1.attn.o.hook_in', 'blocks.1.attn.hook_z', 'blocks.1.attn.o.hook_out', 'blocks.1.attn.hook_out', 'blocks.1.hook_attn_out', 'blocks.1.attn.hook_pattern', 'blocks.1.attn.hook_attn_scores', 'blocks.1.ln2.hook_in', 'blocks.1.hook_resid_mid', 'blocks.1.ln2.hook_scale', 'blocks.1.ln2.hook_normalized', 'blocks.1.ln2.hook_out', 'blocks.1.mlp.hook_in', 'blocks.1.mlp.in.hook_in', 'blocks.1.mlp.in.hook_out', 'blocks.1.mlp.hook_pre', 'blocks.1.mlp.out.hook_in', 'blocks.1.mlp.hook_post', 'blocks.1.mlp.out.hook_out', 'blocks.1.mlp.hook_out', 'blocks.1.hook_mlp_out', 'blocks.1.hook_out', 'blocks.1.hook_resid_post', 'ln_final.hook_in', 'ln_final.hook_scale', 'ln_final.hook_normalized', 'ln_final.hook_out', 'unembed.hook_in', 'unembed.hook_out', 'hook_unembed'])" ] }, - "execution_count": 61, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } diff --git a/demos/doc_sanitize.cfg b/demos/doc_sanitize.cfg index 445e71ae7c..6c1b235a9d 100644 --- a/demos/doc_sanitize.cfg +++ b/demos/doc_sanitize.cfg @@ -12,7 +12,7 @@ replace: HEX-CODE [regex4] regex: ,\s*device='[^']*' -replace: +replace: [regex5] regex: ([1-9]\d*\.\d{3})\d+ @@ -33,3 +33,7 @@ replace: \1 [regex6d] regex: (0\.[1-9]\d{2})\d+ replace: \1 + +[regex7] +regex: /(?:var|tmp|private)/\S* +replace: TMP-PATH diff --git a/docs/source/content/migrating_to_v3.md b/docs/source/content/migrating_to_v3.md index 1766cf70a9..b61b60326f 100644 --- a/docs/source/content/migrating_to_v3.md +++ b/docs/source/content/migrating_to_v3.md @@ -150,7 +150,7 @@ If your code only touches these APIs, the migration is genuinely just the loadin ### BERT Next Sentence Prediction -`BertNextSentencePrediction` is not ported to `TransformerBridge`. Keep using `HookedEncoder` + `BertNextSentencePrediction` for NSP workflows. The bridge's BERT adapter does load NSP HuggingFace checkpoints (it rewires the unembed to `cls.seq_relationship`), but the high-level NSP API – sentence-pair tokenization, `[CLS]` pooling, "sequential"/"not sequential" decoding — is not exposed. If this is feature is something you'd like added to TransformerBridge, please file an issue. +The high-level NSP API (`BertNextSentencePrediction`) is not yet ported: it requires the legacy `HookedEncoder` surface (`encoder_output`, `pooler`, `nsp_head`), which `TransformerBridge` does not expose. The bridge's BERT adapter does load NSP HuggingFace checkpoints (it rewires the unembed to `cls.seq_relationship`), so the underlying weights are available — but sentence-pair tokenization, `[CLS]` pooling, and "sequential"/"not sequential" decoding are not. Until it is ported, NSP workflows need the legacy classes, which are slated for removal; if you rely on this API, please say so on the tracking issue. ### New in 3.x: streaming generation diff --git a/tests/integration/model_bridge/test_bridge_creation_modes.py b/tests/integration/model_bridge/test_bridge_creation_modes.py index afeb84374b..6d4b1b0205 100644 --- a/tests/integration/model_bridge/test_bridge_creation_modes.py +++ b/tests/integration/model_bridge/test_bridge_creation_modes.py @@ -59,6 +59,17 @@ def test_bridge_configuration_persistence(self): assert hasattr(bridge, "cfg"), "Configuration should persist after compatibility mode" assert bridge.cfg is not None, "Configuration should not be None" + def test_audio_model_compat_mode_rejected(self): + """Audio encoders must get a clean NotImplementedError from compat mode. + + The legacy weight processing assumes a text embed/unembed; without the + guard it dies later with an opaque KeyError ('embed.weight'). + """ + bridge = TransformerBridge.boot_transformers("distilgpt2", device="cpu") + bridge.cfg.is_audio_model = True + with pytest.raises(NotImplementedError, match="audio encoder"): + bridge.enable_compatibility_mode() + def test_bridge_device_handling(self, gpt2_bridge): """Test that bridge handles device specification correctly.""" assert ( diff --git a/tests/unit/benchmarks/test_audio_input_prep.py b/tests/unit/benchmarks/test_audio_input_prep.py new file mode 100644 index 0000000000..ebe1ee5074 --- /dev/null +++ b/tests/unit/benchmarks/test_audio_input_prep.py @@ -0,0 +1,54 @@ +"""Tests for the feature-extractor-driven audio input prep.""" + +from types import SimpleNamespace + +import torch + +from transformer_lens.benchmarks.audio import _prepare_audio_encoder_input + + +class _FakeFE: + """Stands in for an AST-style feature extractor (spectrogram features, own rate).""" + + sampling_rate = 8000 + + def __call__(self, waveforms, sampling_rate, return_tensors): + assert sampling_rate == 8000 + batch = len(waveforms) + return {"input_features": torch.ones(batch, 12, 4)} + + +class _Bridge(SimpleNamespace): + pass + + +def _make_bridge(processor): + return _Bridge(processor=processor, cfg=SimpleNamespace(device="cpu", dtype=torch.float32)) + + +def test_uses_feature_extractor_output_and_rate(): + bridge = _make_bridge(_FakeFE()) + prepared = _prepare_audio_encoder_input(bridge) + # feature-space output, not the raw waveform; synthetic waveform used the FE's rate + assert prepared.shape == (1, 12, 4) + assert torch.equal(prepared, torch.ones(1, 12, 4)) + + +def test_unwraps_composite_processor(): + processor = SimpleNamespace(feature_extractor=_FakeFE()) + bridge = _make_bridge(processor) + prepared = _prepare_audio_encoder_input(bridge) + assert prepared.shape == (1, 12, 4) + + +def test_falls_back_to_raw_waveform_without_processor(): + bridge = _make_bridge(None) + prepared = _prepare_audio_encoder_input(bridge) + assert prepared.shape == (1, 16000) + + +def test_existing_waveform_passes_through_feature_extractor(): + bridge = _make_bridge(_FakeFE()) + waveform = torch.randn(2, 8000) + prepared = _prepare_audio_encoder_input(bridge, waveform) + assert prepared.shape == (2, 12, 4) diff --git a/tests/unit/model_bridge/generalized_components/test_mlp_weight_orientation.py b/tests/unit/model_bridge/generalized_components/test_mlp_weight_orientation.py new file mode 100644 index 0000000000..61dcfebdde --- /dev/null +++ b/tests/unit/model_bridge/generalized_components/test_mlp_weight_orientation.py @@ -0,0 +1,79 @@ +"""MLPBridge W_in/W_out/W_gate must resolve to the TL orientation for every wrapper. + +nn.Linear stores [out_features, in_features] (the transpose of the TL +convention) while Conv1D stores [in_features, out_features] (TL as-is) — a raw +``in.weight`` alias silently returned transposed weights for every +nn.Linear-backed model (all of boot_native), in both aspect ratios. +""" + +import pytest +import torch +from transformers.pytorch_utils import Conv1D + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge import TransformerBridge +from transformer_lens.model_bridge.generalized_components.mlp import MLPBridge + + +def _native_bridge(d_model, d_mlp): + cfg = TransformerBridgeConfig( + n_layers=1, + d_model=d_model, + d_head=d_model // 4, + n_heads=4, + d_mlp=d_mlp, + d_vocab=50, + n_ctx=8, + act_fn="gelu", + seed=0, + ) + return TransformerBridge.boot_native(cfg) + + +@pytest.mark.parametrize( + "d_model,d_mlp", + [ + (32, 128), # expansion (the usual shape) + (128, 32), # bottleneck — the aspect ratio a shape heuristic cannot see + ], +) +def test_native_linear_weights_are_tl_oriented(d_model, d_mlp): + bridge = _native_bridge(d_model, d_mlp) + mlp = bridge.blocks[0].mlp + + assert mlp.W_in.shape == (d_model, d_mlp) + assert mlp.W_out.shape == (d_mlp, d_model) + # The stacked bridge-level accessors follow: [n_layers, d_model, d_mlp] etc. + assert bridge.W_in.shape == (1, d_model, d_mlp) + assert bridge.W_out.shape == (1, d_mlp, d_model) + + # Orientation must reflect the actual parameters, not just shapes: the + # nn.Linear weight is [d_mlp, d_model]; the property must be its transpose. + raw = getattr(mlp, "in").weight + assert torch.equal(mlp.W_in, raw.T) + + +def test_conv1d_weight_passes_through(): + """Conv1D (GPT-2 style) already stores [in, out]; no transpose expected.""" + + class _Proj: + def __init__(self, weight, component): + self.weight = weight + self.original_component = component + + mlp = MLPBridge.__new__(MLPBridge) + conv = Conv1D(8, 4) # nf=8 (out), nx=4 (in) -> weight [4, 8] + object.__setattr__(mlp, "_test_proj", _Proj(conv.weight, conv)) + # exercise the resolver directly with a synthetic projection + weight = mlp._tl_oriented_weight("_test_proj") + assert weight.shape == (4, 8) + assert torch.equal(weight, conv.weight) + + +def test_missing_gate_raises_attribute_error(): + """Non-gated MLPs must not appear to have W_gate (hasattr contract).""" + bridge = _native_bridge(32, 128) + mlp = bridge.blocks[0].mlp + assert not hasattr(mlp, "W_gate") + with pytest.raises(AttributeError, match="gate"): + _ = mlp.W_gate diff --git a/tests/unit/model_bridge/sources/test_native_init_semantics.py b/tests/unit/model_bridge/sources/test_native_init_semantics.py new file mode 100644 index 0000000000..69ca1581bf --- /dev/null +++ b/tests/unit/model_bridge/sources/test_native_init_semantics.py @@ -0,0 +1,76 @@ +"""Native init: seeded reproducibility across device/dtype, and initializer_range gain.""" + +import pytest +import torch + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge import TransformerBridge +from transformer_lens.model_bridge.sources.native.init import initialize_native_model + + +def _cfg(**overrides): + base = dict( + n_layers=2, + d_model=64, + d_head=16, + n_heads=4, + d_mlp=128, + d_vocab=100, + n_ctx=16, + act_fn="gelu", + seed=0, + ) + base.update(overrides) + return TransformerBridgeConfig(**base) + + +def test_seeded_reinit_after_dtype_cast_reproduces_boot_weights(): + """Same seed must give the same weights whether init runs before or after + the .to(dtype) cast (boot inits first; init_weights() runs after).""" + cfg = _cfg() + boot_then_cast = TransformerBridge.boot_native(cfg).to(torch.float16) + + recast = TransformerBridge.boot_native(cfg).to(torch.float16) + native = recast.original_model + initialize_native_model(native, cfg) # re-init AFTER the cast + + for (name, a), (_, b) in zip( + boot_then_cast.original_model.named_parameters(), native.named_parameters() + ): + assert torch.equal(a, b), f"{name} diverged between init-before and init-after cast" + + +def test_seeded_init_is_deterministic(): + cfg = _cfg() + a = TransformerBridge.boot_native(cfg) + b = TransformerBridge.boot_native(cfg) + for (name, pa), (_, pb) in zip( + a.original_model.named_parameters(), b.original_model.named_parameters() + ): + assert torch.equal(pa, pb), name + + +@pytest.mark.parametrize("mode", ["xavier_normal", "kaiming_normal"]) +def test_initializer_range_scales_non_gpt2_modes(mode): + """An explicit initializer_range acts as the gain (legacy semantics).""" + plain = TransformerBridge.boot_native(_cfg(init_mode=mode)) + scaled = TransformerBridge.boot_native(_cfg(init_mode=mode, initializer_range=0.5)) + + w_plain = plain.original_model.layers[0].attn.q.weight + w_scaled = scaled.original_model.layers[0].attn.q.weight + ratio = (w_scaled.std() / w_plain.std()).item() + assert ratio == pytest.approx(0.5, rel=0.05), f"gain not applied: ratio {ratio:.4f}" + + +def test_gpt2_mode_default_std_matches_legacy_formula(): + """Unset initializer_range must give N(0, 0.64/d_model) — std 0.8/sqrt(d_model). + + The legacy scheme, not GPT-2's paper 0.02: toy-model training dynamics + (the grokking demo memorizes vs. stalls) depend on this scale. + """ + import math + + cfg = _cfg(d_model=128, d_head=32, d_mlp=512, d_vocab=114) + bridge = TransformerBridge.boot_native(cfg) + std = bridge.original_model.layers[0].attn.q.weight.std().item() + assert std == pytest.approx(0.8 / math.sqrt(128), rel=0.05) diff --git a/tests/unit/model_bridge/supported_architectures/test_rwkv7_adapter.py b/tests/unit/model_bridge/supported_architectures/test_rwkv7_adapter.py index 13844765e5..932d651863 100644 --- a/tests/unit/model_bridge/supported_architectures/test_rwkv7_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_rwkv7_adapter.py @@ -32,6 +32,20 @@ ) +def _subs(component) -> dict: + """component.submodules, asserted non-None for mypy.""" + submodules = component.submodules + assert submodules is not None + return submodules + + +def _mapping(adapter) -> dict: + """adapter.component_mapping, asserted non-None for mypy.""" + mapping = adapter.component_mapping + assert mapping is not None + return mapping + + def _make_cfg( d_model: int = 64, head_dim: int = 64, @@ -132,42 +146,42 @@ def test_head_dim(self, adapter: RWKV7ArchitectureAdapter) -> None: assert adapter.cfg.head_dim == 64 def test_num_heads(self, adapter: RWKV7ArchitectureAdapter) -> None: - assert adapter.cfg.num_heads == 1 + assert getattr(adapter.cfg, "num_heads") == 1 def test_value_dim(self, adapter: RWKV7ArchitectureAdapter) -> None: - assert adapter.cfg.value_dim == [64, 64, 64, 64] + assert getattr(adapter.cfg, "value_dim") == [64, 64, 64, 64] def test_low_rank_dims(self, adapter: RWKV7ArchitectureAdapter) -> None: - assert adapter.cfg.decay_low_rank_dim == 64 - assert adapter.cfg.gate_low_rank_dim == 128 - assert adapter.cfg.a_low_rank_dim == 64 - assert adapter.cfg.v_low_rank_dim == 16 + assert getattr(adapter.cfg, "decay_low_rank_dim") == 64 + assert getattr(adapter.cfg, "gate_low_rank_dim") == 128 + assert getattr(adapter.cfg, "a_low_rank_dim") == 64 + assert getattr(adapter.cfg, "v_low_rank_dim") == 16 def test_norm_flags(self, adapter: RWKV7ArchitectureAdapter) -> None: - assert adapter.cfg.norm_first is True - assert adapter.cfg.norm_bias is True - assert adapter.cfg.fuse_norm is True + assert getattr(adapter.cfg, "norm_first") is True + assert getattr(adapter.cfg, "norm_bias") is True + assert getattr(adapter.cfg, "fuse_norm") is True def test_attn_mode_and_hidden_act(self, adapter: RWKV7ArchitectureAdapter) -> None: - assert adapter.cfg.attn_mode == "chunk" - assert adapter.cfg.hidden_act == "sqrelu" + assert getattr(adapter.cfg, "attn_mode") == "chunk" + assert getattr(adapter.cfg, "hidden_act") == "sqrelu" def test_defaults_when_absent(self) -> None: """Adapter falls back to RWKV-7 defaults if the attrs are missing.""" bare = _make_cfg(inject_shape_attrs=False) a = RWKV7ArchitectureAdapter(bare) - assert a.cfg.head_dim == 64 + assert getattr(a.cfg, "head_dim") == 64 # num_heads defaults to d_model // head_dim. - assert a.cfg.num_heads == 1 + assert getattr(a.cfg, "num_heads") == 1 # value_dim defaults to [d_model] * n_layers. - assert a.cfg.value_dim == [64, 64, 64, 64] - assert a.cfg.decay_low_rank_dim == 64 - assert a.cfg.gate_low_rank_dim == 128 - assert a.cfg.a_low_rank_dim == 64 - assert a.cfg.v_low_rank_dim == 16 - assert a.cfg.norm_first is True - assert a.cfg.fuse_norm is True - assert a.cfg.hidden_act == "sqrelu" + assert getattr(a.cfg, "value_dim") == [64, 64, 64, 64] + assert getattr(a.cfg, "decay_low_rank_dim") == 64 + assert getattr(a.cfg, "gate_low_rank_dim") == 128 + assert getattr(a.cfg, "a_low_rank_dim") == 64 + assert getattr(a.cfg, "v_low_rank_dim") == 16 + assert getattr(a.cfg, "norm_first") is True + assert getattr(a.cfg, "fuse_norm") is True + assert getattr(a.cfg, "hidden_act") == "sqrelu" # --------------------------------------------------------------------------- @@ -179,7 +193,7 @@ class TestRWKV7TopLevelComponents: """component_mapping exposes embed / blocks / ln_final / unembed.""" def test_required_keys(self, adapter: RWKV7ArchitectureAdapter) -> None: - assert set(adapter.component_mapping.keys()) == { + assert set(_mapping(adapter).keys()) == { "embed", "blocks", "ln_final", @@ -187,24 +201,24 @@ def test_required_keys(self, adapter: RWKV7ArchitectureAdapter) -> None: } def test_embed_is_embedding_bridge(self, adapter: RWKV7ArchitectureAdapter) -> None: - assert isinstance(adapter.component_mapping["embed"], EmbeddingBridge) - assert adapter.component_mapping["embed"].name == "model.embeddings" + assert isinstance(_mapping(adapter)["embed"], EmbeddingBridge) + assert _mapping(adapter)["embed"].name == "model.embeddings" def test_blocks_is_ssm_block_bridge(self, adapter: RWKV7ArchitectureAdapter) -> None: # OpaqueBlockBridge delegates the whole recurrent block so the internal # mixing is preserved (a standard BlockBridge assumes a pre-norm attn flow). - blocks = adapter.component_mapping["blocks"] + blocks = _mapping(adapter)["blocks"] assert isinstance(blocks, OpaqueBlockBridge) assert blocks.name == "model.layers" def test_ln_final_is_normalization_bridge(self, adapter: RWKV7ArchitectureAdapter) -> None: - ln_final = adapter.component_mapping["ln_final"] + ln_final = _mapping(adapter)["ln_final"] assert isinstance(ln_final, NormalizationBridge) assert ln_final.name == "model.norm" def test_unembed_is_unembedding_bridge(self, adapter: RWKV7ArchitectureAdapter) -> None: - assert isinstance(adapter.component_mapping["unembed"], UnembeddingBridge) - assert adapter.component_mapping["unembed"].name == "lm_head" + assert isinstance(_mapping(adapter)["unembed"], UnembeddingBridge) + assert _mapping(adapter)["unembed"].name == "lm_head" # --------------------------------------------------------------------------- @@ -216,11 +230,11 @@ class TestRWKV7BlockSubmodules: """Each block wraps attn_norm / attn / ffn_norm / ffn.""" def test_submodule_keys(self, adapter: RWKV7ArchitectureAdapter) -> None: - block = adapter.component_mapping["blocks"] - assert set(block.submodules.keys()) == {"attn_norm", "attn", "ffn_norm", "ffn"} + block = _mapping(adapter)["blocks"] + assert set(_subs(block).keys()) == {"attn_norm", "attn", "ffn_norm", "ffn"} def test_attn_norm_is_normalization_bridge(self, adapter: RWKV7ArchitectureAdapter) -> None: - attn_norm = adapter.component_mapping["blocks"].submodules["attn_norm"] + attn_norm = _subs(_mapping(adapter)["blocks"])["attn_norm"] assert isinstance(attn_norm, NormalizationBridge) assert attn_norm.name == "attn_norm" @@ -228,7 +242,7 @@ def test_ffn_norm_is_delegating_passthrough(self, adapter: RWKV7ArchitectureAdap # Under config.fuse_norm the block calls ffn_norm(x, residual, True) -> # (normed, residual); the reimplementing NormalizationBridge can't express # that, so it is a plain delegating GeneralizedComponent, not a norm bridge. - ffn_norm = adapter.component_mapping["blocks"].submodules["ffn_norm"] + ffn_norm = _subs(_mapping(adapter)["blocks"])["ffn_norm"] assert type(ffn_norm) is GeneralizedComponent assert not isinstance(ffn_norm, NormalizationBridge) assert ffn_norm.name == "ffn_norm" @@ -236,26 +250,26 @@ def test_ffn_norm_is_delegating_passthrough(self, adapter: RWKV7ArchitectureAdap def test_attn_is_generalized_component_with_projections( self, adapter: RWKV7ArchitectureAdapter ) -> None: - attn = adapter.component_mapping["blocks"].submodules["attn"] + attn = _subs(_mapping(adapter)["blocks"])["attn"] assert isinstance(attn, GeneralizedComponent) assert attn.name == "attn" - assert set(attn.submodules.keys()) == {"r_proj", "k_proj", "v_proj", "o_proj"} + assert set(_subs(attn).keys()) == {"r_proj", "k_proj", "v_proj", "o_proj"} for proj in ("r_proj", "k_proj", "v_proj", "o_proj"): - assert isinstance(attn.submodules[proj], LinearBridge) - assert attn.submodules[proj].name == proj + assert isinstance(_subs(attn)[proj], LinearBridge) + assert _subs(attn)[proj].name == proj def test_ffn_is_generalized_component_with_projections( self, adapter: RWKV7ArchitectureAdapter ) -> None: - ffn = adapter.component_mapping["blocks"].submodules["ffn"] + ffn = _subs(_mapping(adapter)["blocks"])["ffn"] assert isinstance(ffn, GeneralizedComponent) assert ffn.name == "ffn" # HF names the up-projection "key" and the (down) output projection "value". - assert set(ffn.submodules.keys()) == {"key", "value"} - assert isinstance(ffn.submodules["key"], LinearBridge) - assert ffn.submodules["key"].name == "key" - assert isinstance(ffn.submodules["value"], LinearBridge) - assert ffn.submodules["value"].name == "value" + assert set(_subs(ffn).keys()) == {"key", "value"} + assert isinstance(_subs(ffn)["key"], LinearBridge) + assert _subs(ffn)["key"].name == "key" + assert isinstance(_subs(ffn)["value"], LinearBridge) + assert _subs(ffn)["value"].name == "value" # --------------------------------------------------------------------------- diff --git a/tests/unit/model_bridge/test_bridge_weight_properties.py b/tests/unit/model_bridge/test_bridge_weight_properties.py index 34cafddbbe..c81c2ca550 100644 --- a/tests/unit/model_bridge/test_bridge_weight_properties.py +++ b/tests/unit/model_bridge/test_bridge_weight_properties.py @@ -1,10 +1,12 @@ """Tests for TransformerBridge._stack_block_params() and weight properties.""" +import pytest import torch from transformer_lens.model_bridge.generalized_components.attention import ( AttentionBridge, ) +from transformer_lens.model_bridge.transformer_bridge import TransformerBridge class TestReshapeBias: @@ -48,3 +50,39 @@ def test_already_2d_bias_returned_as_is(self): result = bridge._reshape_bias(bias) # ndim != 1, so no reshape — returns as-is assert result is bias + + +class TestStackBlockParams: + """Tests for TransformerBridge._stack_block_params() via a duck-typed self.""" + + class _Attn(torch.nn.Module): + def __init__(self, bias): + super().__init__() + self.b_Q = bias + + class _Block(torch.nn.Module): + def __init__(self, bias): + super().__init__() + self.attn = TestStackBlockParams._Attn(bias) + + class _FakeBridge: + def __init__(self, biases): + self.blocks = [TestStackBlockParams._Block(b) for b in biases] + + class Cfg: + n_devices = 1 + device = None + + self.cfg = Cfg() + + def test_stacks_present_params(self): + fake = self._FakeBridge([torch.ones(2, 3), torch.zeros(2, 3)]) + stacked = TransformerBridge._stack_block_params(fake, "attn.b_Q") + assert stacked.shape == (2, 2, 3) + assert torch.equal(stacked[0], torch.ones(2, 3)) + + def test_none_param_raises_actionable_error(self): + """Bias-free checkpoints must get a clear error, not a raw stack TypeError.""" + fake = self._FakeBridge([torch.ones(2, 3), None]) + with pytest.raises(AttributeError, match=r"blocks\[1\]\.attn\.b_Q.*bias-free"): + TransformerBridge._stack_block_params(fake, "attn.b_Q") diff --git a/tests/unit/model_bridge/test_get_params_util.py b/tests/unit/model_bridge/test_get_params_util.py index ad16eabaf4..0f1ffab2b9 100644 --- a/tests/unit/model_bridge/test_get_params_util.py +++ b/tests/unit/model_bridge/test_get_params_util.py @@ -254,3 +254,71 @@ def _create_mock_block(self): block.mlp.b_out = torch.randn(768) return block + + +class TestGQAExpansion: + """Grouped K/V must be expanded to n_heads (legacy HT convention).""" + + def _make_gqa_bridge(self): + mock_bridge = Mock() + mock_bridge.cfg = Mock() + mock_bridge.cfg.n_layers = 1 + mock_bridge.cfg.d_model = 64 + mock_bridge.cfg.n_heads = 4 + mock_bridge.cfg.d_head = 16 + mock_bridge.cfg.d_vocab = 100 + mock_bridge.cfg.n_ctx = 32 + mock_bridge.cfg.d_mlp = 128 + mock_bridge.cfg.device = torch.device("cpu") + + mock_bridge.embed = Mock() + mock_bridge.embed.weight = torch.randn(100, 64) + mock_bridge.pos_embed = Mock() + mock_bridge.pos_embed.weight = torch.randn(32, 64) + mock_bridge.unembed = Mock() + mock_bridge.unembed.weight = torch.randn(100, 64) + + block = Mock() + block.attn = Mock() + block.attn.W_Q = torch.randn(4, 64, 16) + block.attn.W_K = torch.randn(2, 64, 16) # grouped: n_kv_heads=2 + block.attn.W_V = torch.randn(2, 64, 16) + block.attn.W_O = torch.randn(4, 16, 64) + block.attn.b_Q = torch.randn(4, 16) + block.attn.b_K = torch.randn(2, 16) + block.attn.b_V = torch.randn(2, 16) + block.attn.b_O = torch.randn(64) + block.mlp = Mock() + block.mlp.W_in = torch.randn(64, 128) + block.mlp.W_out = torch.randn(128, 64) + block.mlp.b_in = torch.randn(128) + block.mlp.b_out = torch.randn(64) + mock_bridge.blocks = [block] + return mock_bridge + + def test_grouped_kv_expanded_to_n_heads(self): + bridge = self._make_gqa_bridge() + params = get_bridge_params(bridge) + + w_k = params["blocks.0.attn.W_K"] + w_v = params["blocks.0.attn.W_V"] + assert w_k.shape == (4, 64, 16) + assert w_v.shape == (4, 64, 16) + # repeat_interleave semantics: heads 0,1 share kv head 0; heads 2,3 share kv head 1 + assert torch.equal(w_k[0], w_k[1]) + assert torch.equal(w_k[0], bridge.blocks[0].attn.W_K[0]) + assert torch.equal(w_k[2], bridge.blocks[0].attn.W_K[1]) + + def test_grouped_biases_expanded(self): + bridge = self._make_gqa_bridge() + params = get_bridge_params(bridge) + b_k = params["blocks.0.attn.b_K"] + assert b_k.shape == (4, 16) + assert params["blocks.0.attn.b_V"].shape == (4, 16) + # Pairing must be repeat_interleave (blocked), not tiling: heads 0,1 + # share kv head 0 and heads 2,3 share kv head 1. + grouped = bridge.blocks[0].attn.b_K + assert torch.equal(b_k[0], b_k[1]) + assert torch.equal(b_k[0], grouped[0]) + assert torch.equal(b_k[2], grouped[1]) + assert torch.equal(params["blocks.0.attn.b_Q"], bridge.blocks[0].attn.b_Q) diff --git a/tests/unit/model_bridge/test_hook_alias_fallback_chains.py b/tests/unit/model_bridge/test_hook_alias_fallback_chains.py new file mode 100644 index 0000000000..700f7ae1d0 --- /dev/null +++ b/tests/unit/model_bridge/test_hook_alias_fallback_chains.py @@ -0,0 +1,53 @@ +"""Fallback-chain (list-valued) hook alias targets resolve by priority.""" + +from transformer_lens.model_bridge.bridge_core import BridgeCore + + +class _FakeComponent: + def __init__(self, hook_aliases=None, submodules=None): + self.hook_aliases = hook_aliases or {} + self.submodules = submodules or {} + + +def _resolve(mapping, hook_names): + core = BridgeCore.__new__(BridgeCore) + component_aliases = core._collect_component_aliases(mapping) + return dict( + BridgeCore._compute_hook_aliases_cached( + tuple(sorted(hook_names)), tuple(sorted(component_aliases.items())) + ) + ) + + +def test_string_targets_unchanged(): + mapping = {"blocks": _FakeComponent({"hook_out_alias": "mlp.hook_out"})} + aliases = _resolve(mapping, ["blocks.0.mlp.hook_out"]) + assert aliases == {"blocks.0.hook_out_alias": "blocks.0.mlp.hook_out"} + + +def test_list_target_prefers_first_live_candidate(): + mapping = {"blocks": _FakeComponent({"hook_mlp_out": ["ln2_post.hook_out", "mlp.hook_out"]})} + # Both candidates exist (post-norm layer): the first must win. + aliases = _resolve(mapping, ["blocks.0.ln2_post.hook_out", "blocks.0.mlp.hook_out"]) + assert aliases["blocks.0.hook_mlp_out"] == "blocks.0.ln2_post.hook_out" + + +def test_list_target_falls_back_when_first_absent(): + mapping = {"blocks": _FakeComponent({"hook_mlp_out": ["ln2_post.hook_out", "mlp.hook_out"]})} + # Linear-attention layer: no ln2_post hook — the fallback resolves. + aliases = _resolve(mapping, ["blocks.1.mlp.hook_out"]) + assert aliases["blocks.1.hook_mlp_out"] == "blocks.1.mlp.hook_out" + + +def test_per_block_priority_is_independent(): + mapping = {"blocks": _FakeComponent({"hook_mlp_out": ["ln2_post.hook_out", "mlp.hook_out"]})} + aliases = _resolve( + mapping, + [ + "blocks.0.ln2_post.hook_out", + "blocks.0.mlp.hook_out", + "blocks.1.mlp.hook_out", + ], + ) + assert aliases["blocks.0.hook_mlp_out"] == "blocks.0.ln2_post.hook_out" + assert aliases["blocks.1.hook_mlp_out"] == "blocks.1.mlp.hook_out" diff --git a/tests/unit/model_bridge/test_prepare_loading_orchestration.py b/tests/unit/model_bridge/test_prepare_loading_orchestration.py new file mode 100644 index 0000000000..1aaea31ff0 --- /dev/null +++ b/tests/unit/model_bridge/test_prepare_loading_orchestration.py @@ -0,0 +1,111 @@ +"""Adapter prepare_loading orchestration on fake remote modules (raven, rwkv7). + +The shared patch mechanics are unit-tested in test_remote_code_compat.py; these +tests cover each adapter's orchestration — that a modeling module planted in +sys.modules gets both v5 patches applied (tied-weights dict rewrite, guarded +_init_weights) and that the flow is idempotent. Real trust_remote_code loads +are CI-invisible, so this is the only coverage those load-bearing patches get. +""" + +import importlib +import sys +import types + +import pytest +import torch + + +def _make_fake_modeling_module(module_name, causal_name, pretrained_name, tied_to): + module = types.ModuleType(module_name) + + class FakePreTrainedModel: + def _init_weights(self, mod): + mod.initialized = True + + class FakeCausalLM(FakePreTrainedModel): + _tied_weights_keys = ["lm_head.weight"] + + FakePreTrainedModel.__name__ = pretrained_name + FakeCausalLM.__name__ = causal_name + setattr(module, causal_name, FakeCausalLM) + setattr(module, pretrained_name, FakePreTrainedModel) + module._expected_tied_mapping = {"lm_head.weight": tied_to} + return module + + +@pytest.fixture +def planted_module(request, monkeypatch): + """Plant a fake remote modeling module for the requested arch fragment.""" + fragment, causal, pretrained, tied_to = request.param + name = f"transformers_modules.fake.modeling_{fragment}" + module = _make_fake_modeling_module(name, causal, pretrained, tied_to) + monkeypatch.setitem(sys.modules, name, module) + return module + + +_CASES = [ + pytest.param( + ("raven", "RavenForCausalLM", "RavenPreTrainedModel", "transformer.wte.weight"), + id="raven", + ), + pytest.param( + ("rwkv7", "RWKV7ForCausalLM", "RWKV7PreTrainedModel", "model.embeddings.weight"), + id="rwkv7", + ), +] + + +def _adapter_for(fragment): + from transformer_lens.config import TransformerBridgeConfig + + cfg = TransformerBridgeConfig( + n_layers=1, d_model=8, d_head=2, n_heads=4, d_vocab=16, n_ctx=8, act_fn="gelu" + ) + if fragment == "raven": + from transformer_lens.model_bridge.supported_architectures.raven import ( + RavenArchitectureAdapter, + ) + + cfg.architecture = "RavenForCausalLM" + return RavenArchitectureAdapter(cfg) + from transformer_lens.model_bridge.supported_architectures.rwkv7 import ( + RWKV7ArchitectureAdapter, + ) + + cfg.architecture = "RWKV7ForCausalLM" + return RWKV7ArchitectureAdapter(cfg) + + +@pytest.mark.parametrize("planted_module", _CASES, indirect=True) +def test_prepare_loading_applies_both_v5_patches(planted_module, monkeypatch): + fragment = planted_module.__name__.rsplit("modeling_", 1)[-1] + causal = [n for n in vars(planted_module) if n.endswith("ForCausalLM")][0] + pretrained = [n for n in vars(planted_module) if n.endswith("PreTrainedModel")][0] + + # Force the fallback path deterministically: block a real `fla` install from + # short-circuiting the rwkv7 case (and patching genuine fla classes), and + # stub the Hub-dependent force-import (the module is already planted). + monkeypatch.setitem(sys.modules, "fla", None) + adapter_mod = importlib.import_module( + f"transformer_lens.model_bridge.supported_architectures.{fragment}" + ) + monkeypatch.setattr(adapter_mod, "force_import_remote_class", lambda *a, **k: object) + + adapter = _adapter_for(fragment) + adapter.prepare_loading("fake/model", {}) + + causal_cls = getattr(planted_module, causal) + pretrained_cls = getattr(planted_module, pretrained) + + # Patch 1: list-form tied weights rewritten to the v5 dict form. + assert causal_cls._tied_weights_keys == planted_module._expected_tied_mapping + + # Patch 2: _init_weights guarded — materialized modules are not re-initialized. + assert getattr(pretrained_cls, "_tl_patched", False) + loaded = torch.nn.Linear(2, 2) # real (non-meta) params = "loaded from checkpoint" + pretrained_cls._init_weights(pretrained_cls.__new__(pretrained_cls), loaded) + assert not getattr(loaded, "initialized", False) + + # Idempotent: a second run must not double-wrap or crash. + adapter.prepare_loading("fake/model", {}) + assert causal_cls._tied_weights_keys == planted_module._expected_tied_mapping diff --git a/tests/unit/test_doctest_no_hooked_transformer.py b/tests/unit/test_doctest_no_hooked_transformer.py new file mode 100644 index 0000000000..6cf200f816 --- /dev/null +++ b/tests/unit/test_doctest_no_hooked_transformer.py @@ -0,0 +1,64 @@ +"""Guard: surviving modules' doctests must not construct HookedTransformer. + +The v4 removal deletes the Hooked* implementation files; any OTHER module whose +docstring examples build a HookedTransformer would turn the docstring test tier +red the moment those files go. This guard makes the removal safe by +construction: it fails the instant such an example is (re)introduced. + +The Hooked* implementation files themselves (and the legacy train module) are +exempt — their docstrings legitimately describe the class they implement, and +they are deleted wholesale by the removal PR. +""" + +import re +from pathlib import Path + +PACKAGE_ROOT = Path(__file__).parents[2] / "transformer_lens" + +# Deleted wholesale by the Hooked* removal — their own docstrings are exempt. +EXEMPT = { + "HookedTransformer.py", + "HookedRootModule.py", + "HookedEncoder.py", + "HookedEncoderDecoder.py", + "HookedAudioEncoder.py", + "train.py", +} +EXEMPT_DIRS = {"components", "factories"} + +# A doctest example line (>>> or ... continuation) that references the class at +# all — construction, import, or isinstance: any of them turns the docstring +# tier red once the class is gone. The lookahead excludes surviving classes +# whose names extend it (HookedTransformerConfig, ...KeyValueCache). +_CONSTRUCTION = re.compile(r"^\s*(?:>>>|\.\.\.).*HookedTransformer(?![A-Za-z_])") + + +def _doctest_sources(path: Path): + """Yield (lineno, line) for every doctest example line in the file. + + Grep-level by design: example lines are the only place `>>> ` appears, so a + line scan finds them without importing the module or parsing docstrings + (DocTestParser rejects pseudo-examples embedded in comments). + """ + for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + stripped = line.lstrip() + if stripped.startswith(">>>") or stripped.startswith("..."): + yield lineno, line + + +def test_no_surviving_doctest_constructs_hooked_transformer(): + offenders = [] + for path in sorted(PACKAGE_ROOT.rglob("*.py")): + relative = path.relative_to(PACKAGE_ROOT) + if relative.name in EXEMPT and len(relative.parts) == 1: + continue + if relative.parts[0] in EXEMPT_DIRS: + continue + for lineno, source in _doctest_sources(path): + if _CONSTRUCTION.search(source): + offenders.append(f"{relative}:{lineno}: {source.strip()}") + + assert not offenders, ( + "Doctest examples in surviving modules reference HookedTransformer; " + "port them to TransformerBridge before the v4 removal:\n " + "\n ".join(offenders) + ) diff --git a/tests/unit/tools/test_training_config.py b/tests/unit/tools/test_training_config.py new file mode 100644 index 0000000000..1f9ecdcf78 --- /dev/null +++ b/tests/unit/tools/test_training_config.py @@ -0,0 +1,41 @@ +"""train() must not mutate the caller's config (device/wandb defaults).""" + +import torch +from torch import nn +from torch.utils.data import Dataset + +from transformer_lens.tools.training import TrainConfig, train + + +class _OneSample(Dataset): + def __len__(self): + return 1 + + def __getitem__(self, idx): + return {"tokens": torch.zeros(4, dtype=torch.long)} + + +class _TinyLM(nn.Module): + """Minimal model exposing the train() contract: __call__(tokens, return_type="loss").""" + + def __init__(self) -> None: + super().__init__() + self.embed = nn.Embedding(8, 4) + self.head = nn.Linear(4, 8) + + def forward(self, tokens, return_type="loss"): + logits = self.head(self.embed(tokens)) + loss = nn.functional.cross_entropy(logits[:, :-1].reshape(-1, 8), tokens[:, 1:].reshape(-1)) + return loss + + +def test_train_does_not_mutate_callers_config(): + """train() defaults device/wandb settings on an internal copy; the caller's + object must come back exactly as it went in.""" + cfg = TrainConfig(num_epochs=1, batch_size=1, wandb=False) + assert cfg.device is None + + train(_TinyLM(), cfg, _OneSample()) + + assert cfg.device is None, "train() leaked its device default into the caller's config" + assert cfg.wandb_project_name is None diff --git a/transformer_lens/ActivationCache.py b/transformer_lens/ActivationCache.py index 58f8355107..41cb22d6ec 100644 --- a/transformer_lens/ActivationCache.py +++ b/transformer_lens/ActivationCache.py @@ -445,10 +445,11 @@ def accumulated_resid( >>> print(layers_logits.shape) torch.Size([9, 50257]) - >>> # If you want to apply the unembedding bias, add b_U when present: - >>> # b_U = getattr(model, "b_U", None) - >>> # layers_logits = layers_logits + b_U if b_U is not None else layers_logits - >>> # print(layers_logits.shape) + >>> # The unembedding bias can be added on top when the model carries one + >>> # (the rank table below stays on the bias-free logits): + >>> b_U = getattr(model, "b_U", None) + >>> with_bias = layers_logits + b_U if b_U is not None else layers_logits + >>> print(with_bias.shape) torch.Size([9, 50257]) >>> # Get the rank of the correct answer by layer diff --git a/transformer_lens/__init__.py b/transformer_lens/__init__.py index 49fad4cead..d8cdc447e3 100644 --- a/transformer_lens/__init__.py +++ b/transformer_lens/__init__.py @@ -33,6 +33,21 @@ from .SVDInterpreter import SVDInterpreter + +def __getattr__(name: str): + # Lazy: model_bridge is import-heavy and importing it eagerly here would + # risk cycles with modules the bridge itself imports. + if name == "TransformerBridge": + from .model_bridge import TransformerBridge + + return TransformerBridge + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__(): + return sorted(set(globals()) | {"TransformerBridge"}) + + import os as _os # noqa: E402 if _os.environ.get("TRANSFORMERLENS_HF_RETRY") == "1": @@ -42,6 +57,7 @@ __all__ = [ "HookedTransformerConfig", + "TransformerBridge", "TransformerBridgeConfig", "FactoredMatrix", "ActivationCache", diff --git a/transformer_lens/benchmarks/audio.py b/transformer_lens/benchmarks/audio.py index d216d6f54c..c2cfab0426 100644 --- a/transformer_lens/benchmarks/audio.py +++ b/transformer_lens/benchmarks/audio.py @@ -5,7 +5,7 @@ stable representations. """ -from typing import List, Optional +from typing import Any, List, Optional import torch @@ -23,6 +23,37 @@ _AUDIO_PARITY_RTOL = 3e-2 +def _prepare_audio_encoder_input( + bridge: Any, test_audio: Optional[torch.Tensor] = None +) -> torch.Tensor: + """Model-ready audio input via the bridge's feature extractor when available. + + Non-wav2vec2-style architectures (e.g. AST) consume feature-extractor + outputs (spectrograms), not raw waveforms, and declare their own sampling + rate — so input prep must go through ``bridge.processor`` whenever the + boot attached one. Falls back to a raw 16 kHz waveform otherwise. + """ + processor = getattr(bridge, "processor", None) + fe = getattr(processor, "feature_extractor", processor) + sampling_rate = int(getattr(fe, "sampling_rate", 16000) or 16000) + + device = bridge.cfg.device + dtype = bridge.cfg.dtype + if test_audio is None: + test_audio = torch.randn(1, sampling_rate, device=device, dtype=dtype) + + if fe is not None and callable(fe): + try: + waveforms = [w for w in test_audio.detach().cpu().float().numpy()] + out = fe(waveforms, sampling_rate=sampling_rate, return_tensors="pt") + prepared = out.get("input_values", out.get("input_features")) + if prepared is not None: + return prepared.to(device=device, dtype=dtype) + except Exception: + pass # fall through to the raw waveform + return test_audio + + def _prepare_audio_text_inputs(bridge: TransformerBridge): """Build audio-conditioned inputs (synthetic waveform + audio token) for an audio-text decoder; ``(None, None)`` if the processor has no audio path.""" @@ -172,7 +203,7 @@ def benchmark_audio_forward( # Compare against HF reference if available if reference_model is not None: with torch.no_grad(): - ref_output_raw = reference_model(input_values=test_audio) + ref_output_raw = reference_model(test_audio) if output_key == "logits": ref_output = ref_output_raw.logits else: @@ -227,16 +258,18 @@ def benchmark_audio_cache( passed=False, ) - # Check for critical audio-specific hooks - critical_hooks = [ + # Critical hooks: first/last block are universal; the named audio + # front-end hooks are wav2vec2/HuBERT-shaped, so require them only when + # the architecture actually exposes them in its hook registry. + n_layers = bridge.cfg.n_layers + critical_hooks = ["blocks.0.hook_out", f"blocks.{n_layers - 1}.hook_out"] + front_end_hooks = [ "audio_feature_extractor.hook_out", "conv_pos_embed.hook_out", "embed_ln.hook_out", ] - # Also check at least the first and last block - n_layers = bridge.cfg.n_layers - critical_hooks.append("blocks.0.hook_out") - critical_hooks.append(f"blocks.{n_layers - 1}.hook_out") + hook_registry = getattr(bridge, "hook_dict", {}) + critical_hooks.extend(h for h in front_end_hooks if h in hook_registry) missing = [h for h in critical_hooks if h not in cache_keys] found = len(critical_hooks) - len(missing) @@ -253,7 +286,9 @@ def benchmark_audio_cache( name="audio_cache", severity=BenchmarkSeverity.WARNING, message=f"Missing {len(missing)} critical hooks: {missing[:3]}", - passed=found >= 3, # Pass if at least 3 of 5 critical hooks present + passed=not any( + h.startswith("blocks.") for h in missing + ), # block hooks are mandatory details={ "total_cached": len(cache_keys), "critical_found": found, @@ -547,10 +582,7 @@ def run_audio_benchmarks( Returns: List of BenchmarkResult objects """ - if test_audio is None: - device = bridge.cfg.device - dtype = bridge.cfg.dtype - test_audio = torch.randn(1, 16000, device=device, dtype=dtype) + test_audio = _prepare_audio_encoder_input(bridge, test_audio) results = [] diff --git a/transformer_lens/benchmarks/forward_pass.py b/transformer_lens/benchmarks/forward_pass.py index fb6929e2b8..c066b810f8 100644 --- a/transformer_lens/benchmarks/forward_pass.py +++ b/transformer_lens/benchmarks/forward_pass.py @@ -134,10 +134,11 @@ def benchmark_forward_pass( if reference_logits is not None: reference_output = reference_logits.to(bridge_output.device) elif _is_audio and isinstance(test_input, torch.Tensor): - # Audio HF reference model: pass waveform directly + # Audio HF reference model: pass the prepared audio input positionally + # (input_values for wav2vec2-style, input_features for AST-style) assert reference_model is not None with torch.no_grad(): - hf_output = reference_model(input_values=test_input) + hf_output = reference_model(test_input) if hasattr(hf_output, "logits") and hf_output.logits is not None: reference_output = hf_output.logits else: diff --git a/transformer_lens/benchmarks/main_benchmark.py b/transformer_lens/benchmarks/main_benchmark.py index 09627a4cc0..c2f73c1581 100644 --- a/transformer_lens/benchmarks/main_benchmark.py +++ b/transformer_lens/benchmarks/main_benchmark.py @@ -889,7 +889,12 @@ def cleanup_model(model, model_name_str: str): bridge_unprocessed.cfg, "is_audio_model", False ) # Shared waveform for audio model benchmarks (consistent across HF capture and bridge forward) - _test_audio = torch.randn(1, 16000, device=device, dtype=dtype) if _is_audio else None + if _is_audio: + from transformer_lens.benchmarks.audio import _prepare_audio_encoder_input + + _test_audio = _prepare_audio_encoder_input(bridge_unprocessed) + else: + _test_audio = None # Run Phase 1 benchmarks if should_run_phase(1) and bridge_unprocessed: @@ -924,7 +929,7 @@ def cleanup_model(model, model_name_str: str): if _is_audio: # Audio models: use the shared waveform for HF vs bridge comparison with torch.no_grad(): - hf_out = hf_model(input_values=_test_audio) + hf_out = hf_model(_test_audio) # Audio encoders output last_hidden_state, not logits if hasattr(hf_out, "logits") and hf_out.logits is not None: hf_saved_logits = hf_out.logits.detach().cpu().clone() @@ -1321,7 +1326,8 @@ def cleanup_model(model, model_name_str: str): try: from transformer_lens.benchmarks.audio import run_audio_benchmarks - test_audio = torch.randn(1, 16000, device=device, dtype=dtype) + # None → run_audio_benchmarks derives feature-extractor-driven input + test_audio = None audio_results = run_audio_benchmarks( bridge_unprocessed, test_audio=test_audio, diff --git a/transformer_lens/model_bridge/bridge_core.py b/transformer_lens/model_bridge/bridge_core.py index a7119b1709..5a712d24d7 100644 --- a/transformer_lens/model_bridge/bridge_core.py +++ b/transformer_lens/model_bridge/bridge_core.py @@ -222,7 +222,11 @@ def _collect_component_aliases(self, component_mapping: Any, prefix: str = "") - if hasattr(component_mapping, "hook_aliases") and component_mapping.hook_aliases: for alias_name, target in component_mapping.hook_aliases.items(): full_alias = f"{prefix}.{alias_name}" if prefix else alias_name - full_target = f"{prefix}.{target}" if prefix else target + if isinstance(target, list): + # Fallback chain: candidates in declared priority order. + full_target: Any = tuple(f"{prefix}.{t}" if prefix else t for t in target) + else: + full_target = f"{prefix}.{target}" if prefix else target aliases[full_alias] = full_target if hasattr(component_mapping, "submodules") and component_mapping.submodules: for sub_name, sub_component in component_mapping.submodules.items(): @@ -234,16 +238,24 @@ def _collect_component_aliases(self, component_mapping: Any, prefix: str = "") - @lru_cache(maxsize=128) def _compute_hook_aliases_cached( hook_names_tuple: Tuple[str, ...], - component_aliases_tuple: Tuple[Tuple[str, str], ...], + component_aliases_tuple: Tuple[Tuple[str, Union[str, Tuple[str, ...]]], ...], ) -> Tuple[Tuple[str, str], ...]: """Cached computation of hook aliases.""" aliases: dict = {} + # For list-valued (fallback-chain) targets, remember which candidate + # resolved each alias so an earlier (higher-priority) candidate wins. + alias_priority: dict = {} component_aliases = dict(component_aliases_tuple) for hook_name in hook_names_tuple: - for alias_pattern, target_pattern in component_aliases.items(): - if "blocks." in target_pattern and "blocks." in hook_name: - block_match = _BLOCK_PATTERN.search(hook_name) - if block_match: + for alias_pattern, target_patterns in component_aliases.items(): + candidates = ( + target_patterns if isinstance(target_patterns, tuple) else (target_patterns,) + ) + for priority, target_pattern in enumerate(candidates): + if "blocks." in target_pattern and "blocks." in hook_name: + block_match = _BLOCK_PATTERN.search(hook_name) + if not block_match: + continue block_num = block_match.group(1) dynamic_alias_pattern = alias_pattern.replace( "blocks.", f"blocks.{block_num}." @@ -254,11 +266,15 @@ def _compute_hook_aliases_cached( if hook_name.endswith(dynamic_target_pattern): target_len = len(dynamic_target_pattern) alias_name = hook_name[:-target_len] + dynamic_alias_pattern + if alias_priority.get(alias_name, len(candidates)) > priority: + aliases[alias_name] = hook_name + alias_priority[alias_name] = priority + elif hook_name.endswith(target_pattern): + target_len = len(target_pattern) + alias_name = hook_name[:-target_len] + alias_pattern + if alias_priority.get(alias_name, len(candidates)) > priority: aliases[alias_name] = hook_name - elif hook_name.endswith(target_pattern): - target_len = len(target_pattern) - alias_name = hook_name[:-target_len] + alias_pattern - aliases[alias_name] = hook_name + alias_priority[alias_name] = priority return tuple(aliases.items()) def _collect_hook_aliases_from_registry(self) -> dict: diff --git a/transformer_lens/model_bridge/generalized_components/base.py b/transformer_lens/model_bridge/generalized_components/base.py index 316a54393d..76612acd75 100644 --- a/transformer_lens/model_bridge/generalized_components/base.py +++ b/transformer_lens/model_bridge/generalized_components/base.py @@ -96,10 +96,11 @@ def _register_hook(self, name: str, hook: HookPoint) -> None: def _register_aliases(self) -> None: """Register aliases from class-level dictionaries. - This is called ONLY in enable_compatibility_mode() after weight processing. + Called unconditionally at bridge init (see bridge.py); compatibility mode + additionally re-registers after weight processing. It creates actual Python attributes/properties that directly reference the target objects. - Note: This should only be called when compatibility mode is enabled and after + Note: Re-registration expects to run after weight processing is complete to ensure property aliases point to processed weights. """ if self.hook_aliases: diff --git a/transformer_lens/model_bridge/generalized_components/mlp.py b/transformer_lens/model_bridge/generalized_components/mlp.py index afe72332af..4f8013dbbc 100644 --- a/transformer_lens/model_bridge/generalized_components/mlp.py +++ b/transformer_lens/model_bridge/generalized_components/mlp.py @@ -4,6 +4,9 @@ """ from typing import Any, Dict, Optional +import torch +from transformers.pytorch_utils import Conv1D + from transformer_lens.model_bridge.generalized_components.base import ( GeneralizedComponent, ) @@ -17,15 +20,50 @@ class MLPBridge(GeneralizedComponent): """ hook_aliases = {"hook_pre": "in.hook_out", "hook_post": "out.hook_in"} + # W_* are real properties below (layout-aware); only the 1-D biases are + # orientation-free enough for raw passthrough aliases. property_aliases = { - "W_gate": "gate.weight", "b_gate": "gate.bias", - "W_in": "in.weight", "b_in": "in.bias", - "W_out": "out.weight", "b_out": "out.bias", } + def _tl_oriented_weight(self, proj_name: str) -> Any: + """Wrapped projection weight in the TL orientation ([d_model, d_mlp] for + the input side; [d_mlp, d_model] for the output side). + + nn.Linear stores [out_features, in_features] (transpose of TL); Conv1D + (GPT-2 style) stores [in_features, out_features] (TL as-is). Unknown + wrapper types pass the raw weight through unchanged. + """ + proj = getattr(self, proj_name, None) + if proj is None: + raise AttributeError(f"{type(self).__name__} has no '{proj_name}' projection") + weight = proj.weight + if weight.ndim != 2: + return weight + component = getattr(proj, "original_component", None) + if isinstance(component, Conv1D): + return weight + if isinstance(component, torch.nn.Linear): + return weight.T + return weight + + @property + def W_in(self) -> Any: + """W_in in TL orientation [d_model, d_mlp].""" + return self._tl_oriented_weight("in") + + @property + def W_out(self) -> Any: + """W_out in TL orientation [d_mlp, d_model].""" + return self._tl_oriented_weight("out") + + @property + def W_gate(self) -> Any: + """W_gate in TL orientation [d_model, d_mlp].""" + return self._tl_oriented_weight("gate") + def __init__( self, name: Optional[str], diff --git a/transformer_lens/model_bridge/get_params_util.py b/transformer_lens/model_bridge/get_params_util.py index fbfc2625c1..cd680cba6c 100644 --- a/transformer_lens/model_bridge/get_params_util.py +++ b/transformer_lens/model_bridge/get_params_util.py @@ -86,20 +86,31 @@ def _zeros(*shape) -> torch.Tensor: layer_idx, ) else: + # GQA: expand grouped K/V (and their biases below) to n_heads so + # per-head pairings like SVDInterpreter's OV = W_V[h] @ W_O[h] + # line up — the legacy HT convention repeat_interleaved these. + n_kv_heads = w_k.shape[0] + if w_k.ndim == 3 and 0 < n_kv_heads < cfg.n_heads: + if cfg.n_heads % n_kv_heads != 0: + raise ValueError( + f"blocks.{layer_idx}.attn: n_heads ({cfg.n_heads}) is not " + f"divisible by n_kv_heads ({n_kv_heads}); cannot expand " + "grouped K/V to per-query heads." + ) + repeats = cfg.n_heads // n_kv_heads + w_k = torch.repeat_interleave(w_k, repeats, dim=0) + w_v = torch.repeat_interleave(w_v, repeats, dim=0) params_dict[f"blocks.{layer_idx}.attn.W_Q"] = w_q params_dict[f"blocks.{layer_idx}.attn.W_K"] = w_k params_dict[f"blocks.{layer_idx}.attn.W_V"] = w_v params_dict[f"blocks.{layer_idx}.attn.W_O"] = w_o - n_kv_heads = w_k.shape[0] - for bias_name, n in ( - ("b_Q", cfg.n_heads), - ("b_K", n_kv_heads), - ("b_V", n_kv_heads), - ): + for bias_name in ("b_Q", "b_K", "b_V"): bias = _tensor_attr(attn, bias_name) - params_dict[f"blocks.{layer_idx}.attn.{bias_name}"] = ( - bias if bias is not None else _zeros(n, cfg.d_head) - ) + if bias is None: + bias = _zeros(cfg.n_heads, cfg.d_head) + elif bias.ndim == 2 and 0 < bias.shape[0] < cfg.n_heads: + bias = torch.repeat_interleave(bias, cfg.n_heads // bias.shape[0], dim=0) + params_dict[f"blocks.{layer_idx}.attn.{bias_name}"] = bias b_O = _tensor_attr(attn, "b_O") params_dict[f"blocks.{layer_idx}.attn.b_O"] = ( b_O if b_O is not None else _zeros(cfg.d_model) diff --git a/transformer_lens/model_bridge/sources/native/init.py b/transformer_lens/model_bridge/sources/native/init.py index 479b87df84..96d0c19693 100644 --- a/transformer_lens/model_bridge/sources/native/init.py +++ b/transformer_lens/model_bridge/sources/native/init.py @@ -31,13 +31,19 @@ ) # Residual-scaled output is gpt2-specific; other modes treat every weight the -# same. Each entry takes ``(tensor, generator)`` to thread the scoped Generator. -_NonResidualInit = Callable[[torch.Tensor, Optional[torch.Generator]], torch.Tensor] +# same. Each entry takes ``(tensor, generator, gain)`` — gain honors +# ``cfg.initializer_range`` like the legacy init did (which passed it as the +# xavier/kaiming gain); kaiming has no gain kwarg, so scale after. +_NonResidualInit = Callable[[torch.Tensor, Optional[torch.Generator], float], torch.Tensor] _NON_RESIDUAL_MODES: dict[str, _NonResidualInit] = { - "xavier_uniform": lambda t, g: nn.init.xavier_uniform_(t, generator=g), - "xavier_normal": lambda t, g: nn.init.xavier_normal_(t, generator=g), - "kaiming_uniform": lambda t, g: nn.init.kaiming_uniform_(t, nonlinearity="relu", generator=g), - "kaiming_normal": lambda t, g: nn.init.kaiming_normal_(t, nonlinearity="relu", generator=g), + "xavier_uniform": lambda t, g, gain: nn.init.xavier_uniform_(t, gain=gain, generator=g), + "xavier_normal": lambda t, g, gain: nn.init.xavier_normal_(t, gain=gain, generator=g), + "kaiming_uniform": lambda t, g, gain: nn.init.kaiming_uniform_( + t, nonlinearity="relu", generator=g + ).mul_(gain), + "kaiming_normal": lambda t, g, gain: nn.init.kaiming_normal_( + t, nonlinearity="relu", generator=g + ).mul_(gain), } _SUPPORTED_MODES = frozenset({"gpt2", *_NON_RESIDUAL_MODES}) @@ -55,19 +61,30 @@ def initialize_native_model( """Initialize ``model`` weights in-place. Honors ``cfg.init_mode`` and ``cfg.seed``.""" effective_seed = seed if seed is not None else cfg.seed - # Scoped generator on the model's device — None falls back to the global RNG. - try: - gen_device = next(model.parameters()).device - except StopIteration: - gen_device = torch.device("cpu") + # Always generate on CPU/fp32 and copy into the parameter: boot initializes + # before .to(device)/.to(dtype) while init_weights() runs after, and a + # generator seeded on the live parameter device produces a different stream + # — the same seed must reproduce the same weights either way. generator: Optional[torch.Generator] if effective_seed is not None: - g = torch.Generator(device=gen_device) + g = torch.Generator() g.manual_seed(effective_seed) generator = g else: generator = None + def _staged( + fn: Callable[[torch.Tensor], torch.Tensor] + ) -> Callable[[torch.Tensor], torch.Tensor]: + def apply(t: torch.Tensor) -> torch.Tensor: + staging = torch.empty(t.shape, dtype=torch.float32) + fn(staging) + with torch.no_grad(): + t.copy_(staging) + return t + + return apply + init_mode = (cfg.init_mode or "gpt2").lower() if init_mode not in _SUPPORTED_MODES: raise NotImplementedError( @@ -78,7 +95,10 @@ def initialize_native_model( weight_init: Callable[[torch.Tensor], torch.Tensor] output_init: Callable[[torch.Tensor], torch.Tensor] if init_mode == "gpt2": - std = cfg.initializer_range if cfg.initializer_range > 0 else 0.02 + # Default matches the legacy TL scheme: N(0, 0.64/d_model), i.e. + # std = 0.8/sqrt(d_model), not GPT-2's paper 0.02 — toy-model training + # dynamics (e.g. the grokking demo) depend on this scale. + std = cfg.initializer_range if cfg.initializer_range > 0 else 0.8 / math.sqrt(cfg.d_model) residual_scale = 1.0 / math.sqrt(2 * cfg.n_layers) weight_init = lambda t: nn.init.normal_( t, mean=0.0, std=std, generator=generator @@ -88,9 +108,15 @@ def initialize_native_model( ) else: fn = _NON_RESIDUAL_MODES[init_mode] - weight_init = lambda t: fn(t, generator) # noqa: E731 + # Honor an explicitly-set initializer_range as the gain (legacy + # behavior); the sentinel/default keeps plain xavier/kaiming scaling. + gain = cfg.initializer_range if cfg.initializer_range > 0 else 1.0 + weight_init = lambda t: fn(t, generator, gain) # noqa: E731 output_init = weight_init + weight_init = _staged(weight_init) + output_init = _staged(output_init) + tok_embed = cast(nn.Embedding, _unwrap_component(model.tok_embed)) weight_init(tok_embed.weight) if model.pos is not None: diff --git a/transformer_lens/model_bridge/supported_architectures/olmo_hybrid.py b/transformer_lens/model_bridge/supported_architectures/olmo_hybrid.py index 47366fb90b..6fba237857 100644 --- a/transformer_lens/model_bridge/supported_architectures/olmo_hybrid.py +++ b/transformer_lens/model_bridge/supported_architectures/olmo_hybrid.py @@ -40,6 +40,11 @@ def __init__(self, *args: Any, **kwargs: Any): if self.hook_aliases is BlockBridge.hook_aliases: self.hook_aliases = dict(self.hook_aliases) self.hook_aliases.pop("hook_resid_mid", None) + # Full-attention (OLMo2 post-norm) layers route the MLP output through + # post_feedforward_layernorm, so the residual-facing MLP output is + # ln2_post.hook_out there; linear-attention layers have no ln2_post and + # fall back to the raw mlp.hook_out. + self.hook_aliases["hook_mlp_out"] = ["ln2_post.hook_out", "mlp.hook_out"] class OlmoHybridArchitectureAdapter(ArchitectureAdapter): diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index ae71b0b6f4..8d3326f113 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -63,8 +63,8 @@ _BLOCK_PATTERN = re.compile("blocks\\.(\\d+)") -def _resolve_attr_path(obj: nn.Module, attr_path: str) -> torch.Tensor: - """Walk a dot-separated attribute path and return the final tensor.""" +def _resolve_attr_path(obj: nn.Module, attr_path: str) -> Optional[torch.Tensor]: + """Walk a dot-separated attribute path and return the final tensor (None if bias-free).""" result = obj for attr in attr_path.split("."): result = getattr(result, attr) @@ -415,17 +415,33 @@ def __getattr__(self, name: str) -> Any: return getattr(self.__dict__["original_model"], name) except AttributeError: pass # type: ignore[operator,assignment] + # A class property whose fget raised AttributeError lands here with the + # informative message discarded (CPython drops it before __getattr__). + # Re-invoke the property so its own diagnostic (e.g. "bias-free + # projection") surfaces instead of a generic missing-attribute error. + descriptor = getattr(type(self), name, None) + if isinstance(descriptor, property) and descriptor.fget is not None: + descriptor.fget(self) raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") def __str__(self) -> str: - """Get a string representation of the bridge. - # type: ignore[operator] - Returns: - A string describing the bridge's components # type: ignore[operator] + """One-line-per-component summary of the bridge. + + Returns: + A string describing the bridge's components. """ lines = ["TransformerBridge:"] mapping = self.adapter.get_component_mapping() - lines.extend(self._format_component_mapping(mapping, indent=1)) + + def _describe(component_mapping, indent): + pad = " " * indent + for name, component in component_mapping.items(): + lines.append(f"{pad}{name}: {type(component).__name__}") + submodules = getattr(component, "submodules", None) + if submodules: + _describe(submodules, indent + 1) + + _describe(mapping, 1) return "\n".join(lines) def enable_compatibility_mode( @@ -479,6 +495,17 @@ def enable_compatibility_mode( "reference model. Use the default bridge forward instead." ) + if getattr(self.cfg, "is_audio_model", False): + # Audio encoders have no text embed/unembed for the legacy weight + # processing to operate on; without this guard the processing path + # dies later with an opaque KeyError ('embed.weight'). + raise NotImplementedError( + "enable_compatibility_mode() is not supported for audio encoder models: " + "the legacy weight processing (fold_ln/centering) assumes a text " + "embed/unembed, which audio encoders do not have. Use the bridge's " + "native hooks (run_with_cache / run_with_hooks) directly." + ) + self.compatibility_mode = True def set_compatibility_mode(component: Any) -> None: @@ -895,6 +922,11 @@ def stack_params_for( weights: List[torch.Tensor] = [] for idx, block in matching: w = _resolve_attr_path(block, attr_path) + if w is None: + raise AttributeError( + f"blocks[{idx}].{attr_path} is None — this checkpoint has no such " + f"parameter (bias-free projection)." + ) if reshape_fn is not None: w = reshape_fn(w) weights.append(w) @@ -938,8 +970,17 @@ def _stack_block_params( ) weights: List[torch.Tensor] = [] - for _, block in matching_blocks: + for block_idx, block in matching_blocks: w = _resolve_attr_path(block, attr_path) + if w is None: + # Bias-free checkpoints (e.g. qkv_bias=False) expose None here; + # stacking would raise an opaque TypeError. + raise AttributeError( + f"blocks[{block_idx}].{attr_path} is None — this checkpoint has no " + f"such parameter (bias-free projection). Bias-free models expose no " + f"stacked {attr_path.rsplit('.', 1)[-1]}; construct zeros explicitly " + f"if your analysis needs one." + ) if reshape_fn is not None: w = reshape_fn(w) weights.append(w) @@ -1203,8 +1244,13 @@ def all_composition_scores(self, mode: str) -> CompositionScores: def _stack(attr_path: str, reshape_fn: Optional[Callable] = None) -> torch.Tensor: weights: List[torch.Tensor] = [] - for block in blocks_list: + for block_idx, block in zip(indices, blocks_list): w = _resolve_attr_path(block, attr_path) + if w is None: + raise AttributeError( + f"blocks[{block_idx}].{attr_path} is None — this checkpoint has " + f"no such parameter (bias-free projection)." + ) if reshape_fn is not None: w = reshape_fn(w) weights.append(w) diff --git a/transformer_lens/tools/training.py b/transformer_lens/tools/training.py index b52a6e0837..8bb97f7434 100644 --- a/transformer_lens/tools/training.py +++ b/transformer_lens/tools/training.py @@ -5,6 +5,7 @@ work through this loop. """ +import dataclasses from dataclasses import dataclass from typing import Optional, Union @@ -76,6 +77,10 @@ def train( The trained model """ + # Work on a copy: mutating the caller's config (wandb_project_name/device + # defaults below) was a silent side effect the caller never asked for. + config = dataclasses.replace(config) + torch.manual_seed(config.seed) model.train() diff --git a/transformer_lens/utilities/tl_checkpoint_conversion.py b/transformer_lens/utilities/tl_checkpoint_conversion.py index 81a6d2f37b..baa9de85ec 100644 --- a/transformer_lens/utilities/tl_checkpoint_conversion.py +++ b/transformer_lens/utilities/tl_checkpoint_conversion.py @@ -1,5 +1,9 @@ """One-time converter for legacy TL-property-format checkpoints (#1588). +Rotary-model checkpoints are unsupported: their ``rotary_sin``/``rotary_cos`` +buffer keys fail loudly as unrecognized keys (drop them first if you need to +convert one — the bridge recomputes rotary embeddings from the config). + Historical training runs (OthelloGPT, grokking demos, ARENA content) were saved via ``HookedTransformer.state_dict()`` before ``TransformerBridge`` existed, using property-style keys ("blocks.0.attn.W_Q", "embed.W_E", ...) From 7db5f8dcdea87d9b96cd43907af56b588e4ec966 Mon Sep 17 00:00:00 2001 From: Sohan Venkatesh <126096232+sohv@users.noreply.github.com> Date: Sat, 8 Aug 2026 02:37:05 +0100 Subject: [PATCH 42/87] fix(bridge): accept attention_mask in generate() for pre-padded prompts (#1617) * fix(bridge): accept attention_mask in generate() for pre-padded prompts generate() had no way to be told which prompt tokens are padding, so an already-padded token tensor generated as though its pads were real context: every real token's position was shifted and the continuation diverged from the same prompt unpadded. An attention_mask passed by a hopeful caller was absorbed into **multimodal_kwargs, which are merged into the forward kwargs on step 0 only -- so the first token came out right and every later one drifted, which is harder to spot than a uniform failure. attention_mask is now an explicit parameter. It is extended by one attended column per generated token, so every step sees a mask spanning the prompt plus what has been generated, and forward() derives positions from it. The cached step no longer pins position_ids to total_len - 1, which counts pad slots and is wrong for a left-padded prompt; it reads the new token's position off the mask. padding_side is now applied to token input as well. generate() has always documented the argument but only used it when tokenizing string or list input, leaving it inert for a tensor. It is only consulted when explicitly passed: deriving a mask on the default path would change behaviour for every existing caller and would require a real tokenizer where none is needed today. On the inputs_embeds and encoder-decoder paths the mask keeps flowing through to the model untouched, as it did when it arrived via **multimodal_kwargs -- image processors emit one alongside pixel_values and callers forward the lot. Verified across 3 architectures x 3 padding amounts x 2 cache settings, via both the explicit mask and padding_side. Only the mask can express an interior gap or a pad id that also occurs as a real token. Refs #1612 Co-Authored-By: Claude Opus 5 * fix(bridge): raise when generate(padding_side=) cannot be honored Reading the padding off the tokens needs a tokenizer with a pad id. Without one -- a boot_native bridge, say -- the argument was inert, which left exactly the bug this PR fixes, silently, on a different kind of bridge. Since the heuristic only runs when the caller explicitly passed padding_side, say so instead: two ValueErrors, one for a missing tokenizer and one for a missing pad_token_id, each naming attention_mask as the way through. Verified that the alternative works, with a left-padded boot_native prompt generating the same continuation as the unpadded one. Also correct the attention_mask docstring. It still said the parameter was "not supported for encoder-decoder or inputs_embeds generation", left over from the version that raised on those paths; the code forwards the mask to the model as-is there, which is what the test asserts and what processors emitting one alongside pixel_values expect. Refs #1612 Co-Authored-By: Claude Opus 5 * fix(bridge): import PreTrainedTokenizerBase for the padding_side guard The isinstance check added in a81a323 referenced the class without importing it, so mypy failed in CI with "Name PreTrainedTokenizerBase is not defined". It passed locally against a stale incremental cache, which was checking 422 files where CI checks 424; a cleared cache reproduces the error exactly. Refs #1612 Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- .../test_generate_attention_mask.py | 274 ++++++++++++++++++ .../model_bridge/transformer_bridge.py | 104 ++++++- 2 files changed, 377 insertions(+), 1 deletion(-) create mode 100644 tests/integration/model_bridge/test_generate_attention_mask.py diff --git a/tests/integration/model_bridge/test_generate_attention_mask.py b/tests/integration/model_bridge/test_generate_attention_mask.py new file mode 100644 index 0000000000..7fbe82b71c --- /dev/null +++ b/tests/integration/model_bridge/test_generate_attention_mask.py @@ -0,0 +1,274 @@ +"""Generation from an already-padded prompt. + +``generate()`` had no way to be told which prompt tokens are padding, so a +pre-padded tensor generated as though its pads were real context: every real +token's position was shifted and the continuation diverged from the same prompt +unpadded. See #1612. + +Two routes now work. ``attention_mask`` states the padding explicitly, and +``padding_side`` — which the bridge accepted but never applied to token input — +reads it off the pad token. Only the explicit mask can express an interior gap +or a pad id that also occurs as a real token. +""" + +from __future__ import annotations + +import copy + +import pytest +import torch + +GREEDY = dict(max_new_tokens=5, do_sample=False, verbose=False) + + +@pytest.fixture(scope="module") +def prompt(distilgpt2_bridge) -> torch.Tensor: + return distilgpt2_bridge.to_tokens("The capital of France is") + + +@pytest.fixture(scope="module") +def unpadded_continuation(distilgpt2_bridge, prompt) -> list[int]: + return distilgpt2_bridge.generate(prompt, **GREEDY)[0, prompt.shape[1] :].tolist() + + +def _left_pad(bridge, tokens: torch.Tensor, n_pad: int) -> tuple[torch.Tensor, torch.Tensor]: + pad_id = bridge.tokenizer.pad_token_id + if pad_id is None: + pad_id = bridge.tokenizer.eos_token_id + padded = torch.cat([torch.full((1, n_pad), pad_id, dtype=tokens.dtype), tokens], dim=1) + mask = torch.cat( + [torch.zeros(1, n_pad, dtype=torch.long), torch.ones(1, tokens.shape[1], dtype=torch.long)], + dim=1, + ) + return padded, mask + + +@pytest.mark.parametrize("use_past_kv_cache", [True, False]) +@pytest.mark.parametrize("n_pad", [1, 3, 7]) +def test_attention_mask_recovers_the_unpadded_continuation( + distilgpt2_bridge, prompt, unpadded_continuation, n_pad, use_past_kv_cache +) -> None: + """The whole point: padding a prompt must not change what it generates.""" + padded, mask = _left_pad(distilgpt2_bridge, prompt, n_pad) + + out = distilgpt2_bridge.generate( + padded, attention_mask=mask, use_past_kv_cache=use_past_kv_cache, **GREEDY + ) + + assert out[0, n_pad + prompt.shape[1] :].tolist() == unpadded_continuation + + +def test_without_a_mask_the_pads_are_treated_as_context( + distilgpt2_bridge, prompt, unpadded_continuation +) -> None: + """The unfixed behaviour, pinned so a regression is visible rather than silent. + + padding_side defaults to "right", so leading pads are not recognised and the + continuation drifts. This is the case #1612 reported. + """ + padded, _ = _left_pad(distilgpt2_bridge, prompt, 4) + + out = distilgpt2_bridge.generate(padded, **GREEDY) + + assert out[0, 4 + prompt.shape[1] :].tolist() != unpadded_continuation + + +def test_padding_side_left_is_applied_to_token_input( + distilgpt2_bridge, prompt, unpadded_continuation +) -> None: + """generate() has always documented a padding_side argument, but applied it + only when tokenizing string or list input. For a token tensor it was inert.""" + padded, _ = _left_pad(distilgpt2_bridge, prompt, 4) + + out = distilgpt2_bridge.generate(padded, padding_side="left", **GREEDY) + + assert out[0, 4 + prompt.shape[1] :].tolist() == unpadded_continuation + + +def test_padding_side_is_restored_afterwards(distilgpt2_bridge, prompt) -> None: + """The tokenizer is shared across a session, so the override must not leak.""" + before = distilgpt2_bridge.tokenizer.padding_side + padded, _ = _left_pad(distilgpt2_bridge, prompt, 3) + + distilgpt2_bridge.generate(padded, padding_side="left", **GREEDY) + + assert distilgpt2_bridge.tokenizer.padding_side == before + + +def test_explicit_mask_wins_over_the_padding_side_heuristic( + distilgpt2_bridge, prompt, unpadded_continuation +) -> None: + """A caller who states the padding must not be second-guessed by the pad-token + scan, which here would mask nothing because padding_side is "right".""" + padded, mask = _left_pad(distilgpt2_bridge, prompt, 4) + + out = distilgpt2_bridge.generate(padded, attention_mask=mask, padding_side="right", **GREEDY) + + assert out[0, 4 + prompt.shape[1] :].tolist() == unpadded_continuation + + +def test_interior_gap_needs_the_explicit_mask(distilgpt2_bridge, prompt) -> None: + """padding_side can only describe padding at one edge. A masked-out token in + the middle shifts later positions just the same, and only a mask says so.""" + pad_id = distilgpt2_bridge.tokenizer.eos_token_id + gapped = prompt.clone() + gapped[0, 2] = pad_id + mask = torch.ones_like(prompt) + mask[0, 2] = 0 + compact = torch.cat([prompt[:, :2], prompt[:, 3:]], dim=1) + + reference = distilgpt2_bridge.generate(compact, **GREEDY)[0, compact.shape[1] :].tolist() + via_mask = distilgpt2_bridge.generate(gapped, attention_mask=mask, **GREEDY)[ + 0, prompt.shape[1] : + ].tolist() + + assert via_mask == reference + + +def test_rows_padded_to_different_lengths(distilgpt2_bridge) -> None: + """Each row must generate what it would alone, whatever its own pad count.""" + long_prompt = distilgpt2_bridge.to_tokens("The capital of France is") + short_prompt = distilgpt2_bridge.to_tokens("Hello") + width = max(long_prompt.shape[1], short_prompt.shape[1]) + + rows, masks = [], [] + for tokens in (long_prompt, short_prompt): + padded, mask = _left_pad(distilgpt2_bridge, tokens, width - tokens.shape[1]) + rows.append(padded) + masks.append(mask) + + out = distilgpt2_bridge.generate( + torch.cat(rows, dim=0), attention_mask=torch.cat(masks, dim=0), **GREEDY + ) + + for index, tokens in enumerate((long_prompt, short_prompt)): + solo = distilgpt2_bridge.generate(tokens, **GREEDY)[0, tokens.shape[1] :].tolist() + assert out[index, width:].tolist() == solo + + +def test_the_mask_reaches_every_step_not_just_the_first(distilgpt2_bridge, prompt) -> None: + """Before #1612 an attention_mask kwarg was absorbed into **multimodal_kwargs, + which are merged into the forward kwargs on step 0 only. That made the first + token come out right and every later one wrong, which is worse to debug than a + uniform failure. Each step must see a mask covering the prompt plus the tokens + generated so far. + """ + n_pad = 3 + padded, mask = _left_pad(distilgpt2_bridge, prompt, n_pad) + prompt_width = padded.shape[1] + seen: list[torch.Tensor | None] = [] + + original = distilgpt2_bridge.original_model.forward + + def _spy(*args, **kwargs): + seen.append(kwargs.get("attention_mask")) + return original(*args, **kwargs) + + distilgpt2_bridge.original_model.forward = _spy + try: + distilgpt2_bridge.generate(padded, attention_mask=mask, **GREEDY) + finally: + distilgpt2_bridge.original_model.forward = original + + assert len(seen) == GREEDY["max_new_tokens"] + for step, observed in enumerate(seen): + assert observed is not None, f"step {step} received no attention_mask" + assert observed.shape[1] == prompt_width + step + # The prompt's padding stays masked however far generation has run. + assert observed[0, :n_pad].sum() == 0 + assert observed[0, n_pad:].all() + + +def test_unpadded_generation_is_unchanged(distilgpt2_bridge, prompt, unpadded_continuation) -> None: + """An all-ones mask is what the model assumes anyway, so supplying one must + be a no-op rather than a second code path.""" + out = distilgpt2_bridge.generate(prompt, attention_mask=torch.ones_like(prompt), **GREEDY) + + assert out[0, prompt.shape[1] :].tolist() == unpadded_continuation + + +def test_string_and_list_input_still_work(distilgpt2_bridge) -> None: + """The list path builds its own mask; neither route may regress.""" + if distilgpt2_bridge.tokenizer.pad_token_id is None: + distilgpt2_bridge.tokenizer.pad_token = distilgpt2_bridge.tokenizer.eos_token + + solo = distilgpt2_bridge.generate("The capital of France is", **GREEDY) + batched = distilgpt2_bridge.generate(["The capital of France is", "Hi"], **GREEDY) + + assert isinstance(solo, str) and solo.startswith("The capital of France is") + assert batched[0] == solo + + +def test_mask_shape_must_match_the_prompt(distilgpt2_bridge, prompt) -> None: + """generate() extends the mask itself, so a pre-extended one is a mistake + worth naming rather than broadcasting into something unintended.""" + with pytest.raises(ValueError, match="does not match the prompt shape"): + distilgpt2_bridge.generate( + prompt, attention_mask=torch.ones(1, prompt.shape[1] + 5, dtype=torch.long), **GREEDY + ) + + +def test_padding_side_without_a_tokenizer_is_an_error(distilgpt2_bridge, prompt) -> None: + """A bridge booted without a tokenizer has nothing to read the padding from, so + padding_side would be inert — leaving exactly the bug this module is about, but + silently. attention_mask still works there and the message must say so.""" + bridge = copy.copy(distilgpt2_bridge) + bridge.tokenizer = None + assert bridge.tokenizer is None and distilgpt2_bridge.tokenizer is not None + padded, _ = _left_pad(distilgpt2_bridge, prompt, 3) + + with pytest.raises(ValueError, match="this bridge has none"): + bridge.generate(padded, padding_side="left", **GREEDY) + + +def test_padding_side_without_a_pad_token_is_an_error(distilgpt2_bridge, prompt) -> None: + """Same reasoning for a tokenizer that has no pad id to scan for.""" + padded, _ = _left_pad(distilgpt2_bridge, prompt, 3) + tokenizer = distilgpt2_bridge.tokenizer + original_pad = tokenizer.pad_token_id + tokenizer.pad_token_id = None + try: + with pytest.raises(ValueError, match="pad_token_id"): + distilgpt2_bridge.generate(padded, padding_side="left", **GREEDY) + finally: + tokenizer.pad_token_id = original_pad + + +def test_a_tokenizerless_bridge_still_accepts_an_explicit_mask( + distilgpt2_bridge, prompt, unpadded_continuation +) -> None: + """The alternative the error points at has to actually work.""" + bridge = copy.copy(distilgpt2_bridge) + bridge.tokenizer = None + n_pad = 3 + padded, mask = _left_pad(distilgpt2_bridge, prompt, n_pad) + + out = bridge.generate(padded, attention_mask=mask, **GREEDY) + + assert out[0, n_pad + prompt.shape[1] :].tolist() == unpadded_continuation + + +def test_inputs_embeds_forwards_the_mask_untouched(distilgpt2_bridge, prompt) -> None: + """There are no token positions to correct on the embeds path, but processors + emit an attention_mask alongside their other outputs and callers pass the lot + straight through. Before this parameter existed that mask reached the model via + **multimodal_kwargs, so it must still arrive rather than raise. + """ + embeds = distilgpt2_bridge.original_model.get_input_embeddings()(prompt) + mask = torch.ones_like(prompt) + seen: list[torch.Tensor | None] = [] + + original = distilgpt2_bridge.original_model.forward + + def _spy(*args, **kwargs): + seen.append(kwargs.get("attention_mask")) + return original(*args, **kwargs) + + distilgpt2_bridge.original_model.forward = _spy + try: + distilgpt2_bridge.generate(embeds, attention_mask=mask, **GREEDY) + finally: + distilgpt2_bridge.original_model.forward = original + + assert seen and seen[0] is not None + torch.testing.assert_close(seen[0], mask) diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index 8d3326f113..1095fb67b7 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -29,6 +29,7 @@ import tqdm from torch import nn from torch.nn import functional as F +from transformers.tokenization_utils_base import PreTrainedTokenizerBase from transformer_lens import utilities as utils from transformer_lens.ActivationCache import ActivationCache @@ -2070,6 +2071,7 @@ def _generate_tokens( multimodal_kwargs: Dict[str, Any], verbose: bool, stopping_criteria_list: Optional[Any] = None, + initial_attention_mask: Optional[torch.Tensor] = None, ) -> Generator[Tuple[torch.Tensor, torch.Tensor, bool], None, None]: """Core generation loop. Yields (sampled_tokens, final_logits, all_finished) per step. @@ -2104,10 +2106,30 @@ def _generate_tokens( ) else: forward_kwargs: Dict[str, Any] = {} + # A prompt mask covers only the prompt, so extend it by one + # attended column per token generated so far. position_ids are + # left to forward(), which derives them from the mask for the + # models that can take them. + running_attention_mask: Optional[torch.Tensor] = None + if initial_attention_mask is not None: + n_generated = current_tokens.shape[1] - initial_attention_mask.shape[1] + running_attention_mask = torch.cat( + [ + initial_attention_mask.to(current_tokens.device), + torch.ones( + (current_tokens.shape[0], n_generated), + dtype=initial_attention_mask.dtype, + device=current_tokens.device, + ), + ], + dim=1, + ) + forward_kwargs["attention_mask"] = running_attention_mask # Compute attention mask and position_ids for batched # inputs with padding. if ( - _is_batched_list + initial_attention_mask is None + and _is_batched_list and self.tokenizer is not None and self.tokenizer.pad_token_id is not None ): @@ -2180,6 +2202,13 @@ def _generate_tokens( forward_kwargs["position_ids"] = forward_kwargs["position_ids"][ :, -1: ] + elif running_attention_mask is not None: + # total_len - 1 counts pad slots, so it is wrong + # for a left-padded prompt. Derive the new token's + # position from the mask instead. + forward_kwargs["position_ids"] = utils.get_offset_position_ids( + 0, running_attention_mask.long() + )[:, -1:] else: forward_kwargs["position_ids"] = torch.full( (batch_size, 1), @@ -2341,6 +2370,7 @@ def generate( pixel_values: Optional[torch.Tensor] = None, stop_strings: Optional[Union[str, List[str]]] = None, stopping_criteria: Optional[Any] = None, + attention_mask: Optional[torch.Tensor] = None, **multimodal_kwargs, ) -> ( str @@ -2417,6 +2447,20 @@ def generate( Stateful/SSM models raise only when run with use_past_kv_cache=False (the default keeps them on the hooked loop). Each error names the supported alternative. + attention_mask: Optional ``[batch, pos]`` 0/1 mask over the prompt, marking + which prompt tokens are real. Required to generate correctly from an + already-padded token tensor: without it the pad tokens are treated as + real context and every real token's position is shifted, so the + continuation differs from the same prompt unpadded. The mask is extended + by one attended column per generated token. Takes precedence over the + ``padding_side`` heuristic, and unlike it can express an interior gap or + a pad id that also occurs as a real token. Passing ``padding_side`` + instead reads the padding off the pad token, which is enough for the + common single-edge case, and raises if this bridge has no tokenizer + or pad id to read it from. On the encoder-decoder and inputs_embeds + paths the mask is forwarded to the model as-is rather than grown per + step, which is what processors emitting one alongside + ``pixel_values`` expect. Returns: Generated sequence as string, list of strings, or tensor depending on input type and return_type. @@ -2473,6 +2517,63 @@ def generate( input_tokens = input.to(self.cfg.device) input_type = "tokens" + # Without one of these a pre-padded tensor generates as though its pads were + # real context, shifting every real token's position (#1612). An explicit + # mask wins; otherwise the padding is read off the tokens, but only when the + # caller asked for that by passing padding_side. Deriving a mask on the + # default path would silently change behaviour for every existing caller, + # and would demand a real tokenizer where today none is required. + initial_attention_mask: Optional[torch.Tensor] = attention_mask + if initial_attention_mask is not None and ( + _generate_from_embeds + or getattr(getattr(self.original_model, "config", None), "is_encoder_decoder", False) + ): + # Growing the mask per step only means something for decoder-only token + # generation. On these paths the mask used to arrive via + # **multimodal_kwargs and be forwarded to the model untouched — as + # processors emit it alongside pixel_values — so keep doing that rather + # than reject a call that worked before this parameter existed. + multimodal_kwargs = {**multimodal_kwargs, "attention_mask": initial_attention_mask} + initial_attention_mask = None + if initial_attention_mask is not None: + if initial_attention_mask.shape != input_tokens.shape: + raise ValueError( + f"attention_mask shape {tuple(initial_attention_mask.shape)} does not " + f"match the prompt shape {tuple(input_tokens.shape)}. Pass a 0/1 mask " + "covering exactly the prompt tokens; generate() extends it itself." + ) + initial_attention_mask = initial_attention_mask.to(self.cfg.device) + elif padding_side is not None and input_type == "tokens": + # Reading the padding off the tokens needs a tokenizer with a pad id. + # Without one the argument would be inert, leaving exactly the bug this + # fixes — silently, on a bridge booted without a tokenizer. Say so + # rather than generate something quietly wrong. + if not isinstance(self.tokenizer, PreTrainedTokenizerBase): + raise ValueError( + "generate(padding_side=...) reads the padding off the pad token, " + "which needs a tokenizer; this bridge has none. Pass " + "attention_mask=... to state the padding directly instead." + ) + if self.tokenizer.pad_token_id is None: + raise ValueError( + "generate(padding_side=...) reads the padding off the pad token, " + "but this tokenizer has no pad_token_id. Set one, or pass " + "attention_mask=... to state the padding directly instead." + ) + _prepend = self.cfg.default_prepend_bos if prepend_bos is None else prepend_bos + _orig_side = self.tokenizer.padding_side + self.tokenizer.padding_side = padding_side + try: + initial_attention_mask = utils.get_attention_mask( + self.tokenizer, input_tokens, _prepend + ).to(self.cfg.device) + finally: + self.tokenizer.padding_side = _orig_side + # An all-ones mask is what the model assumes anyway; skipping it keeps + # the unpadded path byte-identical to before. + if initial_attention_mask is not None and bool(initial_attention_mask.all()): + initial_attention_mask = None + # Determine return type if return_type == "input": if input_type in ["str", "list"]: @@ -2723,6 +2824,7 @@ def generate( multimodal_kwargs=multimodal_kwargs if multimodal_kwargs else {}, verbose=verbose, stopping_criteria_list=stopping_criteria_list, + initial_attention_mask=initial_attention_mask, ): sampled_tokens_list.append(sampled_tokens.unsqueeze(1)) if logits_seq_list is not None: From 1532412b8daeaf19e64e56928b8211a0a4abfddb Mon Sep 17 00:00:00 2001 From: Nayab_code <147242551+LightWork666@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:25:04 +0530 Subject: [PATCH 43/87] feat(bridge): support disk device_map offload targets (#1615) * feat(bridge): support disk device_map offload targets device_map entries targeting "disk" were rejected: a smoke test in #1459 found that a disk-offloaded GPT-2 block loaded successfully but crashed mid-forward on a meta tensor inside a Bridge-wrapped component. Accelerate's offload hooks materialize real data only around the wrapped module's own forward() call, but Bridge components read raw params directly rather than exclusively calling that forward(), so a read outside that window silently sees the meta placeholder instead. GeneralizedComponent.__call__ now wraps forward() in Accelerate's own align_module_device(original_component), the same pre/post-forward hook Accelerate would have run natively; it's a no-op when nothing is offloaded. JointQKVAttentionBridge's one-time QKV split reads a descendant of its own original_component (c_attn), not the object itself, so it uses a second helper (align_offloaded_subtree) that walks the whole subtree instead. "meta" targets remain rejected under load_weights=True (no real data to offload from). Mixed CPU+GPU maps are untouched, unverified without GPU hardware. Fixes #1280 * update * fix(bridge): address disk-offload review feedback Four requests from review, plus one more instance of the same bug found while addressing the first: - JointGateUpMLPBridge.set_original_component had the same setup-time raw-weight-read as JointQKVAttentionBridge's QKV split (gate_up_proj sliced once at setup); same align_offloaded_subtree fix. - Also found and fixed the identical pattern in UnembeddingBridge: both the bias-construction fallback in set_original_component and the b_U property's fallback build a zero-bias tensor sized/deviced from the wrapped weight, unguarded - under offload, .device reads meta and produces a fake, data-less bias instead of real zeros. - align_module_device only exists in accelerate>=1.1.0; bumped the pyproject floor from >=0.23.0 (previously satisfied by the resolved version, but not guaranteed by the constraint itself). - Mixed disk+GPU maps were slipping through unblocked even though mixed CPU+GPU maps stay rejected as unverified without GPU hardware - disk rides the same offload mechanism, so it needs the same caution. Renamed is_mixed_cpu_gpu -> is_mixed_offload_gpu to match. - enable_compatibility_mode()'s weight processing (fold_ln etc.) reads and rewrites params directly across many components at once, not through any single component's own forward(), so it isn't covered by GeneralizedComponent's per-call materialization and previously crashed deep in weight folding with an opaque meta-device error. Now raises a clear RuntimeError immediately; no_processing=True still works under offload. --------- Co-authored-by: jlarson4 --- docs/source/content/compatibility_mode.md | 6 + pyproject.toml | 2 +- .../model_bridge/test_multi_gpu_bridge.py | 179 +++++++++++++++++- .../test_joint_gate_up_mlp.py | 35 ++++ .../generalized_components/base.py | 60 ++++++ .../joint_gate_up_mlp.py | 9 +- .../joint_qkv_attention.py | 18 +- .../generalized_components/unembedding.py | 23 ++- .../sources/transformers/source.py | 38 ++-- .../model_bridge/transformer_bridge.py | 17 ++ transformer_lens/utilities/multi_gpu.py | 59 +++--- uv.lock | 2 +- 12 files changed, 381 insertions(+), 67 deletions(-) diff --git a/docs/source/content/compatibility_mode.md b/docs/source/content/compatibility_mode.md index e299d59040..28363aa8bf 100644 --- a/docs/source/content/compatibility_mode.md +++ b/docs/source/content/compatibility_mode.md @@ -86,6 +86,12 @@ New integration tests should use the variant that matches the property they're t - **One-shot:** calling it twice re-runs the centering subtractions. Don't. - **Not reversible** from within the bridge — re-boot for raw weights. - **`_setup_hook_compatibility` is idempotent**; only `process_weights` mutates weights. +- **Incompatible with a CPU/disk-offloaded `device_map`** unless `no_processing=True`. Weight + processing (`fold_ln` etc.) reads and rewrites parameters directly across many components at + once, not through a single component's own `forward()` call, so it isn't covered by + `GeneralizedComponent`'s per-call materialization and raises immediately rather than crashing + mid-fold on a raw `meta` tensor. The default (non-compat-mode) forward pass works normally + under offload — this restriction is compat mode's weight processing specifically. ## See also diff --git a/pyproject.toml b/pyproject.toml index 2ce8c7c076..1eca6a11fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] authors=[{name="Neel Nanda", email="77788841+TransformerLensOrg@users.noreply.github.com"}] dependencies=[ - "accelerate>=0.23.0", # Needed for Llama Models + "accelerate>=1.1.0", # align_module_device (device_map disk offload) "beartype>=0.14.1", "better-abc>=0.0.3", "datasets>=2.7.1", diff --git a/tests/acceptance/model_bridge/test_multi_gpu_bridge.py b/tests/acceptance/model_bridge/test_multi_gpu_bridge.py index 42c7d58105..54edc5fbfc 100644 --- a/tests/acceptance/model_bridge/test_multi_gpu_bridge.py +++ b/tests/acceptance/model_bridge/test_multi_gpu_bridge.py @@ -9,8 +9,15 @@ import pytest import torch +from accelerate.hooks import attach_align_device_hook from transformer_lens.model_bridge import TransformerBridge +from transformer_lens.model_bridge.generalized_components.normalization import ( + NormalizationBridge, +) +from transformer_lens.model_bridge.generalized_components.unembedding import ( + UnembeddingBridge, +) from transformer_lens.utilities.multi_gpu import ( cast_floating_params_to_dtype, count_unique_devices, @@ -106,10 +113,11 @@ def test_cpu_value_in_device_map_passes_through(self): assert dm is explicit assert mm is None - def test_disk_value_in_device_map_rejected(self): - bad: Dict[str, Union[str, int]] = {"transformer.h.0": "disk"} - with pytest.raises(ValueError, match="not supported yet"): - resolve_device_map(None, bad, None) + def test_disk_value_in_device_map_passes_through(self): + explicit: Dict[str, Union[str, int]] = {"transformer.h.0": "disk"} + dm, mm = resolve_device_map(None, explicit, None) + assert dm is explicit + assert mm is None def test_meta_value_in_device_map_passes_through(self): device_map: Dict[str, Union[str, int]] = {"transformer.h.0": "meta"} @@ -119,18 +127,33 @@ def test_meta_value_in_device_map_passes_through(self): class TestMixedCpuGpuMapRejection: - """Mixed CPU+GPU maps mean accelerate CPU offload — meta placeholders that - param-reading Bridge components never materialize. Rejected at the resolver - (explicit dicts) so no weights are downloaded before the error.""" - - def test_explicit_mixed_dict_rejected(self): + """Mixed CPU/disk + GPU maps are rejected — not because they're known to be + broken (GeneralizedComponent.__call__ materializes offloaded params for + every component call, the same mechanism either way), but because that's + only been verified on CPU-only hardware, with no GPU to mix in. Rejected at + the resolver (explicit dicts) so no weights are downloaded before the + error. All-CPU, all-disk, and CPU+disk maps involve no GPU and so are fine.""" + + def test_explicit_mixed_cpu_gpu_dict_rejected(self): with pytest.raises(ValueError, match="offload"): resolve_device_map(None, {"transformer.wte": 0, "transformer.ln_f": "cpu"}, None) + def test_explicit_mixed_disk_gpu_dict_rejected(self): + with pytest.raises(ValueError, match="offload"): + resolve_device_map(None, {"transformer.wte": 0, "transformer.h.0": "disk"}, None) + def test_all_cpu_dict_passes(self): dm, _ = resolve_device_map(None, {"transformer.wte": "cpu", "lm_head": "cpu"}, None) assert dm == {"transformer.wte": "cpu", "lm_head": "cpu"} + def test_all_disk_dict_passes(self): + dm, _ = resolve_device_map(None, {"transformer.wte": "disk", "lm_head": "disk"}, None) + assert dm == {"transformer.wte": "disk", "lm_head": "disk"} + + def test_cpu_and_disk_mixed_dict_passes(self): + dm, _ = resolve_device_map(None, {"transformer.wte": "cpu", "lm_head": "disk"}, None) + assert dm == {"transformer.wte": "cpu", "lm_head": "disk"} + def test_all_gpu_dict_passes(self): dm, _ = resolve_device_map(None, {"transformer.wte": 0, "lm_head": "cuda:1"}, None) assert dm == {"transformer.wte": 0, "lm_head": "cuda:1"} @@ -385,6 +408,144 @@ def test_accumulated_bias_handles_cross_device(self, gpt2_bridge): gpt2_bridge.cfg.n_devices = original_n_devices +class TestGeneralizedComponentOffloadMaterialization: + """CPU/disk offload (#1280, #1459, #1493) hits meta tensors inside Bridge + components because Accelerate only materializes real data around a hooked + module's own forward() - components that read raw params directly (or via + a synthetic sub-module built once at setup, e.g. split QKV) bypass that + window. GeneralizedComponent.__call__ wraps every component call in + Accelerate's own align_module_device, so this covers the leaf case without + needing a real (slow) model download.""" + + def _offloaded_layer_norm(self) -> tuple[torch.nn.LayerNorm, dict[str, torch.Tensor]]: + module = torch.nn.LayerNorm(4) + weights_map = {k: v.clone() for k, v in module.state_dict().items()} + attach_align_device_hook( + module, execution_device=torch.device("cpu"), offload=True, weights_map=weights_map + ) + return module, weights_map + + def test_offloaded_leaf_module_starts_as_meta(self): + # Establishes the failure mode this test guards against: Accelerate + # offload really does leave the module's own weight as a meta + # placeholder outside of a forward call. + module, _ = self._offloaded_layer_norm() + assert module.weight.is_meta + + def test_normalization_bridge_materializes_offloaded_params(self): + module, weights_map = self._offloaded_layer_norm() + + class _Cfg: + eps = 1e-5 + rmsnorm_uses_offset = False + + bridge = NormalizationBridge(name="ln", config=_Cfg(), uses_rms_norm=False) + bridge.set_original_component(module) + bridge.eval() + + x = torch.randn(2, 3, 4) + with torch.no_grad(): + out = bridge(x) + assert not torch.isnan(out).any() + + expected = torch.nn.functional.layer_norm( + x, (4,), weight=weights_map["weight"], bias=weights_map["bias"], eps=1e-5 + ) + torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5) + + def test_unembedding_bridge_bias_construction_reads_real_device(self): + # set_original_component synthesizes a zero bias for a bias-less Linear, + # sized/dtyped/deviced from the wrapped weight. Reading .device unguarded + # on an offloaded (meta) weight would silently build a meta-device (i.e. + # fake, data-less) bias instead of real zeros. + module = torch.nn.Linear(4, 6, bias=False) + weights_map = {k: v.clone() for k, v in module.state_dict().items()} + attach_align_device_hook( + module, execution_device=torch.device("cpu"), offload=True, weights_map=weights_map + ) + assert module.weight.is_meta + + bridge = UnembeddingBridge(name="unembed") + bridge.set_original_component(module) + + assert not module.bias.is_meta + assert module.bias.device.type == "cpu" + torch.testing.assert_close(module.bias, torch.zeros(6)) + + +class TestDiskOffloadEndToEnd: + """Real gpt2 end-to-end: disk device_map used to be rejected outright + (#1280); now it should produce identical numerics to a non-offloaded load, + including through gpt2's split-QKV path (JointQKVAttentionBridge builds + q/k/v as one-time slices of c_attn's raw weight at setup - a case the + generic per-call materialization alone doesn't cover, see + set_original_component).""" + + def _device_map(self, disk_layers: set[int]) -> Dict[str, str]: + device_map: Dict[str, str] = { + "transformer.wte": "cpu", + "transformer.wpe": "cpu", + "transformer.ln_f": "cpu", + "lm_head": "cpu", + } + for i in range(12): + device_map[f"transformer.h.{i}"] = "disk" if i in disk_layers else "cpu" + return device_map + + def test_disk_offload_matches_non_offloaded_forward_pass(self, gpt2_bridge, tmp_path): + gpt2_bridge.eval() + torch.manual_seed(0) + tokens = torch.randint(0, 1000, (2, 8)) + with torch.no_grad(): + baseline_logits = gpt2_bridge(tokens).clone() + baseline_loss = gpt2_bridge(tokens, return_type="loss").item() + + offloaded_bridge = TransformerBridge.boot_transformers( + "gpt2", + device_map=self._device_map({0, 3, 6, 9}), + offload_folder=str(tmp_path), + ) + offloaded_bridge.eval() + with torch.no_grad(): + offloaded_logits = offloaded_bridge(tokens) + offloaded_loss = offloaded_bridge(tokens, return_type="loss").item() + + torch.testing.assert_close(offloaded_logits, baseline_logits, atol=1e-4, rtol=1e-4) + assert abs(offloaded_loss - baseline_loss) < 1e-4 + + def test_disk_value_no_longer_raises_at_boot(self, tmp_path): + # Regression guard for the ValueError this used to raise unconditionally. + bridge = TransformerBridge.boot_transformers( + "gpt2", + device_map=self._device_map({0}), + offload_folder=str(tmp_path), + ) + assert bridge.original_model.hf_device_map["transformer.h.0"] == "disk" + + def test_compatibility_mode_raises_clear_error_on_offloaded_bridge(self, tmp_path): + # Weight processing (fold_ln etc.) reads/rewrites params directly across + # many components at once, not through any one component's own forward(), + # so it isn't covered by GeneralizedComponent's per-call materialization - + # this should fail loud and clear, not deep inside weight folding on a raw + # meta tensor. + bridge = TransformerBridge.boot_transformers( + "gpt2", + device_map=self._device_map({0}), + offload_folder=str(tmp_path), + ) + with pytest.raises(RuntimeError, match="not supported on a bridge with an offloaded"): + bridge.enable_compatibility_mode() + + def test_compatibility_mode_no_processing_works_on_offloaded_bridge(self, tmp_path): + bridge = TransformerBridge.boot_transformers( + "gpt2", + device_map=self._device_map({0}), + offload_folder=str(tmp_path), + ) + bridge.enable_compatibility_mode(no_processing=True) + assert bridge.compatibility_mode + + # Real-hardware multi-GPU coverage lives in test_bridge_multigpu.py and # test_bridge_multigpu_device_map.py (`-m multigpu`, >= 2 CUDA devices) — this file # is the CPU/mocked tier: resolver, validation gates, and spoofed-n_devices guards. diff --git a/tests/unit/model_bridge/generalized_components/test_joint_gate_up_mlp.py b/tests/unit/model_bridge/generalized_components/test_joint_gate_up_mlp.py index fe7db5056c..8a4f48d77c 100644 --- a/tests/unit/model_bridge/generalized_components/test_joint_gate_up_mlp.py +++ b/tests/unit/model_bridge/generalized_components/test_joint_gate_up_mlp.py @@ -1,6 +1,7 @@ """Tests for JointGateUpMLPBridge split logic.""" import torch +from accelerate.hooks import attach_align_device_hook from transformer_lens.model_bridge.generalized_components.joint_gate_up_mlp import ( JointGateUpMLPBridge, @@ -61,3 +62,37 @@ def test_split_without_bias(self): assert gate_proj.bias is None assert up_proj.bias is None + + +class TestSetOriginalComponentUnderOffload: + """set_original_component reads gate_up_proj's raw weight once, at setup, to + build the gate/up slices - the same setup-time pattern as + JointQKVAttentionBridge.set_original_component (#1280 review), which needs + real (not meta) data materialized under Accelerate CPU/disk offload.""" + + def test_split_materializes_offloaded_gate_up_proj(self): + d_model, d_mlp = 8, 16 + mock_mlp = _MockMLP(d_model, d_mlp) + weights_map = { + k[len("gate_up_proj.") :]: v.clone() + for k, v in mock_mlp.state_dict().items() + if k.startswith("gate_up_proj.") + } + attach_align_device_hook( + mock_mlp.gate_up_proj, + execution_device=torch.device("cpu"), + offload=True, + weights_map=weights_map, + ) + assert mock_mlp.gate_up_proj.weight.is_meta + + bridge = JointGateUpMLPBridge(name="mlp") + bridge.set_original_component(mock_mlp) + + gate_weight = bridge.gate.original_component.weight + up_weight = getattr(bridge, "in").original_component.weight + assert not gate_weight.is_meta + assert not up_weight.is_meta + torch.testing.assert_close( + torch.cat([gate_weight, up_weight], dim=0), weights_map["weight"] + ) diff --git a/transformer_lens/model_bridge/generalized_components/base.py b/transformer_lens/model_bridge/generalized_components/base.py index 76612acd75..7071064f60 100644 --- a/transformer_lens/model_bridge/generalized_components/base.py +++ b/transformer_lens/model_bridge/generalized_components/base.py @@ -1,6 +1,7 @@ """Base class for generalized transformer components.""" from __future__ import annotations +import contextlib import inspect import warnings from collections.abc import Callable @@ -8,6 +9,7 @@ import torch import torch.nn as nn +from accelerate.utils import align_module_device from transformer_lens.conversion_utils.conversion_steps.base_tensor_conversion import ( BaseTensorConversion, @@ -15,6 +17,26 @@ from transformer_lens.hook_points import HookPoint +def align_offloaded_subtree(module: nn.Module) -> contextlib.ExitStack: + """Materialize every Accelerate-offloaded descendant of ``module`` for the + caller's duration (an ``ExitStack`` of ``align_module_device`` contexts). + + Accelerate attaches offload hooks at leaf level - whichever submodule + directly owns the Parameter (e.g. ``c_attn``, ``c_proj``) - not on + container modules like an attention block as a whole. ``align_module_device`` + on a container alone is therefore a no-op even though its descendants are + offloaded. Walking every descendant and entering each one's + ``align_module_device`` (a cheap no-op for any module that has no hook of + its own) covers both a leaf ``original_component`` and a multi-level + container uniformly, without needing to know in advance which specific + descendant a given architecture adapter's code actually reads from. + """ + stack = contextlib.ExitStack() + for submodule in module.modules(): + stack.enter_context(align_module_device(submodule)) + return stack + + class CloneOutputUnderGradMixin(nn.Module): """Clone the forward output so HF's in-place mutation cannot corrupt it. @@ -88,6 +110,44 @@ def __init__( self.hook_aliases = self.__class__.hook_aliases.copy() self.hook_aliases.update(hook_alias_overrides) + def __call__(self, *args: Any, **kwargs: Any) -> Any: + """Run forward(), materializing the wrapped component's params first if offloaded. + + Bridge components read the wrapped module's raw parameters directly + (self.weight / self.original_component.bias / etc. via __getattr__ or + direct attribute access) rather than exclusively calling the wrapped + module's own forward(). Under an Accelerate CPU/disk device_map, + Accelerate only swaps a meta placeholder for the real, materialized + tensor around the wrapped module's OWN forward() call (its pre/post + forward hooks) - a raw attribute read outside that window silently + sees the meta placeholder instead, with no error. align_module_device + wraps this call in the same pre/post-forward hook Accelerate would + have run, so every read during this call - whichever way the code + reaches it - gets real data. It's a no-op (bare yield) when the + wrapped module has no offload hook, which covers the common case of + no device_map, a single device, or a multi-GPU split with no CPU/disk + offload involved. + + Deliberately just original_component, not align_offloaded_subtree's + whole-subtree walk: Accelerate hooks the leaf modules that actually own + Parameters (e.g. NormalizationBridge/LinearBridge wrap one directly), so + materializing exactly that leaf for exactly its own call keeps the same + one-leaf-at-a-time memory footprint Accelerate's native per-module hooks + would give a plain (non-bridge) forward pass. A component whose + original_component is a container of several separately-hooked leaves + (e.g. an attention module wrapping distinct q/k/v/o projections) doesn't + need this to also materialize here - each of ITS own sub-bridges calls + into its own leaf the same way. align_offloaded_subtree is for the + narrower case of code that reads a specific descendant's raw params + directly during setup, without going through that descendant's own + bridge __call__ at all (see JointQKVAttentionBridge.set_original_component). + """ + original_component = self._modules.get("_original_component") + if original_component is None: + return super().__call__(*args, **kwargs) + with align_module_device(original_component): + return super().__call__(*args, **kwargs) + def _register_hook(self, name: str, hook: HookPoint) -> None: """Register a hook in the component's hook registry.""" hook.name = name diff --git a/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py b/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py index 79573e7a4d..391b68fa7e 100644 --- a/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py +++ b/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py @@ -8,6 +8,7 @@ from transformer_lens.model_bridge.generalized_components.base import ( GeneralizedComponent, + align_offloaded_subtree, ) from transformer_lens.model_bridge.generalized_components.gated_mlp import ( GatedMLPBridge, @@ -108,7 +109,13 @@ def set_original_component(self, original_component: torch.nn.Module) -> None: """Set the original MLP component and split fused projections.""" super().set_original_component(original_component) - gate_proj, up_proj = self.split_gate_up_matrix(original_component) + # Same setup-time raw-weight read as JointQKVAttentionBridge.set_original_component: + # split_gate_up_matrix reads original_component.gate_up_proj directly, once, to build + # independent gate/up slices - needs real (not meta) data materialized for that one read + # under Accelerate offload. See align_offloaded_subtree's docstring for why the whole + # subtree, not just original_component itself, needs materializing here. + with align_offloaded_subtree(original_component): + gate_proj, up_proj = self.split_gate_up_matrix(original_component) self.gate.set_original_component(gate_proj) getattr(self, "in").set_original_component(up_proj) diff --git a/transformer_lens/model_bridge/generalized_components/joint_qkv_attention.py b/transformer_lens/model_bridge/generalized_components/joint_qkv_attention.py index 2490c3ee38..1aefc20d06 100644 --- a/transformer_lens/model_bridge/generalized_components/joint_qkv_attention.py +++ b/transformer_lens/model_bridge/generalized_components/joint_qkv_attention.py @@ -16,6 +16,7 @@ ) from transformer_lens.model_bridge.generalized_components.base import ( GeneralizedComponent, + align_offloaded_subtree, ) from transformer_lens.model_bridge.generalized_components.linear import LinearBridge @@ -284,9 +285,20 @@ def set_original_component(self, original_component: torch.nn.Module) -> None: original_component, "reorder_and_upcast_attn", False ) - q_transformation, k_transformation, v_transformation = self.split_qkv_matrix( - original_component - ) + # split_qkv_matrix reads original_component's raw weight/bias once, here, + # to build independent q/k/v slices - not a live view, so under Accelerate + # offload this needs the real (not meta) data materialized for this one + # read. The resulting slices are real, standalone tensors that stay valid + # afterward regardless of what Accelerate later does to original_component + # (unlike whatever reads original_component itself on every forward call, + # e.g. GeneralizedComponent.__call__ / LinearBridge for "o"/c_proj, split + # q/k/v specifically stay permanently resident rather than re-offloading + # after each forward - a deliberate, small, documented memory trade-off + # for combined-qkv architectures). + with align_offloaded_subtree(original_component): + q_transformation, k_transformation, v_transformation = self.split_qkv_matrix( + original_component + ) self.q.set_original_component(q_transformation) self.k.set_original_component(k_transformation) self.v.set_original_component(v_transformation) diff --git a/transformer_lens/model_bridge/generalized_components/unembedding.py b/transformer_lens/model_bridge/generalized_components/unembedding.py index ac7ab7e740..dde4b23328 100644 --- a/transformer_lens/model_bridge/generalized_components/unembedding.py +++ b/transformer_lens/model_bridge/generalized_components/unembedding.py @@ -5,6 +5,7 @@ from typing import Any, Dict, Optional import torch +from accelerate.utils import align_module_device from transformer_lens.model_bridge.generalized_components.base import ( GeneralizedComponent, @@ -43,9 +44,16 @@ def set_original_component(self, original_component: torch.nn.Module) -> None: # If this is a Linear layer without bias, enable it if isinstance(original_component, torch.nn.Linear) and original_component.bias is None: # Get the output features (vocab size) - vocab_size = original_component.weight.shape[0] - device = original_component.weight.device - dtype = original_component.weight.dtype + vocab_size = original_component.weight.shape[0] # shape is safe on a meta tensor too + dtype = original_component.weight.dtype # dtype is also safe on a meta tensor + + # .device is NOT safe the same way: under Accelerate offload this + # weight is a meta placeholder outside of a materialized window, so + # reading .device unguarded would build a meta-device (i.e. fake, + # data-less) bias instead of real zeros. align_module_device gives + # the real execution device for this one read. + with align_module_device(original_component): + device = original_component.weight.device original_component.bias = torch.nn.Parameter( torch.zeros(vocab_size, device=device, dtype=dtype) @@ -122,7 +130,10 @@ def b_U(self) -> torch.Tensor: ), f"Component {self.name} has no weight attribute" weight = self.original_component.weight assert isinstance(weight, torch.Tensor), f"Weight is not a tensor for {self.name}" - device = weight.device - dtype = weight.dtype - vocab_size: int = int(weight.shape[0]) + dtype = weight.dtype # safe on a meta tensor + vocab_size: int = int(weight.shape[0]) # safe on a meta tensor + # .device is not safe the same way under offload - see + # set_original_component's identical guard above. + with align_module_device(self.original_component): + device = weight.device return torch.zeros(vocab_size, device=device, dtype=dtype) diff --git a/transformer_lens/model_bridge/sources/transformers/source.py b/transformer_lens/model_bridge/sources/transformers/source.py index ac0bb28ad0..3ca6a22876 100644 --- a/transformer_lens/model_bridge/sources/transformers/source.py +++ b/transformer_lens/model_bridge/sources/transformers/source.py @@ -56,6 +56,7 @@ def boot( device_map: str | dict[str, str | int] | None = None, n_devices: int | None = None, max_memory: dict[str | int, str | int] | None = None, + offload_folder: str | None = None, ) -> TransformerBridge: """Boot a model from HuggingFace (exposed as ``TransformerBridge.boot_transformers``). @@ -84,12 +85,20 @@ def boot( models loaded with custom configurations (e.g., quantization via BitsAndBytesConfig). When provided, load_weights is ignored. device_map: HuggingFace-style device map (``"auto"``, ``"balanced"``, dict, etc.) for - dispatched inference. Explicit maps may include CPU targets; disk / meta offload - targets are still rejected because Bridge component wrappers need additional - offload-hook routing work. Mutually exclusive with ``device``. + dispatched inference. Explicit maps may include CPU and disk targets; meta targets + are still rejected when ``load_weights=True`` (meta has no real data to offload from, + unlike disk/cpu). Mixed CPU/disk + GPU maps are rejected too, not because they're + known to be broken but because CPU/disk offload has only been verified on CPU-only + hardware — no GPU to mix in. ``bridge.enable_compatibility_mode()`` (with weight + processing, i.e. not ``no_processing=True``) is unsupported on a CPU/disk-offloaded + bridge and raises immediately rather than mid-fold; the default (non-compat-mode) + forward pass, ``run_with_cache``, and hooks all work normally under offload. Mutually + exclusive with ``device``. n_devices: Convenience: split the model across this many CUDA devices (translated to a ``max_memory`` dict internally). Requires CUDA with at least this many visible devices. max_memory: Optional per-device memory budget for HF's dispatcher. + offload_folder: Directory for disk-offloaded weight shards when ``device_map`` includes + a ``"disk"`` target. Defaults to a temporary directory (HF's own default) if omitted. n_ctx: Optional context length override. The bridge normally uses the model's documented max context from the HF config. Setting this writes to whichever HF field the model uses (n_positions / max_position_embeddings / etc.), so callers don't need to know @@ -212,12 +221,12 @@ def boot( # resolver raises on conflict. If n_devices>1 is passed it's translated into a device_map + # max_memory pair here so downstream code only needs to check the resolved values. from transformer_lens.utilities.multi_gpu import ( - MIXED_CPU_GPU_ERROR, + MIXED_OFFLOAD_GPU_ERROR, cast_floating_params_to_dtype, count_unique_devices, find_embedding_device, find_misplaced_modules, - is_mixed_cpu_gpu, + is_mixed_offload_gpu, resolve_device_map, ) @@ -251,6 +260,8 @@ def boot( model_kwargs["device_map"] = resolved_device_map if resolved_max_memory is not None: model_kwargs["max_memory"] = resolved_max_memory + if offload_folder is not None: + model_kwargs["offload_folder"] = offload_folder if hasattr(adapter.cfg, "attn_implementation") and adapter.cfg.attn_implementation is not None: model_kwargs["attn_implementation"] = adapter.cfg.attn_implementation else: @@ -321,20 +332,19 @@ def boot( # with a resolved device_map AND pre-loaded models with caller-dispatched device_map="auto". hf_device_map_post = getattr(hf_model, "hf_device_map", None) if hf_device_map_post: - # All-CPU placement is supported (real parameters, no offload). Disk / meta — - # and CPU entries in a MIXED map, which accelerate implements as CPU offload — - # are rejected: offload materializes weights via forward hooks that wrapped - # Bridge components bypass (e.g. NormalizationBridge computes from raw params). + # All-CPU placement, and disk offload, are supported (GeneralizedComponent.__call__ + # wraps forward in Accelerate's align_module_device, so wrapped components reading + # raw params directly still see materialized data). meta remains rejected here: it + # has no real data to materialize even under offload/dispatch, unlike disk/cpu. offload_values = {str(v).lower() for v in hf_device_map_post.values() if isinstance(v, str)} - unsupported = offload_values & {"disk", "meta"} + unsupported = offload_values & {"meta"} if unsupported: raise ValueError( f"hf_device_map contains unsupported offload targets: {sorted(unsupported)}. " - "TransformerBridge currently supports CPU device_map targets, but disk / meta " - "offload can bypass Accelerate hooks inside wrapped Bridge components." + "TransformerBridge currently supports CPU and disk device_map targets." ) - if is_mixed_cpu_gpu(hf_device_map_post.values()): - raise ValueError(f"Realized hf_device_map is unsupported: {MIXED_CPU_GPU_ERROR}") + if is_mixed_offload_gpu(hf_device_map_post.values()): + raise ValueError(f"Realized hf_device_map is unsupported: {MIXED_OFFLOAD_GPU_ERROR}") if ( "cpu" in offload_values and device_map is None diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index 1095fb67b7..aeac8a63b4 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -496,6 +496,23 @@ def enable_compatibility_mode( "reference model. Use the default bridge forward instead." ) + hf_device_map = getattr(self.original_model, "hf_device_map", None) + if hf_device_map and not no_processing: + offloaded = {k for k, v in hf_device_map.items() if str(v).lower() in ("cpu", "disk")} + if offloaded: + raise RuntimeError( + "enable_compatibility_mode() with weight processing " + "(fold_ln/center_writing_weights/center_unembed/fold_value_biases) is not " + "supported on a bridge with an offloaded device_map " + f"({sorted(offloaded)} are CPU/disk-offloaded). Weight processing reads and " + "rewrites parameters directly across many components at once, not through a " + "single component's own forward() call the way the default (non-compat-mode) " + "bridge forward does, so it isn't covered by GeneralizedComponent's per-call " + "materialization and hits raw meta tensors. Load without a CPU/disk device_map " + "for compatibility mode, or call enable_compatibility_mode(no_processing=True) " + "for the hook/component compatibility layer without the weight transforms." + ) + if getattr(self.cfg, "is_audio_model", False): # Audio encoders have no text embed/unembed for the legacy weight # processing to operate on; without this guard the processing path diff --git a/transformer_lens/utilities/multi_gpu.py b/transformer_lens/utilities/multi_gpu.py index e29023aedf..98ca8a4717 100644 --- a/transformer_lens/utilities/multi_gpu.py +++ b/transformer_lens/utilities/multi_gpu.py @@ -17,8 +17,6 @@ else: ConfigType = Any -_UNSUPPORTED_OFFLOAD_DEVICE_MAP_VALUES = {"disk"} - AvailableDeviceMemory = list[tuple[int, int]] """ This type is passed around between different CUDA memory operations. @@ -204,48 +202,45 @@ def resolve_device_map( def _validate_device_map_values( device_map: Union[str, Dict[str, Union[str, int]]], ) -> None: - """Reject explicit disk values and mixed CPU+GPU targets in a user-supplied - device_map dict. Meta values are passed through (validated at boot against - load_weights).""" + """Reject mixed CPU/disk + GPU targets in a user-supplied device_map dict. + All-CPU and all-disk-or-CPU maps are accepted (GeneralizedComponent.__call__ + wraps forward in Accelerate's align_module_device, so components reading raw + params directly still see materialized data, verified on CPU-only hardware). + Meta values are passed through (validated at boot against load_weights).""" if isinstance(device_map, str): return - for key, value in device_map.items(): - normalized = str(value).lower() if isinstance(value, str) else None - if normalized in _UNSUPPORTED_OFFLOAD_DEVICE_MAP_VALUES: - raise ValueError( - f"device_map[{key!r}]={value!r} is not supported yet. TransformerBridge " - "currently supports CPU device_map targets, but disk / meta offload can " - "bypass Accelerate hooks inside wrapped Bridge components." - ) - if is_mixed_cpu_gpu(device_map.values()): - raise ValueError(MIXED_CPU_GPU_ERROR) - - -# In a mixed map, accelerate OFFLOADS the CPU entries: weights live in a CPU state -# dict, the modules hold meta placeholders, and an AlignDevicesHook on the original -# module's forward materializes them per-call. Bridge components that compute from raw -# parameters (e.g. NormalizationBridge reads self.weight to expose hook_normalized) -# never trigger that hook, so the forward hits meta tensors. All-CPU maps are fine — -# no offload, real parameters. -MIXED_CPU_GPU_ERROR = ( - "device_map mixes CPU and GPU targets, which accelerate implements as CPU offload " - "(meta placeholders materialized by forward hooks that Bridge components bypass). " - "Use an all-GPU map (or n_devices) for multi-GPU, or an all-CPU map." + if is_mixed_offload_gpu(device_map.values()): + raise ValueError(MIXED_OFFLOAD_GPU_ERROR) + + +# In a mixed map, accelerate OFFLOADS the CPU/disk entries: weights live in a CPU +# state dict or on disk, the modules hold meta placeholders, and an AlignDevicesHook +# on the original module's forward materializes them per-call. +# GeneralizedComponent.__call__ wraps every component call in that same hook, so this +# is likely fine in principle — but it's only been verified on CPU-only hardware (no +# GPU to mix in), so a map that actually puts some weights on a GPU stays rejected +# until that's confirmed. All-CPU, all-disk, or CPU+disk maps are fine — no GPU +# involved, so nothing to leave unverified. +MIXED_OFFLOAD_GPU_ERROR = ( + "device_map mixes CPU/disk offload targets with a GPU target. This is likely fine " + "(GeneralizedComponent.__call__ materializes offloaded params for every component " + "call), but has only been verified on CPU-only hardware — no GPU to mix in. Use an " + "all-GPU map (or n_devices) for multi-GPU, or an all-CPU/all-disk map." ) -def is_mixed_cpu_gpu(values: Any) -> bool: - has_cpu = has_gpu = False +def is_mixed_offload_gpu(values: Any) -> bool: + has_offload = has_gpu = False for value in values: if isinstance(value, int): has_gpu = True elif isinstance(value, str): v = value.lower() - if v == "cpu": - has_cpu = True + if v in ("cpu", "disk"): + has_offload = True elif v.startswith("cuda"): has_gpu = True - return has_cpu and has_gpu + return has_offload and has_gpu def cast_floating_params_to_dtype(model: nn.Module, dtype: torch.dtype) -> None: diff --git a/uv.lock b/uv.lock index 0da72f87bf..ce651de9c7 100644 --- a/uv.lock +++ b/uv.lock @@ -9562,7 +9562,7 @@ quantization = [ [package.metadata] requires-dist = [ - { name = "accelerate", specifier = ">=0.23.0" }, + { name = "accelerate", specifier = ">=1.1.0" }, { name = "beartype", specifier = ">=0.14.1" }, { name = "better-abc", specifier = ">=0.0.3" }, { name = "chardet", marker = "extra == 'evals'", specifier = "<6" }, From 956c989ce29b588c0c164995ad40e0b4ad2ca018 Mon Sep 17 00:00:00 2001 From: emerardd <113128214+emerardd@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:25:35 +0800 Subject: [PATCH 44/87] Fix run_with_cache BOS handling for strings (#1625) --- tests/unit/model_bridge/test_boot_native.py | 29 ++++++++++++++++++++ transformer_lens/model_bridge/bridge_core.py | 6 ++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/unit/model_bridge/test_boot_native.py b/tests/unit/model_bridge/test_boot_native.py index bf3045299e..2848673a1e 100644 --- a/tests/unit/model_bridge/test_boot_native.py +++ b/tests/unit/model_bridge/test_boot_native.py @@ -367,6 +367,35 @@ def test_boot_native_forward_and_cache(): assert "blocks.0.attn.hook_pattern" in cache +@pytest.mark.parametrize("prepend_bos", [True, False]) +def test_run_with_cache_forwards_prepend_bos_for_string_input(monkeypatch, prepend_bos): + cfg = _cfg() + bridge = TransformerBridge.boot_native(cfg) + bridge._tokenizer = object() + tokenization_calls = [] + + def to_tokens(input, prepend_bos=None, padding_side=None): + tokenization_calls.append((input, prepend_bos, padding_side)) + if prepend_bos is None: + prepend_bos = bridge.cfg.default_prepend_bos + tokens = [0, 7] if prepend_bos else [7] + return torch.tensor([tokens]) + + monkeypatch.setattr(bridge, "to_tokens", to_tokens) + bridge.eval() + + with torch.no_grad(): + direct_logits = bridge("hello", prepend_bos=prepend_bos) + cached_logits, cache = bridge.run_with_cache("hello", prepend_bos=prepend_bos) + + assert tokenization_calls == [ + ("hello", prepend_bos, None), + ("hello", prepend_bos, None), + ] + torch.testing.assert_close(cached_logits, direct_logits) + assert cache["hook_embed"].shape[1] == direct_logits.shape[1] + + def test_boot_native_does_not_load_transformers_runtime(): # Sanity that the native path doesn't depend on HuggingFace's `transformers` # for the runtime work — we check that calling boot_native doesn't trigger diff --git a/transformer_lens/model_bridge/bridge_core.py b/transformer_lens/model_bridge/bridge_core.py index 5a712d24d7..620c7dab54 100644 --- a/transformer_lens/model_bridge/bridge_core.py +++ b/transformer_lens/model_bridge/bridge_core.py @@ -1293,14 +1293,16 @@ def grad_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: target_device = self._input_device() if processed_args and isinstance(processed_args[0], str): assert self.tokenizer is not None, "Tokenizer must be set to pass string input." - input_ids = self.to_tokens(processed_args[0]) + prepend_bos = kwargs.pop("prepend_bos", None) + input_ids = self.to_tokens(processed_args[0], prepend_bos=prepend_bos) if target_device is not None: input_ids = input_ids.to(target_device) kwargs["input_ids"] = input_ids processed_args = processed_args[1:] elif "input" in kwargs and isinstance(kwargs["input"], str): assert self.tokenizer is not None, "Tokenizer must be set to pass string input." - input_ids = self.to_tokens(kwargs["input"]) + prepend_bos = kwargs.pop("prepend_bos", None) + input_ids = self.to_tokens(kwargs["input"], prepend_bos=prepend_bos) if target_device is not None: input_ids = input_ids.to(target_device) kwargs["input_ids"] = input_ids From 4574892771a482f870dfe5c1db7807b721ec6055 Mon Sep 17 00:00:00 2001 From: Sohan Venkatesh <126096232+sohv@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:28:52 +0100 Subject: [PATCH 45/87] fix(tokenizer): do not strip a BOS token the tokenizer does not have (#1629) get_tokens_with_bos_removed assumed a bos_token_id exists. Callers gate it on cfg.tokenizer_prepends_bos, which detect_tokenizer_bos_eos only sets when the tokenizer has one, but the flag goes stale on a bridge built via build_bridge_from_module(tokenizer=None) and given a tokenizer afterwards: the setter re-runs configure_tokenizer only on reassignment, so the config default of True survives. Trusting it then does damage in both directions. Under right padding, the default, the helper drops the first token unconditionally, silently removing [CLS] from a BERT tokenizer's output and returning a plausible-looking wrong result. Under left padding it evaluates (tokens == None).int() and raises AttributeError: 'bool' object has no attribute 'int', which names neither the tokenizer nor the flag. Return the tokens unchanged when there is no bos_token_id: with no BOS there is nothing to remove, which is correct however the config got out of sync. Reproduced with two off-the-shelf tokenizers, bert-base-cased and t5-small, both of which have bos_token_id None. The normal boot_transformers path is unaffected, since detection runs there. The root cause is the reassignment test in bridge_core.py, left alone deliberately. Correcting the flag would route to_tokens(prepend_bos=True) into the manual-prepend branch at transformer_bridge.py:716, which calls get_input_with_manually_prepended_bos(tokenizer.bos_token, ...) and raises TypeError on a None bos_token. The stale flag currently masks that, so the root-cause fix needs the prepend path hardened first. Fixes #1628 Co-authored-by: Claude Opus 5 --- .../test_get_tokens_with_bos_removed.py | 82 +++++++++++++++++++ transformer_lens/utilities/tokenize_utils.py | 8 ++ 2 files changed, 90 insertions(+) create mode 100644 tests/unit/utilities/test_get_tokens_with_bos_removed.py diff --git a/tests/unit/utilities/test_get_tokens_with_bos_removed.py b/tests/unit/utilities/test_get_tokens_with_bos_removed.py new file mode 100644 index 0000000000..75e51ad741 --- /dev/null +++ b/tests/unit/utilities/test_get_tokens_with_bos_removed.py @@ -0,0 +1,82 @@ +"""Tests for get_tokens_with_bos_removed when the tokenizer has no BOS token. + +Callers gate this helper on ``cfg.tokenizer_prepends_bos``. That flag is set by +``detect_tokenizer_bos_eos``, which requires a ``bos_token_id`` — so a tokenizer +with none should never reach here. It does when the flag is stale: a bridge built +via ``build_bridge_from_module(tokenizer=None)`` keeps the config default of True, +and the tokenizer setter only re-runs detection on *re*-assignment. + +Trusting a stale flag is not harmless. Under right padding the helper drops the +first token unconditionally, which silently removes ``[CLS]`` from a BERT +tokenizer's output; under left padding it compares tokens against ``None`` and +raises an ``AttributeError`` naming neither the tokenizer nor the flag. +""" + +from __future__ import annotations + +import pytest +import torch +from transformers import AutoTokenizer + +from transformer_lens.utilities.tokenize_utils import get_tokens_with_bos_removed + + +@pytest.fixture(scope="module") +def no_bos_tokenizer(): + """BERT uses [CLS] rather than a BOS token, so bos_token_id is None.""" + tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased") + assert tokenizer.bos_token_id is None + return tokenizer + + +@pytest.fixture(scope="module") +def bos_tokenizer(): + tokenizer = AutoTokenizer.from_pretrained("distilgpt2") + assert tokenizer.bos_token_id is not None + return tokenizer + + +@pytest.mark.parametrize("padding_side", ["left", "right"]) +def test_no_bos_token_returns_tokens_unchanged(no_bos_tokenizer, padding_side) -> None: + """There is no BOS to remove, so the tokens must come back untouched.""" + no_bos_tokenizer.padding_side = padding_side + tokens = torch.tensor([[101, 19082, 1362, 102]]) + + result = get_tokens_with_bos_removed(no_bos_tokenizer, tokens) + + torch.testing.assert_close(result, tokens) + + +def test_no_bos_token_does_not_drop_cls_under_right_padding(no_bos_tokenizer) -> None: + """The damaging case: [CLS] is not a BOS token, and dropping it changes what + the model is asked to encode.""" + no_bos_tokenizer.padding_side = "right" + tokens = no_bos_tokenizer("hello world", return_tensors="pt")["input_ids"] + + result = get_tokens_with_bos_removed(no_bos_tokenizer, tokens) + + assert result.shape == tokens.shape + assert result[0, 0].item() == no_bos_tokenizer.cls_token_id + + +def test_no_bos_token_does_not_raise_under_left_padding(no_bos_tokenizer) -> None: + """Previously `(tokens == None).int()` — a Python bool, not a tensor.""" + no_bos_tokenizer.padding_side = "left" + tokens = torch.tensor([[101, 19082, 1362, 102]]) + + result = get_tokens_with_bos_removed(no_bos_tokenizer, tokens) + + assert result.shape == tokens.shape + + +@pytest.mark.parametrize("padding_side", ["left", "right"]) +def test_a_real_bos_is_still_removed(bos_tokenizer, padding_side) -> None: + """The guard must not disturb the case the helper exists for.""" + bos_tokenizer.padding_side = padding_side + bos = bos_tokenizer.bos_token_id + tokens = torch.tensor([[bos, 15496, 995]]) + + result = get_tokens_with_bos_removed(bos_tokenizer, tokens) + + assert result.shape[-1] == tokens.shape[-1] - 1 + assert bos not in result[0].tolist() diff --git a/transformer_lens/utilities/tokenize_utils.py b/transformer_lens/utilities/tokenize_utils.py index 418694798c..3abe27af93 100644 --- a/transformer_lens/utilities/tokenize_utils.py +++ b/transformer_lens/utilities/tokenize_utils.py @@ -216,6 +216,14 @@ def get_tokens_with_bos_removed( Returns: torch.Tensor: The tokenized input with the bos token removed. """ + if tokenizer.bos_token_id is None: + # Nothing to remove (#1628). Callers reach this when cfg.tokenizer_prepends_bos + # says the tokenizer prepends a BOS but the tokenizer has none — a stale + # flag, since detect_tokenizer_bos_eos() requires a bos_token_id. Trusting + # it here would drop a real first token under right padding ([CLS] for a + # BERT tokenizer), and compare tokens against None under left padding. + return tokens + if tokenizer.padding_side == "right": return tokens[..., 1:] From 0d1259adc70ab0b00da45459739b6aef867a73ba Mon Sep 17 00:00:00 2001 From: emerardd <113128214+emerardd@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:31:02 +0800 Subject: [PATCH 46/87] fix(bridge): restore native residual stopping (#1633) --- tests/unit/model_bridge/test_boot_native.py | 36 +++++++++++++++++++ .../generalized_components/block.py | 6 +--- .../model_bridge/sources/native/model.py | 23 +++++++++--- .../model_bridge/transformer_bridge.py | 5 ++- 4 files changed, 59 insertions(+), 11 deletions(-) diff --git a/tests/unit/model_bridge/test_boot_native.py b/tests/unit/model_bridge/test_boot_native.py index 2848673a1e..d933e0e464 100644 --- a/tests/unit/model_bridge/test_boot_native.py +++ b/tests/unit/model_bridge/test_boot_native.py @@ -75,6 +75,42 @@ def test_boot_native_returns_bridge_over_native_model(): assert isinstance(bridge.original_model, NativeModel) +@pytest.mark.parametrize("stop_at_layer", [0, 2, -1]) +def test_boot_native_direct_stop_matches_cached_stop(stop_at_layer: int): + bridge = TransformerBridge.boot_native(_cfg(n_layers=3)) + bridge.eval() + tokens = torch.tensor([[1, 2, 3]]) + + with torch.no_grad(): + expected, _ = bridge.run_with_cache(tokens, stop_at_layer=stop_at_layer) + actual = bridge(tokens, stop_at_layer=stop_at_layer) + + assert actual.shape == (1, 3, bridge.cfg.d_model) + torch.testing.assert_close(actual, expected) + + +def test_boot_native_input_to_embed_round_trip(): + bridge = TransformerBridge.boot_native(_cfg(n_layers=3)) + bridge.eval() + tokens = torch.tensor([[1, 2, 3]]) + + with torch.no_grad(): + expected = bridge(tokens) + residual, returned_tokens, shortformer_pos_embed, attention_mask = bridge.input_to_embed( + tokens + ) + actual = bridge( + residual, + start_at_layer=0, + attention_mask=attention_mask, + ) + + assert residual.shape == (1, 3, bridge.cfg.d_model) + assert torch.equal(returned_tokens, tokens) + assert shortformer_pos_embed is None + torch.testing.assert_close(actual, expected) + + def test_native_state_dict_round_trip_restores_parameters(): bridge = TransformerBridge.boot_native(_cfg()) diff --git a/transformer_lens/model_bridge/generalized_components/block.py b/transformer_lens/model_bridge/generalized_components/block.py index 89d0347a59..1f89c7dcaf 100644 --- a/transformer_lens/model_bridge/generalized_components/block.py +++ b/transformer_lens/model_bridge/generalized_components/block.py @@ -261,11 +261,7 @@ def _extract_layer_idx(self) -> Optional[int]: """Parse this block's layer index from its name (TL/GPT-2/LLaMA patterns).""" if self.name is None: return None - match = ( - re.search(r"blocks\.(\d+)", self.name) - or re.search(r"\.h\.(\d+)", self.name) - or re.search(r"\.layers\.(\d+)", self.name) - ) + match = re.search(r"(?:^|\.)(?:blocks|h|layers)\.(\d+)", self.name) return int(match.group(1)) if match else None def _check_stop_at_layer(self, *args: Any, **kwargs: Any) -> None: diff --git a/transformer_lens/model_bridge/sources/native/model.py b/transformer_lens/model_bridge/sources/native/model.py index 02a3aa9fbe..30ba363099 100644 --- a/transformer_lens/model_bridge/sources/native/model.py +++ b/transformer_lens/model_bridge/sources/native/model.py @@ -487,15 +487,27 @@ def __init__(self, cfg: TransformerBridgeConfig): def forward( self, - input_ids: torch.Tensor, + input_ids: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, **kwargs, ) -> torch.Tensor: """Returns logits directly.""" + if input_ids is not None and inputs_embeds is not None: + raise ValueError("Exactly one of input_ids or inputs_embeds must be provided.") + if input_ids is not None: + model_input = input_ids + hidden_states = self.tok_embed(input_ids) + elif inputs_embeds is not None: + model_input = inputs_embeds + hidden_states = inputs_embeds + else: + raise ValueError("Exactly one of input_ids or inputs_embeds must be provided.") + # Bounds check up front so both absolute and rotary paths produce a # self-explanatory error rather than IndexError / shape mismatch. - seq_len = input_ids.shape[-1] + seq_len = model_input.shape[1] if seq_len > self.cfg.n_ctx: raise ValueError( f"input length {seq_len} exceeds n_ctx={self.cfg.n_ctx}; " @@ -504,11 +516,12 @@ def forward( # Resolve position_ids before the block loop so rotary sees the caller's # positions, not the dense default. - batch, seq = input_ids.shape + batch, seq = model_input.shape[:2] if position_ids is None: - position_ids = torch.arange(seq, device=input_ids.device).unsqueeze(0).expand(batch, -1) + position_ids = ( + torch.arange(seq, device=model_input.device).unsqueeze(0).expand(batch, -1) + ) - hidden_states = self.tok_embed(input_ids) if self.pos is not None: hidden_states = hidden_states + self.pos(position_ids) diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index aeac8a63b4..331bde2e22 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -1665,8 +1665,11 @@ def forward( "The bridge only supports stop_at_layer on 'blocks'." ) if hasattr(self, "blocks"): + effective_stop_at_layer = ( + len(self.blocks) + stop_at_layer if stop_at_layer < 0 else stop_at_layer + ) for block in self.blocks: - block._stop_at_layer_idx = stop_at_layer + block._stop_at_layer_idx = effective_stop_at_layer # Map HookedEncoderDecoder-style kwargs to HF-compatible names if "decoder_input" in kwargs: From 71e3b30d8d3d264c9e85037a0bc32ae921f3f8d0 Mon Sep 17 00:00:00 2001 From: Chinmayrawat15 <88652081+Chinmayrawat15@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:39:50 -0700 Subject: [PATCH 47/87] fix(tokenizer): do not prepend a BOS token the tokenizer does not have (#1634) get_input_with_manually_prepended_bos concatenated bos_token + input unconditionally, which is None + str for a tokenizer with no BOS token and raises TypeError naming neither the tokenizer nor the flag that caused it. With no BOS token there is nothing to prepend, so return the input unchanged. Reached when a tokenizer skips setup_tokenizer's bos_token = eos_token backfill, which is the initial-assignment branch at bridge_core.py:113, and tokenizer_prepends_bos is then corrected to False. That is the follow-up scoped out of #1628; hardening it here unblocks the root-cause fix. The guard sits in the shared helper, so all three call sites are covered: transformer_bridge.py:717, HookedTransformer.py:850, remote_bridge.py:216. bos_token widens to Optional[str] because beartype rejects None against the old str annotation, so the runtime guard alone would still fail under test. Co-authored-by: Claude Opus 5 (1M context) --- ...t_get_input_with_manually_prepended_bos.py | 70 +++++++++++++++++++ transformer_lens/utilities/tokenize_utils.py | 15 +++- 2 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 tests/unit/utilities/test_get_input_with_manually_prepended_bos.py diff --git a/tests/unit/utilities/test_get_input_with_manually_prepended_bos.py b/tests/unit/utilities/test_get_input_with_manually_prepended_bos.py new file mode 100644 index 0000000000..f11a38bac1 --- /dev/null +++ b/tests/unit/utilities/test_get_input_with_manually_prepended_bos.py @@ -0,0 +1,70 @@ +"""Tests for get_input_with_manually_prepended_bos when the tokenizer has no BOS token. + +``to_tokens`` reaches this helper whenever the caller wants a BOS that the tokenizer +will not add on its own — ``prepend_bos and not cfg.tokenizer_prepends_bos``. For a +tokenizer with no BOS token that condition is *correctly* true rather than stale: +``detect_tokenizer_bos_eos`` requires a ``bos_token_id``, so it reports False for +BERT and T5, and ``prepend_bos`` defaults to True. The helper then evaluated +``None + input`` and raised ``TypeError: unsupported operand type(s) for +: +'NoneType' and 'str'``, naming neither the tokenizer nor the flag. + +This is the prepend-side counterpart of #1628, which covers the removal side. +""" + +from __future__ import annotations + +import pytest +from transformers import AutoTokenizer + +from transformer_lens.utilities.tokenize_utils import ( + get_input_with_manually_prepended_bos, +) + + +@pytest.fixture( + scope="module", + params=["google-bert/bert-base-cased", "google-t5/t5-small"], +) +def no_bos_tokenizer(request): + """BERT opens with [CLS] and T5 with nothing, so bos_token is None for both.""" + tokenizer = AutoTokenizer.from_pretrained(request.param) + assert tokenizer.bos_token is None + return tokenizer + + +@pytest.fixture(scope="module") +def bos_tokenizer(): + tokenizer = AutoTokenizer.from_pretrained("distilgpt2") + assert tokenizer.bos_token is not None + return tokenizer + + +def test_no_bos_token_returns_string_unchanged(no_bos_tokenizer) -> None: + """There is no BOS to prepend, so the string must come back untouched.""" + assert get_input_with_manually_prepended_bos(no_bos_tokenizer.bos_token, "hello world") == ( + "hello world" + ) + + +def test_no_bos_token_returns_list_unchanged(no_bos_tokenizer) -> None: + """Same for the batched form — and no partially-prepended list.""" + inputs = ["hello world", "second string"] + + result = get_input_with_manually_prepended_bos(no_bos_tokenizer.bos_token, inputs) + + assert result == ["hello world", "second string"] + + +def test_a_real_bos_is_still_prepended_to_a_string(bos_tokenizer) -> None: + """The guard must not disturb the case the helper exists for.""" + result = get_input_with_manually_prepended_bos(bos_tokenizer.bos_token, "hello world") + + assert result == bos_tokenizer.bos_token + "hello world" + + +def test_a_real_bos_is_still_prepended_to_a_list(bos_tokenizer) -> None: + bos = bos_tokenizer.bos_token + + result = get_input_with_manually_prepended_bos(bos, ["hello world", "second string"]) + + assert result == [bos + "hello world", bos + "second string"] diff --git a/transformer_lens/utilities/tokenize_utils.py b/transformer_lens/utilities/tokenize_utils.py index 3abe27af93..b0ab21bba9 100644 --- a/transformer_lens/utilities/tokenize_utils.py +++ b/transformer_lens/utilities/tokenize_utils.py @@ -183,18 +183,27 @@ def get_tokenizer_with_bos(tokenizer: PreTrainedTokenizerBase) -> PreTrainedToke def get_input_with_manually_prepended_bos( - bos_token: str, input: str | list[str] + bos_token: Optional[str], input: str | list[str] ) -> str | list[str]: """ Manually prepends the bos token to the input. Args: - bos_token (str): The BOS token to prepend. + bos_token (Optional[str]): The BOS token to prepend, or None for a tokenizer + that has none (e.g. BERT, T5). input (str | list[str]): The input to prepend the bos token to. Returns: - str | list[str]: The input with the bos token manually prepended. + str | list[str]: The input with the bos token manually prepended, or unchanged + when there is no BOS token to prepend. """ + if bos_token is None: + # Nothing to prepend. Callers reach this when prepend_bos is asked for and + # cfg.tokenizer_prepends_bos is False — correctly so for a BOS-less tokenizer, + # since detect_tokenizer_bos_eos() requires a bos_token_id. Concatenating + # would raise a TypeError naming neither the tokenizer nor the flag. + return input + if isinstance(input, str): input = bos_token + input else: From 4f5042ccc7adfcceff8ce3e15cadc0c3476ba4de Mon Sep 17 00:00:00 2001 From: emerardd <113128214+emerardd@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:58:24 +0800 Subject: [PATCH 48/87] Fix TransformerBridge temporary hook cleanup (#1638) * Fix TransformerBridge temporary hook cleanup * Handle lightweight bridge hook contexts --- .../test_hook_orchestration_parity.py | 180 ++++++++++++++++++ transformer_lens/model_bridge/bridge_core.py | 102 ++++++---- 2 files changed, 250 insertions(+), 32 deletions(-) diff --git a/tests/unit/model_bridge/supported_architectures/test_hook_orchestration_parity.py b/tests/unit/model_bridge/supported_architectures/test_hook_orchestration_parity.py index 7cd17ba1f9..1131e5eeeb 100644 --- a/tests/unit/model_bridge/supported_architectures/test_hook_orchestration_parity.py +++ b/tests/unit/model_bridge/supported_architectures/test_hook_orchestration_parity.py @@ -31,6 +31,14 @@ def _noop(value, hook=None): return value +def _record(events, label): + def hook(value, hook=None): + events.append(label) + return value + + return hook + + class TestResetHooks: def test_removes_forward_and_backward_from_every_point(self): """Default direction='both' clears bwd hooks even on non-io points @@ -178,3 +186,175 @@ def recording( with bridge.hooks(fwd_hooks=[("blocks.0.hook_out", _noop)]): pass assert "blocks.0.hook_out" in seen + + +class TestTemporaryHookScopes: + @pytest.mark.parametrize("helper", ["hooks", "run_with_hooks", "run_with_cache"]) + def test_preexisting_forward_hook_survives_helper_cleanup(self, helper): + bridge = _build() + tokens = torch.randint(0, 64, (1, 4)) + hook_name = "blocks.0.hook_out" + hook_point = bridge._hook_registry[hook_name] + events: list[str] = [] + hook_point.add_hook(_record(events, "existing")) + + if helper == "hooks": + with bridge.hooks(fwd_hooks=[(hook_name, _record(events, "temporary"))]): + bridge(tokens) + elif helper == "run_with_hooks": + bridge.run_with_hooks( + tokens, + fwd_hooks=[(hook_name, _record(events, "temporary"))], + ) + else: + bridge.run_with_cache(tokens, names_filter=hook_name) + + events_after_helper = events.copy() + assert len(hook_point.fwd_hooks) == 1 + + bridge(tokens) + + assert events == events_after_helper + ["existing"] + + def test_nested_context_removes_only_inner_hooks(self): + bridge = _build() + tokens = torch.randint(0, 64, (1, 4)) + hook_name = "blocks.0.hook_out" + hook_point = bridge._hook_registry[hook_name] + events: list[str] = [] + + with bridge.hooks(fwd_hooks=[(hook_name, _record(events, "outer"))]): + with bridge.hooks(fwd_hooks=[(hook_name, _record(events, "inner"))]): + bridge(tokens) + assert len(hook_point.fwd_hooks) == 1 + bridge(tokens) + + assert events == ["outer", "inner", "outer"] + assert not hook_point.has_hooks() + + def test_nested_run_with_cache_preserves_outer_hook(self): + bridge = _build() + tokens = torch.randint(0, 64, (1, 4)) + hook_name = "blocks.0.hook_out" + hook_point = bridge._hook_registry[hook_name] + events: list[str] = [] + + with bridge.hooks(fwd_hooks=[(hook_name, _record(events, "outer"))]): + bridge.run_with_cache(tokens, names_filter=hook_name) + assert len(hook_point.fwd_hooks) == 1 + bridge(tokens) + + assert events == ["outer", "outer"] + assert not hook_point.has_hooks() + + def test_run_with_cache_exception_preserves_preexisting_hook(self): + bridge = _build() + tokens = torch.randint(0, 64, (1, 4)) + hook_name = "blocks.0.hook_out" + hook_point = bridge._hook_registry[hook_name] + + def raising_hook(value, hook=None): + raise RuntimeError("existing hook failed") + + hook_point.add_hook(raising_hook) + + with pytest.raises(RuntimeError, match="existing hook failed"): + bridge.run_with_cache(tokens, names_filter=hook_name) + + assert len(hook_point.fwd_hooks) == 1 + + def test_preexisting_backward_hook_survives_context_cleanup(self): + bridge = _build() + tokens = torch.randint(0, 64, (1, 4)) + hook_name = "blocks.0.hook_out" + hook_point = bridge._hook_registry[hook_name] + events: list[str] = [] + + def existing_hook(gradient, hook=None): + events.append("existing") + + def temporary_hook(gradient, hook=None): + events.append("temporary") + + hook_point.add_hook(existing_hook, dir="bwd") + with bridge.hooks(bwd_hooks=[(hook_name, temporary_hook)]): + bridge(tokens).sum().backward() + + events_after_context = events.copy() + assert len(hook_point.bwd_hooks) == 1 + + bridge.zero_grad() + bridge(tokens).sum().backward() + + assert events == events_after_context + ["existing"] + + def test_run_with_cache_incl_bwd_preserves_preexisting_hooks(self): + bridge = _build() + tokens = torch.randint(0, 64, (1, 4)) + hook_name = "blocks.0.hook_out" + hook_point = bridge._hook_registry[hook_name] + hook_point.add_hook(_noop, dir="fwd") + hook_point.add_hook(_noop, dir="bwd") + + bridge.run_with_cache( + tokens, + names_filter=hook_name, + incl_bwd=True, + return_type="loss", + ) + + assert len(hook_point.fwd_hooks) == 1 + assert len(hook_point.bwd_hooks) == 1 + + def test_permanent_hook_survives_temporary_scope_cleanup(self): + bridge = _build() + tokens = torch.randint(0, 64, (1, 4)) + hook_name = "blocks.0.hook_out" + hook_point = bridge._hook_registry[hook_name] + bridge.add_perma_hook(hook_name, _noop) + + bridge.run_with_hooks(tokens, fwd_hooks=[(hook_name, _noop)]) + + assert len(hook_point.fwd_hooks) == 1 + assert hook_point.fwd_hooks[0].is_permanent + + @pytest.mark.parametrize("helper", ["hooks", "run_with_hooks"]) + def test_reset_hooks_end_false_retains_temporary_hooks(self, helper): + bridge = _build() + tokens = torch.randint(0, 64, (1, 4)) + hook_name = "blocks.0.hook_out" + hook_point = bridge._hook_registry[hook_name] + + if helper == "hooks": + with bridge.hooks(fwd_hooks=[(hook_name, _noop)], reset_hooks_end=False): + bridge(tokens) + else: + bridge.run_with_hooks( + tokens, + fwd_hooks=[(hook_name, _noop)], + reset_hooks_end=False, + ) + + assert bridge.context_level == 0 + assert len(hook_point.fwd_hooks) == 1 + + def test_callable_filter_adds_one_hook_for_canonical_and_alias_names(self): + bridge = _build() + tokens = torch.randint(0, 64, (1, 4)) + canonical_name = "blocks.0.mlp.hook_out" + alias_name = "blocks.0.hook_mlp_out" + hook_point = bridge.hook_dict[canonical_name] + events: list[str] = [] + + with bridge.hooks( + fwd_hooks=[ + ( + lambda name: name in {canonical_name, alias_name}, + _record(events, "temporary"), + ) + ] + ): + bridge(tokens) + + assert events == ["temporary"] + assert not hook_point.has_hooks() diff --git a/transformer_lens/model_bridge/bridge_core.py b/transformer_lens/model_bridge/bridge_core.py index 620c7dab54..b5797ebd62 100644 --- a/transformer_lens/model_bridge/bridge_core.py +++ b/transformer_lens/model_bridge/bridge_core.py @@ -88,6 +88,7 @@ def __init__( self._hook_registry_initialized = False self._hook_alias_registry: Dict[str, Union[str, List[str]]] = {} self._property_alias_registry: Dict[str, str] = {} + self.context_level = 0 self._driver = driver if not hasattr(adapter, "component_mapping") or adapter.component_mapping is None: raise ValueError("Adapter must have a component_mapping attribute") @@ -929,6 +930,8 @@ def hooks( @contextmanager def _hooks_context() -> Iterator["BridgeCore"]: added_hooks: List[Tuple[HookPoint, Literal["fwd", "bwd"]]] = [] + context_level = getattr(self, "context_level", 0) + 1 + self.context_level = context_level def add_hook_to_point( hook_point: HookPoint, @@ -942,9 +945,14 @@ def add_hook_to_point( if hook_point.name is not None: alias_names_list.append(hook_point.name) alias_names_list.append(name) - hook_point.add_hook(hook_fn, dir=dir, alias_names=alias_names_list) + hook_point.add_hook( + hook_fn, + dir=dir, + level=context_level, + alias_names=alias_names_list, + ) else: - hook_point.add_hook(hook_fn, dir=dir) + hook_point.add_hook(hook_fn, dir=dir, level=context_level) added_hooks.append((hook_point, dir)) def apply_hooks(hook_list: List[Tuple[Any, Callable]], is_fwd: bool) -> None: @@ -973,9 +981,12 @@ def apply_hooks(hook_list: List[Tuple[Any, Callable]], is_fwd: bool) -> None: apply_hooks(bwd_hooks, False) yield self finally: - if reset_hooks_end: - for hook_point, direction in added_hooks: - hook_point.remove_hooks(dir=direction) + try: + if reset_hooks_end: + for hook_point, direction in added_hooks: + hook_point.remove_hooks(dir=direction, level=context_level) + finally: + self.context_level -= 1 return _hooks_context() @@ -1045,25 +1056,16 @@ def add_hook_to_point( if hook_point.name is not None: alias_names_list.append(hook_point.name) alias_names_list.append(name) - hook_point.add_hook(hook_fn, dir=dir, alias_names=alias_names_list) + hook_point.add_hook( + hook_fn, + dir=dir, + level=context_level, + alias_names=alias_names_list, + ) else: - hook_point.add_hook(hook_fn, dir=dir) + hook_point.add_hook(hook_fn, dir=dir, level=context_level) added_hooks.append((hook_point, dir)) - if stop_at_layer is not None and hasattr(self, "blocks"): - if stop_at_layer < 0: - stop_at_layer = len(self.blocks) + stop_at_layer - if stop_at_layer >= 0 and stop_at_layer < len(self.blocks): - - def stop_hook(tensor: Any, *, hook: Any) -> Any: - raise StopAtLayerException(tensor) - - # Stop at the beginning of the specified block, not at the end of the previous block - block_hook_name = f"blocks.{stop_at_layer}.hook_in" - hook_dict = self.hook_dict - if block_hook_name in hook_dict: - add_hook_to_point(hook_dict[block_hook_name], stop_hook, block_hook_name, "fwd") - def apply_hooks( hook_list: List[Tuple[Union[str, Callable], Callable]], is_fwd: bool ) -> None: @@ -1102,7 +1104,25 @@ def wrapped_hook_fn(tensor, hook, _orig_fn=original_hook_fn): hook_name_to_use = hook_point.name if hook_point.name else n add_hook_to_point(hook_point, hook_fn, hook_name_to_use, direction) + context_level = getattr(self, "context_level", 0) + 1 + self.context_level = context_level try: + if stop_at_layer is not None and hasattr(self, "blocks"): + if stop_at_layer < 0: + stop_at_layer = len(self.blocks) + stop_at_layer + if stop_at_layer >= 0 and stop_at_layer < len(self.blocks): + + def stop_hook(tensor: Any, *, hook: Any) -> Any: + raise StopAtLayerException(tensor) + + # Stop at the beginning of the specified block, not at the end of the previous block + block_hook_name = f"blocks.{stop_at_layer}.hook_in" + hook_dict = self.hook_dict + if block_hook_name in hook_dict: + add_hook_to_point( + hook_dict[block_hook_name], stop_hook, block_hook_name, "fwd" + ) + apply_hooks(fwd_hooks, True) apply_hooks(bwd_hooks, False) if start_at_layer is not None: @@ -1115,9 +1135,12 @@ def wrapped_hook_fn(tensor, hook, _orig_fn=original_hook_fn): output = e.layer_output return output finally: - if reset_hooks_end: - for hook_point, direction in added_hooks: - hook_point.remove_hooks(dir=direction) + try: + if reset_hooks_end: + for hook_point, direction in added_hooks: + hook_point.remove_hooks(dir=direction, level=context_level) + finally: + self.context_level -= 1 # ---- high-level execution: run_with_cache ---- @@ -1202,6 +1225,8 @@ def create_names_filter_fn(filter_input): cache: Dict[str, torch.Tensor] = {} hooks: List[Tuple[HookPoint, str]] = [] visited: set[int] = set() + stop_hook_point: Optional[HookPoint] = None + stop_hook_fn: Optional[Callable] = None # None → no-op .to(None), tensors stay on their current device. cache_device = kwargs.pop("device", None) @@ -1283,10 +1308,6 @@ def grad_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: return grad_hook - for hp, name in hooks: - hp.add_hook(make_cache_hook(name)) - if incl_bwd: - hp.add_hook(make_grad_cache_hook(name), dir="bwd") processed_args = [input] # Driver-aware input placement: torch drivers move input_ids to the model's # device; remote drivers (no local parameters) leave them as-is. @@ -1320,8 +1341,8 @@ def stop_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: block_hook_name = f"blocks.{stop_at_layer}.hook_in" hook_dict = self.hook_dict if block_hook_name in hook_dict: - hook_dict[block_hook_name].add_hook(stop_hook) - hooks.append((hook_dict[block_hook_name], block_hook_name)) + stop_hook_point = hook_dict[block_hook_name] + stop_hook_fn = stop_hook filtered_kwargs = kwargs.copy() # ``cache_device`` is honored by ``make_cache_hook`` above (``tensor.detach().to(cache_device)``); # the model and inputs stay where the caller put them, matching ``ActivationCache.to``. @@ -1338,6 +1359,21 @@ def stop_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: ) if start_at_layer is not None: filtered_kwargs["start_at_layer"] = start_at_layer + context_level = getattr(self, "context_level", 0) + 1 + self.context_level = context_level + try: + for hp, name in hooks: + hp.add_hook(make_cache_hook(name), level=context_level) + if incl_bwd: + hp.add_hook(make_grad_cache_hook(name), dir="bwd", level=context_level) + if stop_hook_point is not None and stop_hook_fn is not None: + stop_hook_point.add_hook(stop_hook_fn, level=context_level) + except Exception: + try: + self.remove_all_hook_fns(level=context_level) + finally: + self.context_level -= 1 + raise try: if ( "output_attentions" not in filtered_kwargs @@ -1377,8 +1413,10 @@ def stop_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: except Exception as e: raise e finally: - for hp, _ in hooks: - hp.remove_hooks(dir="both" if incl_bwd else "fwd") + try: + self.remove_all_hook_fns(level=context_level) + finally: + self.context_level -= 1 if self.compatibility_mode == True: reverse_aliases = {} for old_name, new_name in aliases.items(): From ac4f7f134b12bdf7d9f9a4d351264c98ed301176 Mon Sep 17 00:00:00 2001 From: Jonah Larson Date: Tue, 11 Aug 2026 12:22:21 -0500 Subject: [PATCH 49/87] Claude Code Architecture Adapter Creation Tool (#1641) * migration of the Claude Architecture Adapter creation tool * Improving solo system * improvied the solo system * Updating README, adding implementation information to docs * Readme improvements * accidentally kept a trailing hyphen * Allow the ability to easily override the model being used for the command prompts --- AGENTS.md | 3 +- CLAUDE.md | 1 + devtools/adapter_builder/.env.example | 18 + devtools/adapter_builder/.gitignore | 6 + devtools/adapter_builder/CLAUDE.md | 120 +++ devtools/adapter_builder/README.md | 214 ++++++ .../adapter_builder/agents/check-signals.sh | 49 ++ .../agents/hooks/gate-lint-checks.sh | 103 +++ .../agents/hooks/gate-reviewer-writes-file.sh | 134 ++++ .../adapter_builder/agents/hooks/guard-git.sh | 79 ++ .../agents/hooks/guard-hooked-transformer.sh | 70 ++ .../agents/hooks/guard-review-rounds.sh | 76 ++ .../agents/hooks/guard-verify-models.sh | 114 +++ .../agents/hooks/notify-on-completion.sh | 59 ++ .../agents/hooks/timeline-capture.sh | 79 ++ .../agents/launch-agent-pair.sh | 59 ++ .../agents/launch-solo-pair.sh | 483 ++++++++++++ devtools/adapter_builder/agents/launch.sh | 710 ++++++++++++++++++ devtools/adapter_builder/agents/lib/common.sh | 29 + devtools/adapter_builder/agents/ops.sh | 538 +++++++++++++ .../adapter_builder/agents/orchestrator.md | 71 ++ .../agents/overlord-request.sh | 150 ++++ devtools/adapter_builder/agents/programmer.md | 117 +++ devtools/adapter_builder/agents/progress.sh | 210 ++++++ devtools/adapter_builder/agents/reviewer.md | 81 ++ devtools/adapter_builder/agents/signals.sh | 84 +++ .../agents/solo-coordinator.sh | 281 +++++++ .../adapter_builder/agents/solo-programmer.md | 143 ++++ .../adapter_builder/agents/solo-reviewer.md | 111 +++ .../agents/watch-completion.sh | 160 ++++ .../docs/adapter-specification.md | 296 ++++++++ .../adapter_builder/docs/adapter-template.py | 162 ++++ .../docs/artifact-templates.md | 265 +++++++ .../adapter_builder/docs/cli-reference.md | 263 +++++++ .../docs/hf-model-analysis-guide.md | 168 +++++ .../adapter_builder/docs/hooks-reference.md | 155 ++++ devtools/adapter_builder/docs/memory-lock.md | 58 ++ .../docs/review-specification.md | 152 ++++ .../adapter_builder/docs/scripts-reference.md | 152 ++++ .../scripts/analyze-hf-model.py | 621 +++++++++++++++ .../scripts/compare-adapters.sh | 215 ++++++ .../adapter_builder/scripts/dry-run-test.sh | 404 ++++++++++ .../scripts/format-timeline.py | 73 ++ devtools/adapter_builder/scripts/notify.sh | 82 ++ .../scripts/port-arch-models.py | 81 ++ .../scripts/scan-hf-architecture.py | 140 ++++ .../adapter_builder/scripts/strip-adapter.py | 134 ++++ .../scripts/validate-adapter-deep.py | 330 ++++++++ .../scripts/validate-adapter.sh | 213 ++++++ .../scripts/validate-architecture.py | 183 +++++ .../_static/model_properties_table.jsonl | 493 ++++++------ .../adapter-builder-tool.md | 72 ++ docs/source/content/contributing.md | 2 + 53 files changed, 8819 insertions(+), 247 deletions(-) create mode 100644 devtools/adapter_builder/.env.example create mode 100644 devtools/adapter_builder/.gitignore create mode 100644 devtools/adapter_builder/CLAUDE.md create mode 100644 devtools/adapter_builder/README.md create mode 100755 devtools/adapter_builder/agents/check-signals.sh create mode 100755 devtools/adapter_builder/agents/hooks/gate-lint-checks.sh create mode 100755 devtools/adapter_builder/agents/hooks/gate-reviewer-writes-file.sh create mode 100755 devtools/adapter_builder/agents/hooks/guard-git.sh create mode 100755 devtools/adapter_builder/agents/hooks/guard-hooked-transformer.sh create mode 100755 devtools/adapter_builder/agents/hooks/guard-review-rounds.sh create mode 100755 devtools/adapter_builder/agents/hooks/guard-verify-models.sh create mode 100755 devtools/adapter_builder/agents/hooks/notify-on-completion.sh create mode 100755 devtools/adapter_builder/agents/hooks/timeline-capture.sh create mode 100755 devtools/adapter_builder/agents/launch-agent-pair.sh create mode 100755 devtools/adapter_builder/agents/launch-solo-pair.sh create mode 100755 devtools/adapter_builder/agents/launch.sh create mode 100644 devtools/adapter_builder/agents/lib/common.sh create mode 100755 devtools/adapter_builder/agents/ops.sh create mode 100644 devtools/adapter_builder/agents/orchestrator.md create mode 100755 devtools/adapter_builder/agents/overlord-request.sh create mode 100644 devtools/adapter_builder/agents/programmer.md create mode 100644 devtools/adapter_builder/agents/progress.sh create mode 100644 devtools/adapter_builder/agents/reviewer.md create mode 100644 devtools/adapter_builder/agents/signals.sh create mode 100755 devtools/adapter_builder/agents/solo-coordinator.sh create mode 100644 devtools/adapter_builder/agents/solo-programmer.md create mode 100644 devtools/adapter_builder/agents/solo-reviewer.md create mode 100755 devtools/adapter_builder/agents/watch-completion.sh create mode 100644 devtools/adapter_builder/docs/adapter-specification.md create mode 100644 devtools/adapter_builder/docs/adapter-template.py create mode 100644 devtools/adapter_builder/docs/artifact-templates.md create mode 100644 devtools/adapter_builder/docs/cli-reference.md create mode 100644 devtools/adapter_builder/docs/hf-model-analysis-guide.md create mode 100644 devtools/adapter_builder/docs/hooks-reference.md create mode 100644 devtools/adapter_builder/docs/memory-lock.md create mode 100644 devtools/adapter_builder/docs/review-specification.md create mode 100644 devtools/adapter_builder/docs/scripts-reference.md create mode 100755 devtools/adapter_builder/scripts/analyze-hf-model.py create mode 100755 devtools/adapter_builder/scripts/compare-adapters.sh create mode 100755 devtools/adapter_builder/scripts/dry-run-test.sh create mode 100755 devtools/adapter_builder/scripts/format-timeline.py create mode 100755 devtools/adapter_builder/scripts/notify.sh create mode 100755 devtools/adapter_builder/scripts/port-arch-models.py create mode 100755 devtools/adapter_builder/scripts/scan-hf-architecture.py create mode 100644 devtools/adapter_builder/scripts/strip-adapter.py create mode 100644 devtools/adapter_builder/scripts/validate-adapter-deep.py create mode 100755 devtools/adapter_builder/scripts/validate-adapter.sh create mode 100755 devtools/adapter_builder/scripts/validate-architecture.py create mode 100644 docs/source/content/adapter_development/adapter-builder-tool.md diff --git a/AGENTS.md b/AGENTS.md index a520c1b195..b67d87d796 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,6 +99,7 @@ Python: **>=3.10, <4.0**. CI tests 3.10, 3.11, 3.12. Format/type/docstring check | [demos/](demos/) | Jupyter notebooks; a subset runs in CI under `nbval` with sanitization from [demos/doc_sanitize.cfg](demos/doc_sanitize.cfg) | | [docs/source/content/](docs/source/content/) | Sphinx markdown sources | | [docs/source/content/adapter_development/](docs/source/content/adapter_development/) | Adapter-authoring guides — read these before adding a new architecture | +| [devtools/adapter_builder/](devtools/adapter_builder/) | Contributor-only agent-team adapter builder (not shipped in the package) — see its [README](devtools/adapter_builder/README.md) | | [makefile](makefile) | Canonical test/format/docs targets | | [pyproject.toml](pyproject.toml) | Deps, pytest / mypy / format / build config | | [.github/workflows/checks.yml](.github/workflows/checks.yml) | CI gates | @@ -113,7 +114,7 @@ Prefer Bridge-native names in new code. Raw-HF-forward drivers comparing against ## 6. Adding a model -Adapters are written **per architecture family**, not per individual model — adding `gpt2` registers all GPT-2 variants. Full workflow (starter-adapter table, 4-place registration, common gotchas, anti-patterns): **[supported_architectures/AGENTS.md](transformer_lens/model_bridge/supported_architectures/AGENTS.md)**. Verification flow: **[tools/model_registry/AGENTS.md](transformer_lens/tools/model_registry/AGENTS.md)**. Claude Code users: invoke `/add-model-support `. +Adapters are written **per architecture family**, not per individual model — adding `gpt2` registers all GPT-2 variants. Full workflow (starter-adapter table, 4-place registration, common gotchas, anti-patterns): **[supported_architectures/AGENTS.md](transformer_lens/model_bridge/supported_architectures/AGENTS.md)**. Verification flow: **[tools/model_registry/AGENTS.md](transformer_lens/tools/model_registry/AGENTS.md)**. Claude Code users: invoke `/add-model-support `. For batch/autonomous adapter creation there is an agent harness in [devtools/adapter_builder/](devtools/adapter_builder/README.md) — agent-teams mode needs Claude Code Max; its solo mode works on any tier. ## 7. Prioritization diff --git a/CLAUDE.md b/CLAUDE.md index f075c399b7..f09bae8c31 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,6 +26,7 @@ - [tests/QUARANTINES.md](tests/QUARANTINES.md) — check before debugging any failing test. The macOS-arm64 KV-cache skip is the most common time-sink. - [debugging_numerical_divergence.md](docs/source/content/debugging_numerical_divergence.md) — Bridge-vs-HF logit drift bisection. - [compatibility_mode.md](docs/source/content/compatibility_mode.md) — `bridge.enable_compatibility_mode()` contract; read before adding tests that use it. +- [devtools/adapter_builder/](devtools/adapter_builder/README.md) — autonomous adapter builder (contributor tooling; agent-teams mode needs Max, solo mode runs on any tier); manual path is `/add-model-support`. ## Starter tasks diff --git a/devtools/adapter_builder/.env.example b/devtools/adapter_builder/.env.example new file mode 100644 index 0000000000..64d2a8dcd1 --- /dev/null +++ b/devtools/adapter_builder/.env.example @@ -0,0 +1,18 @@ +HF_TOKEN=your_huggingface_token_here +DEFAULT_BASE_BRANCH=dev +# Target repo override. Optional — defaults to the repo containing this +# checkout (the builder lives in devtools/adapter_builder inside +# TransformerLens). Set only to drive a different checkout. +# DEFAULT_TARGET_REPO=/path/to/TransformerLens +DEFAULT_MAX_MEMORY_GB=48 +# Where agent pair worktrees are created. Optional — defaults to +# "/worktrees" if unset. +# WORKTREE_BASE=/path/to/worktrees +NOTIFICATION_WEBHOOK_URL=https://hooks.slack.com/services/YOUR/WEBHOOK/URL +NOTIFICATION_NUMBER=+15551234567 +# Per-agent model overrides. Optional — defaults come from the agent prompt +# files' frontmatter (agents/*.md). Applies to both modes; ORCHESTRATOR_MODEL +# is agent-teams only. +# PROGRAMMER_MODEL=claude-sonnet-4-6 +# REVIEWER_MODEL=claude-opus-4-6 +# ORCHESTRATOR_MODEL=claude-sonnet-4-6 diff --git a/devtools/adapter_builder/.gitignore b/devtools/adapter_builder/.gitignore new file mode 100644 index 0000000000..61781327dc --- /dev/null +++ b/devtools/adapter_builder/.gitignore @@ -0,0 +1,6 @@ +# Local configuration — contains HF_TOKEN and webhook URLs +.env +# Runtime output: PID files, debug logs, raw tmux dumps, wrapper scripts +.logs/ +__pycache__/ +.DS_Store diff --git a/devtools/adapter_builder/CLAUDE.md b/devtools/adapter_builder/CLAUDE.md new file mode 100644 index 0000000000..d47078eefa --- /dev/null +++ b/devtools/adapter_builder/CLAUDE.md @@ -0,0 +1,120 @@ +# TL Adapter Builder + +## Purpose + +This directory automates creating **Architecture Adapters** for the `TransformerBridge` system. It is a **control plane** — it holds agent definitions, domain knowledge, launch scripts, and tooling. It lives at `devtools/adapter_builder/` inside the TransformerLens repo but is contributor tooling, not part of the shipped package. Agents work in **git worktrees of this repo** created outside the checkout; nothing runs against the main working tree. + +For the manual (non-agent-team) path to the same outcome, see the repo's `/add-model-support` slash command — the adapter spec and registration checklist in the repo docs are canonical; this tool's docs cover orchestration only. + +## Project Layout + +```text +agents/ Agent definitions and orchestration + launch-agent-pair.sh User-facing dispatcher — routes to launch.sh, launch-solo-pair.sh, or ops.sh + launch.sh Launch flow — agent-teams mode (Max tier, orchestrator + subagents) + launch-solo-pair.sh Launch flow — solo mode (any tier, two independent sessions, file-based coordination) + ops.sh Ops subcommands (status/logs/attach/send/stop/clean) + watch-completion.sh Background watcher — auto-sends /exit on genuine completion + solo-coordinator.sh Solo-mode signal router daemon (consumes signals, wakes panes) + check-signals.sh Non-blocking signal state snapshot (solo re-orientation) + solo-programmer.md Programmer prompt for solo mode (signal + end turn) + solo-reviewer.md Reviewer prompt for solo mode (message-driven) + lib/common.sh Shared bash helpers (log/ok/warn/err/require_cmd) + orchestrator.md Orchestration prompt template ({{PLACEHOLDER}} tokens) + programmer.md Programmer agent (3-step lifecycle) + reviewer.md Reviewer agent (5-phase review, Opus model) + signals.sh Signal protocol — single source of truth + progress.sh Progress tracking for crash recovery + overlord-request.sh flock-based memory lock for heavy operations + hooks/ Claude Code hooks for auto-enforcement + timeline-capture.sh All events — structured JSONL logging + guard-hooked-transformer.sh PreToolUse — blocks edits to deprecated files + guard-git.sh PreToolUse — blocks `git commit`, `git push`, `gh pr create` + guard-review-rounds.sh PreToolUse — blocks review files past round 3 + guard-verify-models.sh PreToolUse — blocks verify_models on >7B or unregistered models + gate-reviewer-writes-file.sh SubagentStop — blocks reviewer results that have no file artifact + gate-lint-checks.sh Stop — blocks exit until mypy+format pass + notify-on-completion.sh SessionEnd — fires Slack notification on success +docs/ Domain knowledge for agents + adapter-specification.md What an adapter is and how to build one + adapter-template.py Skeleton adapter (Llama-style pattern) + artifact-templates.md File templates for all build artifacts (brief, plan, reviews, etc.) + memory-lock.md Memory lock protocol (flock-based, run subcommand only) + hf-model-analysis-guide.md How to analyze an HF model for adapter creation + review-specification.md Five-phase review methodology (source of truth) +scripts/ Tooling + analyze-hf-model.py Analyze HF model config, generate scaffold adapters + validate-architecture.py Pre-flight check: is this arch real? (transformers + HF Hub) + scan-hf-architecture.py Exhaustive HF scan for all models of an arch class + port-arch-models.py Merge per-arch model list into supported_models.json + validate-adapter.sh Structural + deep validation of adapters + validate-adapter-deep.py Semantic validation against real models (meta device) + compare-adapters.sh Structured diff between two existing adapters + format-timeline.py Render timeline.jsonl entries for status/logs + notify.sh Slack/iMessage/macOS notification on completion + strip-adapter.py Remove an adapter + registrations + registry entries (golden-master rebuild tests) + dry-run-test.sh Self-test suite for this project +``` + +## TransformerLens Repo + +- **Path:** the repo containing this checkout, derived automatically (`git rev-parse --show-toplevel`). Override with `--target-repo` or `DEFAULT_TARGET_REPO` in `.env` to drive a different checkout. +- **Key paths:** + - `transformer_lens/model_bridge/architecture_adapter.py` — base class + - `transformer_lens/model_bridge/supported_architectures/` — all existing adapters + - `transformer_lens/model_bridge/generalized_components/` — bridge components + - `transformer_lens/factories/architecture_adapter_factory.py` — adapter registry + - `transformer_lens/config/transformer_bridge_config.py` — TransformerBridgeConfig + +## Agent System + +Uses Claude Code experimental agent teams (`CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1`). + +**Lifecycle:** + +1. `launch-agent-pair.sh` creates a git worktree in the TransformerLens repo +2. **Step 1a:** Programmer analyzes architecture, writes brief (`.adapter-workspace/adapter-brief.md`), Reviewer fact-checks +3. **Step 1b:** Programmer writes phased plan (`.adapter-workspace/adapter-plan.md`), Reviewer approves +4. **Step 2:** Programmer implements phase-by-phase, Reviewer reviews each phase +5. **Step 3:** Programmer runs verify_models one model at a time, then mypy + format checks +6. Reviewer writes completion report, notification sent to Slack + +**Key constraints:** + +- Max 3 review rounds per checkpoint before escalating to user +- No git commits or pushes by agents — user commits manually +- `# type: ignore` not acceptable for mypy errors +- New bridge components only when forward pass is fundamentally different + +**Auto-enforced by Claude Code hooks** (in `.claude/settings.json` per worktree): + +- Timeline capture: every tool call + session event → `.adapter-workspace/timeline.jsonl` +- HookedTransformer guardrail: edits to deprecated files are blocked at the framework level +- Lint gate: session can't end until `mypy` + `make check-format` pass +- Completion notifier: Slack fires automatically on `verification_passed: true` + +**Memory lock:** flock-based (`overlord-request.sh`) at `/tmp/tl-adapter-builder.lock`. Only for verify_models and full model loading. 30-minute TTL for stale lock cleanup. + +**Crash recovery:** `.adapter-progress.json` in worktree tracks lifecycle step, completed phases, and verification attempts. Relaunching resumes from saved state. `--retry` flag is safe for planning/programming crashes; verification crashes require manual intervention. + +**Background mode (tmux):** `--background` runs the session inside a detached tmux session named `tl-adapter-`. This gives a real pty (so Claude's interactive mode works), a persistent process that survives terminal disconnects, and lets you attach/inject at any time via: + +- `./agents/launch-agent-pair.sh attach ` — interactive tmux attach +- `./agents/launch-agent-pair.sh send "message"` — inject a prompt as if you typed it + +Nothing is copied into the TransformerLens worktree except `.claude/agents/` (required by Claude Code agent teams) and `.claude/settings.json` (registers hooks). All docs, scripts, and tooling are accessed via absolute paths using the `TL_ADAPTER_BUILDER_ROOT` env var. + +## Environment + +- `.env` — see `.env.example` for all variables; gitignored, never commit it +- `HF_TOKEN` — HuggingFace API token (falls back to the repo root `.env` if unset here) +- `DEFAULT_TARGET_REPO` — optional override; defaults to the containing repo +- `DEFAULT_BASE_BRANCH` — default branch (dev-4.x) +- `DEFAULT_MAX_MEMORY_GB` — memory limit for verification (96) +- `WORKTREE_BASE` — where agent pair worktrees live; optional, defaults to `/worktrees` +- `NOTIFICATION_WEBHOOK_URL` — Slack webhook for notifications +- `NOTIFICATION_NUMBER` — iMessage fallback +- `PROGRAMMER_MODEL` / `REVIEWER_MODEL` / `ORCHESTRATOR_MODEL` — optional per-agent model overrides; default is the agent prompt frontmatter +- Python managed via `uv` + +Runtime output (PID files, debug logs, raw tmux dumps, wrapper scripts, orchestration prompts) is written to `.logs/` in the project root and is gitignored. diff --git a/devtools/adapter_builder/README.md b/devtools/adapter_builder/README.md new file mode 100644 index 0000000000..86ed252a28 --- /dev/null +++ b/devtools/adapter_builder/README.md @@ -0,0 +1,214 @@ +# TL Adapter Builder + +An AI agent team system that automates creating **Architecture Adapters** for the TransformerBridge system. This is **contributor tooling** — it lives in `devtools/adapter_builder/`, is not part of the shipped `transformer_lens` package, and is not supported API. For the manual path to the same outcome, use the repo's `/add-model-support` slash command. + +Give it a HuggingFace architecture class name, and a pair of Claude Code agents will plan the adapter, implement it phase-by-phase with code review at every step, and verify it against real models — all autonomously. It runs in one of two coordination modes: **agent-teams** (an orchestrator routes messages between agents; requires Max tier) or **solo** (two independent sessions coordinated by a signal-routing daemon; works on any tier). + +## Quick Start + +Run from anywhere inside the repo (paths shown from the repo root): + +```bash +# Interactive (foreground, agent-teams mode — Max tier) +devtools/adapter_builder/agents/launch-agent-pair.sh --architecture CohereForCausalLM + +# Solo mode (two sessions + coordinator daemon — works on any tier) +devtools/adapter_builder/agents/launch-agent-pair.sh --mode solo --architecture CohereForCausalLM + +# Pre-flight only: run all checks, create nothing +devtools/adapter_builder/agents/launch-agent-pair.sh --architecture CohereForCausalLM --dry-run + +# Background (detached tmux) — run multiple in parallel +devtools/adapter_builder/agents/launch-agent-pair.sh --architecture CohereForCausalLM --background +devtools/adapter_builder/agents/launch-agent-pair.sh --architecture CodeGenForCausalLM --background + +# Check status of all running pairs +devtools/adapter_builder/agents/launch-agent-pair.sh status + +# Tail the structured timeline for a specific architecture +devtools/adapter_builder/agents/launch-agent-pair.sh logs codegen + +# Clean up one worktree, or all verified-complete ones +devtools/adapter_builder/agents/launch-agent-pair.sh clean CodeGenForCausalLM +devtools/adapter_builder/agents/launch-agent-pair.sh clean +``` + +If the architecture already has an adapter in TransformerLens, the launcher exits immediately. If a worktree already exists for the branch, it resumes from `.adapter-progress.json`. Full CLI details in [`docs/cli-reference.md`](docs/cli-reference.md). + +## How It Works + +This directory is a **control plane**. It does not contain adapter code itself — it launches Claude Code agent teams that work in isolated git worktrees of this repository, created outside the checkout (default: `/worktrees`). The main working tree is never touched. **Nothing is copied into the worktree** except `.claude/agents/` (Claude Code tool config) and `.claude/settings.json` (hook registration). All docs, scripts, and tooling are accessed via absolute paths using `$TL_ADAPTER_BUILDER_ROOT`, which the launcher derives from its own location. + +### Coordination Modes + +Both modes run the same Programmer/Reviewer roles, lifecycle, and hooks — only the plumbing between the agents differs. + +- **Agent-teams** (default): a single Claude Code session with an Orchestrator routing messages to Programmer/Reviewer subagents. Requires Max tier and `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1`. +- **Solo** (`--mode solo`): two independent Claude Code sessions in one tmux window, coordinated by [`agents/solo-coordinator.sh`](agents/solo-coordinator.sh) — a daemon that consumes signal files from `.adapter-workspace/signals/` (archiving each one, so multi-round loops never see stale state) and wakes the counterpart pane with an injected message. Agents never poll or block: they touch a signal, end their turn, and get messaged when there's work. Per-role models are pinned from the solo prompt files' frontmatter. Full protocol in [`docs/cli-reference.md § Solo Mode`](docs/cli-reference.md#solo-mode---mode-solo). + +### The Lifecycle + +Every adapter goes through a strict sequence of checkpoints: + +1. **Step 1a — Architecture Analysis.** Programmer reads the HF Transformers source, extracts module paths and architectural properties per [`docs/hf-model-analysis-guide.md`](docs/hf-model-analysis-guide.md), generates a scaffold adapter via `analyze-hf-model.py --scaffold`, scans HuggingFace for every model of the target architecture via `scan-hf-architecture.py`, and writes an architecture brief. Reviewer independently re-reads the HF source and fact-checks the brief before any plan exists. +2. **Step 1b — Planning.** Using the validated brief, Programmer writes a phased implementation plan. New bridge components get their own phase. Reviewer checks design, completeness, and consistency with the brief. +3. **Step 2 — Programming.** Programmer implements one phase at a time, starting from the scaffold. Each phase goes through the Reviewer's five-phase review process ([`docs/review-specification.md`](docs/review-specification.md)) before the next begins. +4. **Step 3 — Verification.** Programmer ports the architecture's models into `supported_models.json` via `port-arch-models.py`, then runs `verify_models` one model at a time on up to 5 targets ≤7B, selected for **config diversity, not popularity** — always including the smallest model of the architecture (tiny models expose numerical-path divergence fastest), verified smallest-first so bugs fail in seconds rather than after a long large-model run. Verification means full HuggingFace parity (registry `status: 1`); structural-only runs (`--no-hf-reference`, status 4) are blocked by a hook and rejected by the final review. Any failure halts and returns to planning. On success, `mypy` and `make check-format` must pass before the session can end. + +The agent prompts are authoritative and point at the `docs/` files rather than inlining their content — see [`agents/programmer.md`](agents/programmer.md) and [`agents/reviewer.md`](agents/reviewer.md) for the full workflow. + +### The Agent Team + +| Agent | Role | Model | +| ----- | ---- | ----- | +| **Orchestrator** | Routes messages, drives the lifecycle, enforces iteration limits | claude-sonnet-4-6 | +| **Programmer** | Analyzes architectures, plans, writes code, runs verification | claude-sonnet-4-6 | +| **Reviewer** | Reviews briefs, plans, and code using a 5-phase review process | claude-opus-4-6 | + +In solo mode there is no Orchestrator — the coordinator daemon does the routing, and the Programmer/Reviewer prompts are the message-driven variants ([`agents/solo-programmer.md`](agents/solo-programmer.md), [`agents/solo-reviewer.md`](agents/solo-reviewer.md)). + +Agents **DO NOT** make git commits, push to remotes, or create pull requests. All changes stay uncommitted for manual review. This is enforced at the framework level by `guard-git.sh` (see below), not just by the prompts. The intention is that, even if Claude is helping you write the code, you should be reviewing every file & ensuring the code behaves as expected. **DO NOT** trust the LLM to do everything perfectly on the first try, it rarely does. + +### Runtime Enforcement via Claude Code Hooks + +Critical rules are enforced by Claude Code hooks, not by asking agents to remember them. The launcher installs `.claude/settings.json` in each worktree registering the hooks listed below. Full details, matchers, and block conditions are in [`docs/hooks-reference.md`](docs/hooks-reference.md). + +| Hook | Enforces | +| ---- | -------- | +| **guard-hooked-transformer** | Blocks writes to deprecated `HookedTransformer.py`, `loading_from_pretrained.py`, `components/`, `pretrained/weight_conversions/` | +| **guard-git** | Blocks `git commit`, `git push`, `gh pr create`, `gh release create` — agents cannot publish changes | +| **guard-review-rounds** | Blocks review files past round 3 per checkpoint, forcing escalation to the user when loops stall | +| **guard-verify-models** | Blocks `verify_models --model …` invocations targeting unregistered models, anything above `MAX_VERIFY_PARAMS` (default 7.5B), or `--no-hf-reference` (structural-only runs are not verification) | +| **gate-reviewer-writes-file** | Blocks reviewer subagent returns that didn't persist a review file to `.adapter-workspace/reviews/` (or `completion-report.md`) | +| **gate-lint-checks** | Refuses to let the session end until `mypy` and `make check-format` both pass | +| **timeline-capture** | Logs every tool call and session event to `.adapter-workspace/timeline.jsonl` (grepable, structured) | +| **notify-on-completion** | Fires the Slack notification automatically when `verification_passed: true` | + +All hooks are installed in both modes except **gate-reviewer-writes-file**, which hangs off the `SubagentStop` event and only applies in agent-teams mode (solo mode has no subagents; the reviewer prompt enforces file-before-signal instead). + +All hooks are installed in both modes except **gate-reviewer-writes-file**, which hangs off the `SubagentStop` event and only applies in agent-teams mode (solo mode has no subagents; the reviewer prompt enforces file-before-signal instead). + +The effect is that the prompt-level rules (no commits, 3-round limit, ≤7B verification) are also runtime-level stops. An agent that misreads the prompt gets blocked by the framework with a message pointing at the exact next action. + +### Crash Recovery + +`.adapter-progress.json` in the worktree tracks lifecycle step, plan approval status, current/completed phases, verification attempts, and the final result. Relaunching with the same `--architecture` reads this file and constructs a resume prompt telling agents exactly where they left off. + +Use `--retry` to explicitly resume a crashed session. It's safe for planning and programming crashes but refuses to auto-resume verification crashes (the registry may be in a partial state, and reruns can waste hours). Verification crashes trigger a "needs human review" notification and require manual inspection. + +### Background Mode + +`--background` launches Claude Code inside a detached `tl-adapter-` tmux session — real pty (so interactive mode works), persistent across terminal disconnects, attachable and injectable. Full details in [`docs/cli-reference.md § Background Mode`](docs/cli-reference.md#background-mode-tmux). + +## Project Structure + +```text +. +├── agents/ +│ ├── launch-agent-pair.sh # User-facing dispatcher — routes to launch.sh, launch-solo-pair.sh, or ops.sh +│ ├── launch.sh # Agent-teams launch flow — creates worktree, boots orchestrated pair +│ ├── launch-solo-pair.sh # Solo launch flow — two sessions + coordinator daemon +│ ├── solo-coordinator.sh # Solo signal router — consumes signals, wakes panes, handles completion +│ ├── check-signals.sh # Non-blocking signal state snapshot (solo re-orientation) +│ ├── ops.sh # Ops subcommands: status / logs / attach / send / stop / clean +│ ├── watch-completion.sh # Teams-mode completion watcher (solo mode: coordinator handles this) +│ ├── lib/common.sh # Shared bash helpers (log/ok/warn/err, require_cmd, iso_now) +│ ├── orchestrator.md # Teams orchestration prompt template ({{PLACEHOLDER}} tokens) +│ ├── programmer.md # Programmer agent, teams mode (3-step lifecycle) +│ ├── reviewer.md # Reviewer agent, teams mode (references docs/review-specification.md) +│ ├── solo-programmer.md # Programmer prompt, solo mode (signal + end turn) +│ ├── solo-reviewer.md # Reviewer prompt, solo mode (message-driven) +│ ├── signals.sh # Teams signal protocol — single source of truth +│ ├── progress.sh # Progress tracking for crash recovery +│ ├── overlord-request.sh # flock-based memory lock +│ └── hooks/ # Runtime-enforcement hooks (see docs/hooks-reference.md) +│ ├── timeline-capture.sh +│ ├── guard-hooked-transformer.sh +│ ├── guard-git.sh +│ ├── guard-review-rounds.sh +│ ├── guard-verify-models.sh +│ ├── gate-reviewer-writes-file.sh +│ ├── gate-lint-checks.sh +│ └── notify-on-completion.sh +├── docs/ +│ ├── adapter-specification.md # Full adapter spec, all bridge components, common patterns +│ ├── adapter-template.py # Skeleton adapter with TODOs (Llama-style) +│ ├── hf-model-analysis-guide.md # How to analyze an HF model for adapter creation +│ ├── review-specification.md # Five-phase review methodology (source of truth) +│ ├── cli-reference.md # Complete CLI reference (subcommands, flags, env vars, tmux) +│ ├── scripts-reference.md # Detailed script docs +│ └── hooks-reference.md # Hook catalog with triggers and rationale +├── scripts/ +│ ├── analyze-hf-model.py # Analyze HF model config, generate scaffold adapters +│ ├── validate-architecture.py # Pre-flight: does this architecture exist? (transformers + HF Hub) +│ ├── scan-hf-architecture.py # Exhaustive HF scan for all models of an arch class +│ ├── port-arch-models.py # Merge per-arch list into supported_models.json +│ ├── validate-adapter.sh # Structural + deep validation of adapters +│ ├── validate-adapter-deep.py # Semantic validation against real models (meta device) +│ ├── compare-adapters.sh # Structured diff between two existing adapters +│ ├── format-timeline.py # Render timeline.jsonl entries for status/logs +│ ├── notify.sh # Slack/iMessage/macOS notification on completion +│ ├── strip-adapter.py # Remove an adapter + registrations (golden-master rebuild tests) +│ └── dry-run-test.sh # Self-test suite for this project +├── .logs/ # Runtime output (gitignored) — PIDs, raw tmux dumps, debug logs +├── .env # Local defaults (see .env.example) +├── .env.example # Template with placeholder values +├── CLAUDE.md # Project context for Claude Code +└── README.md # This file +``` + +## Configuration + +Local defaults live in `.env`; see [`.env.example`](.env.example) for a template. + +| Variable | Default | Description | +| -------- | ------- | ----------- | +| `DEFAULT_TARGET_REPO` | containing repo | Optional override; defaults to the repo holding this checkout | +| `DEFAULT_BASE_BRANCH` | `dev-4.x` | Base branch for worktrees | +| `DEFAULT_MAX_MEMORY_GB` | `96` | Memory limit in GB for verify_models | +| `WORKTREE_BASE` | `/worktrees` | Where agent pair worktrees are created | +| `NOTIFICATION_WEBHOOK_URL` | — | Slack/Discord webhook URL for notifications | +| `NOTIFICATION_NUMBER` | — | Phone number for iMessage fallback (macOS only) | +| `HF_TOKEN` | repo root `.env` | HuggingFace API token; falls back to the repo root `.env` | +| `PROGRAMMER_MODEL` | prompt frontmatter | Override the Programmer agent's model (both modes) | +| `REVIEWER_MODEL` | prompt frontmatter | Override the Reviewer agent's model (both modes) | +| `ORCHESTRATOR_MODEL` | session default | Override the Orchestrator session's model (agent-teams only) | + +Runtime hook tunables (optional, for advanced use): `MAX_REVIEW_ROUNDS` (default 3) and `MAX_VERIFY_PARAMS` (default 7.5B). See [`docs/cli-reference.md § Hook Tunables`](docs/cli-reference.md#hook-tunables-environment-variables). + +## Prerequisites + +- **Claude Code** — agent-teams mode requires a Max subscription (experimental agent teams; sustained multi-agent sessions exceed Pro/Team limits). **Solo mode works on any tier**, including Pro/Team. +- **TransformerLens** repo cloned locally (this directory ships inside it) +- **git**, **claude**, **jq**, and **tmux** (required for solo mode and for teams `--background` mode) on `PATH` +- **Python 3** with `transformers` and `huggingface_hub` +- **uv** for running TransformerLens tests and verify_models +- **Sufficient system memory** — defaults to 96GB (`DEFAULT_MAX_MEMORY_GB`); adjustable via `--max-memory` + +## How Adapters Work + +An Architecture Adapter maps between a HuggingFace model's internal structure and TransformerLens's canonical component names. Every adapter defines: + +1. **Config attributes** — normalization type, positional embedding type, GQA support, etc. +2. **Component mapping** — maps TL names (`embed`, `blocks`, `attn`, `mlp`, etc.) to HF module paths via Bridge components +3. **Weight processing conversions** — tensor reshaping rules for loading HF weights into TL format + +For models with Grouped Query Attention (GQA), the adapter sets `n_key_value_heads` and weight conversions use `n_kv_heads` (not `n_heads`) for K/V rearrangement. + +The full spec is in [`docs/adapter-specification.md`](docs/adapter-specification.md). The skeleton template is in [`docs/adapter-template.py`](docs/adapter-template.py). TransformerLens currently supports 150+ architectures — see the [factory](https://github.com/TransformerLensOrg/TransformerLens/blob/main/transformer_lens/factories/architecture_adapter_factory.py) for the full list. + +## Testing + +- **Self-test suite**: `scripts/dry-run-test.sh` validates script syntax, signal/routing consistency (including the solo coordinator's full routing table and consume-on-read semantics via `solo-coordinator.sh --route ` and `--once `), launcher pre-flight behavior, and adapter validation against the live repo — without launching any agents. +- **Golden-master rebuild tests**: `scripts/strip-adapter.py` removes an existing adapter (module, registrations, unit tests, registry entries) from a test branch so the builder can be pointed at the architecture as if it were unsupported; the result is then diffed against the original. Use `--base-branch ` at launch — the launcher's already-supported check reads the factory file from the base branch, so stripped branches are handled correctly. +- **Protocol tests**: the solo launcher's `--programmer-task` / `--reviewer-task` overrides let you script short synthetic runs (e.g. forcing a `changes-N` review round) to exercise coordination paths end-to-end. + +## Further Reading + +- **[`docs/cli-reference.md`](docs/cli-reference.md)** — all subcommands, flags, env vars, tmux details, signal protocol +- **[`docs/scripts-reference.md`](docs/scripts-reference.md)** — every tool in `scripts/` with examples +- **[`docs/hooks-reference.md`](docs/hooks-reference.md)** — every hook in `agents/hooks/`, what it blocks, and why +- **[`docs/adapter-specification.md`](docs/adapter-specification.md)** — authoritative adapter spec (bridge components, patterns, registration checklist) +- **[`docs/hf-model-analysis-guide.md`](docs/hf-model-analysis-guide.md)** — how to extract architectural facts from an HF model +- **[`docs/review-specification.md`](docs/review-specification.md)** — five-phase review methodology used by the Reviewer agent +- **[`CLAUDE.md`](CLAUDE.md)** — project context file read by Claude Code when you run it in this repo diff --git a/devtools/adapter_builder/agents/check-signals.sh b/devtools/adapter_builder/agents/check-signals.sh new file mode 100755 index 0000000000..e6217b80a6 --- /dev/null +++ b/devtools/adapter_builder/agents/check-signals.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# ============================================================================= +# check-signals.sh — Non-blocking signal state snapshot for solo mode +# +# Agents never wait or poll for signals (the coordinator wakes them with an +# injected message). This helper is for re-orientation on wake-up or after a +# crash-resume: one call, instant output, no blocking. +# +# Prints: +# - pending signals (not yet consumed by the coordinator) +# - the most recent processed signals from the coordinator's archive +# - review round counts per checkpoint (from .adapter-workspace/reviews/) +# +# Usage (from the worktree root): +# "$TL_ADAPTER_BUILDER_ROOT/agents/check-signals.sh" +# ============================================================================= + +set -euo pipefail + +WORKSPACE="$(pwd)/.adapter-workspace" +SIGNALS_DIR="$WORKSPACE/signals" +ARCHIVE_DIR="$SIGNALS_DIR/.archive" +REVIEWS_DIR="$WORKSPACE/reviews" + +echo "== Pending signals (awaiting coordinator) ==" +if [[ -d "$SIGNALS_DIR" ]] && ls "$SIGNALS_DIR" 2>/dev/null | grep -q .; then + ls -tr "$SIGNALS_DIR" +else + echo "(none)" +fi + +echo "" +echo "== Recently processed (newest last) ==" +if [[ -d "$ARCHIVE_DIR" ]] && ls "$ARCHIVE_DIR" 2>/dev/null | grep -q .; then + # Archive names are .; show the last 10 in order. + ls "$ARCHIVE_DIR" | sort -n | tail -10 | sed -E 's/^[0-9]+\.//' +else + echo "(none)" +fi + +echo "" +echo "== Review rounds per checkpoint ==" +if [[ -d "$REVIEWS_DIR" ]] && ls "$REVIEWS_DIR"/*.md >/dev/null 2>&1; then + # brief-review-2.md -> "brief 2"; keep the highest round per checkpoint. + ls "$REVIEWS_DIR" | sed -nE 's/^(.+)-review-([0-9]+)\.md$/\1 \2/p' \ + | sort -k1,1 -k2,2n | awk '{last[$1]=$2} END {for (c in last) print c ": round " last[c]}' | sort +else + echo "(none)" +fi diff --git a/devtools/adapter_builder/agents/hooks/gate-lint-checks.sh b/devtools/adapter_builder/agents/hooks/gate-lint-checks.sh new file mode 100755 index 0000000000..b57f35472b --- /dev/null +++ b/devtools/adapter_builder/agents/hooks/gate-lint-checks.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# ============================================================================= +# gate-lint-checks.sh — Stop hook +# +# RUNTIME CONTEXT: +# This script lives under devtools/adapter_builder/agents/hooks/ but is executed +# by Claude Code *inside the target TransformerLens worktree* — the +# .claude/settings.json written by launch.sh references this file via an +# absolute path. `pwd` at runtime is the worktree, which is why +# WORKTREE_DIR="$(pwd)" and `uv run mypy .` resolve against the worktree, +# not this file's directory. +# +# Runs mypy and make check-format before allowing the session to end. +# Only runs when the progress file indicates verification has passed +# (we don't want to block mid-build stops for legitimate reasons). +# +# Exits with code 2 (deny stop) if checks fail, causing the agent to keep +# working until it fixes the issues. +# ============================================================================= + +set -euo pipefail + +# We run from the worktree (cwd when the session exits) +WORKTREE_DIR="$(pwd)" +PROGRESS_FILE="$WORKTREE_DIR/.adapter-progress.json" + +allow_stop() { + echo '{"continue": true}' + exit 0 +} + +# If no progress file, this isn't an adapter build — let it exit +[[ -f "$PROGRESS_FILE" ]] || allow_stop + +# Only gate when verification has passed; earlier stops are legitimate +verification_passed=$(jq -r '.verification_passed // false' "$PROGRESS_FILE" 2>/dev/null || echo "false") +if [[ "$verification_passed" != "true" ]]; then + allow_stop +fi + +# If we've already gated this session (final_review_passed), let it exit +final_review=$(jq -r '.final_review_passed // false' "$PROGRESS_FILE" 2>/dev/null || echo "false") +if [[ "$final_review" == "true" ]]; then + allow_stop +fi + +# Run mypy +mypy_output=$(uv run mypy . 2>&1) || mypy_exit=$? +mypy_exit="${mypy_exit:-0}" + +_lint_block() { + local reason="$1" + local tl="$WORKTREE_DIR/.adapter-workspace/timeline.jsonl" + if [[ -f "$tl" ]]; then + jq -n --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg hook "gate-lint-checks" --arg reason "$reason" \ + '{ts:$ts, event:"HookBlocked", tool:"Stop", hook:$hook, reason:$reason}' >> "$tl" 2>/dev/null || true + fi + python3 -c " +import json +print(json.dumps({'continue': False, 'decision': 'block', 'reason': '''$reason'''})) +" + exit 2 +} + +if [[ "$mypy_exit" != "0" ]]; then + mypy_tail=$(echo "$mypy_output" | tail -30) + _lint_block "BLOCKED: mypy checks failing. Fix type errors before completing. + +$mypy_tail + +Remember: # type: ignore is NOT an acceptable fix." +fi + +# Run format check +format_output=$(make check-format 2>&1) || format_exit=$? +format_exit="${format_exit:-0}" + +if [[ "$format_exit" != "0" ]]; then + format_tail=$(echo "$format_output" | tail -30) + _lint_block "BLOCKED: Format check failing. Run make format to fix. + +$format_tail" +fi + +# Both checks pass — mark final_review_passed so we don't re-run on subsequent stops. +# Uses fcntl.flock on the same co-located lock file as progress.sh. +python3 -c " +import json, fcntl +lockfile = '${PROGRESS_FILE%.json}.lock' +lockfd = open(lockfile, 'w') +try: + fcntl.flock(lockfd, fcntl.LOCK_EX) + p = json.load(open('$PROGRESS_FILE')) + p['final_review_passed'] = True + p['last_updated'] = '$(date -u +%Y-%m-%dT%H:%M:%SZ)' + json.dump(p, open('$PROGRESS_FILE','w'), indent=2) +finally: + fcntl.flock(lockfd, fcntl.LOCK_UN) + lockfd.close() +" 2>/dev/null + +echo '{"continue": true}' +exit 0 diff --git a/devtools/adapter_builder/agents/hooks/gate-reviewer-writes-file.sh b/devtools/adapter_builder/agents/hooks/gate-reviewer-writes-file.sh new file mode 100755 index 0000000000..abce946a80 --- /dev/null +++ b/devtools/adapter_builder/agents/hooks/gate-reviewer-writes-file.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# ============================================================================= +# gate-reviewer-writes-file.sh — SubagentStop hook for reviewer drift detection +# +# RUNTIME CONTEXT: +# This script lives under devtools/adapter_builder/agents/hooks/ but is executed +# by Claude Code *inside the target TransformerLens worktree* via the +# .claude/settings.json written by launch.sh. The hook payload is read +# from stdin and `pwd` at runtime is the worktree. +# +# reviewer.md § Feedback Format mandates that EVERY review must be written +# to a file in .adapter-workspace/reviews/ (or .adapter-workspace/completion-report.md +# for the final review) before the reviewer subagent returns. Empirically, +# reviewers have been returning review text in their subagent result WITHOUT +# persisting to disk, which leaves reviews/ empty, defeats the audit trail, +# and bypasses guard-review-rounds.sh (which counts review files). +# +# This hook fires on SubagentStop. When the stopped subagent is a reviewer +# AND no review file has been written in .adapter-workspace/reviews/ OR +# .adapter-workspace/completion-report.md since the matching SubagentStart +# timestamp, it: +# +# 1. Appends a `ReviewerFileDrift` event to timeline.jsonl so the drift +# is visible in subsequent `status`, `logs`, and post-hoc inspection. +# 2. Emits a `{continue: false, decision: "block", reason: …}` response +# pointing the orchestrator at the exact remediation (resume the +# reviewer with a `Resuming you — ` message telling it to write the +# review file before returning). +# +# The `continue: false` signal on SubagentStop may or may not be strictly +# blocking in Claude Code's agent-teams model, but the appended timeline +# event is durable and the `reason` field surfaces to the orchestrator via +# the hook output — either way the drift becomes visible and actionable. +# ============================================================================= + +set -euo pipefail + +payload=$(cat) + +event=$(echo "$payload" | jq -r '.hook_event_name // empty') +agent_type=$(echo "$payload" | jq -r '.agent_type // empty') +agent_id=$(echo "$payload" | jq -r '.agent_id // empty') + +allow() { + echo '{"continue": true}' + exit 0 +} + +# Only fire for reviewer SubagentStop events. +[[ "$event" != "SubagentStop" ]] && allow +[[ "$agent_type" != "reviewer" ]] && allow +[[ -z "$agent_id" ]] && allow + +WORKTREE_DIR="$(pwd)" +TIMELINE="$WORKTREE_DIR/.adapter-workspace/timeline.jsonl" +REVIEWS_DIR="$WORKTREE_DIR/.adapter-workspace/reviews" +COMPLETION_REPORT="$WORKTREE_DIR/.adapter-workspace/completion-report.md" + +# If there's no timeline we can't correlate — let it through. +[[ -f "$TIMELINE" ]] || allow + +# Find the matching SubagentStart timestamp for this agent_id. This is the +# earliest moment during which the reviewer could have written its file. +start_ts=$(python3 -c " +import json +last = None +for line in open('$TIMELINE'): + try: + e = json.loads(line) + except Exception: + continue + if e.get('event') == 'SubagentStart' and e.get('agent_id') == '$agent_id': + last = e.get('ts') # keep the latest matching SubagentStart +print(last or '') +" 2>/dev/null || echo "") + +# No matching start — can't tell if drift happened. Let it through. +[[ -z "$start_ts" ]] && allow + +# Check whether any review file or completion-report.md has been written +# or modified since start_ts. Uses file mtime compared against the parsed +# ISO 8601 timestamp. +new_files_written=$(python3 -c " +import pathlib +from datetime import datetime, timezone +try: + start = datetime.fromisoformat('$start_ts'.replace('Z', '+00:00')).timestamp() +except Exception: + print('unknown') + raise SystemExit(0) + +def any_modified_since(p, since): + if not p.exists(): + return False + if p.is_file(): + return p.stat().st_mtime >= since + if p.is_dir(): + for child in p.iterdir(): + if child.is_file() and child.stat().st_mtime >= since: + return True + return False + +reviews = pathlib.Path('$REVIEWS_DIR') +report = pathlib.Path('$COMPLETION_REPORT') +print('yes' if any_modified_since(reviews, start) or any_modified_since(report, start) else 'no') +" 2>/dev/null || echo "unknown") + +# If we found a new file, or can't determine, let it through. +[[ "$new_files_written" == "yes" ]] && allow +[[ "$new_files_written" == "unknown" ]] && allow + +# --- Drift detected ------------------------------------------------------- + +# Append a visible drift marker to the timeline so `status` and `logs` see it. +now_ts=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +python3 -c " +import json +with open('$TIMELINE', 'a') as f: + f.write(json.dumps({ + 'ts': '$now_ts', + 'event': 'ReviewerFileDrift', + 'agent_id': '$agent_id', + 'agent_type': 'reviewer', + 'tool': None, + 'tool_input': { + 'description': 'Reviewer subagent stopped without writing any review file since its SubagentStart at $start_ts', + }, + }) + '\n') +" 2>/dev/null || true + +reason="REVIEWER DRIFT: reviewer subagent $agent_id stopped without writing any file to .adapter-workspace/reviews/ or .adapter-workspace/completion-report.md between its SubagentStart at $start_ts and this SubagentStop. Per reviewer.md § Feedback Format, every review must be persisted to a file before returning — reviews that only exist in the subagent's result text are not valid and must not be accepted as approvals. REMEDIATION: (1) do NOT treat this reviewer's returned text as a decision; (2) re-engage the reviewer via SendMessage to=$agent_id with the body prefixed 'Resuming you — You did not write your review to a file. Write it to .adapter-workspace/reviews/.md (or completion-report.md for final reviews) BEFORE returning a decision. Your previous result is void.'; (3) then route the file-backed review through the normal approval flow." + +jq -n --arg reason "$reason" '{continue: false, decision: "block", reason: $reason}' +exit 2 diff --git a/devtools/adapter_builder/agents/hooks/guard-git.sh b/devtools/adapter_builder/agents/hooks/guard-git.sh new file mode 100755 index 0000000000..35fdd0966d --- /dev/null +++ b/devtools/adapter_builder/agents/hooks/guard-git.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# ============================================================================= +# guard-git.sh — PreToolUse hook for Bash +# +# RUNTIME CONTEXT: +# This script lives under devtools/adapter_builder/agents/hooks/ but is executed +# by Claude Code *inside the target TransformerLens worktree* via the +# .claude/settings.json written by launch.sh. The hook payload is read +# from stdin and the cwd at runtime is the worktree, not this file's +# directory. +# +# Blocks any Bash command that commits, pushes, or publishes changes. The +# user commits manually after reviewing the agent's uncommitted work — +# agents must never commit on their own. This is a belt-and-braces +# enforcement of the prompt-level "no git commits by agents" rule. +# +# Denied operations (exit 2 with {continue: false}): +# - git commit (any form, including -m, --amend, -a, etc.) +# - git push (any form) +# - gh pr create / merge / close / review +# - gh release create +# +# Everything else passes, including read-only git operations like status, +# diff, log, show, branch -l, worktree list. +# ============================================================================= + +set -euo pipefail + +payload=$(cat) + +tool_name=$(echo "$payload" | jq -r '.tool_name // empty') +command=$(echo "$payload" | jq -r '.tool_input.command // empty') + +allow() { + echo '{"continue": true}' + exit 0 +} + +block() { + # Append a HookBlocked marker to the timeline so the orchestrator can + # detect that a subagent's tool call was denied by a hook. + local tl="$(pwd)/.adapter-workspace/timeline.jsonl" + if [[ -f "$tl" ]]; then + jq -n --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg hook "guard-git" --arg reason "$1" \ + '{ts:$ts, event:"HookBlocked", tool:"Bash", hook:$hook, reason:$reason}' >> "$tl" 2>/dev/null || true + fi + jq -n --arg reason "$1" '{continue: false, decision: "block", reason: $reason}' + exit 2 +} + +# Only intercept Bash tool calls with a non-empty command. +[[ "$tool_name" != "Bash" ]] && allow +[[ -z "$command" ]] && allow + +# Word-boundary regexes. The leading anchor allows the command to appear at +# the start of the string, after whitespace, after a shell separator +# (;, &, |, (, `), or after &&/||. This catches chained invocations like +# `make test && git commit -m ...` without false-positives on things like +# `gitignore` or "commit" appearing in a commit-message string (those don't +# form a `git commit` token). +lead='(^|[[:space:];&|(`])' + +if echo "$command" | grep -qE "${lead}git[[:space:]]+commit([[:space:]]|$)"; then + block "BLOCKED: 'git commit' is not allowed. Agents must leave all changes uncommitted; the user commits manually after reviewing the diff. See agents/programmer.md § What NOT to Do." +fi + +if echo "$command" | grep -qE "${lead}git[[:space:]]+push([[:space:]]|$)"; then + block "BLOCKED: 'git push' is not allowed. Agents must not publish changes to any remote." +fi + +if echo "$command" | grep -qE "${lead}gh[[:space:]]+pr[[:space:]]+(create|merge|close|review|ready|edit)([[:space:]]|$)"; then + block "BLOCKED: Creating/merging/closing/reviewing PRs via 'gh pr' is not allowed. Agents must not publish changes." +fi + +if echo "$command" | grep -qE "${lead}gh[[:space:]]+release[[:space:]]+(create|delete|edit)([[:space:]]|$)"; then + block "BLOCKED: 'gh release' operations are not allowed. Agents must not publish releases." +fi + +allow diff --git a/devtools/adapter_builder/agents/hooks/guard-hooked-transformer.sh b/devtools/adapter_builder/agents/hooks/guard-hooked-transformer.sh new file mode 100755 index 0000000000..e413705dea --- /dev/null +++ b/devtools/adapter_builder/agents/hooks/guard-hooked-transformer.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# ============================================================================= +# guard-hooked-transformer.sh — PreToolUse hook +# +# RUNTIME CONTEXT: +# This script lives under devtools/adapter_builder/agents/hooks/ but is executed +# by Claude Code *inside the target TransformerLens worktree* — the +# .claude/settings.json written by launch.sh references this file via an +# absolute path. `pwd` at runtime is the worktree, NOT the directory +# containing this file. The deny_patterns below match against paths +# inside that worktree. +# +# Denies any Edit/Write tool call targeting deprecated HookedTransformer files. +# Agents may read these files (for reference) but must not modify them. +# +# The hook receives the tool call payload on stdin. If the tool is Edit/Write +# and the target file is a deprecated path, we exit with code 2 and print a +# message to stderr — Claude Code interprets this as a deny decision. +# ============================================================================= + +set -euo pipefail + +# Read the hook payload from stdin +payload=$(cat) + +# Extract the tool name and file path using jq (fall back gracefully if missing) +tool_name=$(echo "$payload" | jq -r '.tool_name // empty') +file_path=$(echo "$payload" | jq -r '.tool_input.file_path // .tool_input.path // empty') + +allow_response() { + echo '{"continue": true}' + exit 0 +} + +# Only intercept Edit, Write, MultiEdit, NotebookEdit operations +case "$tool_name" in + Edit|Write|MultiEdit|NotebookEdit) ;; + *) allow_response ;; +esac + +# No file path — let it through (some tools may not have one) +[[ -z "$file_path" ]] && allow_response + +# Check against the deprecated path patterns. Paths are matched as suffixes +# (the file_path from tool_input is typically absolute, so we check whether +# it ends with the pattern). This prevents false positives on unrelated paths +# like "my_transformer_lens_backup/foo.py". +deny_patterns=( + "transformer_lens/HookedTransformer.py" + "transformer_lens/loading_from_pretrained.py" + "transformer_lens/components/" + "transformer_lens/pretrained/weight_conversions/" +) + +for pattern in "${deny_patterns[@]}"; do + # Match: path ends with the pattern, or contains /pattern (subpath match). + # The leading / ensures we match at a directory boundary, not mid-word. + if [[ "$file_path" == */"$pattern"* ]] || [[ "$file_path" == "$pattern"* ]]; then + local reason="BLOCKED: This file is part of the deprecated HookedTransformer system. Adapter work must only modify files under transformer_lens/model_bridge/ and transformer_lens/factories/architecture_adapter_factory.py. HookedTransformer files may be READ for reference but not modified." + local tl="$(pwd)/.adapter-workspace/timeline.jsonl" + if [[ -f "$tl" ]]; then + jq -n --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg hook "guard-hooked-transformer" --arg reason "$reason" --arg path "$file_path" \ + '{ts:$ts, event:"HookBlocked", tool:"Edit", hook:$hook, file_path:$path, reason:$reason}' >> "$tl" 2>/dev/null || true + fi + jq -n --arg reason "$reason" '{continue: false, decision: "block", reason: $reason}' + exit 2 + fi +done + +allow_response diff --git a/devtools/adapter_builder/agents/hooks/guard-review-rounds.sh b/devtools/adapter_builder/agents/hooks/guard-review-rounds.sh new file mode 100755 index 0000000000..df1bf1d601 --- /dev/null +++ b/devtools/adapter_builder/agents/hooks/guard-review-rounds.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# ============================================================================= +# guard-review-rounds.sh — PreToolUse hook for Edit/Write/MultiEdit/NotebookEdit +# +# RUNTIME CONTEXT: +# This script lives under devtools/adapter_builder/agents/hooks/ but is executed +# by Claude Code *inside the target TransformerLens worktree* via the +# .claude/settings.json written by launch.sh. The hook payload is read +# from stdin. +# +# Enforces the 3-round review iteration limit from agents/orchestrator.md. +# Blocks any Write/Edit targeting a review file at round 4 or higher: +# .adapter-workspace/reviews/brief-review-4.md +# .adapter-workspace/reviews/plan-review-4.md +# .adapter-workspace/reviews/phase-A-review-4.md +# ... +# +# When triggered, the orchestrator must escalate unresolved issues to the +# user and terminate the session instead of looping further. This turns a +# prompt-level rule ("do not exceed 3 rounds") into a runtime stop. +# ============================================================================= + +set -euo pipefail + +# Hard ceiling of 5 prevents env override from disabling the guard entirely. +_env_rounds="${MAX_REVIEW_ROUNDS:-3}" +MAX_ROUNDS=$(( _env_rounds > 5 ? 5 : _env_rounds )) + +payload=$(cat) + +tool_name=$(echo "$payload" | jq -r '.tool_name // empty') +file_path=$(echo "$payload" | jq -r '.tool_input.file_path // .tool_input.path // empty') + +allow() { + echo '{"continue": true}' + exit 0 +} + +block() { + local tl="$(pwd)/.adapter-workspace/timeline.jsonl" + if [[ -f "$tl" ]]; then + jq -n --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg hook "guard-review-rounds" --arg reason "$1" \ + '{ts:$ts, event:"HookBlocked", tool:"Write", hook:$hook, reason:$reason}' >> "$tl" 2>/dev/null || true + fi + jq -n --arg reason "$1" '{continue: false, decision: "block", reason: $reason}' + exit 2 +} + +case "$tool_name" in + Write|Edit|MultiEdit|NotebookEdit) ;; + *) allow ;; +esac + +[[ -z "$file_path" ]] && allow + +# Only care about review files under .adapter-workspace/reviews/. +[[ "$file_path" != *".adapter-workspace/reviews/"* ]] && allow + +# Expected filename patterns (set by reviewer.md § Feedback Format): +# brief-review-.md +# plan-review-.md +# phase--review-.md +fname=$(basename "$file_path") +round=$(echo "$fname" | sed -nE 's/.*-review-([0-9]+)\.md$/\1/p') + +# Unrecognized filename → let it through (probably an unrelated write into +# the reviews/ directory; the reviewer owns the naming convention). +[[ -z "$round" ]] && allow + +if (( round > MAX_ROUNDS )); then + checkpoint=$(echo "$fname" | sed -nE 's/^(.*)-review-[0-9]+\.md$/\1/p') + reason=$(printf '%s' "BLOCKED: Review round ${round} exceeds the ${MAX_ROUNDS}-round iteration limit for '${checkpoint:-this checkpoint}'. Per agents/orchestrator.md § Iteration Limits, the orchestrator must now: (1) collect unresolved CRITICAL issues from the previous reviews, (2) write a summary to .adapter-workspace/stuck-report.md, (3) run the 'stuck' notification, and (4) terminate the session. Do NOT continue the review loop.") + block "$reason" +fi + +allow diff --git a/devtools/adapter_builder/agents/hooks/guard-verify-models.sh b/devtools/adapter_builder/agents/hooks/guard-verify-models.sh new file mode 100755 index 0000000000..d32a18251d --- /dev/null +++ b/devtools/adapter_builder/agents/hooks/guard-verify-models.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# ============================================================================= +# guard-verify-models.sh — PreToolUse hook for Bash +# +# RUNTIME CONTEXT: +# This script lives under devtools/adapter_builder/agents/hooks/ but is executed +# by Claude Code *inside the target TransformerLens worktree* via the +# .claude/settings.json written by launch.sh. The hook payload is read +# from stdin and the cwd at runtime is the worktree root, so relative +# paths to supported_models.json resolve against the worktree's +# transformer_lens/tools/model_registry/data/. +# +# Enforces two rules on verify_models invocations, turning prompt-level +# guidance into runtime stops: +# +# 1. The target model must be registered in supported_models.json. +# Otherwise verify_models silently skips it and the agent gets no +# signal (documented failure mode in agents/programmer.md Step 3.0). +# +# 2. The target model must be ≤ MAX_VERIFY_PARAMS (default 7.5B). Anything +# larger is a waste of verification time per the ≤7B rule in +# agents/programmer.md Step 3.1. +# +# Dry-run invocations (--dry-run) and bulk --architectures runs without a +# specific --model arg pass through: dry runs don't load weights, and bulk +# runs have verify_models' own memory-based skip. +# ============================================================================= + +set -euo pipefail + +MAX_PARAMS="${MAX_VERIFY_PARAMS:-7500000000}" # 7.5B gives a small buffer above "7B" models +REGISTRY="transformer_lens/tools/model_registry/data/supported_models.json" + +payload=$(cat) + +tool_name=$(echo "$payload" | jq -r '.tool_name // empty') +command=$(echo "$payload" | jq -r '.tool_input.command // empty') + +allow() { + echo '{"continue": true}' + exit 0 +} + +block() { + local tl="$(pwd)/.adapter-workspace/timeline.jsonl" + if [[ -f "$tl" ]]; then + jq -n --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg hook "guard-verify-models" --arg reason "$1" \ + '{ts:$ts, event:"HookBlocked", tool:"Bash", hook:$hook, reason:$reason}' >> "$tl" 2>/dev/null || true + fi + jq -n --arg reason "$1" '{continue: false, decision: "block", reason: $reason}' + exit 2 +} + +[[ "$tool_name" != "Bash" ]] && allow +[[ -z "$command" ]] && allow + +# Only intercept verify_models invocations. This matches both +# `verify_models` directly and `python -m transformer_lens.tools.model_registry.verify_models`. +echo "$command" | grep -qE 'verify_models(\b|[[:space:]]|$)' || allow + +# Dry runs are harmless — they don't load weights. +if echo "$command" | grep -qE '(^|[[:space:]])--dry-run(\b|[[:space:]]|=|$)'; then + allow +fi + +# Structural-only mode is never acceptable for adapter verification — it +# skips the HuggingFace parity comparison and writes status 4 (provisional), +# which the review gate does not count as verified. +if echo "$command" | grep -qE '(^|[[:space:]])--no-hf-reference(\b|[[:space:]]|=|$)'; then + block "BLOCKED: --no-hf-reference downgrades verification to structural-only (status 4 provisional). Adapter verification requires the HuggingFace parity comparison — rerun without this flag (the HF reference is on by default)." +fi + +# Extract --model or --model=. If absent, this is likely a bulk +# --architectures run; defer to verify_models' own filtering. +model=$(echo "$command" | sed -nE 's/.*--model[[:space:]]+([^[:space:]]+).*/\1/p' | head -1) +if [[ -z "$model" ]]; then + model=$(echo "$command" | sed -nE 's/.*--model=([^[:space:]]+).*/\1/p' | head -1) +fi +[[ -z "$model" ]] && allow + +if [[ ! -f "$REGISTRY" ]]; then + block "BLOCKED: $REGISTRY is missing. Run scripts/port-arch-models.py (Step 3.0 in agents/programmer.md) to populate the registry before calling verify_models." +fi + +# Look up the model's total_params. Returns one of: +# NOT_FOUND — not in registry +# — params count +# 0 — registered but no param metadata +lookup=$(python3 -c " +import json +try: + data = json.load(open('$REGISTRY')) +except Exception: + print('NOT_FOUND') + raise SystemExit(0) +for m in data.get('models', []): + if m.get('model_id') == '$model': + md = m.get('metadata') or {} + print(md.get('total_params') or 0) + raise SystemExit(0) +print('NOT_FOUND') +" 2>/dev/null) + +if [[ "$lookup" == "NOT_FOUND" ]]; then + block "BLOCKED: Model '$model' is not registered in $REGISTRY. Run scripts/scan-hf-architecture.py then scripts/port-arch-models.py to port this architecture's models into the registry (Step 1a.2 + Step 3.0 in agents/programmer.md), then retry." +fi + +if [[ "$lookup" =~ ^[0-9]+$ ]] && (( lookup > MAX_PARAMS )); then + params_b=$(python3 -c "print(f'{$lookup/1e9:.1f}B')" 2>/dev/null || echo "${lookup}") + cap_b=$(python3 -c "print(f'{$MAX_PARAMS/1e9:.1f}B')" 2>/dev/null || echo "${MAX_PARAMS}") + block "BLOCKED: Model '$model' has ${params_b} parameters, exceeding the ${cap_b} verification ceiling. Large models waste verification time without additional signal. Pick a smaller model from $REGISTRY, or if no models of this architecture fit, document the situation in .adapter-workspace/verification-note.md and signal 'VERIFICATION SKIPPED — ALL MODELS TOO LARGE' per agents/programmer.md Step 3.1." +fi + +allow diff --git a/devtools/adapter_builder/agents/hooks/notify-on-completion.sh b/devtools/adapter_builder/agents/hooks/notify-on-completion.sh new file mode 100755 index 0000000000..5c1acb6352 --- /dev/null +++ b/devtools/adapter_builder/agents/hooks/notify-on-completion.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# ============================================================================= +# notify-on-completion.sh — SessionEnd hook +# +# RUNTIME CONTEXT: +# This script lives under devtools/adapter_builder/agents/hooks/ but is executed +# by Claude Code *inside the target TransformerLens worktree* — the +# .claude/settings.json written by launch.sh references this file via an +# absolute path. `pwd` at runtime is the worktree, which is why +# WORKTREE_DIR="$(pwd)" points at .adapter-progress.json inside it. We use +# the TL_ADAPTER_BUILDER_ROOT env var (exported by launch.sh) to find the +# notify.sh helper back in the control-plane repo. +# +# Fires a Slack notification when the session ends IF the progress file +# shows verification has passed. Uses a sentinel file to avoid double-sending +# notifications on resumed sessions. +# ============================================================================= + +set -euo pipefail + +WORKTREE_DIR="$(pwd)" +PROGRESS_FILE="$WORKTREE_DIR/.adapter-progress.json" +SENT_MARKER="$WORKTREE_DIR/.adapter-workspace/notification-sent" + +# No progress file = not an adapter build +[[ -f "$PROGRESS_FILE" ]] || exit 0 + +# Already notified = skip +[[ -f "$SENT_MARKER" ]] && exit 0 + +verification_passed=$(jq -r '.verification_passed // false' "$PROGRESS_FILE" 2>/dev/null || echo "false") +architecture=$(jq -r '.architecture // "unknown"' "$PROGRESS_FILE" 2>/dev/null || echo "unknown") + +if [[ "$verification_passed" == "true" ]]; then + # Find the notify script via TL_ADAPTER_BUILDER_ROOT or relative + if [[ -n "${TL_ADAPTER_BUILDER_ROOT:-}" ]]; then + notify_script="$TL_ADAPTER_BUILDER_ROOT/scripts/notify.sh" + else + # Fall back: search upward for the scripts directory + notify_script="" + candidate="$WORKTREE_DIR" + for _ in 1 2 3 4; do + candidate=$(dirname "$candidate") + if [[ -f "$candidate/scripts/notify.sh" ]]; then + notify_script="$candidate/scripts/notify.sh" + break + fi + done + fi + + if [[ -x "$notify_script" ]]; then + "$notify_script" "Adapter completed for $architecture" >/dev/null 2>&1 || true + touch "$SENT_MARKER" + fi +fi + +# Emit a valid JSON response for Claude Code's hook protocol +echo '{"continue": true}' +exit 0 diff --git a/devtools/adapter_builder/agents/hooks/timeline-capture.sh b/devtools/adapter_builder/agents/hooks/timeline-capture.sh new file mode 100755 index 0000000000..73fb9a5f71 --- /dev/null +++ b/devtools/adapter_builder/agents/hooks/timeline-capture.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# ============================================================================= +# timeline-capture.sh — Generic hook for capturing events to the timeline +# +# RUNTIME CONTEXT: +# This script lives under devtools/adapter_builder/agents/hooks/ but is executed +# by Claude Code *inside the target TransformerLens worktree* via the +# .claude/settings.json written by launch.sh. When this script runs, `pwd` +# is the worktree, NOT the directory containing this file. All paths below +# are resolved relative to the worktree intentionally. +# +# Reads the hook payload from stdin and appends a JSON line to the worktree's +# .adapter-workspace/timeline.jsonl file. Captures timestamp, event name, +# agent info, tool name, and tool input (including bash commands). +# +# Registered by launch.sh on: SessionStart, SessionEnd, SubagentStart, +# SubagentStop, PreToolUse, PostToolUse. +# +# IMPORTANT: downstream hooks (gate-reviewer-writes-file.sh) depend on the +# timeline existing and being current. If this hook fails silently, those +# guards lose their data source and drift detection stops working. We +# therefore validate prerequisites and emit a warning to stderr (visible in +# the debug log) on failure rather than swallowing errors. +# +# Claude Code expects hook output on stdout to be valid JSON (typically +# {"continue": true}). We always emit that — a broken timeline must not +# block the agent's work, but we do log the failure. +# ============================================================================= + +set -euo pipefail + +WORKSPACE="$(pwd)/.adapter-workspace" +TIMELINE_FILE="$WORKSPACE/timeline.jsonl" + +ts=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +# Read stdin once (hooks can only read payload once) +payload=$(cat) + +# --- Validate prerequisites ------------------------------------------------ + +# The workspace directory must exist (launch.sh creates it at boot). +# If it's missing, something went wrong during setup. +if [[ ! -d "$WORKSPACE" ]]; then + echo "[timeline-capture] ERROR: $WORKSPACE does not exist — timeline cannot be written. Downstream guards (gate-reviewer-writes-file.sh) will not function." >&2 + echo '{"continue": true}' + exit 0 +fi + +# jq must be available for structured JSON extraction. +if ! command -v jq &>/dev/null; then + # Fall back to appending the raw payload with a timestamp prefix so we + # don't lose the event entirely. Downstream hooks that parse timeline + # fields may get malformed lines, but at least the file exists and has + # entries (which prevents false "empty timeline" conditions). + echo "[timeline-capture] WARNING: jq is not installed — appending raw payload to timeline. Install jq for proper structured capture." >&2 + echo "{\"ts\":\"$ts\",\"raw\":$(echo "$payload" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '""')}" >> "$TIMELINE_FILE" 2>/dev/null || true + echo '{"continue": true}' + exit 0 +fi + +# --- Append the structured timeline entry ---------------------------------- + +if ! echo "$payload" | jq -c --arg ts "$ts" '{ + ts: $ts, + event: .hook_event_name, + agent_id: .agent_id, + agent_type: .agent_type, + tool: .tool_name, + tool_input: .tool_input +}' >> "$TIMELINE_FILE" 2>/tmp/tl-timeline-err.$$; then + echo "[timeline-capture] ERROR: jq failed to process payload — $(cat /tmp/tl-timeline-err.$$ 2>/dev/null)" >&2 + rm -f /tmp/tl-timeline-err.$$ +fi +rm -f /tmp/tl-timeline-err.$$ 2>/dev/null + +# Always allow the tool call to proceed regardless of timeline errors. +echo '{"continue": true}' +exit 0 diff --git a/devtools/adapter_builder/agents/launch-agent-pair.sh b/devtools/adapter_builder/agents/launch-agent-pair.sh new file mode 100755 index 0000000000..757b2b58db --- /dev/null +++ b/devtools/adapter_builder/agents/launch-agent-pair.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# ============================================================================= +# launch-agent-pair.sh — User-facing entry point (dispatcher) +# +# Routes subcommands and launch modes: +# - Operational subcommands → ops.sh +# - --mode solo → launch-solo-pair.sh (file-based, any tier) +# - Default → launch.sh (agent-teams, Max tier) +# +# Usage: +# ./launch-agent-pair.sh --architecture [options] # agent-teams (Max tier) +# ./launch-agent-pair.sh --mode solo --architecture [opts] # file-based (any tier) +# ./launch-agent-pair.sh status # running pairs +# ./launch-agent-pair.sh logs [--raw] # tail timeline +# ./launch-agent-pair.sh attach # tmux attach +# ./launch-agent-pair.sh send # inject text +# ./launch-agent-pair.sh stop [architecture|all] # kill pair(s) +# ./launch-agent-pair.sh clean [architecture] # remove worktree +# ============================================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export TL_CMD="$0" + +# Check for --mode flag before dispatching +MODE="" +REMAINING_ARGS=() +for arg in "$@"; do + if [[ "$MODE" == "NEXT" ]]; then + MODE="$arg" + elif [[ "$arg" == "--mode" ]]; then + MODE="NEXT" + else + REMAINING_ARGS+=("$arg") + fi +done +# If --mode was the last arg with no value +[[ "$MODE" == "NEXT" ]] && MODE="" + +case "${1:-}" in + status|logs|stop|attach|send|clean) + exec "$SCRIPT_DIR/ops.sh" "$@" + ;; +esac + +case "$MODE" in + solo) + exec "$SCRIPT_DIR/launch-solo-pair.sh" "${REMAINING_ARGS[@]}" + ;; + team|"") + exec "$SCRIPT_DIR/launch.sh" "$@" + ;; + *) + echo "Unknown mode: $MODE (valid: solo, team)" >&2 + exit 1 + ;; +esac diff --git a/devtools/adapter_builder/agents/launch-solo-pair.sh b/devtools/adapter_builder/agents/launch-solo-pair.sh new file mode 100755 index 0000000000..8ee53eaea9 --- /dev/null +++ b/devtools/adapter_builder/agents/launch-solo-pair.sh @@ -0,0 +1,483 @@ +#!/usr/bin/env bash +# ============================================================================= +# launch-solo-pair.sh +# +# Launches two independent Claude Code sessions sharing a worktree: +# - Programmer: builds the adapter (analysis, planning, coding, verification) +# - Reviewer: reviews at each checkpoint (brief, plan, phases, final) +# +# The two sessions communicate through signal files in +# .adapter-workspace/signals/, routed by solo-coordinator.sh — a daemon that +# consumes each signal and wakes the counterpart pane with an injected +# message. Agents never block or poll. No agent-teams, no +# CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS. Works on Pro/Team tier. +# +# The same hooks, scripts, progress tracking, and memory lock infrastructure +# from the agent-teams mode apply — only the coordination mechanism changes. +# +# Usage: +# ./agents/launch-solo-pair.sh --architecture [options] +# +# Options: +# [--seed-model ] Known model for scaffolding +# [--target-repo ] Path to TransformerLens repo (default from .env) +# [--base-branch ] Base branch (default from .env) +# [--new-branch ] Feature branch (default: feature/-adapter) +# [--max-memory ] Memory limit in GB (default from .env) +# [--worktree-dir ] Where to create the worktree +# [--skip-arch-check] Skip architecture existence check +# [--auto-approve] Skip Claude Code permission prompts +# [--dry-run] Run all pre-flight checks, then exit without +# creating a worktree or launching sessions +# [--programmer-model ] Model for the programmer session +# (default: PROGRAMMER_MODEL env, then prompt frontmatter) +# [--reviewer-model ] Model for the reviewer session +# (default: REVIEWER_MODEL env, then prompt frontmatter) +# ============================================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +LOGS_DIR="$PROJECT_ROOT/.logs" + +TL_LOG_TAG="solo" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/lib/common.sh" + +TL_CMD="${TL_CMD:-$0}" + +usage() { + grep '^#' "$0" | sed 's/^# \{0,2\}//' | tail -n +2 + exit 1 +} + +# --------------------------------------------------------------------------- # +# Argument parsing +# --------------------------------------------------------------------------- # +TARGET_REPO="" +ARCHITECTURE="" +SEED_MODEL="" +BASE_BRANCH="" +NEW_BRANCH="" +MAX_MEMORY_GB="" +WORKTREE_DIR="" +AUTO_APPROVE=false +SKIP_ARCH_CHECK=false +DRY_RUN=false +PROGRAMMER_TASK_OVERRIDE="" +REVIEWER_TASK_OVERRIDE="" +PROGRAMMER_MODEL_ARG="" +REVIEWER_MODEL_ARG="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --target-repo) TARGET_REPO="$2"; shift 2 ;; + --architecture) ARCHITECTURE="$2"; shift 2 ;; + --seed-model) SEED_MODEL="$2"; shift 2 ;; + --base-branch) BASE_BRANCH="$2"; shift 2 ;; + --new-branch) NEW_BRANCH="$2"; shift 2 ;; + --max-memory) MAX_MEMORY_GB="$2"; shift 2 ;; + --worktree-dir) WORKTREE_DIR="$2"; shift 2 ;; + --skip-arch-check) SKIP_ARCH_CHECK=true; shift ;; + --auto-approve) AUTO_APPROVE=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; + --programmer-task) PROGRAMMER_TASK_OVERRIDE="$2"; shift 2 ;; + --reviewer-task) REVIEWER_TASK_OVERRIDE="$2"; shift 2 ;; + --programmer-model) PROGRAMMER_MODEL_ARG="$2"; shift 2 ;; + --reviewer-model) REVIEWER_MODEL_ARG="$2"; shift 2 ;; + -h|--help) usage ;; + *) err "Unknown argument: $1" ;; + esac +done + +# Load defaults from .env +ENV_FILE="$PROJECT_ROOT/.env" +if [[ -f "$ENV_FILE" ]]; then + # shellcheck disable=SC1090 + source "$ENV_FILE" +fi + +# The builder lives in devtools/adapter_builder inside TransformerLens, so the +# containing repo is the default target. --target-repo / DEFAULT_TARGET_REPO +# remain as overrides for driving a different checkout. +TARGET_REPO="${TARGET_REPO:-${DEFAULT_TARGET_REPO:-}}" +if [[ -z "$TARGET_REPO" ]]; then + TARGET_REPO="$(git -C "$PROJECT_ROOT" rev-parse --show-toplevel 2>/dev/null)" \ + || err "Could not derive the TransformerLens repo root (pass --target-repo or set DEFAULT_TARGET_REPO in .env)." +fi +[[ -z "$ARCHITECTURE" ]] && err "--architecture is required" + +BASE_BRANCH="${BASE_BRANCH:-${DEFAULT_BASE_BRANCH:-dev-4.x}}" +MAX_MEMORY_GB="${MAX_MEMORY_GB:-${DEFAULT_MAX_MEMORY_GB:-96}}" + +if [[ -z "$NEW_BRANCH" ]]; then + ARCH_SHORT=$(echo "$ARCHITECTURE" | sed -E 's/(For(Causal|Conditional|Masked).*|LMHead.*)$//' | tr '[:upper:]' '[:lower:]') + NEW_BRANCH="feature/${ARCH_SHORT}-adapter" +fi + +# --------------------------------------------------------------------------- # +# Sanity checks +# --------------------------------------------------------------------------- # +require_cmd git +require_cmd claude +require_cmd tmux +require_cmd jq + +REPO_ROOT="$(cd "$TARGET_REPO" && git rev-parse --show-toplevel 2>/dev/null)" \ + || err "--target-repo '$TARGET_REPO' is not a git repository." + +WORKTREE_BASE="${WORKTREE_BASE:-$(dirname "$REPO_ROOT")/worktrees}" + +# --------------------------------------------------------------------------- # +# Architecture validation +# --------------------------------------------------------------------------- # +if [[ -n "$SEED_MODEL" && "$SKIP_ARCH_CHECK" == false ]]; then + log "Seed model '${SEED_MODEL}' provided — skipping architecture existence check." + SKIP_ARCH_CHECK=true +fi + +# The worktree is created from BASE_BRANCH, so the "already supported" check +# must read the factory file from that branch — not the main checkout's +# working tree, which may differ (e.g. golden-master test branches with the +# adapter stripped). Fall back to the on-disk file if the branch ref can't +# be read; the branch is hard-validated later. +FACTORY_REL="transformer_lens/factories/architecture_adapter_factory.py" +FACTORY_CONTENT="$(git -C "$REPO_ROOT" show "${BASE_BRANCH}:${FACTORY_REL}" 2>/dev/null \ + || cat "$REPO_ROOT/$FACTORY_REL" 2>/dev/null || true)" +if [[ -n "$FACTORY_CONTENT" ]] && grep -q "\"${ARCHITECTURE}\"" <<< "$FACTORY_CONTENT"; then + ok "Architecture '${ARCHITECTURE}' already has an adapter on branch '${BASE_BRANCH}'." + log "No work needed. Exiting." + exit 0 +fi + +if [[ "$SKIP_ARCH_CHECK" == true ]]; then + warn "Skipping architecture existence check." +else + log "Validating '${ARCHITECTURE}'..." + set +e + env -u VIRTUAL_ENV uv run --project "$REPO_ROOT" \ + python "$PROJECT_ROOT/scripts/validate-architecture.py" "$ARCHITECTURE" + validate_exit=$? + set -e + case "$validate_exit" in + 0) ok "Architecture validated." ;; + 1) err "Architecture '${ARCHITECTURE}' does not exist. Check spelling or use --skip-arch-check." ;; + 2) warn "Could not fully verify '${ARCHITECTURE}'. Proceeding." ;; + esac +fi + +# --------------------------------------------------------------------------- # +# Worktree setup +# --------------------------------------------------------------------------- # +if [[ -z "$WORKTREE_DIR" ]]; then + SAFE_BRANCH=$(echo "$NEW_BRANCH" | tr '/' '-') + WORKTREE_DIR="${WORKTREE_BASE}/${SAFE_BRANCH}" +fi + +git -C "$REPO_ROOT" rev-parse --verify "$BASE_BRANCH" &>/dev/null \ + || err "Base branch '$BASE_BRANCH' does not exist." + +# Per-role model resolution, highest precedence first: CLI flag > env var +# (settable in .env) > prompt-file frontmatter default. The frontmatter is +# stripped before the content is passed as a prompt, so the model must be +# re-applied via --model or both roles silently run the session default. +_frontmatter_model() { sed -n 's/^model:[[:space:]]*//p' "$1" | head -1; } +PROG_MODEL="${PROGRAMMER_MODEL_ARG:-${PROGRAMMER_MODEL:-$(_frontmatter_model "$SCRIPT_DIR/solo-programmer.md")}}" +REV_MODEL="${REVIEWER_MODEL_ARG:-${REVIEWER_MODEL:-$(_frontmatter_model "$SCRIPT_DIR/solo-reviewer.md")}}" +PROG_MODEL_FLAG=""; [[ -n "$PROG_MODEL" ]] && PROG_MODEL_FLAG="--model $PROG_MODEL" +REV_MODEL_FLAG=""; [[ -n "$REV_MODEL" ]] && REV_MODEL_FLAG="--model $REV_MODEL" + +if [[ "$DRY_RUN" == true ]]; then + ok "Pre-flight passed (dry run) — no worktree created, no sessions launched." + log " would create : $WORKTREE_DIR (branch $NEW_BRANCH from $BASE_BRANCH)" + log " architecture : $ARCHITECTURE" + log " seed model : ${SEED_MODEL:-none}" + log " programmer : ${PROG_MODEL:-session default}" + log " reviewer : ${REV_MODEL:-session default}" + exit 0 +fi + +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/progress.sh" + +REUSE_WORKTREE=false +if [[ -d "$WORKTREE_DIR" ]] && git -C "$REPO_ROOT" rev-parse --verify "$NEW_BRANCH" &>/dev/null; then + ok "Existing worktree found — resuming." + REUSE_WORKTREE=true +else + log "Creating worktree at: $WORKTREE_DIR" + + _cleanup_partial_worktree() { + warn "Cleaning up partial worktree..." + git -C "$REPO_ROOT" worktree remove --force "$WORKTREE_DIR" 2>/dev/null || true + git -C "$REPO_ROOT" branch -D "$NEW_BRANCH" 2>/dev/null || true + } + + if ! git -C "$REPO_ROOT" worktree add -b "$NEW_BRANCH" "$WORKTREE_DIR" "$BASE_BRANCH" 2>/tmp/tl-wt-err.$$; then + wt_err=$(cat /tmp/tl-wt-err.$$ 2>/dev/null); rm -f /tmp/tl-wt-err.$$ + err "Failed to create worktree: $wt_err" + fi + rm -f /tmp/tl-wt-err.$$ + trap '_cleanup_partial_worktree' ERR + + ok "Worktree created." + progress_init "$WORKTREE_DIR" "$ARCHITECTURE" + ok "Progress tracking initialized." + trap - ERR +fi + +# --------------------------------------------------------------------------- # +# Workspace and signals directory +# --------------------------------------------------------------------------- # +mkdir -p "$WORKTREE_DIR/.adapter-workspace/reviews" +mkdir -p "$WORKTREE_DIR/.adapter-workspace/phase-reports" +mkdir -p "$WORKTREE_DIR/.adapter-workspace/signals" +touch "$WORKTREE_DIR/.adapter-workspace/adapter-brief.md" 2>/dev/null || true +touch "$WORKTREE_DIR/.adapter-workspace/adapter-plan.md" 2>/dev/null || true +touch "$WORKTREE_DIR/.adapter-workspace/adapter-scaffold.py" 2>/dev/null || true +touch "$WORKTREE_DIR/.adapter-workspace/timeline.jsonl" 2>/dev/null || true + +# --------------------------------------------------------------------------- # +# Install hooks (same as agent-teams mode) +# --------------------------------------------------------------------------- # +SETTINGS_FILE="$WORKTREE_DIR/.claude/settings.json" +mkdir -p "$WORKTREE_DIR/.claude" + +_hook_cmd() { echo "'$SCRIPT_DIR/hooks/$1'"; } + +TIMELINE_HOOK=$(_hook_cmd timeline-capture.sh) +GUARD_HT_HOOK=$(_hook_cmd guard-hooked-transformer.sh) +GUARD_GIT_HOOK=$(_hook_cmd guard-git.sh) +GUARD_REVIEW_ROUNDS_HOOK=$(_hook_cmd guard-review-rounds.sh) +GUARD_VERIFY_MODELS_HOOK=$(_hook_cmd guard-verify-models.sh) +GATE_LINT_HOOK=$(_hook_cmd gate-lint-checks.sh) +NOTIFY_HOOK=$(_hook_cmd notify-on-completion.sh) + +_h() { jq -n --arg cmd "$1" '{"type":"command","command":$cmd}'; } + +jq -n \ + --argjson session_start "[{\"hooks\":[ $(_h "$TIMELINE_HOOK") ]}]" \ + --argjson session_end "[{\"hooks\":[ $(_h "$TIMELINE_HOOK"), $(_h "$NOTIFY_HOOK") ]}]" \ + --argjson pre_edit "{\"matcher\":\"Edit|Write|MultiEdit|NotebookEdit\",\"hooks\":[ $(_h "$GUARD_HT_HOOK"), $(_h "$GUARD_REVIEW_ROUNDS_HOOK") ]}" \ + --argjson pre_bash "{\"matcher\":\"Bash\",\"hooks\":[ $(_h "$GUARD_GIT_HOOK"), $(_h "$GUARD_VERIFY_MODELS_HOOK") ]}" \ + --argjson pre_all "{\"matcher\":\"\",\"hooks\":[ $(_h "$TIMELINE_HOOK") ]}" \ + --argjson post_all "[{\"matcher\":\"\",\"hooks\":[ $(_h "$TIMELINE_HOOK") ]}]" \ + --argjson stop "[{\"hooks\":[ $(_h "$GATE_LINT_HOOK") ]}]" \ + '{hooks:{ + SessionStart: $session_start, + SessionEnd: $session_end, + PreToolUse: [$pre_edit, $pre_bash, $pre_all], + PostToolUse: $post_all, + Stop: $stop + }}' > "$SETTINGS_FILE" + +ok "Hooks installed." + +# --------------------------------------------------------------------------- # +# Pre-seed workspace trust for the worktree +# +# Claude Code shows a "Do you trust this folder?" prompt on first launch in a +# new directory. `--dangerously-skip-permissions` does NOT bypass it (only +# `-p`/print mode does). A fresh worktree is an unknown path, so without +# pre-seeding, both panes hang at the trust prompt forever and coordination +# never starts. We atomically merge a trust entry into ~/.claude.json. +# --------------------------------------------------------------------------- # +CLAUDE_JSON="$HOME/.claude.json" +if [[ -f "$CLAUDE_JSON" ]]; then + TRUST_TMP=$(mktemp) + jq --arg path "$WORKTREE_DIR" ' + .projects //= {} | + .projects[$path] = ((.projects[$path] // {}) + { + hasTrustDialogAccepted: true, + hasClaudeMdExternalIncludesApproved: true, + hasClaudeMdExternalIncludesWarningShown: true, + projectOnboardingSeenCount: 1 + }) + ' "$CLAUDE_JSON" > "$TRUST_TMP" && mv "$TRUST_TMP" "$CLAUDE_JSON" + ok "Workspace trust pre-seeded for $WORKTREE_DIR" +else + warn "~/.claude.json not found — trust dialog may appear in tmux panes." +fi + +# --------------------------------------------------------------------------- # +# Build prompts +# --------------------------------------------------------------------------- # +SEED_MODEL_BLOCK="" +if [[ -n "$SEED_MODEL" ]]; then + SEED_MODEL_BLOCK=" +Seed model: ${SEED_MODEL} +Use this model for scaffold generation and as the primary HF source reference. +Include it in your verification targets." +fi + +PROGRAMMER_TASK="Create an Architecture Adapter for: ${ARCHITECTURE} +${SEED_MODEL_BLOCK} +Memory limit: ${MAX_MEMORY_GB}GB" +[[ -n "$PROGRAMMER_TASK_OVERRIDE" ]] && PROGRAMMER_TASK="$PROGRAMMER_TASK_OVERRIDE" + +REVIEWER_TASK="Review the adapter for architecture: ${ARCHITECTURE}" +[[ -n "$REVIEWER_TASK_OVERRIDE" ]] && REVIEWER_TASK="$REVIEWER_TASK_OVERRIDE" + +# Read base prompts and append session-specific instructions +SAFE_BRANCH_LOG=$(echo "$NEW_BRANCH" | tr '/' '-') +mkdir -p "$LOGS_DIR" + +PROG_PROMPT_FILE="$LOGS_DIR/${SAFE_BRANCH_LOG}.programmer-prompt.txt" +REV_PROMPT_FILE="$LOGS_DIR/${SAFE_BRANCH_LOG}.reviewer-prompt.txt" + +# Strip the YAML frontmatter (first `---` to next `---`) before writing the +# prompt file. The frontmatter is only meaningful when the .md is read as +# an agent definition by Claude Code's agent-teams mode. When we pass the +# file content directly as a prompt string, the leading `---` is parsed as +# a command-line flag and claude exits with "error: unknown option '---". +_strip_frontmatter() { + # Delete lines from the first `---` through the next `---`. + awk 'BEGIN{in_fm=0; done=0} /^---$/ && done==0 { if(in_fm==0){in_fm=1; next} else {in_fm=0; done=1; next} } in_fm==0 && done==1 {print} in_fm==0 && done==0 && !/^---$/ {print}' "$1" +} + +_strip_frontmatter "$SCRIPT_DIR/solo-programmer.md" > "$PROG_PROMPT_FILE" +printf '\n---\n## Session Task\n\n%s\n' "$PROGRAMMER_TASK" >> "$PROG_PROMPT_FILE" + +_strip_frontmatter "$SCRIPT_DIR/solo-reviewer.md" > "$REV_PROMPT_FILE" +printf '\n---\n## Session Task\n\n%s\n' "$REVIEWER_TASK" >> "$REV_PROMPT_FILE" + +# --------------------------------------------------------------------------- # +# Launch two Claude Code sessions in tmux +# --------------------------------------------------------------------------- # +TMUX_SESSION="tl-adapter-$SAFE_BRANCH_LOG" +PID_FILE="$LOGS_DIR/${SAFE_BRANCH_LOG}.pid" +PROG_LOG="$LOGS_DIR/${SAFE_BRANCH_LOG}.programmer.raw" +REV_LOG="$LOGS_DIR/${SAFE_BRANCH_LOG}.reviewer.raw" +WATCH_LOG="$LOGS_DIR/${SAFE_BRANCH_LOG}.watch.log" + +# Use a dedicated tmux socket (`-L tl-adapter`) so our sessions don't share a +# server with any tmux session the user already has running. Without this, +# if a pane crashes or emits an unexpected control sequence, it could kill +# the user's entire tmux session. TM() wraps tmux with the socket flag. +TMUX_SOCKET="tl-adapter" +TM() { tmux -L "$TMUX_SOCKET" "$@"; } + +# Kill any stale session on our dedicated socket +TM kill-session -t "$TMUX_SESSION" 2>/dev/null || true + +CLAUDE_FLAGS=( + "--dangerously-skip-permissions" + "--permission-mode" "bypassPermissions" +) +if [[ "$AUTO_APPROVE" == true ]]; then + CLAUDE_FLAGS+=("--allowedTools" "bash" "read" "write" "edit" "glob" "grep") +fi + +# Common environment exports. +# +# IMPORTANT: the outer shell may itself be running inside a Claude Code +# session (e.g. the user is running this launcher from within Claude Code). +# If so, CLAUDECODE, CLAUDE_CODE_ENTRYPOINT, CLAUDE_CODE_EXECPATH, and +# MCP_CONNECTION_NONBLOCKING leak into the spawned claude and confuse it — +# a nested Claude Code may try to connect to the parent's SSE port or +# behave oddly during startup. Unset them to guarantee a clean session. +EXPORT_BLOCK=" +unset CLAUDECODE CLAUDE_CODE_ENTRYPOINT CLAUDE_CODE_EXECPATH CLAUDE_CODE_SSE_PORT CLAUDE_CODE_ENABLE_SDK_FILE_CHECKPOINTING MCP_CONNECTION_NONBLOCKING +export TL_ADAPTER_BUILDER_ROOT='$PROJECT_ROOT' +export MAX_MEMORY_GB='$MAX_MEMORY_GB' +export NOTIFICATION_WEBHOOK_URL='${NOTIFICATION_WEBHOOK_URL:-}' +export NOTIFICATION_NUMBER='${NOTIFICATION_NUMBER:-}' +export HF_TOKEN='${HF_TOKEN:-}' +export HUGGING_FACE_HUB_TOKEN='${HF_TOKEN:-}' +cd '$WORKTREE_DIR' +" + +# Build wrapper scripts for each pane +PROG_WRAPPER="$LOGS_DIR/${SAFE_BRANCH_LOG}.programmer-wrapper.sh" +cat > "$PROG_WRAPPER" < "$REV_WRAPPER" </dev/null || true + +# Pipe each pane's output to its own raw log +TM pipe-pane -t "$TMUX_SESSION:0.0" -O "python3 -u -c \" +import sys, os +f = open('$PROG_LOG', 'a', buffering=1) +n = 0 +for line in sys.stdin: + f.write(line) + n += 1 + if n % 500 == 0: + try: + if os.path.getsize('$PROG_LOG') > 50_000_000: + f.close() + os.replace('$PROG_LOG', '$PROG_LOG.1') + f = open('$PROG_LOG', 'a', buffering=1) + except OSError: + pass +\"" + +TM pipe-pane -t "$TMUX_SESSION:0.1" -O "python3 -u -c \" +import sys, os +f = open('$REV_LOG', 'a', buffering=1) +n = 0 +for line in sys.stdin: + f.write(line) + n += 1 + if n % 500 == 0: + try: + if os.path.getsize('$REV_LOG') > 50_000_000: + f.close() + os.replace('$REV_LOG', '$REV_LOG.1') + f = open('$REV_LOG', 'a', buffering=1) + except OSError: + pass +\"" + +# Capture PIDs +PROG_PID=$(TM list-panes -t "$TMUX_SESSION:0" -F '#{pane_pid}' | head -1) +echo "${PROG_PID:-0}" > "$PID_FILE" + +# Start the signal coordinator (routes signals between panes; also handles +# completion — it subsumes watch-completion.sh in solo mode) +"$SCRIPT_DIR/solo-coordinator.sh" "$WORKTREE_DIR" "$TMUX_SESSION" "$ARCHITECTURE" \ + >> "$WATCH_LOG" 2>&1 & +COORDINATOR_PID=$! +echo "$COORDINATOR_PID" > "$LOGS_DIR/${SAFE_BRANCH_LOG}.coordinator.pid" + +ok "Solo pair launched in tmux session '$TMUX_SESSION' (dedicated socket '$TMUX_SOCKET')" +ok " Left pane: programmer (PID: ${PROG_PID:-?}, model: ${PROG_MODEL:-default})" +ok " Right pane: reviewer (model: ${REV_MODEL:-default})" +ok " Coordinator: PID $COORDINATOR_PID → $WATCH_LOG" +ok " Attach: tmux -L $TMUX_SOCKET attach -t $TMUX_SESSION" +ok " Status: ./agents/launch-agent-pair.sh status" +ok " Timeline: ./agents/launch-agent-pair.sh logs $SAFE_BRANCH_LOG" +ok " Signals: ls $WORKTREE_DIR/.adapter-workspace/signals/" diff --git a/devtools/adapter_builder/agents/launch.sh b/devtools/adapter_builder/agents/launch.sh new file mode 100755 index 0000000000..2743be4ea3 --- /dev/null +++ b/devtools/adapter_builder/agents/launch.sh @@ -0,0 +1,710 @@ +#!/usr/bin/env bash +# ============================================================================= +# launch.sh +# +# Creates a git worktree on a new branch in the TransformerLens repo and kicks +# off a Programmer/Reviewer Claude Code agent pair inside it. This is the +# launch path only; operational subcommands (status, logs, attach, send, stop, +# clean) live in ops.sh. The user-facing entry point is launch-agent-pair.sh. +# +# Usage: +# ./launch.sh --architecture [options] +# +# Options: +# [--seed-model ] Known model for scaffolding (e.g. Qwen/Qwen3.5-9B) +# [--target-repo ] Path to TransformerLens repo (default from .env) +# [--base-branch ] Base branch (default: DEFAULT_BASE_BRANCH from .env) +# [--new-branch ] Feature branch (default: feature/-adapter) +# [--max-memory ] Memory limit in GB (default: DEFAULT_MAX_MEMORY_GB from .env) +# [--programmer-prompt ] Override the auto-generated programmer prompt +# [--reviewer-prompt ] Extra review criteria/focus for the Reviewer +# [--worktree-dir ] Where to create the worktree (default: $WORKTREE_BASE/) +# [--background] Run detached inside a tmux session +# [--retry] Resume from saved progress (safe for planning/programming) +# [--auto-approve] Run agents in auto-approve mode (no permission prompts) +# [--skip-arch-check] Skip the pre-flight architecture existence check +# [--dry-run] Run all pre-flight checks, then exit without +# creating a worktree or launching sessions +# [--programmer-model ] Model for the Programmer agent (default: PROGRAMMER_MODEL env, then frontmatter) +# [--reviewer-model ] Model for the Reviewer agent (default: REVIEWER_MODEL env, then frontmatter) +# [--orchestrator-model ] Model for the Orchestrator session (default: ORCHESTRATOR_MODEL env, then session default) +# +# Example: +# # Interactive (foreground) +# ./launch.sh --architecture CohereForCausalLM +# +# # Background (detached) — run multiple in parallel +# ./launch.sh --architecture CohereForCausalLM --background +# ./launch.sh --architecture CodeGenForCausalLM --background +# ============================================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +LOGS_DIR="$PROJECT_ROOT/.logs" + +# Shared helpers (log/ok/warn/err/require_cmd/iso_now) +TL_LOG_TAG="launch" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/lib/common.sh" + +# Friendly command name for user-facing hints. +TL_CMD="${TL_CMD:-$0}" + +usage() { + grep '^#' "$0" | sed 's/^# \{0,2\}//' | tail -n +2 + exit 1 +} + +# --------------------------------------------------------------------------- # +# Argument parsing +# --------------------------------------------------------------------------- # +TARGET_REPO="" +ARCHITECTURE="" +SEED_MODEL="" +BASE_BRANCH="" +NEW_BRANCH="" +MAX_MEMORY_GB="" +PROGRAMMER_PROMPT="" +REVIEWER_PROMPT="" +WORKTREE_DIR="" +AUTO_APPROVE=false +BACKGROUND=false +RETRY=false +SKIP_ARCH_CHECK=false +DRY_RUN=false +PROGRAMMER_MODEL_ARG="" +REVIEWER_MODEL_ARG="" +ORCHESTRATOR_MODEL_ARG="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --target-repo) TARGET_REPO="$2"; shift 2 ;; + --architecture) ARCHITECTURE="$2"; shift 2 ;; + --seed-model) SEED_MODEL="$2"; shift 2 ;; + --base-branch) BASE_BRANCH="$2"; shift 2 ;; + --new-branch) NEW_BRANCH="$2"; shift 2 ;; + --max-memory) MAX_MEMORY_GB="$2"; shift 2 ;; + --programmer-prompt) PROGRAMMER_PROMPT="$2"; shift 2 ;; + --reviewer-prompt) REVIEWER_PROMPT="$2"; shift 2 ;; + --worktree-dir) WORKTREE_DIR="$2"; shift 2 ;; + --background) BACKGROUND=true; shift ;; + --retry) RETRY=true; shift ;; + --auto-approve) AUTO_APPROVE=true; shift ;; + --skip-arch-check) SKIP_ARCH_CHECK=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; + --programmer-model) PROGRAMMER_MODEL_ARG="$2"; shift 2 ;; + --reviewer-model) REVIEWER_MODEL_ARG="$2"; shift 2 ;; + --orchestrator-model) ORCHESTRATOR_MODEL_ARG="$2"; shift 2 ;; + -h|--help) usage ;; + *) err "Unknown argument: $1" ;; + esac +done + +# Load defaults from .env if present +ENV_FILE="$PROJECT_ROOT/.env" +if [[ -f "$ENV_FILE" ]]; then + # shellcheck disable=SC1090 + source "$ENV_FILE" +fi + +# Model precedence: CLI flag > env var (possibly from .env) > frontmatter. +[[ -n "$PROGRAMMER_MODEL_ARG" ]] && PROGRAMMER_MODEL="$PROGRAMMER_MODEL_ARG" +[[ -n "$REVIEWER_MODEL_ARG" ]] && REVIEWER_MODEL="$REVIEWER_MODEL_ARG" +[[ -n "$ORCHESTRATOR_MODEL_ARG" ]] && ORCHESTRATOR_MODEL="$ORCHESTRATOR_MODEL_ARG" + +# The builder lives in devtools/adapter_builder inside TransformerLens, so the +# containing repo is the default target. --target-repo / DEFAULT_TARGET_REPO +# remain as overrides for driving a different checkout. +TARGET_REPO="${TARGET_REPO:-${DEFAULT_TARGET_REPO:-}}" +if [[ -z "$TARGET_REPO" ]]; then + TARGET_REPO="$(git -C "$PROJECT_ROOT" rev-parse --show-toplevel 2>/dev/null)" \ + || err "Could not derive the TransformerLens repo root (pass --target-repo or set DEFAULT_TARGET_REPO in .env)." +fi +[[ -z "$ARCHITECTURE" ]] && err "--architecture is required (e.g., CohereForCausalLM)" + +# Apply defaults: base branch from .env, new branch from architecture name +BASE_BRANCH="${BASE_BRANCH:-${DEFAULT_BASE_BRANCH:-dev-4.x}}" +MAX_MEMORY_GB="${MAX_MEMORY_GB:-${DEFAULT_MAX_MEMORY_GB:-96}}" + +if [[ -z "$NEW_BRANCH" ]]; then + # Convert architecture class to lowercase branch name. + # e.g. CohereForCausalLM -> cohere, Gemma3ForConditionalGeneration -> gemma3 + ARCH_SHORT=$(echo "$ARCHITECTURE" | sed -E 's/(For(Causal|Conditional|Masked).*|LMHead.*)$//' | tr '[:upper:]' '[:lower:]') + NEW_BRANCH="feature/${ARCH_SHORT}-adapter" +fi + +# Auto-generate programmer prompt from architecture if not explicitly provided +if [[ -z "$PROGRAMMER_PROMPT" ]]; then + SEED_MODEL_BLOCK="" + if [[ -n "$SEED_MODEL" ]]; then + SEED_MODEL_BLOCK=" +Seed model: ${SEED_MODEL} +Use this model for scaffold generation (analyze-hf-model.py --scaffold) and +as the primary source for reading modeling_*.py / configuration_*.py. Include +it in your verification targets." + fi + + PROGRAMMER_PROMPT="Create an Architecture Adapter for the HuggingFace architecture: ${ARCHITECTURE} +${SEED_MODEL_BLOCK} +Follow your 3-step lifecycle (Plan, Program, Verify): + +Step 1 — Planning: +1. Find models using this architecture on HuggingFace +2. Read the HF model source (modeling_*.py, configuration_*.py) to understand the architecture +3. Use \$TL_ADAPTER_BUILDER_ROOT/scripts/analyze-hf-model.py on a representative model to extract config details +4. Read docs/adapter-specification.md for the full specification +5. Find the closest existing adapter in transformer_lens/model_bridge/supported_architectures/ +6. Write a phased plan and submit for review + +Step 2 — Programming: +7. Implement each phase, getting reviewer approval before moving to the next + +Step 3 — Verification: +8. Run verify_models on the top 5 models of this architecture that fit in ${MAX_MEMORY_GB}GB +9. If failures, return to planning and fix + +Architecture class: ${ARCHITECTURE}" +fi + +REVIEWER_PROMPT="${REVIEWER_PROMPT:-Review the programmer changes thoroughly for correctness, edge cases, code style, and maintainability.}" + +# --------------------------------------------------------------------------- # +# Sanity checks +# --------------------------------------------------------------------------- # +require_cmd git +require_cmd claude + +# Use --target-repo as the repo root (the TransformerLens repo) +REPO_ROOT="$(cd "$TARGET_REPO" && git rev-parse --show-toplevel 2>/dev/null)" \ + || err "--target-repo '$TARGET_REPO' is not a git repository." + +# The repo-level .env may hold HF_TOKEN (repo convention for HF-Hub-hitting +# commands); the builder's own .env wins if both define it. +if [[ -z "${HF_TOKEN:-}" && -f "$REPO_ROOT/.env" ]]; then + HF_TOKEN="$({ . "$REPO_ROOT/.env" >/dev/null 2>&1 || true; printf '%s' "${HF_TOKEN:-}"; })" +fi + +# Where to create worktrees. WORKTREE_BASE may be set in .env; otherwise +# fall back to "/worktrees" (matches the original layout). +WORKTREE_BASE="${WORKTREE_BASE:-$(dirname "$REPO_ROOT")/worktrees}" + +# --------------------------------------------------------------------------- # +# Early opt-out: check if architecture is already supported +# --------------------------------------------------------------------------- # +# The worktree is created from BASE_BRANCH, so the "already supported" check +# must read the factory file from that branch — not the main checkout's +# working tree, which may differ (e.g. golden-master test branches with the +# adapter stripped). Fall back to the on-disk file if the branch ref can't +# be read; the branch is hard-validated later. +FACTORY_REL="transformer_lens/factories/architecture_adapter_factory.py" +FACTORY_CONTENT="$(git -C "$REPO_ROOT" show "${BASE_BRANCH}:${FACTORY_REL}" 2>/dev/null \ + || cat "$REPO_ROOT/$FACTORY_REL" 2>/dev/null || true)" +if [[ -n "$FACTORY_CONTENT" ]] && grep -q "\"${ARCHITECTURE}\"" <<< "$FACTORY_CONTENT"; then + ok "Architecture '${ARCHITECTURE}' already has an adapter on branch '${BASE_BRANCH}'." + log "No work needed. Exiting." + exit 0 +fi + +# --------------------------------------------------------------------------- # +# Pre-flight: does the architecture actually exist? +# +# Checks the installed transformers package first (fast, local), then falls +# back to a bounded HuggingFace Hub scan. Aborts on typos before any worktree +# is created. Runs in the target repo's uv env so it sees the project's +# transformers and huggingface_hub installs. +# +# Bypass with --skip-arch-check for genuinely new architectures that haven't +# made it into any of the above yet. +# --------------------------------------------------------------------------- # +# --seed-model implies --skip-arch-check (the user knows the arch exists) +if [[ -n "$SEED_MODEL" && "$SKIP_ARCH_CHECK" == false ]]; then + log "Seed model '${SEED_MODEL}' provided — skipping architecture existence check." + SKIP_ARCH_CHECK=true +fi + +if [[ "$SKIP_ARCH_CHECK" == true ]]; then + warn "Skipping architecture existence check." +else + log "Validating '${ARCHITECTURE}' exists in transformers or on HuggingFace Hub..." + set +e + # Unset VIRTUAL_ENV so uv doesn't warn when the user has a different venv + # active in their shell — the validator needs the TransformerLens project + # env, not whatever happens to be on PATH. + env -u VIRTUAL_ENV uv run --project "$REPO_ROOT" \ + python "$PROJECT_ROOT/scripts/validate-architecture.py" "$ARCHITECTURE" + validate_exit=$? + set -e + case "$validate_exit" in + 0) + ok "Architecture validated." + ;; + 1) + err "Architecture '${ARCHITECTURE}' does not exist in the installed transformers package or on HuggingFace Hub. Check the spelling (class names are case-sensitive, e.g. 'CohereForCausalLM'). If this is a genuinely new architecture not yet on HF, rerun with --skip-arch-check." + ;; + 2) + warn "Could not fully verify '${ARCHITECTURE}' (missing deps or network error). Proceeding anyway — rerun with --skip-arch-check to suppress this warning." + ;; + *) + err "validate-architecture.py returned unexpected exit code: $validate_exit" + ;; + esac +fi + +if [[ -z "$WORKTREE_DIR" ]]; then + SAFE_BRANCH=$(echo "$NEW_BRANCH" | tr '/' '-') + WORKTREE_DIR="${WORKTREE_BASE}/${SAFE_BRANCH}" +fi + +git -C "$REPO_ROOT" rev-parse --verify "$BASE_BRANCH" &>/dev/null \ + || err "Base branch '$BASE_BRANCH' does not exist in this repository." + +if [[ "$DRY_RUN" == true ]]; then + ok "Pre-flight passed (dry run) — no worktree created, no sessions launched." + log " would create : $WORKTREE_DIR (branch $NEW_BRANCH from $BASE_BRANCH)" + log " architecture : $ARCHITECTURE" + log " seed model : ${SEED_MODEL:-none}" + exit 0 +fi + +# Source progress tracking +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/progress.sh" + +# --------------------------------------------------------------------------- # +# Create or reuse the worktree — atomic with rollback on partial failure. +# +# Instead of check-then-create (TOCTOU), we attempt creation directly and +# interpret the exit code: +# - Success → new worktree, proceed with init +# - "already exists" → resume path +# - Other failure → abort +# +# A cleanup trap removes a partially-created worktree + branch if anything +# between creation and the first successful progress_init fails. +# --------------------------------------------------------------------------- # +RESUME_CONTEXT="" +REUSE_WORKTREE=false + +if [[ -d "$WORKTREE_DIR" ]] && git -C "$REPO_ROOT" rev-parse --verify "$NEW_BRANCH" &>/dev/null; then + # Both directory and branch exist — this is a resume, not a race. + ok "Existing worktree found at $WORKTREE_DIR (branch: $NEW_BRANCH) — resuming." + REUSE_WORKTREE=true +else + # Attempt atomic creation. git-worktree-add will fail if the branch or + # directory already exists (concurrent launch lost the race). + log "Creating worktree at: $WORKTREE_DIR" + log " base branch : $BASE_BRANCH" + log " new branch : $NEW_BRANCH" + + _cleanup_partial_worktree() { + warn "Cleaning up partial worktree after failure..." + git -C "$REPO_ROOT" worktree remove --force "$WORKTREE_DIR" 2>/dev/null || true + git -C "$REPO_ROOT" branch -D "$NEW_BRANCH" 2>/dev/null || true + } + + if ! git -C "$REPO_ROOT" worktree add -b "$NEW_BRANCH" "$WORKTREE_DIR" "$BASE_BRANCH" 2>/tmp/tl-worktree-err.$$; then + wt_err=$(cat /tmp/tl-worktree-err.$$ 2>/dev/null) + rm -f /tmp/tl-worktree-err.$$ + if echo "$wt_err" | grep -qiE "already exists|already checked out"; then + err "Worktree or branch already exists (concurrent launch?): $wt_err" + else + err "Failed to create worktree: $wt_err" + fi + fi + rm -f /tmp/tl-worktree-err.$$ + + # Trap: if anything between here and the end of init fails, roll back. + trap '_cleanup_partial_worktree' ERR + + ok "Worktree created." + + # Initialize fresh progress file + progress_init "$WORKTREE_DIR" "$ARCHITECTURE" + ok "Progress tracking initialized." + + # Clear the ERR trap — init succeeded, worktree is fully set up. + trap - ERR +fi + +if [[ "$REUSE_WORKTREE" == true ]]; then + # Handle --retry: resume planning/programming, but NOT verification + if [[ "$RETRY" == true ]]; then + progress_read "$WORKTREE_DIR" + if [[ "$PROGRESS_EXISTS" == "true" && "$PROGRESS_STEP" == "verification" ]]; then + warn "Last session crashed during verification." + "$PROJECT_ROOT/scripts/notify.sh" "Crash occurred for ${ARCHITECTURE} during verification, please manually resume" + log "Verification requires manual resume. Exiting." + exit 1 + elif [[ "$PROGRESS_EXISTS" == "true" ]]; then + ok "Progress is at '${PROGRESS_STEP}' — retrying from there." + fi + fi + + # Generate resume context from saved progress + RESUME_CONTEXT=$(generate_resume_context "$WORKTREE_DIR") + if [[ -n "$RESUME_CONTEXT" ]]; then + progress_read "$WORKTREE_DIR" + ok "Resuming from: step=${PROGRESS_STEP}, completed_phases=${PROGRESS_COMPLETED_PHASES:-none}" + fi +fi + +# --------------------------------------------------------------------------- # +# Inject agent definitions into the worktree (.claude/agents/ is tool config, +# required by Claude Code agent teams — no repo code is copied) +# --------------------------------------------------------------------------- # +AGENTS_DIR="$WORKTREE_DIR/.claude/agents" +mkdir -p "$AGENTS_DIR" + +# Create .adapter-workspace/ for brief/plan/scaffold/timeline/reviews — NOT in +# .claude/ because Claude Code treats .claude/ paths specially and prompts on +# writes even with --dangerously-skip-permissions. +mkdir -p "$WORKTREE_DIR/.adapter-workspace/reviews" +mkdir -p "$WORKTREE_DIR/.adapter-workspace/phase-reports" +touch "$WORKTREE_DIR/.adapter-workspace/adapter-brief.md" 2>/dev/null || true +touch "$WORKTREE_DIR/.adapter-workspace/adapter-plan.md" 2>/dev/null || true +touch "$WORKTREE_DIR/.adapter-workspace/adapter-scaffold.py" 2>/dev/null || true +touch "$WORKTREE_DIR/.adapter-workspace/timeline.jsonl" 2>/dev/null || true + +# Install Claude Code hooks in the worktree so every tool call, subagent spawn, +# and session event is automatically logged to the timeline. No agent +# cooperation needed — hooks fire at the framework level. +TIMELINE_PATH="$WORKTREE_DIR/.adapter-workspace/timeline.jsonl" +SETTINGS_FILE="$WORKTREE_DIR/.claude/settings.json" +mkdir -p "$WORKTREE_DIR/.claude" + +# Absolute paths to the hook scripts. Claude Code invokes hook commands via +# /bin/sh -c "", so paths with spaces need shell quoting. We build the +# command strings as '' (single-quoted) and let jq handle all JSON +# escaping — this prevents broken JSON if paths contain backslashes, +# double-quotes, or other special characters. +_hook_cmd() { echo "'$SCRIPT_DIR/hooks/$1'"; } + +TIMELINE_HOOK=$(_hook_cmd timeline-capture.sh) +GUARD_HT_HOOK=$(_hook_cmd guard-hooked-transformer.sh) +GUARD_GIT_HOOK=$(_hook_cmd guard-git.sh) +GUARD_REVIEW_ROUNDS_HOOK=$(_hook_cmd guard-review-rounds.sh) +GUARD_VERIFY_MODELS_HOOK=$(_hook_cmd guard-verify-models.sh) +GATE_REVIEWER_WRITES_HOOK=$(_hook_cmd gate-reviewer-writes-file.sh) +GATE_LINT_HOOK=$(_hook_cmd gate-lint-checks.sh) +NOTIFY_HOOK=$(_hook_cmd notify-on-completion.sh) + +# Build settings.json via jq so all hook paths are properly JSON-escaped. +# This is the only reliable way to handle paths with spaces, backslashes, +# or other characters that would break a naive heredoc interpolation. +_h() { jq -n --arg cmd "$1" '{"type":"command","command":$cmd}'; } + +jq -n \ + --argjson session_start "[{\"hooks\":[ $(_h "$TIMELINE_HOOK") ]}]" \ + --argjson session_end "[{\"hooks\":[ $(_h "$TIMELINE_HOOK"), $(_h "$NOTIFY_HOOK") ]}]" \ + --argjson subagent_start "[{\"hooks\":[ $(_h "$TIMELINE_HOOK") ]}]" \ + --argjson subagent_stop "[{\"hooks\":[ $(_h "$TIMELINE_HOOK"), $(_h "$GATE_REVIEWER_WRITES_HOOK") ]}]" \ + --argjson pre_edit "{\"matcher\":\"Edit|Write|MultiEdit|NotebookEdit\",\"hooks\":[ $(_h "$GUARD_HT_HOOK"), $(_h "$GUARD_REVIEW_ROUNDS_HOOK") ]}" \ + --argjson pre_bash "{\"matcher\":\"Bash\",\"hooks\":[ $(_h "$GUARD_GIT_HOOK"), $(_h "$GUARD_VERIFY_MODELS_HOOK") ]}" \ + --argjson pre_all "{\"matcher\":\"\",\"hooks\":[ $(_h "$TIMELINE_HOOK") ]}" \ + --argjson post_all "[{\"matcher\":\"\",\"hooks\":[ $(_h "$TIMELINE_HOOK") ]}]" \ + --argjson stop "[{\"hooks\":[ $(_h "$GATE_LINT_HOOK") ]}]" \ + '{hooks:{ + SessionStart: $session_start, + SessionEnd: $session_end, + SubagentStart: $subagent_start, + SubagentStop: $subagent_stop, + PreToolUse: [$pre_edit, $pre_bash, $pre_all], + PostToolUse: $post_all, + Stop: $stop + }}' > "$SETTINGS_FILE" +ok "Hooks installed: timeline, HookedTransformer guardrail, git guardrail, review-rounds limit, verify-models gate, reviewer-writes-file gate, lint gate, completion notifier" + +# Pre-seed workspace trust for the worktree. Without this, a first launch in +# a fresh worktree hangs at Claude Code's "Do you trust this folder?" prompt +# because --dangerously-skip-permissions does NOT bypass the trust dialog +# (only -p/print mode does). We atomically merge a trust entry into +# ~/.claude.json so the spawned claude sees the worktree as pre-approved. +CLAUDE_JSON="$HOME/.claude.json" +if [[ -f "$CLAUDE_JSON" ]]; then + TRUST_TMP=$(mktemp) + jq --arg path "$WORKTREE_DIR" ' + .projects //= {} | + .projects[$path] = ((.projects[$path] // {}) + { + hasTrustDialogAccepted: true, + hasClaudeMdExternalIncludesApproved: true, + hasClaudeMdExternalIncludesWarningShown: true, + projectOnboardingSeenCount: 1 + }) + ' "$CLAUDE_JSON" > "$TRUST_TMP" && mv "$TRUST_TMP" "$CLAUDE_JSON" + ok "Workspace trust pre-seeded for $WORKTREE_DIR" +fi + +MAIN_AGENTS_DIR="$SCRIPT_DIR" + +# Memory lock reference (injected into both agents) +memory_lock_block() { + cat < "$tmp" && mv "$tmp" "$dest" + log "Model override: $(basename "$dest") → $model" +} + +inject_agent() { + local role="$1" + local extension="$2" + local dest="$AGENTS_DIR/${role}.md" + + if [[ -f "$MAIN_AGENTS_DIR/${role}.md" ]]; then + cp "$MAIN_AGENTS_DIR/${role}.md" "$dest" + log "Copied base ${role}.md template." + else + log "No base ${role}.md found — generating minimal default." + cat > "$dest" <> "$dest" + + # Append memory lock reference + memory_lock_block >> "$dest" + + ok "Wrote ${role}.md to worktree." +} + +inject_agent "programmer" "$PROGRAMMER_PROMPT" +inject_agent "reviewer" "$REVIEWER_PROMPT" +_apply_model_override "$AGENTS_DIR/programmer.md" "${PROGRAMMER_MODEL:-}" +_apply_model_override "$AGENTS_DIR/reviewer.md" "${REVIEWER_MODEL:-}" + +# --------------------------------------------------------------------------- # +# Build the orchestration prompt from template + signals +# --------------------------------------------------------------------------- # +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/signals.sh" + +PROTOCOL_BLOCK=$(generate_protocol_block) + +# Render orchestrator.md template with placeholder substitution +ORCHESTRATOR_TEMPLATE="$SCRIPT_DIR/orchestrator.md" +[[ -f "$ORCHESTRATOR_TEMPLATE" ]] || err "orchestrator.md not found at $ORCHESTRATOR_TEMPLATE" + +ORCHESTRATION_PROMPT=$(sed \ + -e "s|{{ARCHITECTURE}}|${ARCHITECTURE}|g" \ + -e "s|{{NEW_BRANCH}}|${NEW_BRANCH}|g" \ + -e "s|{{BASE_BRANCH}}|${BASE_BRANCH}|g" \ + -e "s|{{MAX_MEMORY_GB}}|${MAX_MEMORY_GB}|g" \ + -e "s|{{PROJECT_ROOT}}|${PROJECT_ROOT}|g" \ + -e "s|{{SIG_CODE_APPROVED}}|${SIG_CODE_APPROVED}|g" \ + "$ORCHESTRATOR_TEMPLATE") + +# Replace multi-line placeholders that sed can't handle +ORCHESTRATION_PROMPT="${ORCHESTRATION_PROMPT//\{\{PROTOCOL_BLOCK\}\}/$PROTOCOL_BLOCK}" +ORCHESTRATION_PROMPT="${ORCHESTRATION_PROMPT//\{\{PROGRAMMER_PROMPT\}\}/$PROGRAMMER_PROMPT}" +ORCHESTRATION_PROMPT="${ORCHESTRATION_PROMPT//\{\{REVIEWER_PROMPT\}\}/$REVIEWER_PROMPT}" +ORCHESTRATION_PROMPT="${ORCHESTRATION_PROMPT//\{\{RESUME_CONTEXT\}\}/$RESUME_CONTEXT}" + +# --------------------------------------------------------------------------- # +# Launch Claude Code inside the worktree +# --------------------------------------------------------------------------- # +SAFE_BRANCH_LOG=$(echo "$NEW_BRANCH" | tr '/' '-') +mkdir -p "$LOGS_DIR" + +CLAUDE_FLAGS=( + "--dangerously-skip-permissions" + "--permission-mode" "bypassPermissions" + "--debug-file" "$LOGS_DIR/${SAFE_BRANCH_LOG}.debug.log" + "--debug" "hooks" +) +if [[ "$AUTO_APPROVE" == true ]]; then + CLAUDE_FLAGS+=("--allowedTools" "bash" "read" "write" "edit" "glob" "grep") +fi +# Orchestrator session model override (ORCHESTRATOR_MODEL, settable in .env). +# Programmer/Reviewer models come from their agent-definition frontmatter, +# overridable via PROGRAMMER_MODEL / REVIEWER_MODEL (see _apply_model_override). +[[ -n "${ORCHESTRATOR_MODEL:-}" ]] && CLAUDE_FLAGS+=("--model" "$ORCHESTRATOR_MODEL") + +MODE="interactive" +[[ "$BACKGROUND" == true ]] && MODE="background" + +log "Launching agent pair..." +log " worktree : $WORKTREE_DIR" +log " project root: $PROJECT_ROOT" +log " memory lock : /tmp/tl-adapter-builder.lock (flock)" +log " hooks debug : $LOGS_DIR/${SAFE_BRANCH_LOG}.debug.log" +log " mode : $MODE" + +cd "$WORKTREE_DIR" + +if [[ "$BACKGROUND" == true ]]; then + require_cmd tmux + + LOG_FILE="$LOGS_DIR/${SAFE_BRANCH_LOG}.log" + PID_FILE="$LOGS_DIR/${SAFE_BRANCH_LOG}.pid" + RAW_LOG="${LOG_FILE%.log}.raw" + TMUX_SESSION="tl-adapter-$SAFE_BRANCH_LOG" + WRAPPER="$LOGS_DIR/${SAFE_BRANCH_LOG}.wrapper.sh" + PROMPT_FILE="$LOGS_DIR/${SAFE_BRANCH_LOG}.prompt.txt" + WATCH_LOG="$LOGS_DIR/${SAFE_BRANCH_LOG}.watch.log" + + log " tmux session: $TMUX_SESSION" + log " raw log : $RAW_LOG" + log " watch log : $WATCH_LOG" + echo "" + + # Kill any stale session with the same name + # Use a dedicated tmux socket so our sessions don't share a server with + # any tmux session the user already has running. If a pane crashes or + # emits an unexpected control sequence, it can't kill the user's tmux. + TMUX_SOCKET="tl-adapter" + TM() { tmux -L "$TMUX_SOCKET" "$@"; } + + TM kill-session -t "$TMUX_SESSION" 2>/dev/null || true + + # Write the orchestration prompt to a file (too big for shell quoting) + printf '%s' "$ORCHESTRATION_PROMPT" > "$PROMPT_FILE" + + # Build the wrapper script tmux will execute. This avoids shell quoting + # nightmares inside `tmux new-session -d`'s command string. + cat > "$WRAPPER" <> "$WATCH_LOG" 2>&1 & +WATCHER_PID=\$! +trap 'kill \$WATCHER_PID 2>/dev/null || true' EXIT + +# Read the orchestration prompt from the file +PROMPT="\$(cat "$PROMPT_FILE")" +claude "\$@" "\$PROMPT" +CLAUDE_EXIT=\$? + +# Claude has exited (either via /exit from the watcher or a manual exit). +# Kill the watcher if it's still running; the trap will handle the cleanup +# in the abnormal-exit case. +kill \$WATCHER_PID 2>/dev/null || true + +echo "" +echo "[session ended with exit code \$CLAUDE_EXIT — tmux pane will stay open for 1 hour]" +echo "Press Ctrl-b then d to detach, or Ctrl-b then x to close." +sleep 3600 +WRAPPER_EOF + chmod +x "$WRAPPER" + + # Launch Claude inside a detached tmux session. tmux provides a real pty, + # so claude's interactive mode works as expected. The session persists in + # the background and can be attached to interactively at any time. + TM new-session -d -s "$TMUX_SESSION" -c "$WORKTREE_DIR" \ + "$WRAPPER" "${CLAUDE_FLAGS[@]}" + + # Pipe all pane output to the raw log via a python rotator that caps + # the file at ~50MB. When exceeded, the current log is moved to .1 + # (one backup kept). This prevents unbounded growth on long sessions. + TM pipe-pane -t "$TMUX_SESSION" -O \ + "python3 -u -c \" +import sys, os +f = open('$RAW_LOG', 'a', buffering=1) +n = 0 +for line in sys.stdin: + f.write(line) + n += 1 + if n % 500 == 0: + try: + if os.path.getsize('$RAW_LOG') > 50_000_000: + f.close() + os.replace('$RAW_LOG', '$RAW_LOG.1') + f = open('$RAW_LOG', 'a', buffering=1) + except OSError: + pass +\"" + + # Capture the tmux session's top-level pane PID so stop/status works + CLAUDE_PID=$(TM list-panes -t "$TMUX_SESSION" -F '#{pane_pid}' 2>/dev/null | head -1) + echo "${CLAUDE_PID:-0}" > "$PID_FILE" + + ok "Agent pair launched in tmux session '$TMUX_SESSION' (PID: ${CLAUDE_PID:-?})" + ok "Raw log: $RAW_LOG" + ok "Attach: $TL_CMD attach $SAFE_BRANCH_LOG" + ok "Send msg: $TL_CMD send $SAFE_BRANCH_LOG \"\"" + ok "Detach from tmux: Ctrl-b d" + ok "Status: $TL_CMD status" + ok "Timeline: $TL_CMD logs $SAFE_BRANCH_LOG" +else + echo "" + + # Interactive foreground mode + CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1 \ + TL_ADAPTER_BUILDER_ROOT="$PROJECT_ROOT" \ + MAX_MEMORY_GB="$MAX_MEMORY_GB" \ + NOTIFICATION_WEBHOOK_URL="${NOTIFICATION_WEBHOOK_URL:-}" \ + NOTIFICATION_NUMBER="${NOTIFICATION_NUMBER:-}" \ + HF_TOKEN="${HF_TOKEN:-}" \ + HUGGING_FACE_HUB_TOKEN="${HF_TOKEN:-}" \ + claude "${CLAUDE_FLAGS[@]}" \ + "$ORCHESTRATION_PROMPT" + + CLAUDE_EXIT=$? +fi + +# --------------------------------------------------------------------------- # +# Post-session cleanup (foreground only — background cleans up via `clean`) +# --------------------------------------------------------------------------- # +if [[ "$BACKGROUND" == false && -f "$WORKTREE_DIR/.adapter-progress.json" ]]; then + VERIFIED=$(python3 -c "import json; print(json.load(open('$WORKTREE_DIR/.adapter-progress.json')).get('verification_passed', False))" 2>/dev/null || echo "False") + if [[ "$VERIFIED" == "True" ]]; then + ok "Adapter completed successfully." + log "Worktree preserved for manual commit: $WORKTREE_DIR" + log " Clean up when done: $TL_CMD clean $ARCHITECTURE" + else + log "Session ended but adapter not verified. Worktree preserved for resume." + log " Resume: $TL_CMD --architecture $ARCHITECTURE" + log " Clean: $TL_CMD clean $ARCHITECTURE" + fi +fi diff --git a/devtools/adapter_builder/agents/lib/common.sh b/devtools/adapter_builder/agents/lib/common.sh new file mode 100644 index 0000000000..9bef5994f6 --- /dev/null +++ b/devtools/adapter_builder/agents/lib/common.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# ============================================================================= +# lib/common.sh — Shared bash helpers for TL Adapter Builder scripts +# +# Provides colored logging, command presence checks, and a UTC timestamp. +# Sourced by launch.sh, ops.sh, overlord-request.sh, and any helper that +# needs consistent terminal output. +# +# The [TAG] prefix for `log` is controlled by TL_LOG_TAG. Callers should set +# it before sourcing: +# +# TL_LOG_TAG="launch" +# source "$SCRIPT_DIR/lib/common.sh" +# +# This file intentionally does NOT set `set -euo pipefail` — that is the +# caller's responsibility, so sourcing into an existing shell is side-effect +# free beyond the helper definitions. +# ============================================================================= + +: "${TL_LOG_TAG:=tl}" + +log() { echo -e "\033[1;34m[${TL_LOG_TAG}]\033[0m $*"; } +ok() { echo -e "\033[1;32m[ ok ]\033[0m $*"; } +warn() { echo -e "\033[1;33m[ warn ]\033[0m $*"; } +err() { echo -e "\033[1;31m[ err ]\033[0m $*" >&2; exit 1; } + +require_cmd() { command -v "$1" &>/dev/null || err "'$1' is not installed or not in PATH."; } + +iso_now() { date -u +"%Y-%m-%dT%H:%M:%SZ"; } diff --git a/devtools/adapter_builder/agents/ops.sh b/devtools/adapter_builder/agents/ops.sh new file mode 100755 index 0000000000..0e0b860e5b --- /dev/null +++ b/devtools/adapter_builder/agents/ops.sh @@ -0,0 +1,538 @@ +#!/usr/bin/env bash +# ============================================================================= +# ops.sh — Operational subcommands for the TL Adapter Builder +# +# Handles status / logs / attach / send / stop / clean. The launch path lives +# in launch.sh, and the user-facing entry point is launch-agent-pair.sh which +# routes subcommands here. +# +# Usage: +# ./ops.sh status +# ./ops.sh logs [--raw] +# ./ops.sh attach +# ./ops.sh send +# ./ops.sh stop [architecture|all] +# ./ops.sh clean [architecture] +# ============================================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +LOGS_DIR="$PROJECT_ROOT/.logs" +FORMAT_TIMELINE="$PROJECT_ROOT/scripts/format-timeline.py" + +# Load .env once so subcommands can share WORKTREE_BASE / DEFAULT_TARGET_REPO. +ENV_FILE="$PROJECT_ROOT/.env" +if [[ -f "$ENV_FILE" ]]; then + # shellcheck disable=SC1090 + source "$ENV_FILE" +fi + +# Where worktrees live. Override via WORKTREE_BASE in .env; otherwise fall +# back to "/worktrees" — outside the repo, matching launch.sh. +REPO_ROOT="$(git -C "$PROJECT_ROOT" rev-parse --show-toplevel 2>/dev/null || dirname "$PROJECT_ROOT")" +WORKTREE_BASE="${WORKTREE_BASE:-$(dirname "$REPO_ROOT")/worktrees}" + +# Friendly command name shown in user-facing messages (e.g. "$TL_CMD attach X"). +# launch-agent-pair.sh exports this so users see the dispatcher name rather +# than ops.sh when subcommands are invoked through it. +TL_CMD="${TL_CMD:-$0}" + +# Shared helpers (log/ok/warn/err/require_cmd/iso_now). +TL_LOG_TAG="ops" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/lib/common.sh" + +# Dedicated tmux socket for all TL Adapter Builder sessions. Isolates our +# tmux server from any tmux session the user might have running, so a crashed +# pane or escape sequence can't kill the user's tmux. +TMUX_SOCKET="tl-adapter" +TM() { tmux -L "$TMUX_SOCKET" "$@"; } + +# --------------------------------------------------------------------------- # +# last_event_age — compute a human-readable "time since" string from the +# last JSONL entry in a timeline file. Emits forms like "12s ago", "3m 40s +# ago", "2h 15m ago", "1d 4h ago", or "?" on error. Uses python3 for +# cross-platform ISO 8601 parsing (macOS `date -j` vs GNU `date -d`). +# --------------------------------------------------------------------------- # +last_event_age() { + local filepath="$1" + tail -1 "$filepath" 2>/dev/null | python3 -c " +import json, sys +from datetime import datetime, timezone +try: + ts = json.loads(sys.stdin.read()).get('ts', '') + dt = datetime.fromisoformat(ts.replace('Z', '+00:00')) + s = int(max((datetime.now(timezone.utc) - dt).total_seconds(), 0)) + if s < 60: + out = f'{s}s ago' + elif s < 3600: + out = f'{s//60}m {s%60}s ago' + elif s < 86400: + out = f'{s//3600}h {(s%3600)//60}m ago' + else: + out = f'{s//86400}d {(s%86400)//3600}h ago' + print(out) +except Exception: + print('?') +" 2>/dev/null || echo "?" +} + +# --------------------------------------------------------------------------- # +# status — show all running agent pairs and their progress + memory-lock state +# --------------------------------------------------------------------------- # +cmd_status() { + echo "" + echo "==========================================" + echo " TL Adapter Builder — Running Pairs" + echo "==========================================" + echo "" + + local found=false + if [[ -d "$LOGS_DIR" ]]; then + for pidfile in "$LOGS_DIR"/*.pid; do + [[ -f "$pidfile" ]] || continue + # Daemon PID files (solo coordinator, teams watcher) are not pairs. + case "$pidfile" in *.coordinator.pid|*.watcher.pid) continue ;; esac + found=true + local name pid worktree_dir progressfile="" timelinefile="" + name=$(basename "$pidfile" .pid) + pid=$(cat "$pidfile") + worktree_dir="$WORKTREE_BASE/$name" + + [[ -f "$worktree_dir/.adapter-progress.json" ]] \ + && progressfile="$worktree_dir/.adapter-progress.json" + [[ -f "$worktree_dir/.adapter-workspace/timeline.jsonl" ]] \ + && timelinefile="$worktree_dir/.adapter-workspace/timeline.jsonl" + + if kill -0 "$pid" 2>/dev/null; then + echo -e " \033[1;32m[running]\033[0m $name (PID: $pid)" + else + echo -e " \033[1;31m[stopped]\033[0m $name (PID: $pid — process dead)" + fi + + if [[ -n "$progressfile" ]]; then + local step current_phase completed + step=$(python3 -c "import json; print(json.load(open('$progressfile')).get('step','?'))" 2>/dev/null || echo "?") + current_phase=$(python3 -c "import json; print(json.load(open('$progressfile')).get('current_phase','?'))" 2>/dev/null || echo "?") + completed=$(python3 -c "import json; print(','.join(json.load(open('$progressfile')).get('completed_phases',[])))" 2>/dev/null || echo "?") + echo " step: $step | phase: $current_phase | completed: ${completed:-none}" + fi + + if [[ -n "$timelinefile" ]]; then + local event_count last_event last_age + event_count=$(wc -l < "$timelinefile" 2>/dev/null | tr -d ' ') + if [[ "${event_count:-0}" -gt 0 ]]; then + last_event=$(tail -1 "$timelinefile" 2>/dev/null | python3 "$FORMAT_TIMELINE" 2>/dev/null || echo "?") + last_age=$(last_event_age "$timelinefile") + echo " timeline: $event_count events — last: ${last_event:-?}" + echo " time since last event: ${last_age:-?}" + else + echo " timeline: empty" + fi + fi + + local tmux_session="tl-adapter-$name" + if command -v tmux &>/dev/null && TM has-session -t "$tmux_session" 2>/dev/null; then + echo " tmux: attach with '$TL_CMD attach $name'" + fi + echo "" + done + fi + + if [[ "$found" == false ]]; then + echo " No running pairs found." + echo "" + fi + + # Show memory lock status + "$SCRIPT_DIR/overlord-request.sh" status 2>/dev/null || true + echo "" +} + +# --------------------------------------------------------------------------- # +# logs — tail the structured timeline (default) or the raw ANSI log (--raw) +# --------------------------------------------------------------------------- # +cmd_logs() { + local arch="${1:-}" + [[ -z "$arch" ]] && { echo "Usage: $TL_CMD logs [--raw]"; exit 1; } + + local mode="timeline" + [[ "${2:-}" == "--raw" ]] && mode="raw" + + arch=$(echo "$arch" | tr '[:upper:]' '[:lower:]') + + if [[ "$mode" == "timeline" ]]; then + local worktree_dir="" + for d in "$WORKTREE_BASE"/*"${arch}"*; do + [[ -d "$d" ]] && worktree_dir="$d" && break + done + + if [[ -z "$worktree_dir" ]]; then + echo "No worktree found matching '$arch'" + exit 1 + fi + + local timeline_file="$worktree_dir/.adapter-workspace/timeline.jsonl" + if [[ ! -f "$timeline_file" ]]; then + echo "No timeline file at: $timeline_file" + echo "Falling back to raw log. Use '--raw' to skip this fallback." + mode="raw" + else + echo "Tailing structured timeline: $timeline_file (Ctrl+C to stop)" + echo "(use '$TL_CMD logs $arch --raw' for raw terminal log)" + echo "" + tail -f "$timeline_file" | python3 "$FORMAT_TIMELINE" + return + fi + fi + + # Raw mode (or timeline fallback) + local logfile="" + for f in "$LOGS_DIR"/*"${arch}"*.raw; do + [[ -f "$f" ]] && logfile="$f" && break + done + if [[ -z "$logfile" ]]; then + for f in "$LOGS_DIR"/*"${arch}"*.log; do + [[ -f "$f" ]] && logfile="$f" && break + done + fi + + if [[ -z "$logfile" ]]; then + echo "No log file found matching '$arch' in $LOGS_DIR/" + ls "$LOGS_DIR"/*.raw "$LOGS_DIR"/*.log 2>/dev/null || echo "(no log files exist)" + exit 1 + fi + + echo "Tailing raw log: $logfile (Ctrl+C to stop, ANSI codes stripped)" + tail -f "$logfile" | LC_ALL=C sed -u $'s/\x1b\[[0-9;?]*[a-zA-Z]//g; s/\x1b[][><=()][^\x1b]*//g; s/\r//g' +} + +# --------------------------------------------------------------------------- # +# clean — remove a worktree (by architecture) or all verified-completed ones +# --------------------------------------------------------------------------- # +cmd_clean() { + local arch="${1:-}" + local force=false + local incomplete_only=false + + # Accept flags in either position: `clean --force` or `clean --force` + for a in "$@"; do + case "$a" in + --force|--all) force=true; arch="" ;; + --incomplete) incomplete_only=true; arch="" ;; + esac + done + + # Needed for `git -C ` operations on the branch being cleaned. + # DEFAULT_TARGET_REPO (from .env) overrides the containing-repo default. + local repo="${DEFAULT_TARGET_REPO:-$REPO_ROOT}" + [[ -z "$repo" ]] && { echo "Error: could not derive repo root (set DEFAULT_TARGET_REPO in .env)"; exit 1; } + + if [[ -n "$arch" ]]; then + # Clean a specific architecture + local branch_name + branch_name=$(echo "$arch" | sed -E 's/(For(Causal|Conditional|Masked).*|LMHead.*)$//' | tr '[:upper:]' '[:lower:]') + local safe_branch="feature-${branch_name}-adapter" + local worktree_dir="${WORKTREE_BASE}/${safe_branch}" + + echo "" + echo "Cleaning up: $arch" + echo " worktree : $worktree_dir" + echo " branch : feature/${branch_name}-adapter" + echo "" + + if [[ -d "$worktree_dir" ]]; then + git -C "$repo" worktree remove --force "$worktree_dir" 2>/dev/null && \ + echo " Worktree removed." || echo " Worktree removal failed (may not exist)." + else + echo " No worktree found at $worktree_dir" + fi + + git -C "$repo" branch -D "feature/${branch_name}-adapter" 2>/dev/null && \ + echo " Branch deleted." || echo " Branch not found (may already be deleted)." + + rm -f "$LOGS_DIR/${safe_branch}".{log,pid,raw,debug.log,wrapper.sh,prompt.txt,watch.log} 2>/dev/null + echo " Logs cleaned." + echo "" + else + # Bulk clean. Default: verification_passed: true only. With --force/--all: + # every adapter worktree regardless of state. With --incomplete: only the + # ones where verification_passed is NOT true (useful after finishing + # adapters manually outside the agent pipeline). + echo "" + if [[ "$force" == true ]]; then + echo "Cleaning ALL adapter worktrees (--force)..." + elif [[ "$incomplete_only" == true ]]; then + echo "Cleaning incomplete adapter worktrees (--incomplete)..." + else + echo "Cleaning completed adapter worktrees (verification_passed: true)..." + fi + echo "" + + local cleaned=0 skipped=0 + if [[ -d "$WORKTREE_BASE" ]]; then + for wt in "$WORKTREE_BASE"/feature-*-adapter; do + [[ -d "$wt" ]] || continue + local wt_name branch_ref passed="unknown" + wt_name=$(basename "$wt") + branch_ref=$(echo "$wt_name" | sed 's/^feature-//;s/-adapter$//') + local progress="$wt/.adapter-progress.json" + if [[ -f "$progress" ]]; then + passed=$(python3 -c "import json; print(json.load(open('$progress')).get('verification_passed', False))" 2>/dev/null || echo "False") + fi + + local should_clean=false + if [[ "$force" == true ]]; then + should_clean=true + elif [[ "$incomplete_only" == true ]]; then + [[ "$passed" != "True" ]] && should_clean=true + else + [[ "$passed" == "True" ]] && should_clean=true + fi + + if [[ "$should_clean" == true ]]; then + # Kill any live tmux session for this pair first so the worktree + # remove doesn't fail on locked files. + local tmux_session="tl-adapter-$wt_name" + if command -v tmux &>/dev/null && TM has-session -t "$tmux_session" 2>/dev/null; then + TM kill-session -t "$tmux_session" 2>/dev/null || true + echo " Killed tmux session: $tmux_session" + fi + echo " Removing: $wt_name (verification_passed=$passed)" + git -C "$repo" worktree remove --force "$wt" 2>/dev/null || true + git -C "$repo" branch -D "feature/${branch_ref}-adapter" 2>/dev/null || true + rm -f "$LOGS_DIR/${wt_name}".{log,pid,raw,debug.log,wrapper.sh,prompt.txt,watch.log,programmer.raw,reviewer.raw,programmer-wrapper.sh,reviewer-wrapper.sh,programmer-prompt.txt,reviewer-prompt.txt,watcher.pid} 2>/dev/null + cleaned=$((cleaned + 1)) + else + echo " Skipping: $wt_name (verification_passed=$passed)" + skipped=$((skipped + 1)) + fi + done + # Prune any dangling worktree refs whose directories are gone. + git -C "$repo" worktree prune 2>/dev/null || true + fi + + echo "" + echo "Cleaned $cleaned worktree(s), skipped $skipped." + echo "" + fi +} + +# --------------------------------------------------------------------------- # +# stop — kill a running pair (tmux session or bare PID) and keep its worktree +# --------------------------------------------------------------------------- # +cmd_stop() { + local target="${1:-all}" + + # Kill companion daemons (solo coordinator, teams watcher) for a pair. + # Both self-exit when the tmux session vanishes, but killing them directly + # avoids a dangling poll cycle and cleans up their PID files. + _stop_daemons() { + local name="$1" daemon dpid + for daemon in "$LOGS_DIR/$name.coordinator.pid" "$LOGS_DIR/$name.watcher.pid"; do + [[ -f "$daemon" ]] || continue + dpid=$(cat "$daemon" 2>/dev/null || echo 0) + if kill -0 "$dpid" 2>/dev/null; then + kill "$dpid" 2>/dev/null || true + echo " Stopped: $(basename "$daemon" .pid) (PID $dpid)" + fi + rm -f "$daemon" + done + } + + _stop_pair() { + local pidfile="$1" + local name pid tmux_session + name=$(basename "$pidfile" .pid) + pid=$(cat "$pidfile" 2>/dev/null || echo 0) + tmux_session="tl-adapter-$name" + + # Prefer tmux session termination if we can find one. + if command -v tmux &>/dev/null && TM has-session -t "$tmux_session" 2>/dev/null; then + TM kill-session -t "$tmux_session" 2>/dev/null + echo " Stopped: $name (tmux session killed)" + rm -f "$pidfile" + _stop_daemons "$name" + return 0 + fi + + # Fall back to killing by PID (legacy non-tmux sessions). + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null + echo " Stopped: $name (PID $pid)" + else + echo " Already dead: $name (PID $pid)" + fi + rm -f "$pidfile" + _stop_daemons "$name" + } + + if [[ "$target" == "all" ]]; then + echo "" + echo "Stopping all background agent pairs..." + echo "" + local stopped=0 + for pidfile in "$LOGS_DIR"/*.pid; do + [[ -f "$pidfile" ]] || continue + # Daemon PID files are cleaned up by _stop_daemons via their pair. + case "$pidfile" in *.coordinator.pid|*.watcher.pid) continue ;; esac + _stop_pair "$pidfile" + stopped=$((stopped + 1)) + done + echo "" + echo "Stopped $stopped process(es). Worktrees preserved for resume." + echo "" + else + local arch + arch=$(echo "$target" | sed -E 's/(For(Causal|Conditional|Masked).*|LMHead.*)$//' | tr '[:upper:]' '[:lower:]') + + local matched=false + for pidfile in "$LOGS_DIR"/*"${arch}"*.pid; do + [[ -f "$pidfile" ]] || continue + case "$pidfile" in *.coordinator.pid|*.watcher.pid) continue ;; esac + matched=true + _stop_pair "$pidfile" + done + + if [[ "$matched" == false ]]; then + echo "No running process found matching '$target'" + ls "$LOGS_DIR"/*.pid 2>/dev/null || echo "(no PID files exist)" + else + echo "Worktree preserved for resume." + fi + fi +} + +# --------------------------------------------------------------------------- # +# _find_tmux_session — resolve an architecture name to an exact tmux session. +# +# Matching priority (to avoid "cohere" matching "cohere2"): +# 1. Exact: tl-adapter-feature--adapter +# 2. Unique prefix: only one session starts with tl-adapter-** +# 3. Ambiguous → error listing all matches +# --------------------------------------------------------------------------- # +_find_tmux_session() { + local arch="$1" + local exact="tl-adapter-feature-${arch}-adapter" + + # Try exact match first + if TM has-session -t "$exact" 2>/dev/null; then + echo "$exact" + return 0 + fi + + # Fall back to prefix match, but require uniqueness + local matches=() + for s in $(tmux list-sessions -F '#{session_name}' 2>/dev/null); do + if [[ "$s" == tl-adapter-*"${arch}"* ]]; then + matches+=("$s") + fi + done + + if [[ ${#matches[@]} -eq 1 ]]; then + echo "${matches[0]}" + return 0 + elif [[ ${#matches[@]} -gt 1 ]]; then + echo "Ambiguous: '$arch' matches ${#matches[@]} sessions:" >&2 + printf " %s\n" "${matches[@]}" >&2 + echo "Use a more specific name." >&2 + return 1 + fi + + return 1 +} + +# --------------------------------------------------------------------------- # +# attach — interactively attach to the tmux session for an architecture +# --------------------------------------------------------------------------- # +cmd_attach() { + local arch="${1:-}" + [[ -z "$arch" ]] && { echo "Usage: $TL_CMD attach "; exit 1; } + + command -v tmux &>/dev/null || { echo "tmux not installed"; exit 1; } + + arch=$(echo "$arch" | sed -E 's/(For(Causal|Conditional|Masked).*|LMHead.*)$//' | tr '[:upper:]' '[:lower:]') + + local session + session=$(_find_tmux_session "$arch") || { + echo "No tmux session matching '$arch'. Active sessions:" + TM list-sessions 2>/dev/null || echo "(none)" + exit 1 + } + + echo "Attaching to $session (detach with Ctrl-b d)" + sleep 0.5 + exec TM attach-session -t "$session" +} + +# --------------------------------------------------------------------------- # +# send — inject a message into a running tmux session as if typed at the prompt +# --------------------------------------------------------------------------- # +cmd_send() { + local arch="${1:-}" + shift || true + local message="$*" + + [[ -z "$arch" || -z "$message" ]] && { + echo "Usage: $TL_CMD send " + echo "" + echo "Sends a message to a running Claude Code session as if you typed it" + echo "in the interactive prompt. Useful for injecting mid-run corrections" + echo "without stopping the session." + echo "" + echo "Example:" + echo " $TL_CMD send qwen3moe \"Use float32 instead of bfloat16 for verification\"" + exit 1 + } + + command -v tmux &>/dev/null || { echo "tmux not installed"; exit 1; } + + arch=$(echo "$arch" | sed -E 's/(For(Causal|Conditional|Masked).*|LMHead.*)$//' | tr '[:upper:]' '[:lower:]') + + local session + session=$(_find_tmux_session "$arch") || { + echo "No tmux session matching '$arch'" + exit 1 + } + + # Claude Code's interactive prompt treats large TM send-keys payloads as + # a bracketed paste, which stages them as `[Pasted text #1]` and swallows + # any `Enter` sent in the same tmux call. The fix is a two-step submission: + # + # 1. Deliver the message body via a dedicated paste buffer + # (tmux load-buffer + paste-buffer) so bracketed-paste markers are + # emitted cleanly. This is more reliable than `send-keys -l "$msg"` + # for multi-line / very long payloads. + # 2. Sleep briefly so tmux finalizes the paste before the submit key. + # 3. Send `Enter` as a standalone send-keys call so it reaches the prompt + # as a discrete key press (Claude Code submits the staged paste). + # + # Empirically, the old single-shot `send-keys "$msg"; send-keys Enter` + # pattern left long messages (≳200 chars) unsubmitted with `[Pasted text #1]` + # visible in the pane, forcing the user to press Enter manually. + local tmp_buffer + tmp_buffer="tl-send-$$-$(date +%s%N 2>/dev/null || date +%s)" + printf '%s' "$message" | TM load-buffer -b "$tmp_buffer" - + TM paste-buffer -d -b "$tmp_buffer" -t "$session" + sleep 0.3 + TM send-keys -t "$session" Enter + + echo "Message sent to $session:" + echo " $message" +} + +# --------------------------------------------------------------------------- # +# Dispatch +# --------------------------------------------------------------------------- # +case "${1:-}" in + status) shift; cmd_status "$@" ;; + logs) shift; cmd_logs "$@" ;; + stop) shift; cmd_stop "$@" ;; + attach) shift; cmd_attach "$@" ;; + send) shift; cmd_send "$@" ;; + clean) shift; cmd_clean "$@" ;; + "") echo "Usage: $TL_CMD {status|logs|stop|attach|send|clean} [args...]" >&2; exit 1 ;; + *) echo "Unknown subcommand: $1" >&2; exit 1 ;; +esac diff --git a/devtools/adapter_builder/agents/orchestrator.md b/devtools/adapter_builder/agents/orchestrator.md new file mode 100644 index 0000000000..806e53adf5 --- /dev/null +++ b/devtools/adapter_builder/agents/orchestrator.md @@ -0,0 +1,71 @@ +Orchestrator for a Programmer/Reviewer pair building an Architecture Adapter. + +## Team +- **Programmer** — Plan → Program (phase-by-phase) → Verify +- **Reviewer** — reviews at each checkpoint + +{{PROTOCOL_BLOCK}} + +## How to spawn and message subagents + +**First contact:** use the `Agent` tool with `subagent_type="programmer"` or `subagent_type="reviewer"` and the task in the `prompt` field. This creates a new subagent and returns its result when it finishes. + +**Subsequent messages:** use `SendMessage`. But apply the **Subagent resume protocol** from the Protocol section above — `SendMessage to=` is a silent no-op against a stopped subagent. Always track `agent_id` from `SubagentStart` events and use `SendMessage to=` with `Resuming you — ` prefix when the target has stopped. + +**Lifecycle flow after BRIEF APPROVED:** +1. Resume Programmer by agent_id → tell it to write the plan (Step 1b) +2. Wait for `PLAN READY FOR REVIEW` → spawn/resume Reviewer for plan review +3. After `PLAN APPROVED` → resume Programmer → tell it to begin implementation (Step 2, Phase A) +4. Wait for `PHASE A READY FOR REVIEW` → spawn/resume Reviewer for code review +5. After `APPROVED` → resume Programmer → next phase (or verification if all done) +6. Repeat 4-5 for each phase +7. After `VERIFICATION COMPLETE` + `READY FOR REVIEW` → spawn/resume Reviewer for final review +8. After final `APPROVED` → completion procedure (see "On completion" below) + +## Brief before Plan +Step 1 has two checkpoints: +- **1a:** Programmer writes brief → `PROGRAMMER: BRIEF READY FOR REVIEW` → Reviewer fact-checks +- **1b:** Only after `REVIEWER: BRIEF APPROVED` → Programmer writes plan + +## Iteration limits +Max **3 rounds** per review loop (brief, plan, or phase). After round 3 with unresolved CRITICALs: +1. Write issues to `.adapter-workspace/stuck-report.md` +2. Run: `{{PROJECT_ROOT}}/scripts/notify.sh "{{ARCHITECTURE}} stuck at after 3 review rounds"` +3. Stop. Do NOT keep looping. + +Limit resets per checkpoint. A hook also blocks review files past round 3. + +## Session +Architecture: {{ARCHITECTURE}} | Branch: `{{NEW_BRANCH}}` (from `{{BASE_BRANCH}}`) | Memory: {{MAX_MEMORY_GB}}GB + +### Programmer task +{{PROGRAMMER_PROMPT}} + +### Reviewer focus +{{REVIEWER_PROMPT}} +{{RESUME_CONTEXT}} + +## Progress tracking +Programmer updates `.adapter-progress.json` after each milestone: +```bash +python3 -c " +import json; p = json.load(open('.adapter-progress.json')) +p['step'] = '' +p['plan_approved'] = True # after plan approval +p['current_phase'] = 'C'; p['completed_phases'].append('A+B') +p['verification_attempts'] = 1 # increment per verify_models run +p.setdefault('verified_models', []).append('model/id') # after each model passes +p['last_updated'] = '$(date -u +%Y-%m-%dT%H:%M:%SZ)' +json.dump(p, open('.adapter-progress.json','w'), indent=2) +" +``` +Use exact phase labels from the plan (e.g. `A+B` not `A`). Enables crash recovery. + +## On completion +After final `{{SIG_CODE_APPROVED}}`: +1. Verify `.adapter-workspace/completion-report.md` exists (ask Reviewer if missing) +2. Set `verification_passed: true` in `.adapter-progress.json` +3. Terminate — SessionEnd hook fires Slack notification automatically + +If stuck (3 rounds exceeded): write stuck-report, notify, terminate. +If `VERIFICATION SKIPPED — ALL MODELS TOO LARGE`: write stuck-report, notify, terminate. Do NOT loop. diff --git a/devtools/adapter_builder/agents/overlord-request.sh b/devtools/adapter_builder/agents/overlord-request.sh new file mode 100755 index 0000000000..cb6cb9ce78 --- /dev/null +++ b/devtools/adapter_builder/agents/overlord-request.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# ============================================================================= +# overlord-request.sh — Simple flock-based memory slot lock +# +# Ensures only one memory-intensive operation (verify_models, benchmarks, +# full model loading) runs at a time across all agent pairs. +# +# Usage (preferred — single command, atomic acquire/run/release): +# ./overlord-request.sh run "verify_models" -- uv run python -m ... +# +# Usage (status check): +# ./overlord-request.sh status +# +# NOTE: when `source`d by another script, err() must not call `exit` or it +# kills the parent. We override err() locally for the sourced case. +# ============================================================================= + +set -euo pipefail + +LOCK_FILE="/tmp/tl-adapter-builder.lock" +STATUS_FILE="/tmp/tl-adapter-builder.status" +LOCK_TTL_SECONDS=1800 # 30 minutes — auto-expire stale locks + +# Shared helpers. Override err() to use return instead of exit when sourced, +# so a lock timeout doesn't kill the calling script. +OVERLORD_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TL_LOG_TAG="memory-lock" +# shellcheck disable=SC1091 +source "$OVERLORD_SCRIPT_DIR/lib/common.sh" + +# If sourced, redefine err() to return 1 instead of exit 1. +if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then + err() { echo -e "\033[1;31m[memory-lock ERROR]\033[0m $*" >&2; return 1; } +fi + +iso_now() { date -u +"%Y-%m-%dT%H:%M:%SZ"; } + +# --------------------------------------------------------------------------- # +# Stale lock cleanup — runs AFTER flock, not before. This avoids the TOCTOU +# where cleanup removes the lock file between another process's flock and +# its status write. Instead, we acquire the lock first (blocking), then +# check if the status file is stale (meaning the previous holder crashed +# without releasing). If stale, we already hold the lock so no race. +# --------------------------------------------------------------------------- # +_cleanup_stale_status() { + # Only meaningful when we already hold the lock. + if [[ -f "$STATUS_FILE" ]]; then + local file_age + if [[ "$(uname)" == "Darwin" ]]; then + file_age=$(( $(date +%s) - $(stat -f %m "$STATUS_FILE") )) + else + file_age=$(( $(date +%s) - $(stat -c %Y "$STATUS_FILE") )) + fi + if (( file_age > LOCK_TTL_SECONDS )); then + local stale_holder + stale_holder=$(python3 -c "import json; print(json.load(open('$STATUS_FILE')).get('operation','unknown'))" 2>/dev/null || echo "unknown") + log "Stale status detected (${file_age}s old, holder: ${stale_holder}). Clearing." + rm -f "$STATUS_FILE" + fi + fi +} + +# --------------------------------------------------------------------------- # +# acquire: take the lock, clean stale status, write new status +# --------------------------------------------------------------------------- # +overlord_acquire() { + local operation="${1:-unspecified}" + local timeout="${2:-300}" + + log "Requesting memory slot for: $operation" + + # Open the lock file on fd 9 + exec 9>"$LOCK_FILE" + + # Block until lock is available (or timeout) + if ! flock -w "$timeout" 9; then + err "Timeout (${timeout}s) waiting for memory slot. Another heavy operation is running." + return 1 + fi + + # Now that we hold the lock, clean any stale status from a crashed holder. + _cleanup_stale_status + + # Write status so others can see who holds the lock + cat > "$STATUS_FILE" <&- 2>/dev/null || true + log "Released memory slot." +} + +# --------------------------------------------------------------------------- # +# status: show who holds the lock (if anyone) +# --------------------------------------------------------------------------- # +cmd_status() { + # Acquire the lock briefly to check stale status atomically, then release. + if exec 9>"$LOCK_FILE" && flock -n 9 2>/dev/null; then + _cleanup_stale_status + exec 9>&- 2>/dev/null || true + fi + + if [[ -f "$STATUS_FILE" ]]; then + log "Memory slot is HELD:" + cat "$STATUS_FILE" + else + log "Memory slot is FREE." + fi +} + +# --------------------------------------------------------------------------- # +# run: acquire lock, run a command, release lock (all in one process) +# --------------------------------------------------------------------------- # +cmd_run() { + local operation="${1:-unspecified}" + shift + [[ "${1:-}" == "--" ]] && shift + + overlord_acquire "$operation" + local exit_code=0 + "$@" || exit_code=$? + overlord_release + return $exit_code +} + +# --------------------------------------------------------------------------- # +# Direct CLI invocation +# --------------------------------------------------------------------------- # +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + case "${1:-}" in + acquire) shift; overlord_acquire "$@" ;; + release) overlord_release ;; + status) cmd_status ;; + run) shift; cmd_run "$@" ;; + *) echo "Usage: $0 {run -- |status|acquire |release}" >&2; exit 1 ;; + esac +fi diff --git a/devtools/adapter_builder/agents/programmer.md b/devtools/adapter_builder/agents/programmer.md new file mode 100644 index 0000000000..61b617b376 --- /dev/null +++ b/devtools/adapter_builder/agents/programmer.md @@ -0,0 +1,117 @@ +--- +name: programmer +description: Implements Architecture Adapters for TransformerLens TransformerBridge. +model: claude-sonnet-4-6 +--- + +# Programmer — Adapter Builder + +Build Architecture Adapters for TransformerBridge. Strict lifecycle: +Plan → Program → Verify. No skipping. No advancing without Reviewer approval. + +## Design principle +When making critical design decisions, ask: **which approach is most +beneficial for Mech Interp research?** Take that approach, even if it is +more difficult. This means: maximize hook granularity, preserve internal +activations faithfully, prefer explicit component decomposition over +opaque fused operations, and ensure every intermediate representation is +accessible for inspection. + +## Read first (from `$TL_ADAPTER_BUILDER_ROOT`) +- `docs/adapter-specification.md` — adapter spec, bridge components, patterns +- `docs/hf-model-analysis-guide.md` — HF model analysis procedure +- `docs/artifact-templates.md` — templates for all output files +- `docs/memory-lock.md` — lock protocol (read before Step 3) + +## Constraints (hook-enforced) +- HookedTransformer is read-only (hook blocks edits) +- No `git commit`/`push`/`gh pr` (hook blocks) +- `verify_models` blocked on >7.5B or unregistered models (hook blocks) +- All output → files in `.adapter-workspace/`, not terminal +- Signal orchestrator with file pointer only, never full content + +--- + +## Step 1a: Analysis → Brief + +1. Read HF source: `modeling_.py`, `configuration_.py`. Follow `docs/hf-model-analysis-guide.md`. +2. Generate scaffold: + ```bash + python "$TL_ADAPTER_BUILDER_ROOT/scripts/analyze-hf-model.py" \ + --scaffold --scaffold-out .adapter-workspace/adapter-scaffold.py + ``` + Cross-check scaffold's detected module paths against what you read in the HF source. +3. Scan HF for all models of this architecture: + ```bash + uv run python "$TL_ADAPTER_BUILDER_ROOT/scripts/scan-hf-architecture.py" \ + --arch-short + ``` +4. Find closest reference adapter. Use `scripts/compare-adapters.sh`. +5. Write brief → `.adapter-workspace/adapter-brief.md` (template: `docs/artifact-templates.md`) +6. Signal: `PROGRAMMER: BRIEF READY FOR REVIEW` + +Iterate until `REVIEWER: BRIEF APPROVED`. + +## Step 1b: Plan + +Write plan → `.adapter-workspace/adapter-plan.md` (template: `docs/artifact-templates.md`). +Typical phases: A=config+weights, B=mapping, C=overrides, D=registration. +Merge coupled phases as `Phase A+B` — use that label everywhere. + +Signal: `PROGRAMMER: PLAN READY FOR REVIEW`. Iterate until approved. + +## Step 2: Implement + +One phase at a time. Per phase: +1. Implement +2. Test +3. Write report → `.adapter-workspace/phase-reports/phase--report.md` (use plan's phase label: `phase-A-report.md`, `phase-A+B-report.md`) +4. Update `.adapter-progress.json`: set `current_phase` to this phase's label +5. Signal: `PROGRAMMER: PHASE READY FOR REVIEW` +6. Address feedback +7. After approval: append this phase to `completed_phases`, set `current_phase` to next → proceed + +New bridge components: only when `forward()` must differ. Document why. +Place in `generalized_components/`, export, test, own phase. + +Tests: required, CI-friendly, no tautologies. Use `@pytest.mark.skip` for genuinely CI-incompatible tests. + +**Do not** modify files outside the current phase's scope without flagging it. +**Do not** duplicate `docs/` content into artifacts — reference the doc file. + +## Step 3: Verify + +**3.0** Port models: `uv run python "$TL_ADAPTER_BUILDER_ROOT/scripts/port-arch-models.py" --arch-short ` +Skip this → verify_models silently skips everything. + +**3.1** Select up to 5 models, all ≤7B, chosen for **config diversity, not popularity**: +always include the smallest model of the architecture (tiny models expose +numerical-path divergence fastest and verify in seconds) and the most-downloaded +that fits; fill remaining slots with config variants (different head_dim, +rotary_pct, bias flags, GQA settings) over download rank. Verify **smallest +first** — a numerical bug should fail in seconds, not after a 30-minute large-model +run. If none fit → signal `VERIFICATION SKIPPED — ALL MODELS TOO LARGE`, stop. + +**3.2** One model at a time. Read `docs/memory-lock.md`, then: +```bash +"$TL_ADAPTER_BUILDER_ROOT/agents/overlord-request.sh" run "verify_models: " -- \ + uv run python -m transformer_lens.tools.model_registry.verify_models \ + --model --max-memory $MAX_MEMORY_GB --device cpu --dtype float32 +``` +Check `status` in `supported_models.json`: 1=next, 2=note+next, 3=**stop and fix**, 4 (provisional, "Structural only")=**wrong invocation — rerun with the HF reference on**. +After each passing model, append its ID to `verified_models` in `.adapter-progress.json`. +On crash-resume, skip models already in `verified_models`. + +Default `--dtype float32`. **Never pass `--no-hf-reference`** — it skips the HuggingFace parity comparison and writes status 4 (provisional), which does not count as verified (a hook blocks it). If verify_models rejects a flag, read `--help` and understand a replacement before using it — never substitute a similar-looking flag. If a model fails: +- **status=3 (phase score failure):** read the `note` and phase scores. This is an adapter bug — investigate root cause, fix, re-verify. +- **OOM / MemoryError / killed:** retry that single model with `--dtype bfloat16`. If it still OOMs, skip it (note in verification results) and move to the next model. Do not loop. +- **status=2 (SKIPPED by verify_models):** the model exceeded `--max-memory` pre-check. Note why and move on — this is not an adapter bug. + +**3.3** `uv run mypy .` + `make check-format` must pass. No `# type: ignore`. + +**3.4** Write results → `.adapter-workspace/verification-results.md`. +Signal: `PROGRAMMER: VERIFICATION COMPLETE` then `PROGRAMMER: READY FOR REVIEW`. +On failure → write failure analysis, signal `PROGRAMMER: VERIFICATION FAILED`, and return to planning. + +--- + diff --git a/devtools/adapter_builder/agents/progress.sh b/devtools/adapter_builder/agents/progress.sh new file mode 100644 index 0000000000..db0770cf3a --- /dev/null +++ b/devtools/adapter_builder/agents/progress.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# ============================================================================= +# progress.sh — Lightweight adapter build progress tracking +# +# Writes/reads .adapter-progress.json in the worktree to track lifecycle state. +# Used by launch-agent-pair.sh to construct resume prompts after crashes. +# +# All writes use Python's fcntl.flock to prevent concurrent +# orchestrator/programmer/hook updates from clobbering each other. This +# works on both Linux and macOS (unlike bash flock which isn't on macOS). +# +# Usage (sourced by launch-agent-pair.sh): +# source agents/progress.sh +# progress_init "$WORKTREE_DIR" "$ARCHITECTURE" +# progress_update "$WORKTREE_DIR" "step" "planning" +# progress_update "$WORKTREE_DIR" "current_phase" "B" +# progress_add_completed_phase "$WORKTREE_DIR" "A" +# progress_read "$WORKTREE_DIR" # sets PROGRESS_* vars +# ============================================================================= + +PROGRESS_FILE=".adapter-progress.json" + +# --------------------------------------------------------------------------- # +# Locked JSON write via Python fcntl.flock — works on Linux and macOS. +# The lock file is co-located with the progress file as .adapter-progress.lock +# so it's per-worktree, not global. The Python script is fed via heredoc to +# avoid bash variable expansion mangling multiline strings. +# --------------------------------------------------------------------------- # + +# --------------------------------------------------------------------------- # +# Initialize a new progress file +# --------------------------------------------------------------------------- # +progress_init() { + local worktree="$1" + local architecture="$2" + local filepath="$worktree/$PROGRESS_FILE" + local lockfile="${filepath%.json}.lock" + local now + now="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + + python3 - "$filepath" "$lockfile" "$architecture" "$now" <<'PYEOF' +import json, fcntl, sys +filepath, lockfile, arch, now = sys.argv[1:5] +lockfd = open(lockfile, 'w') +try: + fcntl.flock(lockfd, fcntl.LOCK_EX) + json.dump({ + "architecture": arch, + "step": "planning", + "plan_approved": False, + "current_phase": None, + "completed_phases": [], + "verification_attempts": 0, + "verified_models": [], + "verification_passed": False, + "final_review_passed": False, + "last_updated": now, + "started_at": now, + }, open(filepath, "w"), indent=2) +finally: + fcntl.flock(lockfd, fcntl.LOCK_UN) + lockfd.close() +PYEOF +} + +# --------------------------------------------------------------------------- # +# Update a field in the progress file +# --------------------------------------------------------------------------- # +progress_update() { + local worktree="$1" + local key="$2" + local value="$3" + local filepath="$worktree/$PROGRESS_FILE" + local lockfile="${filepath%.json}.lock" + + [[ ! -f "$filepath" ]] && return 1 + + python3 - "$filepath" "$lockfile" "$key" "$value" "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" <<'PYEOF' +import json, fcntl, sys +filepath, lockfile, key, value, now = sys.argv[1:6] +lockfd = open(lockfile, 'w') +try: + fcntl.flock(lockfd, fcntl.LOCK_EX) + with open(filepath) as f: + p = json.load(f) + if value == 'true': value = True + elif value == 'false': value = False + elif value == 'null': value = None + elif value.isdigit(): value = int(value) + p[key] = value + p['last_updated'] = now + with open(filepath, 'w') as f: + json.dump(p, f, indent=2) +finally: + fcntl.flock(lockfd, fcntl.LOCK_UN) + lockfd.close() +PYEOF +} + +# --------------------------------------------------------------------------- # +# Add a completed phase +# --------------------------------------------------------------------------- # +progress_add_completed_phase() { + local worktree="$1" + local phase="$2" + local filepath="$worktree/$PROGRESS_FILE" + local lockfile="${filepath%.json}.lock" + + [[ ! -f "$filepath" ]] && return 1 + + python3 - "$filepath" "$lockfile" "$phase" "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" <<'PYEOF' +import json, fcntl, sys +filepath, lockfile, phase, now = sys.argv[1:5] +lockfd = open(lockfile, 'w') +try: + fcntl.flock(lockfd, fcntl.LOCK_EX) + with open(filepath) as f: + p = json.load(f) + if phase not in p['completed_phases']: + p['completed_phases'].append(phase) + p['last_updated'] = now + with open(filepath, 'w') as f: + json.dump(p, f, indent=2) +finally: + fcntl.flock(lockfd, fcntl.LOCK_UN) + lockfd.close() +PYEOF +} + +# --------------------------------------------------------------------------- # +# Read progress into shell variables (no lock needed — read-only) +# --------------------------------------------------------------------------- # +progress_read() { + local worktree="$1" + local filepath="$worktree/$PROGRESS_FILE" + + if [[ ! -f "$filepath" ]]; then + PROGRESS_EXISTS=false + return 1 + fi + + PROGRESS_EXISTS=true + PROGRESS_STEP=$(python3 -c "import json; print(json.load(open('$filepath')).get('step','unknown'))" 2>/dev/null) + PROGRESS_PLAN_APPROVED=$(python3 -c "import json; print(str(json.load(open('$filepath')).get('plan_approved',False)).lower())" 2>/dev/null) + PROGRESS_CURRENT_PHASE=$(python3 -c "import json; print(json.load(open('$filepath')).get('current_phase','none'))" 2>/dev/null) + PROGRESS_COMPLETED_PHASES=$(python3 -c "import json; print(','.join(json.load(open('$filepath')).get('completed_phases',[])))" 2>/dev/null) + PROGRESS_VERIFICATION_ATTEMPTS=$(python3 -c "import json; print(json.load(open('$filepath')).get('verification_attempts',0))" 2>/dev/null) + PROGRESS_VERIFIED_MODELS=$(python3 -c "import json; print(','.join(json.load(open('$filepath')).get('verified_models',[])))" 2>/dev/null) + PROGRESS_LAST_UPDATED=$(python3 -c "import json; print(json.load(open('$filepath')).get('last_updated','unknown'))" 2>/dev/null) +} + +# --------------------------------------------------------------------------- # +# Generate a resume prompt from progress state +# --------------------------------------------------------------------------- # +generate_resume_context() { + local worktree="$1" + progress_read "$worktree" + + if [[ "$PROGRESS_EXISTS" != "true" ]]; then + echo "" + return + fi + + cat <.py`, `configuration_.py`). +Cross-reference every claim in `.adapter-workspace/adapter-brief.md`. +Write review → `.adapter-workspace/reviews/brief-review-.md`. +Signal `REVIEWER: BRIEF APPROVED` or `REVIEWER: BRIEF CHANGES REQUESTED` with file pointer. + +## Plan review + +On `PROGRAMMER: PLAN READY FOR REVIEW`: +Brief is approved — use it as factual reference. Check design, completeness, consistency. +Write → `.adapter-workspace/reviews/plan-review-.md`. +Signal `REVIEWER: PLAN APPROVED` or `REVIEWER: PLAN CHANGES REQUESTED`. + +## Phase code review + +On `PROGRAMMER: PHASE READY FOR REVIEW`: +**Scope: only the files changed in this phase.** Do not re-review prior phases. +Follow `docs/review-specification.md` P0–P5, but scope P1/P2/P3/P4 to this phase's changes only. +P0 (HF source reference) is done once and reused across phases. +Also verify adapter-specific items from `docs/adapter-specification.md` for the components this phase touches. +Write → `.adapter-workspace/reviews/phase--review-.md`. +Signal `REVIEWER: APPROVED` or `REVIEWER: CHANGES REQUESTED`. + +Give feedback, not solutions — do not rewrite the adapter yourself. +Verify against code and HF source, not the Programmer's summary. + +## Final review + +On `PROGRAMMER: READY FOR REVIEW` (after verification passes): +**Scope: the complete adapter across all phases.** +**Gate on verification quality first:** every verified model must have `status: 1` +in `supported_models.json`. Status 4 with "Structural only (no HF reference)" means +the HuggingFace parity comparison never ran — that is NOT verification; request +changes and have the Programmer rerun verify_models with the HF reference on. This is a holistic review — check cross-phase consistency, end-to-end correctness, and anything that individual phase reviews couldn't catch in isolation. Run P5 (differential review) fully: plan-to-code match across ALL phases, all prior findings resolved, test quality across the whole test suite. +Write completion report → `.adapter-workspace/completion-report.md` (template: `docs/artifact-templates.md`). +Signal `REVIEWER: APPROVED` with file pointer. + +## Review file protocol + +File must exist BEFORE you signal. A hook detects missing files and voids your result. +1. Write file with Write tool +2. Emit signal with pointer to file path + +Check `ls .adapter-workspace/reviews/` to pick next round number. + +--- + diff --git a/devtools/adapter_builder/agents/signals.sh b/devtools/adapter_builder/agents/signals.sh new file mode 100644 index 0000000000..0cd16300c8 --- /dev/null +++ b/devtools/adapter_builder/agents/signals.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# ============================================================================= +# signals.sh — Single source of truth for the agent team protocol +# +# Defines all signals, their meanings, and the state machine transitions. +# Sourced by launch-agent-pair.sh and injected into agent prompts. +# ============================================================================= + +# --------------------------------------------------------------------------- # +# Programmer signals +# --------------------------------------------------------------------------- # +SIG_BRIEF_READY="PROGRAMMER: BRIEF READY FOR REVIEW" +SIG_PLAN_READY="PROGRAMMER: PLAN READY FOR REVIEW" +SIG_PHASE_READY="PROGRAMMER: PHASE READY FOR REVIEW" # replaced at runtime +SIG_VERIFICATION_COMPLETE="PROGRAMMER: VERIFICATION COMPLETE" +SIG_VERIFICATION_FAILED="PROGRAMMER: VERIFICATION FAILED" +SIG_VERIFICATION_SKIPPED="PROGRAMMER: VERIFICATION SKIPPED — ALL MODELS TOO LARGE" +SIG_FINAL_READY="PROGRAMMER: READY FOR REVIEW" + +# --------------------------------------------------------------------------- # +# Reviewer signals +# --------------------------------------------------------------------------- # +SIG_BRIEF_APPROVED="REVIEWER: BRIEF APPROVED" +SIG_BRIEF_CHANGES="REVIEWER: BRIEF CHANGES REQUESTED" +SIG_PLAN_APPROVED="REVIEWER: PLAN APPROVED" +SIG_PLAN_CHANGES="REVIEWER: PLAN CHANGES REQUESTED" +SIG_CODE_APPROVED="REVIEWER: APPROVED" +SIG_CODE_CHANGES="REVIEWER: CHANGES REQUESTED" + +# --------------------------------------------------------------------------- # +# State machine (for injection into orchestrator prompt) +# --------------------------------------------------------------------------- # +generate_protocol_block() { + cat <<'PROTOCOL' +## Protocol + +### Signals +Programmer: `BRIEF READY FOR REVIEW`, `PLAN READY FOR REVIEW`, `PHASE READY FOR REVIEW`, `VERIFICATION COMPLETE`, `VERIFICATION FAILED`, `VERIFICATION SKIPPED — ALL MODELS TOO LARGE`, `READY FOR REVIEW` +Reviewer: `BRIEF APPROVED`/`CHANGES REQUESTED`, `PLAN APPROVED`/`CHANGES REQUESTED`, `APPROVED`/`CHANGES REQUESTED` + +### Subagent resume (CRITICAL) +`SendMessage to=` is a **silent no-op** against a stopped subagent. Always: +1. Record each subagent's `agent_id` from its `SubagentStart` event +2. Before `SendMessage`, check if target is running (last event = `SubagentStart`, not `SubagentStop`) +3. If stopped → `SendMessage to=` with body prefixed `Resuming you — ` +4. If running → `SendMessage to=` is fine +5. To find an ID: `tail -r .adapter-workspace/timeline.jsonl | jq -r 'select(.event=="SubagentStart" and .agent_type=="programmer") | .agent_id' | head -1` +Never send pings. Never retry by type name against a stopped subagent. Fresh spawn via `Task` is also valid but loses prior context. + +### Review file check +After every reviewer turn, verify a new file exists in `.adapter-workspace/reviews/` (or `completion-report.md`). A hook also enforces this. If no file → review is VOID. Resume reviewer by ID: `Resuming you — write your review to a file before returning.` + +### Hook block monitoring +All blocking hooks append a `HookBlocked` event to `.adapter-workspace/timeline.jsonl` with `hook`, `reason`, and `tool` fields. If a subagent seems stuck or repeating the same action, check for recent `HookBlocked` events: +```bash +grep HookBlocked .adapter-workspace/timeline.jsonl | tail -3 +``` +If you see repeated blocks, the subagent is retrying a denied action. Resume it with explicit guidance on what to do instead (the `reason` field explains the constraint). + +### Subagent activity monitoring +If a subagent has been running for >15 minutes with no new timeline events, it may be hung. Check: +```bash +tail -1 .adapter-workspace/timeline.jsonl | jq -r .ts +``` +If the last event is old AND the subagent is still `SubagentStart`'d (no `SubagentStop`), consider sending a status ping via `SendMessage` to the agent_id. If doing legitimate long work (verify_models downloading a model), the timeline will show a long-running Bash PreToolUse without a PostToolUse — that's normal. True hangs show no events at all. + +### Routing rules +All routing applies the resume protocol (use agent_id for stopped subagents) and the file check (verify file exists before accepting reviewer decisions). + +1. `BRIEF READY FOR REVIEW` → spawn Reviewer +2. `BRIEF CHANGES REQUESTED` → route to Programmer +3. `BRIEF APPROVED` → tell Programmer: start planning (Step 1b) +4. `PLAN READY FOR REVIEW` → spawn Reviewer +5. `PLAN CHANGES REQUESTED` → route to Programmer +6. `PLAN APPROVED` → tell Programmer: begin implementation (Step 2) +7. `PHASE READY FOR REVIEW` → spawn Reviewer +8. `CHANGES REQUESTED` → route to Programmer +9. `APPROVED` → tell Programmer: next phase (or verification if all done) +10. `VERIFICATION FAILED` → Programmer writes a **new** fix plan (`.adapter-workspace/adapter-plan.md`), signals `PLAN READY FOR REVIEW`. Reviewer reviews the fix plan (rule 4-6). Then re-implement + re-verify. Do NOT skip the plan review — the failure may reveal architectural issues that need reviewer input. +11. `VERIFICATION COMPLETE` + `READY FOR REVIEW` → spawn Reviewer (final) +12. `VERIFICATION SKIPPED — ALL MODELS TOO LARGE` → notify, write stuck-report, terminate +13. Final `APPROVED` → notify, summarize, exit +PROTOCOL +} diff --git a/devtools/adapter_builder/agents/solo-coordinator.sh b/devtools/adapter_builder/agents/solo-coordinator.sh new file mode 100755 index 0000000000..b12af12957 --- /dev/null +++ b/devtools/adapter_builder/agents/solo-coordinator.sh @@ -0,0 +1,281 @@ +#!/usr/bin/env bash +# ============================================================================= +# solo-coordinator.sh — Signal router for solo-mode agent pairs +# +# Solo mode runs two independent Claude Code sessions (programmer pane :0.0, +# reviewer pane :0.1) that cannot block on each other: long waits inside an +# agent's Bash tool die at the tool timeout, and polling burns turns. This +# daemon is the solo analogue of the agent-teams orchestrator — agents only +# ever `touch` a signal file and end their turn; the coordinator consumes +# each signal and wakes the counterpart with an injected tmux message. +# +# Lifecycle per signal: +# 1. Signal file appears in .adapter-workspace/signals/ +# 2. Coordinator archives it to signals/.archive/. (consume — +# this is what makes multi-round loops re-runnable; stale signals never +# linger to satisfy a later wait) +# 3. Routes it: a message is pasted into the target pane as if typed +# +# Routing table (see route_signal): +# *-ready, verification-complete → reviewer (do the matching review) +# *-approved, *-changes-N → programmer (proceed / address review) +# verification-failed → reviewer (informational standby) +# verification-skipped, stuck → notify-human (Slack/iMessage, panes +# informed, sessions left alive) +# done → complete (flags + idle check, +# then /exit to BOTH panes) +# +# Completion also has a passive fallback: if verification_passed and +# final_review_passed are both true and the timeline has been idle for +# $WATCH_IDLE_SECS, the coordinator completes even without a `done` signal +# (same safeguard logic as watch-completion.sh, which this replaces in solo +# mode). +# +# Usage: +# solo-coordinator.sh [architecture] # daemon +# solo-coordinator.sh --route # print "\t" +# solo-coordinator.sh --once # one scan: archive + print +# # actions, no tmux (for tests) +# +# Environment: +# COORD_POLL_SECS — poll interval (default 10) +# WATCH_IDLE_SECS — required timeline idle before /exit (default 60) +# ============================================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +POLL="${COORD_POLL_SECS:-10}" +IDLE_GRACE="${WATCH_IDLE_SECS:-60}" + +TMUX_SOCKET="tl-adapter" +TM() { tmux -L "$TMUX_SOCKET" "$@"; } + +# --------------------------------------------------------------------------- # +# route_signal — pure routing decision, no side effects. +# Prints "\t" where target is one of: +# reviewer | programmer | notify-human | complete | unknown +# Kept side-effect-free so tests can assert the table via --route. +# --------------------------------------------------------------------------- # +route_signal() { + local sig="$1" + local target="" msg="" + + case "$sig" in + brief-ready) + target="reviewer" + msg="Signal 'brief-ready' received. Review .adapter-workspace/adapter-brief.md per your 'Brief review' step: verify against HF source, write .adapter-workspace/reviews/brief-review-.md, then touch signals/brief-approved or signals/brief-changes- and end your turn." ;; + plan-ready) + target="reviewer" + msg="Signal 'plan-ready' received. Review .adapter-workspace/adapter-plan.md per your 'Plan review' step, write .adapter-workspace/reviews/plan-review-.md, then touch signals/plan-approved or signals/plan-changes- and end your turn." ;; + phase-*-ready) + local phase="${sig#phase-}"; phase="${phase%-ready}" + target="reviewer" + msg="Signal 'phase-${phase}-ready' received. Review phase ${phase} per your 'Phase code review' step (scope: this phase's changes only; report at .adapter-workspace/phase-reports/phase-${phase}-report.md), write .adapter-workspace/reviews/phase-${phase}-review-.md, then touch signals/phase-${phase}-approved or signals/phase-${phase}-changes- and end your turn." ;; + verification-complete) + target="reviewer" + msg="Signal 'verification-complete' received. Run your 'Final review' step (holistic, all phases; results at .adapter-workspace/verification-results.md), write .adapter-workspace/completion-report.md, then touch signals/final-approved and end your turn." ;; + verification-failed) + target="reviewer" + msg="Signal 'verification-failed' received. The Programmer is analyzing the failure and re-planning — no action needed from you now. You will be messaged again at the next checkpoint (plan-ready)." ;; + brief-approved) + target="programmer" + msg="Signal 'brief-approved' received. The brief is approved — proceed to Step 1b (plan), then touch signals/plan-ready and end your turn." ;; + plan-approved) + target="programmer" + msg="Signal 'plan-approved' received. The plan is approved — proceed to Step 2 (implement the first phase), then touch signals/phase--ready and end your turn." ;; + phase-*-approved) + local phase="${sig#phase-}"; phase="${phase%-approved}" + target="programmer" + msg="Signal 'phase-${phase}-approved' received. Append '${phase}' to completed_phases in .adapter-progress.json, then implement the next phase (touch signals/phase--ready) or, if all phases are done, run Step 3 (verification) and touch signals/verification-complete. End your turn after signaling." ;; + final-approved) + target="programmer" + msg="Signal 'final-approved' received. The final review passed. Set verification_passed to true in .adapter-progress.json, touch signals/done, and end your turn — the coordinator will close both sessions." ;; + brief-changes-*|plan-changes-*|phase-*-changes-*) + local checkpoint="${sig%-changes-*}" + local round="${sig##*-}" + target="programmer" + msg="Signal '${sig}' received. Read .adapter-workspace/reviews/${checkpoint}-review-${round}.md, address every finding, then re-touch signals/${checkpoint}-ready and end your turn. If this was round 3, write .adapter-workspace/stuck-report.md and touch signals/stuck instead." ;; + verification-skipped) + target="notify-human" + msg="Adapter built but verification skipped — all models exceed the memory limit. Human review needed: inspect the worktree, verify on bigger hardware or approve manually." ;; + stuck) + target="notify-human" + msg="Review loop hit the 3-round limit. See .adapter-workspace/stuck-report.md. Human intervention needed — attach to the tmux session to unblock." ;; + done) + target="complete" + msg="Programmer signaled done — verifying completion flags, then closing both sessions." ;; + *) + target="unknown" + msg="Unrecognized signal '${sig}' — archived without routing." ;; + esac + + printf '%s\t%s\n' "$target" "$msg" +} + +# --------------------------------------------------------------------------- # +# --route mode: print the routing decision and exit (test hook) +# --------------------------------------------------------------------------- # +if [[ "${1:-}" == "--route" ]]; then + route_signal "${2:?--route requires a signal name}" + exit 0 +fi + +# --------------------------------------------------------------------------- # +# Shared scan: archive each pending signal, emit "\t\t" +# --------------------------------------------------------------------------- # +scan_signals() { + local signals_dir="$1" + local archive_dir="$signals_dir/.archive" + mkdir -p "$archive_dir" + + # Oldest first (ls -tr) so multi-signal bursts route in causal order. + # Signal names are coordinator-controlled (no spaces); .archive is hidden + # so plain ls skips it. + local name + while IFS= read -r name; do + [[ -n "$name" && -f "$signals_dir/$name" ]] || continue + mv "$signals_dir/$name" "$archive_dir/$(date +%s).$name" + printf '%s\t' "$name" + route_signal "$name" + done < <(ls -tr "$signals_dir" 2>/dev/null) +} + +# --------------------------------------------------------------------------- # +# --once mode: single scan against a worktree, print actions, no tmux (tests) +# --------------------------------------------------------------------------- # +if [[ "${1:-}" == "--once" ]]; then + WORKTREE_DIR="${2:?--once requires a worktree dir}" + scan_signals "$WORKTREE_DIR/.adapter-workspace/signals" + exit 0 +fi + +# --------------------------------------------------------------------------- # +# Daemon mode +# --------------------------------------------------------------------------- # +WORKTREE_DIR="${1:?Usage: solo-coordinator.sh [architecture]}" +TMUX_SESSION="${2:?Usage: solo-coordinator.sh [architecture]}" +ARCHITECTURE="${3:-unknown}" + +SIGNALS_DIR="$WORKTREE_DIR/.adapter-workspace/signals" +PROGRESS="$WORKTREE_DIR/.adapter-progress.json" +TIMELINE="$WORKTREE_DIR/.adapter-workspace/timeline.jsonl" +TAG="[solo-coordinator:${ARCHITECTURE}]" + +log() { echo "$TAG $(date -u +%H:%M:%S) $*"; } + +# Panes as created by launch-solo-pair.sh: left=programmer, right=reviewer. +PANE_PROGRAMMER="$TMUX_SESSION:0.0" +PANE_REVIEWER="$TMUX_SESSION:0.1" + +# Paste-buffer injection (same two-step idiom as ops.sh send: long payloads +# staged via send-keys get stuck as an unsubmitted bracketed paste, so load +# a buffer, paste it, then send Enter as a discrete key press). +# +# Do NOT send control characters (e.g. Ctrl-U) before the paste to "clear" +# the input: Claude Code's TUI is not readline — empirically C-u at an idle +# prompt triggers a session-level reset (SessionStart fires, permission mode +# drops to manual) and the paste that follows is swallowed. Stale input-box +# text (ghost suggestions) is harmless: it prefixes the message on submit +# and agents parse past it. +inject() { + local pane="$1" message="$2" + local buf="tl-coord-$$-$(date +%s)" + printf '%s' "[coordinator] $message" | TM load-buffer -b "$buf" - + TM paste-buffer -d -b "$buf" -t "$pane" + sleep 0.3 + TM send-keys -t "$pane" Enter +} + +notify_human() { + local message="$1" + "$PROJECT_ROOT/scripts/notify.sh" "[$ARCHITECTURE] $message" >/dev/null 2>&1 || true + # Inform both panes; sessions stay alive so the user can attach and act. + inject "$PANE_PROGRAMMER" "$message The coordinator has notified the user; hold for instructions." || true + inject "$PANE_REVIEWER" "$message The coordinator has notified the user; hold for instructions." || true +} + +flags_complete() { + python3 -c " +import json +try: + p = json.load(open('$PROGRESS')) + print('yes' if p.get('verification_passed') is True and p.get('final_review_passed') is True else 'no') +except Exception: + print('no') +" 2>/dev/null || echo "no" +} + +timeline_idle() { + python3 -c " +import os, time +try: + print('yes' if time.time() - os.path.getmtime('$TIMELINE') >= $IDLE_GRACE else 'no') +except Exception: + print('no') +" 2>/dev/null || echo "no" +} + +# Wait for flags + quiet timeline, then /exit both panes. The double idle +# check closes the race where an agent writes between check and /exit. +complete_run() { + log "Completion requested — waiting for progress flags + ${IDLE_GRACE}s timeline idle." + while TM has-session -t "$TMUX_SESSION" 2>/dev/null; do + if [[ "$(flags_complete)" == "yes" && "$(timeline_idle)" == "yes" ]]; then + sleep 2 + [[ "$(timeline_idle)" == "yes" ]] || continue + log "Flags set and timeline idle (confirmed twice) — sending /exit to both panes." + for pane in "$PANE_REVIEWER" "$PANE_PROGRAMMER"; do + TM send-keys -t "$pane" "/exit" Enter 2>/dev/null || true + sleep 5 + # Retry once if the pane's session half is still alive. + if TM list-panes -t "$pane" &>/dev/null; then + TM send-keys -t "$pane" Enter 2>/dev/null || true + sleep 1 + TM send-keys -t "$pane" "/exit" Enter 2>/dev/null || true + sleep 5 + fi + done + log "Completion sequence done. Session alive? $(TM has-session -t "$TMUX_SESSION" 2>/dev/null && echo yes || echo no)" + return 0 + fi + sleep "$POLL" + done + log "tmux session vanished during completion wait." +} + +log "Started. Poll ${POLL}s, idle grace ${IDLE_GRACE}s. Watching $SIGNALS_DIR → $TMUX_SESSION." +trap 'log "Received termination signal — exiting."; exit 0' TERM INT + +while true; do + sleep "$POLL" + + if ! TM has-session -t "$TMUX_SESSION" 2>/dev/null; then + log "tmux session '$TMUX_SESSION' is gone — coordinator exiting." + exit 0 + fi + + # Passive completion fallback: agents finished but forgot/failed to signal + # done (same protection watch-completion.sh gave the teams mode). + if [[ "$(flags_complete)" == "yes" && "$(timeline_idle)" == "yes" ]]; then + log "Progress flags complete without 'done' signal — entering completion." + complete_run + exit 0 + fi + + [[ -d "$SIGNALS_DIR" ]] || continue + + while IFS=$'\t' read -r name target msg; do + [[ -n "${name:-}" ]] || continue + log "signal='$name' → $target" + case "$target" in + reviewer) inject "$PANE_REVIEWER" "$msg" || log "WARN: inject to reviewer pane failed" ;; + programmer) inject "$PANE_PROGRAMMER" "$msg" || log "WARN: inject to programmer pane failed" ;; + notify-human) notify_human "$msg" ;; + complete) complete_run; exit 0 ;; + unknown) log "WARN: $msg" ;; + esac + done < <(scan_signals "$SIGNALS_DIR") +done diff --git a/devtools/adapter_builder/agents/solo-programmer.md b/devtools/adapter_builder/agents/solo-programmer.md new file mode 100644 index 0000000000..13e473f5c2 --- /dev/null +++ b/devtools/adapter_builder/agents/solo-programmer.md @@ -0,0 +1,143 @@ +--- +name: solo-programmer +description: Builds Architecture Adapters for TransformerLens TransformerBridge. Coordinates with a separate reviewer session via signal files routed by a coordinator daemon. +model: claude-sonnet-4-6 +--- + +# Programmer — Adapter Builder (Solo Mode) + +Build Architecture Adapters for TransformerBridge. Strict lifecycle: +Plan → Program → Verify. A separate Reviewer session runs alongside you +in the same worktree. A coordinator daemon relays between you — you never +wait, poll, or block on the Reviewer. + +## Design principle +When making critical design decisions, ask: **which approach is most +beneficial for Mech Interp research?** Take that approach, even if it is +more difficult. This means: maximize hook granularity, preserve internal +activations faithfully, prefer explicit component decomposition over +opaque fused operations, and ensure every intermediate representation is +accessible for inspection. + +## Read first (from `$TL_ADAPTER_BUILDER_ROOT`) +- `docs/adapter-specification.md` — adapter spec, bridge components, patterns +- `docs/hf-model-analysis-guide.md` — HF model analysis procedure +- `docs/artifact-templates.md` — templates for all output files +- `docs/memory-lock.md` — lock protocol (read before Step 3) + +## Constraints (hook-enforced) +- HookedTransformer is read-only (hook blocks edits) +- No `git commit`/`push`/`gh pr` (hook blocks) +- `verify_models` blocked on >7.5B or unregistered models (hook blocks) +- All output → files in `.adapter-workspace/`, not terminal + +## Signal protocol + +A coordinator daemon watches `.adapter-workspace/signals/` and relays +between you and the Reviewer. + +**To signal:** `touch .adapter-workspace/signals/` — then +**end your turn immediately**. Do not wait, poll, sleep, or loop for a +response; blocking calls die at the tool timeout and polling wastes the +session. The coordinator will send you a message (prefixed +`[coordinator]`) when the Reviewer has responded — act on it then. + +**On wake-up or resume**, re-orient with a single non-blocking call: +```bash +"$TL_ADAPTER_BUILDER_ROOT/agents/check-signals.sh" +``` +It prints pending signals, recently processed ones, and review round +counts. + +**Iteration limit:** rounds are counted by review files. Before +re-signaling a checkpoint after addressing changes, count its reviews: +```bash +ls .adapter-workspace/reviews/-review-*.md 2>/dev/null | wc -l +``` +If 3 reviews exist and the third requested changes, do NOT re-signal: +write `.adapter-workspace/stuck-report.md`, `touch +.adapter-workspace/signals/stuck`, and end your turn. + +--- + +## Step 1a: Analysis → Brief + +1. Read HF source: `modeling_.py`, `configuration_.py`. Follow `docs/hf-model-analysis-guide.md`. +2. Generate scaffold: + ```bash + python "$TL_ADAPTER_BUILDER_ROOT/scripts/analyze-hf-model.py" \ + --scaffold --scaffold-out .adapter-workspace/adapter-scaffold.py + ``` + Cross-check scaffold's detected module paths against HF source. +3. Scan HF for all models: + ```bash + uv run python "$TL_ADAPTER_BUILDER_ROOT/scripts/scan-hf-architecture.py" \ + --arch-short + ``` +4. Find closest reference adapter. Use `scripts/compare-adapters.sh`. +5. Write brief → `.adapter-workspace/adapter-brief.md` (template: `docs/artifact-templates.md`) +6. `touch .adapter-workspace/signals/brief-ready` and end your turn. +7. On a `brief-changes-` message → read `.adapter-workspace/reviews/brief-review-.md`, address every finding, re-touch `brief-ready`, end your turn. +8. On a `brief-approved` message → proceed to Step 1b. + +## Step 1b: Plan + +Write plan → `.adapter-workspace/adapter-plan.md` (template: `docs/artifact-templates.md`). +Typical phases: A=config+weights, B=mapping, C=overrides, D=registration. +Merge coupled phases as `Phase A+B` — use that label everywhere. + +`touch .adapter-workspace/signals/plan-ready` and end your turn. +On changes → read the review, fix, re-touch `plan-ready`, end your turn. +On `plan-approved` → proceed to Step 2. + +## Step 2: Implement + +One phase at a time. Per phase: +1. Implement +2. Test +3. Write report → `.adapter-workspace/phase-reports/phase--report.md` +4. Update `.adapter-progress.json`: set `current_phase` to this phase's label +5. `touch .adapter-workspace/signals/phase--ready` and end your turn. +6. On changes → read the review, fix, re-touch `phase--ready`, end your turn. + On `phase--approved` → append to `completed_phases`, next phase (or Step 3 after the last phase). + +New bridge components: only when `forward()` must differ. Own phase. Document why. +Tests: required, CI-friendly, no tautologies. `@pytest.mark.skip` for CI-incompatible. +**Do not** modify files outside the current phase's scope without flagging it. + +## Step 3: Verify + +**3.0** Port models: `uv run python "$TL_ADAPTER_BUILDER_ROOT/scripts/port-arch-models.py" --arch-short ` + +**3.1** Select up to 5 models, all ≤7B, chosen for **config diversity, not popularity**: +always include the smallest model of the architecture (tiny models expose +numerical-path divergence fastest and verify in seconds) and the most-downloaded +that fits; fill remaining slots with config variants (different head_dim, +rotary_pct, bias flags, GQA settings) over download rank. Verify **smallest +first** — a numerical bug should fail in seconds, not after a 30-minute large-model +run. If none fit → `touch .adapter-workspace/signals/verification-skipped`, end your turn (the coordinator escalates to the user). + +**3.2** One model at a time. Read `docs/memory-lock.md`, then: +```bash +"$TL_ADAPTER_BUILDER_ROOT/agents/overlord-request.sh" run "verify_models: " -- \ + uv run python -m transformer_lens.tools.model_registry.verify_models \ + --model --max-memory $MAX_MEMORY_GB --device cpu --dtype float32 +``` +After each: check `status` in `supported_models.json`: 1=next, 2=note+next, 3=**stop and fix**, 4 (provisional, "Structural only")=**wrong invocation — rerun with the HF reference on**. +After each passing model, append its ID to `verified_models` in `.adapter-progress.json`. +On crash-resume, skip models already in `verified_models`. + +Default `--dtype float32`. **Never pass `--no-hf-reference`** — it skips the HuggingFace parity comparison and writes status 4 (provisional), which does not count as verified (a hook blocks it). If verify_models rejects a flag, read `--help` and understand a replacement before using it — never substitute a similar-looking flag. If a model fails: +- **status=3 (phase score failure):** adapter bug — investigate, fix, re-verify. +- **OOM / MemoryError / killed:** retry with `--dtype bfloat16`. If still OOMs, skip and note. +- **status=2 (SKIPPED):** not an adapter bug, note and move on. + +**3.3** `uv run mypy .` + `make check-format` must pass. No `# type: ignore`. + +**3.4** Write results → `.adapter-workspace/verification-results.md`. +`touch .adapter-workspace/signals/verification-complete` and end your turn. +On a `final-approved` message → set `verification_passed: true` in `.adapter-progress.json`, `touch .adapter-workspace/signals/done`, end your turn. The coordinator closes both sessions. +On verification failure → write failure analysis, `touch .adapter-workspace/signals/verification-failed`, then return to Step 1b (re-plan) and signal `plan-ready` when the revised plan is written. + +--- + diff --git a/devtools/adapter_builder/agents/solo-reviewer.md b/devtools/adapter_builder/agents/solo-reviewer.md new file mode 100644 index 0000000000..465b1abf0a --- /dev/null +++ b/devtools/adapter_builder/agents/solo-reviewer.md @@ -0,0 +1,111 @@ +--- +name: solo-reviewer +description: Reviews Architecture Adapters for TransformerLens. Woken by coordinator messages when a separate programmer session signals a checkpoint. +model: claude-opus-4-6 +--- + +# Reviewer — Adapter Review (Solo Mode) + +Review Architecture Adapters at every checkpoint. A separate Programmer +session runs alongside you in the same worktree. A coordinator daemon +watches the signal directory and messages you when there is work — you +never wait, poll, or block. + +## Design principle +Evaluate every design decision against this question: **which approach is +most beneficial for Mech Interp research?** Prefer that approach even if +harder. Flag designs that fuse operations opaquely, drop intermediate +activations, or reduce hook granularity — these harm interpretability +research even if they produce correct outputs. + +## Read first (from `$TL_ADAPTER_BUILDER_ROOT`) +- `docs/review-specification.md` — five-phase methodology (P0–P5), follow verbatim for code reviews +- `docs/adapter-specification.md` — adapter spec, checklists for config/mapping/weights/registration +- `docs/hf-model-analysis-guide.md` — HF model analysis (used in brief review + Phase 0) +- `docs/artifact-templates.md` — templates for review files and completion report + +## Constraints (hook-enforced) +- HookedTransformer changes → flag as CRITICAL +- No git commits/pushes +- Review files past round 3 are blocked (a hook enforces the iteration limit) + +## How you are driven (message-driven, no polling) + +You are idle between reviews. When the Programmer signals a checkpoint, +the coordinator sends you a message prefixed `[coordinator]` naming the +signal (`brief-ready`, `plan-ready`, `phase--ready`, +`verification-complete`). On each message: + +1. Do the matching review from the sections below. +2. Write the review file FIRST. +3. `touch` your response signal in `.adapter-workspace/signals/`. +4. **End your turn immediately.** Never wait, sleep, or loop for the next + checkpoint — blocking calls die at the tool timeout. You will be + messaged again when there is work. + +**Round numbers:** for checkpoint ``, the round is +`N = (number of existing .adapter-workspace/reviews/-review-*.md) + 1`. +Use the same `N` in the review filename and any `-changes-` signal. + +**On wake-up or resume**, re-orient with a single non-blocking call: +```bash +"$TL_ADAPTER_BUILDER_ROOT/agents/check-signals.sh" +``` + +After your initial read-through of the docs above, end your turn and wait +to be messaged. + +## Brief review — on `brief-ready` + +Read HF source yourself (`modeling_.py`, `configuration_.py`). +Cross-reference every claim in `.adapter-workspace/adapter-brief.md`. +Write review → `.adapter-workspace/reviews/brief-review-.md`. +Then signal one of: +```bash +touch .adapter-workspace/signals/brief-approved +touch .adapter-workspace/signals/brief-changes- +``` +End your turn. + +## Plan review — on `plan-ready` + +Brief is approved — use it as factual reference. Check design, completeness, consistency. +Write → `.adapter-workspace/reviews/plan-review-.md`. +Signal `plan-approved` or `plan-changes-`. End your turn. + +## Phase code review — on `phase--ready` + +`` is the plan's phase label (A, B, A+B, …) — it is named in the +coordinator's message. +**Scope: only the files changed in this phase.** Do not re-review prior phases. +Follow `docs/review-specification.md` P0–P5, scoped to this phase's changes. +P0 (HF source reference) is done once and reused across phases. +Also verify adapter-specific items from `docs/adapter-specification.md`. +Write → `.adapter-workspace/reviews/phase--review-.md`. +Signal `phase--approved` or `phase--changes-`. End your turn. + +Give feedback, not solutions — do not rewrite the adapter yourself. +Verify against code and HF source, not the Programmer's summary. + +## Final review — on `verification-complete` + +**Scope: the complete adapter across all phases.** +**Gate on verification quality first:** every verified model must have `status: 1` +in `supported_models.json`. Status 4 with "Structural only (no HF reference)" means +the HuggingFace parity comparison never ran — that is NOT verification; request +changes and have the Programmer rerun verify_models with the HF reference on. Holistic review — check +cross-phase consistency, end-to-end correctness. Run P5 fully: plan-to-code +match across ALL phases, all prior findings resolved, test quality. +Read `.adapter-workspace/verification-results.md` for the verification record. +Write completion report → `.adapter-workspace/completion-report.md` (template: `docs/artifact-templates.md`). +Signal `final-approved`. End your turn — the coordinator closes both +sessions once the Programmer finishes. + +## Informational messages + +On a `verification-failed` message: no action — the Programmer is +re-planning and you will be messaged at the next `plan-ready`. End your +turn. + +--- + diff --git a/devtools/adapter_builder/agents/watch-completion.sh b/devtools/adapter_builder/agents/watch-completion.sh new file mode 100755 index 0000000000..d3a5f348df --- /dev/null +++ b/devtools/adapter_builder/agents/watch-completion.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# ============================================================================= +# watch-completion.sh — Background watcher for --background tmux sessions +# +# In interactive Claude Code, an orchestrator has no tool to self-terminate. +# It can print "Session terminating — SessionEnd hook will fire the Slack +# notification" but that just ends its turn; the session then sits at the +# `❯` prompt waiting for user input forever. The SessionEnd hook never fires +# (because the session hasn't actually ended) and the Slack notification +# never gets sent. +# +# This watcher closes the gap. It's launched in the background by launch.sh's +# --background wrapper, runs for the lifetime of the session, and polls the +# worktree's .adapter-progress.json file. When: +# +# 1. verification_passed == true, AND +# 2. final_review_passed == true, AND +# 3. the timeline.jsonl has been idle for at least $WATCH_IDLE_SECS seconds +# (default 60), confirming the orchestrator has finished printing its +# farewell and isn't still spawning subagents, +# +# …the watcher sends `/exit` to the tmux session via `tmux send-keys`. +# Claude Code processes /exit as a clean session terminate, which fires the +# SessionEnd hook chain (timeline-capture → notify-on-completion → notify.sh +# → Slack), marks the session exit code 0, and lets the wrapper script +# proceed to its post-session tail. +# +# The idle-grace guard is the critical protection against false positives. +# xglm-style cases where the flags get flipped prematurely while the reviewer +# is still actively working will keep the timeline hot, so the watcher waits +# for genuine quiescence before acting. +# +# Usage (invoked by launch.sh's background wrapper): +# watch-completion.sh [architecture] +# +# Environment: +# WATCH_POLL_SECS — poll interval (default 30) +# WATCH_IDLE_SECS — required timeline idle before /exit (default 60) +# ============================================================================= + +set -euo pipefail + +WORKTREE_DIR="${1:?Usage: watch-completion.sh [architecture]}" +TMUX_SESSION="${2:?Usage: watch-completion.sh [architecture]}" +ARCHITECTURE="${3:-unknown}" + +POLL_INTERVAL="${WATCH_POLL_SECS:-30}" +IDLE_GRACE="${WATCH_IDLE_SECS:-60}" + +# All TL Adapter Builder tmux sessions live on a dedicated socket so they +# don't share a server with any tmux session the user has running. +TMUX_SOCKET="tl-adapter" +TM() { tmux -L "$TMUX_SOCKET" "$@"; } + +PROGRESS="$WORKTREE_DIR/.adapter-progress.json" +TIMELINE="$WORKTREE_DIR/.adapter-workspace/timeline.jsonl" +TAG="[watch-completion:${ARCHITECTURE}]" + +log() { + echo "$TAG $(date -u +%H:%M:%S) $*" +} + +log "Started. Poll interval ${POLL_INTERVAL}s, idle grace ${IDLE_GRACE}s. Watching $TMUX_SESSION." + +# The watcher exits cleanly under any of three conditions: +# - it successfully sends /exit (normal completion path) +# - the tmux session disappears (claude was killed externally, or we already sent /exit) +# - its parent wrapper signals it via SIGTERM (cleanup trap) +trap 'log "Received termination signal — watcher exiting."; exit 0' TERM INT + +while true; do + sleep "$POLL_INTERVAL" + + # If tmux is gone or the session vanished, we're done. + if ! command -v tmux &>/dev/null || ! TM has-session -t "$TMUX_SESSION" 2>/dev/null; then + log "tmux session '$TMUX_SESSION' is gone — watcher exiting." + exit 0 + fi + + [[ -f "$PROGRESS" ]] || continue + + # Are both completion flags set? + flags_ok=$(python3 -c " +import json +try: + p = json.load(open('$PROGRESS')) +except Exception: + print('no') + raise SystemExit(0) +verif = p.get('verification_passed') is True +review = p.get('final_review_passed') is True +print('yes' if verif and review else 'no') +" 2>/dev/null || echo "no") + + if [[ "$flags_ok" != "yes" ]]; then + continue + fi + + # Flags set. Has the timeline been quiet long enough to assume the + # orchestrator is done printing its farewell and not mid-subagent? + # + # IMPORTANT: if the timeline file doesn't exist at all, that means hooks + # haven't fired yet (session just started, or workspace wasn't set up). + # Default to NOT idle — we must never send /exit before work begins. + if [[ ! -f "$TIMELINE" ]]; then + log "Completion flags set but timeline file missing — deferring (session may not have started)." + continue + fi + + idle_ok=$(python3 -c " +import os, time +try: + age = time.time() - os.path.getmtime('$TIMELINE') + print('yes' if age >= $IDLE_GRACE else 'no') +except Exception: + print('no') +" 2>/dev/null || echo "no") + + if [[ "$idle_ok" != "yes" ]]; then + log "Completion flags set but timeline still active — deferring /exit." + continue + fi + + # Double-check the mtime again after a short sleep to close the race + # window where an agent could write to the timeline between our check + # and the /exit send. + sleep 2 + idle_recheck=$(python3 -c " +import os, time +try: + age = time.time() - os.path.getmtime('$TIMELINE') + print('yes' if age >= $IDLE_GRACE else 'no') +except Exception: + print('no') +" 2>/dev/null || echo "no") + + if [[ "$idle_recheck" != "yes" ]]; then + log "Timeline became active during recheck — deferring /exit." + continue + fi + + log "Adapter complete + timeline idle ≥${IDLE_GRACE}s (confirmed twice). Sending /exit to $TMUX_SESSION." + TM send-keys -t "$TMUX_SESSION" "/exit" Enter + + # Give Claude Code a moment to process the /exit command and run hooks. + sleep 10 + + # If the session is still alive, try one more /exit (in case the first + # one landed in a paste buffer or Claude was mid-tool-call). + if TM has-session -t "$TMUX_SESSION" 2>/dev/null; then + log "Session still alive 10s after /exit — retrying once." + TM send-keys -t "$TMUX_SESSION" Enter + sleep 1 + TM send-keys -t "$TMUX_SESSION" "/exit" Enter + sleep 10 + fi + + log "Watcher done. tmux session alive? $(TM has-session -t "$TMUX_SESSION" 2>/dev/null && echo yes || echo no)" + exit 0 +done diff --git a/devtools/adapter_builder/docs/adapter-specification.md b/devtools/adapter_builder/docs/adapter-specification.md new file mode 100644 index 0000000000..6bf729482b --- /dev/null +++ b/devtools/adapter_builder/docs/adapter-specification.md @@ -0,0 +1,296 @@ +# Architecture Adapter Specification + +This document is the primary reference for building Architecture Adapters for the TransformerLens TransformerBridge system. + +## What Is an Architecture Adapter? + +An Architecture Adapter is a Python class that extends `ArchitectureAdapter` (from `transformer_lens.model_bridge.architecture_adapter`). It maps between a HuggingFace model's internal structure and TransformerLens's canonical component names. Every adapter must define three things: + +1. **Config attributes** — set on `self.cfg` in `__init__` +2. **Component mapping** — `self.component_mapping` dict mapping TL names to Bridge instances +3. **Weight processing conversions** — `self.weight_processing_conversions` dict for tensor reshaping + +## File Location and Naming + +- **Adapter file:** `transformer_lens/model_bridge/supported_architectures/.py` +- **Class name:** `ArchitectureAdapter` (e.g., `LlamaArchitectureAdapter`) +- **Module name:** lowercase, underscores (e.g., `llama.py`, `qwen2.py`, `granite_moe.py`) + +## Registration Checklist + +After creating the adapter, register it in these files: + +1. **`transformer_lens/model_bridge/supported_architectures/__init__.py`** + - Add import: `from transformer_lens.model_bridge.supported_architectures. import ` + - Add to `__all__` list + +2. **`transformer_lens/factories/architecture_adapter_factory.py`** + - Add import (in the existing import block from `supported_architectures`) + - Add entry to `SUPPORTED_ARCHITECTURES` dict: `"": ` + +## Config Attributes + +Set these on `self.cfg` in `__init__` before building the component mapping: + +| Attribute | Type | Description | Examples | +|-----------|------|-------------|----------| +| `normalization_type` | `str` | `"RMS"` or `"LN"` | Llama="RMS", GPT2="LN" | +| `positional_embedding_type` | `str` | `"rotary"` or `"standard"` | Llama="rotary", GPT2="standard" | +| `final_rms` | `bool` | Whether final layer norm is RMS | Llama=True, GPT2=False | +| `gated_mlp` | `bool` | Whether MLP uses gate projection | Llama=True, GPT2=False | +| `attn_only` | `bool` | Whether model has no MLP layers | Usually False | +| `uses_rms_norm` | `bool` | Redundant with normalization_type but needed | Match normalization_type | +| `eps_attr` | `str` | Attribute name for norm epsilon | `"variance_epsilon"`, `"layer_norm_eps"` | + +### GQA (Grouped Query Attention) + +If the model uses GQA (n_key_value_heads < n_heads), set: +```python +if hasattr(cfg, "n_key_value_heads") and cfg.n_key_value_heads is not None: + self.cfg.n_key_value_heads = cfg.n_key_value_heads +``` + +## Component Mapping + +`self.component_mapping` is a `dict[str, GeneralizedComponent]` mapping TransformerLens canonical names to Bridge instances. The Bridge `name=` parameter is the HuggingFace module path. + +### Standard Mapping (Llama-style decoder-only) + +```python +self.component_mapping = { + "embed": EmbeddingBridge(name="model.embed_tokens"), + "rotary_emb": RotaryEmbeddingBridge(name="model.rotary_emb"), + "blocks": BlockBridge( + name="model.layers", + submodules={ + "ln1": RMSNormalizationBridge(name="input_layernorm", config=self.cfg), + "ln2": RMSNormalizationBridge(name="post_attention_layernorm", config=self.cfg), + "attn": PositionEmbeddingsAttentionBridge( + name="self_attn", + config=self.cfg, + submodules={ + "q": LinearBridge(name="q_proj"), + "k": LinearBridge(name="k_proj"), + "v": LinearBridge(name="v_proj"), + "o": LinearBridge(name="o_proj"), + }, + requires_attention_mask=True, + requires_position_embeddings=True, + ), + "mlp": GatedMLPBridge( + name="mlp", + config=self.cfg, + submodules={ + "gate": LinearBridge(name="gate_proj"), + "in": LinearBridge(name="up_proj"), + "out": LinearBridge(name="down_proj"), + }, + ), + }, + ), + "ln_final": RMSNormalizationBridge(name="model.norm", config=self.cfg), + "unembed": UnembeddingBridge(name="lm_head", config=self.cfg), +} +``` + +### GPT2-style Mapping (standard positional embeddings, combined QKV) + +```python +self.component_mapping = { + "embed": EmbeddingBridge(name="transformer.wte"), + "pos_embed": PosEmbedBridge(name="transformer.wpe"), + "blocks": BlockBridge( + name="transformer.h", + config=self.cfg, + submodules={ + "ln1": NormalizationBridge(name="ln_1", config=self.cfg), + "attn": JointQKVAttentionBridge( + name="attn", + config=self.cfg, + submodules={ + "qkv": LinearBridge(name="c_attn"), + "o": LinearBridge(name="c_proj"), + }, + ), + "ln2": NormalizationBridge(name="ln_2", config=self.cfg), + "mlp": MLPBridge( + name="mlp", + submodules={ + "in": LinearBridge(name="c_fc"), + "out": LinearBridge(name="c_proj"), + }, + ), + }, + ), + "ln_final": NormalizationBridge(name="transformer.ln_f", config=self.cfg), + "unembed": UnembeddingBridge(name="lm_head"), +} +``` + +> **Note:** GPT2's `MLPBridge` and `UnembeddingBridge` do not pass `config=`. The `config` parameter is optional on these bridges — match the existing adapter's pattern. + +## Weight Processing Conversions + +`self.weight_processing_conversions` maps TransformerLens weight paths to `ParamProcessingConversion` instances that handle tensor reshaping during weight loading. + +### Standard QKVO Conversions (most models) + +For models with separate Q/K/V/O projections, use the built-in helper: + +```python +self.weight_processing_conversions = { + **self._qkvo_weight_conversions(), +} +``` + +This generates rearrangement rules for: +- `blocks.{i}.attn.q.weight` — `(n h) m -> n m h` with `n=n_heads` +- `blocks.{i}.attn.k.weight` — `(n h) m -> n m h` with `n=n_kv_heads` +- `blocks.{i}.attn.v.weight` — `(n h) m -> n m h` with `n=n_kv_heads` +- `blocks.{i}.attn.o.weight` — `m (n h) -> n h m` with `n=n_heads` + +### Custom Conversions + +For models with non-standard weight layouts (e.g., combined QKV), define custom `ParamProcessingConversion` or `RearrangeTensorConversion` instances. See `gpt2.py` for the `QKVSplitRearrangeConversion` example. + +## Available Bridge Components + +### Core Components + +| Component | Use When | +|-----------|----------| +| `EmbeddingBridge` | Token embeddings | +| `UnembeddingBridge` | Output head (lm_head) | +| `BlockBridge` | Transformer block container (always named "blocks") | +| `LinearBridge` | Any linear/projection layer | + +### Normalization + +| Component | Use When | +|-----------|----------| +| `NormalizationBridge` | LayerNorm | +| `RMSNormalizationBridge` | RMSNorm | + +### Attention + +| Component | Use When | +|-----------|----------| +| `AttentionBridge` | Basic attention (no positional embeddings passed) | +| `PositionEmbeddingsAttentionBridge` | Attention that receives position embeddings (RoPE models) | +| `JointQKVAttentionBridge` | Combined QKV single linear layer (GPT-2 style) | +| `JointQKVPositionEmbeddingsAttentionBridge` | Combined QKV with position embeddings | + +### MLP + +| Component | Use When | +|-----------|----------| +| `MLPBridge` | Standard 2-layer MLP (in/out) or with separate gate | +| `GatedMLPBridge` | Gated MLP with gate/up/down projections (SwiGLU) | +| `JointGateUpMLPBridge` | MLP where gate and up projections are fused | + +### Position Embeddings + +| Component | Use When | +|-----------|----------| +| `PosEmbedBridge` | Learned positional embeddings (GPT-2 style) | +| `RotaryEmbeddingBridge` | Rotary position embeddings (RoPE) | + +### Specialized + +| Component | Use When | +|-----------|----------| +| `MoEBridge` | Mixture of Experts routing | +| `SymbolicBridge` | Placeholder/container with no direct HF module | +| `Conv1DBridge` | 1D convolution layers | +| `T5BlockBridge` | T5-specific block structure | +| `CLIPVisionEncoderBridge` | CLIP vision encoder (multimodal) | +| `CLIPVisionEncoderLayerBridge` | Individual CLIP vision encoder layer | +| `SiglipVisionEncoderBridge` | Siglip vision encoder (multimodal) | +| `SiglipVisionEncoderLayerBridge` | Individual Siglip vision encoder layer | +| `VisionProjectionBridge` | Vision-to-text projection (multimodal) | + +### Architecture-Specific (Bloom/Falcon) + +These exist for architectures with non-standard internal structures. Discover them by reading the reference adapter. + +| Component | Use When | +|-----------|----------| +| `BloomBlockBridge` | BLOOM transformer blocks | +| `BloomAttentionBridge` | BLOOM attention mechanism | +| `BloomMLPBridge` | BLOOM MLP | +| `AudioFeatureExtractorBridge` | Audio feature extraction (HuBERT) | +| `ConvPosEmbedBridge` | Convolutional positional embeddings (HuBERT) | + +## Optional Overrides + +### `setup_component_testing(hf_model, bridge_model=None)` + +Called after adapter creation. Use to set up model-specific references for component testing. Required for RoPE models to set rotary embedding references: + +```python +def setup_component_testing(self, hf_model, bridge_model=None): + rotary_emb = hf_model.model.rotary_emb + if bridge_model is not None and hasattr(bridge_model, "blocks"): + for block in bridge_model.blocks: + if hasattr(block, "attn"): + block.attn.set_rotary_emb(rotary_emb) + attn_bridge = self.get_generalized_component("blocks.0.attn") + attn_bridge.set_rotary_emb(rotary_emb) +``` + +### `preprocess_weights(state_dict)` + +Apply architecture-specific weight transformations before standard processing. Example: Gemma scales embeddings by `sqrt(d_model)`. + +### `prepare_loading(model_name, model_kwargs)` + +Called before `from_pretrained()`. Use to patch HF model classes. + +### `prepare_model(hf_model)` + +Called after model loading but before bridge creation. Use for post-load fixups. + +## Common Architecture Patterns + +### Pattern 1: Llama-like (most modern models) + +RoPE + RMSNorm + GatedMLP + separate Q/K/V/O. Uses `GatedMLPBridge`. Used by: Llama, Mistral, Gemma, OLMo, Granite, StableLM. + +**Qwen2 variant:** Nearly identical to Llama but uses `MLPBridge` instead of `GatedMLPBridge` (while still setting `gated_mlp = True` and having gate/in/out submodules). Used by: Qwen2, Qwen3. + +### Pattern 2: GPT2-like + +Standard positional embeddings + LayerNorm + standard MLP + combined QKV. Used by: GPT-2, GPT-J, GPT-Neo/NeoX. + +### Pattern 3: MoE (Mixture of Experts) + +Similar to Llama-like but with `MoEBridge` replacing the MLP. Used by: Mixtral, GraniteMoE, OLMoE. + +### Pattern 4: Multimodal + +Extends a text-only pattern with vision encoder and projection bridges. Used by: LLaVA, LLaVA-Next, Gemma3 Multimodal. + +## Imports Template + +```python +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.generalized_components import ( + BlockBridge, + EmbeddingBridge, + GatedMLPBridge, # or MLPBridge for non-gated + LinearBridge, + PositionEmbeddingsAttentionBridge, # or JointQKVAttentionBridge + RMSNormalizationBridge, # or NormalizationBridge for LayerNorm + RotaryEmbeddingBridge, # only for RoPE models + UnembeddingBridge, +) +``` + +## Testing + +After creating an adapter, verify it by: + +1. Running the adapter-specific unit tests +2. Loading a small model variant with `boot_transformers(model_name)` +3. Verifying hook names resolve correctly +4. Checking that weight shapes match expectations diff --git a/devtools/adapter_builder/docs/adapter-template.py b/devtools/adapter_builder/docs/adapter-template.py new file mode 100644 index 0000000000..7b14928b02 --- /dev/null +++ b/devtools/adapter_builder/docs/adapter-template.py @@ -0,0 +1,162 @@ +""" architecture adapter. + +TODO: Replace with the actual model name throughout this file. +""" + +from typing import Any + +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.generalized_components import ( + BlockBridge, + EmbeddingBridge, + GatedMLPBridge, + LinearBridge, + PositionEmbeddingsAttentionBridge, + RMSNormalizationBridge, + RotaryEmbeddingBridge, + UnembeddingBridge, +) + + +class ModelNameArchitectureAdapter(ArchitectureAdapter): + """Architecture adapter for models. + + TODO: Document which parameters are optional (missing biases, etc.) + + Optional Parameters (may not exist in state_dict): + ------------------------------------------------- + TODO: List parameters that may not exist. Example for models without biases: + + - blocks.{i}.attn.b_Q - No bias on query projection + - blocks.{i}.attn.b_K - No bias on key projection + - blocks.{i}.attn.b_V - No bias on value projection + - blocks.{i}.attn.b_O - No bias on output projection + - blocks.{i}.mlp.b_in - No bias on MLP input + - blocks.{i}.mlp.b_gate - No bias on MLP gate projection + - blocks.{i}.mlp.b_out - No bias on MLP output + - blocks.{i}.ln1.b - RMSNorm has no bias + - blocks.{i}.ln2.b - RMSNorm has no bias + - ln_final.b - RMSNorm has no bias + """ + + def __init__(self, cfg: Any) -> None: + """Initialize the architecture adapter.""" + super().__init__(cfg) + + # ===================================================================== + # 1. CONFIG ATTRIBUTES + # Set these based on the HuggingFace model's architecture. + # ===================================================================== + + # TODO: Set normalization type + # "RMS" for RMSNorm (Llama, Qwen, Gemma, etc.) + # "LN" for LayerNorm (GPT-2, GPT-J, etc.) + self.cfg.normalization_type = "RMS" + + # TODO: Set positional embedding type + # "rotary" for RoPE (Llama, Qwen, Mistral, etc.) + # "standard" for learned positional embeddings (GPT-2) + self.cfg.positional_embedding_type = "rotary" + + # TODO: Set these flags + self.cfg.final_rms = True # True if final layer norm is RMSNorm + self.cfg.gated_mlp = True # True if MLP has gate projection (SwiGLU) + self.cfg.attn_only = False # True only for attention-only models (rare) + self.cfg.uses_rms_norm = True # Should match normalization_type + + # TODO: Set the epsilon attribute name used by this model's normalization + # Check the HF model's norm layer to find the correct attribute name + self.cfg.eps_attr = "variance_epsilon" # or "layer_norm_eps", "rms_norm_eps", etc. + + # TODO: Handle GQA if applicable + # If the model uses Grouped Query Attention (n_key_value_heads < n_heads): + if hasattr(cfg, "n_key_value_heads") and cfg.n_key_value_heads is not None: + self.cfg.n_key_value_heads = cfg.n_key_value_heads + + # ===================================================================== + # 2. WEIGHT PROCESSING CONVERSIONS + # Defines how to reshape weights from HF format to TL format. + # For most models with separate Q/K/V/O, use the built-in helper. + # ===================================================================== + + self.weight_processing_conversions = { + **self._qkvo_weight_conversions(), + # TODO: Add any model-specific weight conversions here + } + + # ===================================================================== + # 3. COMPONENT MAPPING + # Maps TransformerLens canonical names to HuggingFace module paths. + # The `name=` parameter is the HF path relative to the model root + # (for top-level) or relative to the block (for block submodules). + # ===================================================================== + + # TODO: Replace all HF paths (name="...") with actual paths from the model. + # Inspect the HF model's named_modules() or config to find the correct paths. + self.component_mapping = { + # Token embedding + "embed": EmbeddingBridge(name="model.embed_tokens"), + # Rotary position embeddings (remove if model uses standard pos embeddings) + "rotary_emb": RotaryEmbeddingBridge(name="model.rotary_emb"), + # Transformer blocks + "blocks": BlockBridge( + name="model.layers", # TODO: HF path to the layer list + submodules={ + # Pre-attention layer norm + "ln1": RMSNormalizationBridge( + name="input_layernorm", # TODO: HF name within block + config=self.cfg, + ), + # Post-attention layer norm + "ln2": RMSNormalizationBridge( + name="post_attention_layernorm", # TODO: HF name within block + config=self.cfg, + ), + # Self-attention + "attn": PositionEmbeddingsAttentionBridge( + name="self_attn", # TODO: HF name within block + config=self.cfg, + submodules={ + "q": LinearBridge(name="q_proj"), # TODO: HF projection names + "k": LinearBridge(name="k_proj"), + "v": LinearBridge(name="v_proj"), + "o": LinearBridge(name="o_proj"), + }, + requires_attention_mask=True, + requires_position_embeddings=True, + ), + # MLP (gated) + "mlp": GatedMLPBridge( + name="mlp", # TODO: HF name within block + config=self.cfg, + submodules={ + "gate": LinearBridge(name="gate_proj"), # TODO: HF projection names + "in": LinearBridge(name="up_proj"), + "out": LinearBridge(name="down_proj"), + }, + ), + }, + ), + # Final layer norm + "ln_final": RMSNormalizationBridge(name="model.norm", config=self.cfg), + # Output head (unembedding) + "unembed": UnembeddingBridge(name="lm_head", config=self.cfg), + } + + def setup_component_testing(self, hf_model: Any, bridge_model: Any = None) -> None: + """Set up model-specific references for component testing. + + TODO: Required for RoPE models. Remove if model uses standard positional embeddings. + """ + # Get rotary embedding instance from the HF model + rotary_emb = hf_model.model.rotary_emb # TODO: Adjust path if different + + # Set rotary_emb on actual bridge instances + if bridge_model is not None and hasattr(bridge_model, "blocks"): + for block in bridge_model.blocks: + if hasattr(block, "attn"): + block.attn.set_rotary_emb(rotary_emb) + + # Set on template for get_generalized_component() calls + attn_bridge = self.get_generalized_component("blocks.0.attn") + attn_bridge.set_rotary_emb(rotary_emb) diff --git a/devtools/adapter_builder/docs/artifact-templates.md b/devtools/adapter_builder/docs/artifact-templates.md new file mode 100644 index 0000000000..5d26184e38 --- /dev/null +++ b/devtools/adapter_builder/docs/artifact-templates.md @@ -0,0 +1,265 @@ +# Artifact Templates + +All structured output files produced during an adapter build. Both the +Programmer and Reviewer reference these templates when writing artifacts to +`.adapter-workspace/`. Copy the relevant template, fill in the placeholders, +and write to the specified path. + +--- + +## Architecture Brief + +**Path:** `.adapter-workspace/adapter-brief.md` +**Written by:** Programmer (Step 1a) +**Reviewed by:** Reviewer (Brief Review Mode) + +```markdown +# Architecture Brief: + +## Source Files +- modeling: `models//modeling_.py` +- config: `models//configuration_.py` + +## Module Hierarchy (from __init__ methods) +- Embedding: `model.embed_tokens` (line N) +- Blocks: `model.layers` (line N) + - Attention: `self_attn` → q_proj, k_proj, v_proj, o_proj + - Norm 1: `input_layernorm` + - Norm 2: `post_attention_layernorm` + - MLP: `mlp` → gate_proj, up_proj, down_proj +- Final norm: `model.norm` (line N) +- LM head: `lm_head` (line N) + +## Config Fields +hidden_size, num_attention_heads, num_key_value_heads, intermediate_size, +num_hidden_layers, vocab_size, max_position_embeddings, rms_norm_eps, ... + +## Architectural Properties +- Normalization: , eps attr: (line N) +- Position embeddings: , module: +- Attention: , n_kv_heads= +- MLP: , activation: +- Biases: + +## Forward Pass Flow + + +## Reference Adapter +- Closest match: +- Key differences from this architecture: + +## Representative Models (≤7B only) +| Model | Params | Est. Memory | Notes | +|-------|--------|-------------|-------| +| ... | ... | ... | ... | +``` + +--- + +## Implementation Plan + +**Path:** `.adapter-workspace/adapter-plan.md` +**Written by:** Programmer (Step 1b) +**Reviewed by:** Reviewer (Plan Review Mode) + +```markdown +# Adapter Plan: + +Brief: .adapter-workspace/adapter-brief.md + +## Phase A: +<details — reference brief for module paths and config fields> + +## Phase B: <title> +<details> + +## Phase C: <title> (if needed) +<details> + +## Phase D: Registration +<details> + +## Verification Strategy +- Models: <from brief, all ≤7B> +- Expected phase scores: <targets> +``` + +Phase merges are fine: name them `Phase A+B: ...` and use that label +everywhere (progress file, phase report filename, signals). + +--- + +## Phase Report + +**Path:** `.adapter-workspace/phase-reports/phase-<X>-report.md` +**Written by:** Programmer (Step 2, after each phase) + +`<X>` is the plan's phase label: `phase-A-report.md` for Phase A, +`phase-A+B-report.md` for a merged Phase A+B. One report per plan phase, +regardless of merge — never split a merged phase into separate report files. + +```markdown +# Phase <letter>: <title> + +## Changes made +- <file>: <what changed and why> + +## Tests run +- <what was tested and the outcome> + +## Verification against HF source +- <module paths, config fields, weight shapes confirmed> +``` + +--- + +## Verification Results + +**Path:** `.adapter-workspace/verification-results.md` +**Written by:** Programmer (Step 3.4, after all models pass) + +```markdown +# Verification Results: <Architecture> + +## Models verified +| Model | Params | Memory (GB) | P1 | P2 | P3 | P4 | Status | +|-------|--------|-------------|----|----|----|----|--------| +| ... | ... | ... | ...| ...| ...| ...| ... | + +## Summary +- Total: N models tested +- Passed: N/N +- All phase scores meet thresholds +- mypy: clean +- format: clean +``` + +--- + +## Verification Failure Analysis + +**Path:** `.adapter-workspace/verification-failure.md` +**Written by:** Programmer (Step 3, when models fail) + +```markdown +# Verification Failure Analysis: <Architecture> + +## Failed models +| Model | Phase | Score | Threshold | Failed Tests | +|-------|-------|-------|-----------|-------------| +| ... | ... | ... | ... | ... | + +## Root cause analysis +- <what went wrong and why> + +## Planned fix approach +- <what needs to change> +``` + +--- + +## Review Files + +**Path:** `.adapter-workspace/reviews/<type>-review-<N>.md` +**Written by:** Reviewer (all review modes) + +File naming: +- Brief reviews: `brief-review-<N>.md` +- Plan reviews: `plan-review-<N>.md` +- Phase reviews: `phase-<letter>-review-<N>.md` + +`<N>` is the round number — check existing files with +`ls .adapter-workspace/reviews/` before writing. + +### Requesting changes + +```markdown +# <Brief|Plan|Phase A|...> Review — Round <N> + +## Overall assessment +<1-3 sentence summary> + +## Issues + +### CRITICAL (blocks approval — crashes or wrong results) +1. **[File: path, ~line N]** Description. + - Verified against: <what source you checked> + - Why it matters: ... + - Suggestion: ... + +### MEDIUM (works but adds risk or maintenance cost) +... + +### LOW (documentation, style, optional improvements) +... + +## What was done well +- ... +``` + +### Approving a phase + +```markdown +# Phase <letter> Review — Round <N> — APPROVED + +## Review summary +<what was reviewed> + +## Verification performed +- Phase 1: <factual verification against HF source> +- Phase 2: <numerical/semantic checks> +- Phase 3: <design assessment> +- Phase 4: <completeness check> +- Phase 5: <plan-to-code match> +``` + +--- + +## Completion Report (Final Review) + +**Path:** `.adapter-workspace/completion-report.md` +**Written by:** Reviewer (final review after verification passes) + +```markdown +# Adapter Completion Report: <Architecture Name> + +### Architecture Overview +- Architecture class: <HF class name> +- Transformers source: <modeling file path> +- Pattern: <Llama-like/GPT2-like/...> +- Key features: <normalization, attention, MLP, etc.> + +### Adapter Implementation +- Adapter file: <path to new adapter .py> +- Closest reference used: <existing adapter it was based on> +- Unique challenges: <what was non-trivial> +- Solutions: <how they were resolved> + +### New Bridge Components (if any) +- <name>: <what it does, why it was needed, file path> +- Tests: <test file path, what is covered> + +### New Utilities (if any) +- <name>: <what it does, file path, tests> + +### Files Changed +- <complete list of files created or modified> + +### Verification Results +| Model | Params | Memory (GB) | P1 | P2 | P3 | P4 | Status | +|-------|--------|-------------|----|----|----|----|--------| +| ... | ... | ... | ...| ...| ...| ...| ... | + +- Total models tested: <N> +- Pass rate: <N/N> +- All phase score thresholds met: yes/no + +### Review Summary +- Planning iterations: <count> +- Implementation phases: <count> +- Review iterations per phase: <summary> +- Total issues found and resolved: <count by severity> + +### Notes for Future Work (optional) +- ... +``` diff --git a/devtools/adapter_builder/docs/cli-reference.md b/devtools/adapter_builder/docs/cli-reference.md new file mode 100644 index 0000000000..313346c7ae --- /dev/null +++ b/devtools/adapter_builder/docs/cli-reference.md @@ -0,0 +1,263 @@ +# CLI Reference + +Complete reference for `./agents/launch-agent-pair.sh` — the user-facing entry point. The dispatcher routes operational subcommands to [`agents/ops.sh`](../agents/ops.sh) and the default launch flow to [`agents/launch.sh`](../agents/launch.sh). Shared bash helpers live in [`agents/lib/common.sh`](../agents/lib/common.sh). + +## Subcommands + +```text +./agents/launch-agent-pair.sh --architecture <class> [flags] # Launch an adapter build +./agents/launch-agent-pair.sh status # Show all running/stopped pairs +./agents/launch-agent-pair.sh logs <name> [--raw] # Tail structured timeline (or raw tty log) +./agents/launch-agent-pair.sh attach <architecture> # Attach to a running tmux session +./agents/launch-agent-pair.sh send <architecture> <message> # Inject a message into a running session +./agents/launch-agent-pair.sh stop [architecture|all] # Cleanly kill running session(s) +./agents/launch-agent-pair.sh clean [architecture] # Remove worktree + branch + logs +``` + +### Default (launch) + +```bash +./agents/launch-agent-pair.sh --architecture CohereForCausalLM # interactive +./agents/launch-agent-pair.sh --architecture CohereForCausalLM --background # detached tmux +``` + +If the architecture already has an adapter in TransformerLens, the script exits immediately — no wasted work. If a worktree already exists for the branch, the launcher resumes the existing session using the `.adapter-progress.json` crash-recovery state. + +### `status` + +Shows every running or recently-stopped pair with: + +- PID and `[running]` / `[stopped]` state +- Lifecycle step (`planning` / `programming` / `verification`) +- Current phase and completed phases +- Timeline event count and the formatted last event +- **Time since the last timeline event** — human-readable age like `12s ago`, `3m 40s ago`, `2h 15m ago`, `1d 4h ago`. Useful for spotting stuck sessions at a glance: a pair with no new events in hours is likely blocked or idling. +- tmux attach command for the session +- Memory-lock status (held or free) + +The last-event column is rendered by piping the final line of the worktree's `.adapter-workspace/timeline.jsonl` through [`scripts/format-timeline.py`](../scripts/format-timeline.py). The age is computed from the same entry's `ts` field. + +### `logs <arch> [--raw]` + +Matches partial architecture names — `logs codegen` finds the running codegen session. + +- **Default (no flag):** tails the structured JSONL timeline through `format-timeline.py`, showing `HH:MM:SS EventName label` lines. This is the preferred view — it skips terminal spinner noise. +- **`--raw`:** tails the raw pty dump from `.logs/<branch>.raw` with ANSI escapes stripped. Use when you need to see exactly what Claude Code rendered in the session. + +Both modes are `tail -f` — Ctrl-C to stop. + +### `attach <arch>` + +Opens an interactive tmux connection to a running background session. You can watch the agents work in real time and type directly into Claude's interactive prompt. Detach with `Ctrl-b d` — the session keeps running. + +Requires the session to have been launched with `--background`. + +### `send <arch> "<message>"` + +Injects a text message into a running session as if you typed it at the Claude prompt. Useful for mid-run corrections without stopping the agents: + +```bash +./agents/launch-agent-pair.sh send qwen3moe "Use float32 instead of bfloat16 for verification" +./agents/launch-agent-pair.sh send codegen "Skip the 13B variant — start with the 2B model" +``` + +### `stop [arch|all]` + +- `stop` (no arg) or `stop all` — cleanly kills every running background pair +- `stop <arch>` — stops the pair matching the given architecture name + +Worktrees and `.adapter-progress.json` are preserved so you can resume later. Prefer `stop` over `kill -9` — `stop` cleanly tears down the tmux session, which lets the SessionEnd hooks fire. + +### `clean [arch]` + +- `clean` (no arg) — removes every worktree with `verification_passed: true` in its progress file. Skips in-progress builds. +- `clean <arch>` — removes that specific worktree, branch, and log files regardless of state. + +Use this after you've committed the adapter from a successful run and want to tidy up. + +## Launch Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--architecture <class>` | (required) | HF architecture class name, e.g. `CohereForCausalLM`, `CodeGenForCausalLM` | +| `--target-repo <path>` | `DEFAULT_TARGET_REPO` from `.env` | Path to TransformerLens repository | +| `--base-branch <branch>` | `DEFAULT_BASE_BRANCH` from `.env` (`dev-4.x`) | Base branch to create the worktree from | +| `--new-branch <branch>` | `feature/<arch>-adapter` | Feature branch name (auto-derived from architecture) | +| `--max-memory <gb>` | `DEFAULT_MAX_MEMORY_GB` from `.env` (`96`) | Memory limit in GB for verify_models | +| `--worktree-dir <path>` | `${WORKTREE_BASE}/<branch>` | Custom worktree location | +| `--programmer-prompt <text>` | auto-generated from architecture | Override the task prompt sent to the Programmer | +| `--reviewer-prompt <text>` | generic review focus | Extra review criteria for the Reviewer | +| `--background` | off | Run detached inside a tmux session — enables parallel pairs, persistent sessions, and `attach`/`send` | +| `--retry` | off | Resume a crashed session. Planning/programming crashes resume normally; verification crashes require manual inspection and trigger a notification instead | +| `--auto-approve` | off | Skip Claude Code permission prompts | +| `--skip-arch-check` | off | Skip the pre-flight architecture existence check (see below). Use when launching an architecture that's so new it isn't yet in the installed `transformers` package *and* isn't on HuggingFace Hub | +| `--dry-run` | off | Run all pre-flight checks (adapter-exists on the base branch, architecture validation, base-branch exists), print what would be created, and exit without creating a worktree or launching sessions | +| `--programmer-task <text>` | auto-generated from architecture | **Solo mode only.** Override the Session Task appended to the programmer prompt — used for scripted protocol tests | +| `--reviewer-task <text>` | generic review focus | **Solo mode only.** Override the Session Task appended to the reviewer prompt — used for scripted protocol tests | +| `--programmer-model <id>` | `PROGRAMMER_MODEL` env, then frontmatter | Model for the Programmer agent (both modes) | +| `--reviewer-model <id>` | `REVIEWER_MODEL` env, then frontmatter | Model for the Reviewer agent (both modes) | +| `--orchestrator-model <id>` | `ORCHESTRATOR_MODEL` env, then session default | Model for the Orchestrator session (agent-teams only) | +| `-h`, `--help` | — | Print the full usage header from `launch.sh` | + +## Pre-flight: Architecture Existence Check + +Before creating a worktree, the launcher validates that the architecture class name actually exists. This catches typos (e.g. `CohoreForCausalLM`) that would otherwise burn a worktree, an agent startup, ~3 brief-review cycles, and a stuck-report notification before failing. + +The check runs [`scripts/validate-architecture.py`](../scripts/validate-architecture.py) in the target repo's uv env and looks in two places in order: + +1. **`transformers` package** — `getattr(transformers, <arch>)`. Fast, local, ~1–2s including cold import. +2. **HuggingFace Hub** — bounded scan of the top 500 models by download count, checking each `config.architectures` field for a match. Only runs if the transformers check comes up empty. ~1–3s on top of the transformers import. + +**Exit codes from the validator:** + +- `0` — found in transformers or on HF Hub → launcher proceeds +- `1` — definitively not found in either → launcher aborts with a "does not exist" error pointing at common causes (case sensitivity, typos) +- `2` — could not verify (missing deps, network error) → launcher emits a warning and proceeds + +**Bypass:** pass `--skip-arch-check` to skip the validation entirely. Useful for architectures not yet released on HuggingFace Hub or not yet in the pinned `transformers` version. + +## Configuration (`.env`) + +Defaults are set in `.env`; see [`.env.example`](../.env.example) for a template. + +| Variable | Default | Description | +|----------|---------|-------------| +| `DEFAULT_TARGET_REPO` | — | Path to the TransformerLens repo | +| `DEFAULT_BASE_BRANCH` | `dev-4.x` | Base branch for worktrees | +| `DEFAULT_MAX_MEMORY_GB` | `96` | Memory limit in GB for verify_models | +| `WORKTREE_BASE` | `<parent-of-TransformerLens>/worktrees` | Where agent pair worktrees are created | +| `NOTIFICATION_WEBHOOK_URL` | — | Slack/Discord/custom webhook URL for notifications | +| `NOTIFICATION_NUMBER` | — | Phone number for iMessage fallback (macOS only) | +| `HF_TOKEN` | — | HuggingFace API token for model access | + +## Hook Tunables (environment variables) + +These control the runtime-enforcement hooks. See [hooks-reference.md](hooks-reference.md) for what each hook does. + +| Variable | Default | Consumed by | Effect | +|----------|---------|-------------|--------| +| `MAX_REVIEW_ROUNDS` | `3` | `guard-review-rounds.sh` | Max review iterations per checkpoint before the hook blocks writes and forces escalation | +| `MAX_VERIFY_PARAMS` | `7500000000` (7.5B) | `guard-verify-models.sh` | Ceiling for models passed to `verify_models --model …` | + +## Runtime Environment Variables + +Per-agent model selection resolves with precedence **CLI flag > env var > +prompt frontmatter**: `--programmer-model`/`PROGRAMMER_MODEL` and +`--reviewer-model`/`REVIEWER_MODEL` apply in both modes (solo passes them via +`--model`; agent-teams rewrites the copied agent definition's frontmatter); +`--orchestrator-model`/`ORCHESTRATOR_MODEL` sets the orchestrator session's +model in agent-teams mode. Env vars are settable in `.env` as durable +defaults; flags are for per-run overrides. Unset means the checked-in +frontmatter (or session default) applies. `--dry-run` prints the resolved +models without launching anything. + +Set automatically by `launch.sh` and exported into every agent session: + +| Variable | Description | +|----------|-------------| +| `TL_ADAPTER_BUILDER_ROOT` | Absolute path to this project (used by agents and hooks to find docs/, scripts/, and `overlord-request.sh`) | +| `MAX_MEMORY_GB` | Memory limit for verify_models | +| `NOTIFICATION_WEBHOOK_URL` | Webhook URL for notifications | +| `NOTIFICATION_NUMBER` | Phone number for iMessage fallback | +| `HF_TOKEN` / `HUGGING_FACE_HUB_TOKEN` | HuggingFace auth (both names exported for compatibility with `huggingface_hub` and `transformers`) | +| `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS` | Always `1` | + +## Background Mode (tmux) + +`--background` launches the Claude Code session inside a detached tmux session named `tl-adapter-<branch>`. This provides: + +- **A real pty** — Claude Code's interactive mode works properly; piped stdio breaks subagent spawning. +- **Process persistence** — the session survives terminal disconnects, SSH drops, and laptop sleeps. +- **Interactive attach** — jump into a running session at any time and watch the agents work. +- **Message injection** — send text to the orchestrator mid-run without stopping anything. + +Under the hood each session's pty output is piped to `.logs/<branch>.raw` via `tmux pipe-pane`, so the raw archive is always available for grepping. The structured timeline at `.adapter-workspace/timeline.jsonl` (inside the worktree) is the preferred source — it skips spinner noise and is what `logs` tails by default. + +**Requirements:** `tmux` must be on `PATH` (`brew install tmux` on macOS, `apt install tmux` on Linux). + +### Artifacts written under `.logs/` + +For each background session `<branch>`: + +| File | Purpose | +|------|---------| +| `<branch>.pid` | Top-level tmux pane PID; read by `status` and `stop` | +| `<branch>.log` | Legacy log target (still emitted for compatibility) | +| `<branch>.raw` | Full pty capture from `tmux pipe-pane` (ANSI-preserved) | +| `<branch>.debug.log` | Claude Code's `--debug hooks` debug output | +| `<branch>.wrapper.sh` | The small bash wrapper tmux invokes (exports env, reads prompt file, calls `claude`) | +| `<branch>.prompt.txt` | The rendered orchestration prompt (too large for shell quoting) | +| `<branch>.watch.log` | Poll log from the completion watcher (see below) | + +`.logs/` is gitignored. Files accumulate across sessions — clean them up manually when needed. + +### Completion watcher + +The wrapper launches [`agents/watch-completion.sh`](../agents/watch-completion.sh) as a background process alongside `claude`. It closes a gap in interactive Claude Code: an orchestrator has no tool to self-terminate — it can print "Session terminating — SessionEnd hook will fire the Slack notification" but that only ends its turn, leaving the pane at the `❯` prompt. Without the watcher, the session sits forever, the SessionEnd hook never fires, and no Slack notification ever gets sent. + +The watcher polls `.adapter-progress.json` every `WATCH_POLL_SECS` seconds (default `30`) and fires `tmux send-keys "/exit" Enter` when: + +1. `verification_passed == true` +2. `final_review_passed == true` +3. The worktree's `.adapter-workspace/timeline.jsonl` has been idle for at least `WATCH_IDLE_SECS` seconds (default `60`) + +The idle-grace guard (#3) is the critical protection against false positives. Cases where a progress flag gets flipped prematurely while the reviewer is still actively working will keep the timeline hot, so the watcher defers until genuine quiescence. Once `/exit` is sent, Claude Code processes it as a clean session terminate, which fires the `SessionEnd` hook chain (`timeline-capture.sh` → `notify-on-completion.sh` → `notify.sh` → Slack). + +**Tunables** (both optional, exported via the wrapper): + +| Variable | Default | Effect | +|---|---|---| +| `WATCH_POLL_SECS` | `30` | Poll interval | +| `WATCH_IDLE_SECS` | `60` | Required timeline idleness before sending `/exit` | + +The watcher logs each poll outcome to `.logs/<branch>.watch.log` for debugging. It exits automatically when the tmux session disappears, when it successfully sends `/exit`, or when the wrapper kills it on claude exit (via `trap EXIT` + explicit `kill`). Foreground mode (`--background` not set) does not launch the watcher — in foreground you're expected to type `/exit` manually when you see the orchestrator print its "terminating" farewell. + +## Signal Protocol + +All agent signals are defined in [`agents/signals.sh`](../agents/signals.sh) — the single source of truth. The orchestrator prompt is generated from `signals.sh` at launch, so the protocol can't drift. + +**Programmer signals**: `BRIEF READY FOR REVIEW`, `PLAN READY FOR REVIEW`, `PHASE <X> READY FOR REVIEW`, `VERIFICATION COMPLETE`, `VERIFICATION FAILED`, `VERIFICATION SKIPPED — ALL MODELS TOO LARGE`, `READY FOR REVIEW`. + +**Reviewer signals**: `BRIEF APPROVED` / `BRIEF CHANGES REQUESTED`, `PLAN APPROVED` / `PLAN CHANGES REQUESTED`, `APPROVED` / `CHANGES REQUESTED`. + +The full state machine diagram and routing rules are in [`agents/signals.sh`](../agents/signals.sh) and get injected into the orchestrator prompt as the `{{PROTOCOL_BLOCK}}` placeholder during `launch.sh` rendering. + +## Solo Mode (`--mode solo`) + +```bash +./agents/launch-agent-pair.sh --mode solo --architecture CohereForCausalLM +``` + +For accounts without agent-teams access (Pro/Team tier). Launches two +independent Claude Code sessions in one tmux session — programmer (left +pane `:0.0`) and reviewer (right pane `:0.1`) — plus a +**coordinator daemon** ([`agents/solo-coordinator.sh`](../agents/solo-coordinator.sh)). + +Coordination is file-based but never blocking: an agent `touch`es a signal +file in `.adapter-workspace/signals/` and ends its turn. The coordinator +consumes the signal (archived to `signals/.archive/<epoch>.<name>`) and +wakes the counterpart pane with an injected `[coordinator]` message. +Agents must never poll or wait — in-agent blocking calls die at the Bash +tool timeout, which is why the routing lives in a daemon. + +File-signal names (distinct from the agent-teams message signals above): + +| Signal | Emitted by | Routed to | +| ------ | ---------- | --------- | +| `brief-ready`, `plan-ready`, `phase-<X>-ready`, `verification-complete` | programmer | reviewer | +| `<checkpoint>-approved`, `<checkpoint>-changes-<N>`, `final-approved` | reviewer | programmer | +| `verification-failed` | programmer | reviewer (informational) | +| `verification-skipped`, `stuck` | programmer | user notification; sessions stay alive | +| `done` | programmer | completion: `/exit` to both panes after progress flags + timeline idle | + +The coordinator also completes passively (flags set + idle timeline, no +`done` signal), subsuming the teams-mode completion watcher. Its log is +`.logs/<branch>.watch.log`; its PID file `.logs/<branch>.coordinator.pid` +(cleaned by `stop`). Agents can snapshot signal state non-blockingly with +[`agents/check-signals.sh`](../agents/check-signals.sh). Per-role models +come from the solo prompt files' frontmatter and are passed via `--model`. + +Testing hooks: `solo-coordinator.sh --route <signal>` prints the routing +decision; `solo-coordinator.sh --once <worktree>` does a single +scan-archive-print pass with no tmux. Both are exercised by +`scripts/dry-run-test.sh`. diff --git a/devtools/adapter_builder/docs/hf-model-analysis-guide.md b/devtools/adapter_builder/docs/hf-model-analysis-guide.md new file mode 100644 index 0000000000..78b62c35a0 --- /dev/null +++ b/devtools/adapter_builder/docs/hf-model-analysis-guide.md @@ -0,0 +1,168 @@ +# HuggingFace Model Analysis Guide + +This guide explains how to analyze a HuggingFace model to extract the information needed to build a TransformerLens Architecture Adapter. + +## Step 1: Read the Model's config.json + +Every HF model has a `config.json` that contains architecture details. You can access it via: + +```python +from transformers import AutoConfig +config = AutoConfig.from_pretrained("model-name-or-path") +print(config) +``` + +Or via the HuggingFace API: +```bash +curl -s "https://huggingface.co/model-name/resolve/main/config.json" | python -m json.tool +``` + +### Key Config Fields to Extract + +| HF Config Field | TL Config Field | Description | +|-----------------|-----------------|-------------| +| `hidden_size` | `d_model` | Model dimension | +| `num_attention_heads` | `n_heads` | Number of attention heads | +| `num_key_value_heads` | `n_key_value_heads` | KV heads (for GQA; if absent or equal to n_heads, not GQA) | +| `intermediate_size` | `d_mlp` | MLP intermediate dimension | +| `num_hidden_layers` | `n_layers` | Number of transformer blocks | +| `vocab_size` | `d_vocab` | Vocabulary size | +| `max_position_embeddings` | `n_ctx` | Maximum sequence length | +| `rms_norm_eps` | `eps` | Normalization epsilon | +| `model_type` | — | Architecture family (e.g., "llama", "gpt2", "mistral") | +| `architectures` | `architecture` | HF class name (e.g., `["LlamaForCausalLM"]`) | + +## Step 2: Determine Architecture Characteristics + +### Normalization Type + +Check the model code or config: +- **RMSNorm** → `normalization_type = "RMS"` — Look for `RMSNorm` in the model code, or `rms_norm_eps` in config +- **LayerNorm** → `normalization_type = "LN"` — Look for `LayerNorm`, or `layer_norm_eps` / `layer_norm_epsilon` in config + +Also identify the epsilon attribute name: +- `"variance_epsilon"` (Llama) +- `"rms_norm_eps"` (some models expose this directly) +- `"layer_norm_eps"` (GPT-2, BERT) +- `"eps"` (generic) + +### Positional Embedding Type + +- **Rotary (RoPE)** → `positional_embedding_type = "rotary"` — Most modern models (Llama, Mistral, Qwen, Gemma) +- **Learned/Standard** → `positional_embedding_type = "standard"` — GPT-2, OPT +- Check for `RotaryEmbedding` class in the model code + +### Attention Type + +- **Multi-Head Attention (MHA)** — `n_key_value_heads == n_heads` or field absent +- **Grouped Query Attention (GQA)** — `n_key_value_heads < n_heads` (e.g., Llama 3, Mistral) +- **Multi-Query Attention (MQA)** — `n_key_value_heads == 1` (e.g., Falcon) + +### MLP Type + +- **Gated MLP (SwiGLU)** → `gated_mlp = True` — Has gate/up/down projections (Llama, Qwen, Gemma) +- **Standard MLP** → `gated_mlp = False` — Has fc1/fc2 or c_fc/c_proj (GPT-2) + +### QKV Layout + +- **Separate Q/K/V** — Most models: `q_proj`, `k_proj`, `v_proj` +- **Combined QKV** — GPT-2 style: single `c_attn` or `query_key_value` linear layer + +## Step 3: Inspect Module Names + +To find the exact HuggingFace module paths for the component mapping: + +```python +from transformers import AutoModelForCausalLM +model = AutoModelForCausalLM.from_pretrained("model-name", torch_dtype="auto") + +# Print all named modules +for name, module in model.named_modules(): + print(f"{name}: {type(module).__name__}") +``` + +### What to Look For + +Map these HF module paths to TL component mapping entries: + +| TL Name | Look for in HF | Common HF Paths | +|---------|----------------|-----------------| +| `embed` | Token embedding | `model.embed_tokens`, `transformer.wte` | +| `pos_embed` | Position embedding (if standard) | `transformer.wpe` | +| `rotary_emb` | Rotary embedding (if RoPE) | `model.rotary_emb`, `model.layers.0.self_attn.rotary_emb` | +| `blocks` | Layer list | `model.layers`, `transformer.h`, `model.decoder.layers` | +| `ln1` | Pre-attention norm | `input_layernorm`, `ln_1` | +| `ln2` | Post-attention norm | `post_attention_layernorm`, `ln_2` | +| `attn` | Self-attention module | `self_attn`, `attn` | +| `attn.q` | Query projection | `q_proj`, `query` | +| `attn.k` | Key projection | `k_proj`, `key` | +| `attn.v` | Value projection | `v_proj`, `value` | +| `attn.o` | Output projection | `o_proj`, `out_proj`, `dense`, `c_proj` | +| `attn.qkv` | Combined QKV (if used) | `c_attn`, `query_key_value` | +| `mlp` | MLP module | `mlp`, `feed_forward` | +| `mlp.gate` | Gate projection (if gated) | `gate_proj`, `w1` | +| `mlp.in` | Up/input projection | `up_proj`, `c_fc`, `fc1`, `w3` | +| `mlp.out` | Down/output projection | `down_proj`, `c_proj`, `fc2`, `w2` | +| `ln_final` | Final layer norm | `model.norm`, `transformer.ln_f`, `model.final_layernorm` | +| `unembed` | LM head | `lm_head`, `embed_out` | + +## Step 4: Check for Biases + +```python +# Check if a specific layer has bias +layer = model.model.layers[0] +print(f"Q bias: {layer.self_attn.q_proj.bias is not None}") +print(f"MLP in bias: {layer.mlp.up_proj.bias is not None}") +``` + +Document which layers lack biases — this affects the "Optional Parameters" section of the adapter docstring. + +## Step 5: Examine State Dict Keys + +```python +# Print all parameter names and shapes +for key, param in model.state_dict().items(): + print(f"{key}: {param.shape}") +``` + +This helps verify: +- Weight naming patterns match your component mapping +- Tensor shapes match expected dimensions +- No unexpected parameters that need special handling + +## Step 6: Find an Existing Similar Adapter + +Check if a similar architecture already has an adapter. Most new models are variants of existing patterns: + +| If your model is like... | Start from adapter... | +|--------------------------|----------------------| +| Llama, Mistral, Qwen2, Gemma | `llama.py` | +| GPT-2, GPT-J | `gpt2.py` | +| BLOOM, Falcon | `bloom.py` or `falcon.py` | +| T5, encoder-decoder | `t5.py` | +| MoE model | `mixtral.py` or `granite_moe.py` | +| Multimodal (vision+text) | `llava.py` or `gemma3_multimodal.py` | + +## Quick Reference: Decision Tree + +``` +1. Does the model use RMSNorm or LayerNorm? + → RMSNorm: normalization_type="RMS", use RMSNormalizationBridge + → LayerNorm: normalization_type="LN", use NormalizationBridge + +2. Does the model use RoPE or learned positional embeddings? + → RoPE: positional_embedding_type="rotary", add RotaryEmbeddingBridge, use PositionEmbeddingsAttentionBridge + → Learned: positional_embedding_type="standard", add PosEmbedBridge + +3. Are Q/K/V separate or combined? + → Separate: use PositionEmbeddingsAttentionBridge with q/k/v/o submodules + → Combined: use JointQKVAttentionBridge with qkv/o submodules + +4. Does the MLP have a gate projection? + → Yes (gate+up+down): gated_mlp=True, use GatedMLPBridge + → No (in+out): gated_mlp=False, use MLPBridge + +5. Is n_key_value_heads < n_heads? + → Yes: GQA — set n_key_value_heads on cfg + → No: standard MHA — no special handling needed +``` diff --git a/devtools/adapter_builder/docs/hooks-reference.md b/devtools/adapter_builder/docs/hooks-reference.md new file mode 100644 index 0000000000..554d78efb6 --- /dev/null +++ b/devtools/adapter_builder/docs/hooks-reference.md @@ -0,0 +1,155 @@ +# Hooks Reference + +Critical rules are enforced at the framework level via Claude Code hooks, not by asking agents to remember them. `launch.sh` writes a `.claude/settings.json` in each worktree that registers absolute paths to the hook scripts under [`agents/hooks/`](../agents/hooks/). When Claude Code fires the relevant event, the hook reads the payload from stdin, decides whether to allow or block, and emits a JSON response on stdout. + +Every hook includes a `RUNTIME CONTEXT:` header reminding the reader that the script *lives* under `devtools/adapter_builder/agents/hooks/` but *runs* inside the target TransformerLens worktree (so `pwd` is the worktree, not the hook's own directory). + +## Hook summary + +| Hook | Event | Matcher | Can block? | +|------|-------|---------|------------| +| [timeline-capture.sh](../agents/hooks/timeline-capture.sh) | `SessionStart`, `SessionEnd`, `SubagentStart`, `SubagentStop`, `PreToolUse`, `PostToolUse` | (all) | No — logging only | +| [guard-hooked-transformer.sh](../agents/hooks/guard-hooked-transformer.sh) | `PreToolUse` | `Edit\|Write\|MultiEdit\|NotebookEdit` | Yes | +| [guard-review-rounds.sh](../agents/hooks/guard-review-rounds.sh) | `PreToolUse` | `Edit\|Write\|MultiEdit\|NotebookEdit` | Yes | +| [guard-git.sh](../agents/hooks/guard-git.sh) | `PreToolUse` | `Bash` | Yes | +| [guard-verify-models.sh](../agents/hooks/guard-verify-models.sh) | `PreToolUse` | `Bash` | Yes | +| [gate-reviewer-writes-file.sh](../agents/hooks/gate-reviewer-writes-file.sh) | `SubagentStop` | (all) | Yes | +| [gate-lint-checks.sh](../agents/hooks/gate-lint-checks.sh) | `Stop` | (all) | Yes | +| [notify-on-completion.sh](../agents/hooks/notify-on-completion.sh) | `SessionEnd` | (all) | No — side effect only | + +Multiple hooks may register on the same matcher; they run in order and any one block denies the tool call. + +## timeline-capture.sh + +Appends a structured JSON line to `.adapter-workspace/timeline.jsonl` on every tool call, subagent spawn, and session event. Captures timestamp, event name, agent id/type, tool name, and `tool_input` (including bash commands). The `logs` and `status` subcommands format this file via [`scripts/format-timeline.py`](../scripts/format-timeline.py). + +Never blocks — always emits `{"continue": true}`. If `jq` or the timeline file isn't available, the append silently fails and the tool call still proceeds. + +## guard-hooked-transformer.sh + +Blocks edits to deprecated HookedTransformer files. Agents may read them for reference but must not modify them — their work must only touch the TransformerBridge system. + +**Blocks writes** whose `tool_input.file_path` contains any of: + +- `transformer_lens/HookedTransformer.py` +- `transformer_lens/loading_from_pretrained.py` +- `transformer_lens/components/` +- `transformer_lens/pretrained/weight_conversions/` + +On block, emits `{"continue": false, "decision": "block", "reason": …}` with a message explaining the adapter should only touch `transformer_lens/model_bridge/` and the factory. + +## guard-review-rounds.sh + +Enforces the 3-round review-iteration limit from [`agents/orchestrator.md`](../agents/orchestrator.md) § Iteration Limits. Turns a prompt-level rule into a runtime stop so runaway loops can't happen. + +**Blocks writes** to review files at round 4 or higher: + +- `.adapter-workspace/reviews/brief-review-<N>.md` where N > 3 +- `.adapter-workspace/reviews/plan-review-<N>.md` where N > 3 +- `.adapter-workspace/reviews/phase-<letter>-review-<N>.md` where N > 3 + +On block, the error message tells the orchestrator to: (1) collect unresolved CRITICAL issues from previous reviews, (2) write `.adapter-workspace/stuck-report.md`, (3) fire the "stuck" notification via `scripts/notify.sh`, and (4) terminate the session. + +Unrecognized files under `.adapter-workspace/reviews/` pass through — the reviewer owns the naming convention and unknown files are presumed unrelated. + +**Tunable:** `MAX_REVIEW_ROUNDS` (default `3`). Exported from the session environment via the wrapper script. + +## guard-git.sh + +Blocks any Bash command that commits, pushes, or publishes changes. Agents must leave everything uncommitted for manual review. + +**Blocks** commands matching (with a shell-separator lead anchor so chained invocations like `make test && git commit -m …` are caught): + +- `git commit` (any flags — `-m`, `--amend`, `-a`) +- `git push` (any form) +- `gh pr create | merge | close | review | ready | edit` +- `gh release create | delete | edit` + +**Passes through** read-only git operations (`status`, `diff`, `log`, `show`, `branch -l`, `worktree list`) and unrelated substrings like `gitignore` — the word-boundary regex requires `git` followed by whitespace followed by the forbidden subcommand, so `cat .gitignore` is not affected. + +On block, the error message references `agents/programmer.md § What NOT to Do`. + +## guard-verify-models.sh + +Enforces the ≤7B verification rule from [`agents/programmer.md`](../agents/programmer.md) Step 3.1. Turns the "skip large models" guidance into a runtime stop so a misread prompt can't waste hours on a 70B model load. + +**Intercepts** Bash commands invoking `verify_models` (matches both `verify_models` directly and `python -m transformer_lens.tools.model_registry.verify_models`). + +**Always allowed:** + +- `--dry-run` invocations — they don't load weights +- Bulk `--architectures` runs without a specific `--model` arg — `verify_models` has its own memory-based skip for bulk mode + +**Blocks** when the command specifies `--model <id>` (or `--model=<id>`) and either: + +1. The model is not registered in `transformer_lens/tools/model_registry/data/supported_models.json`. The error message tells the agent to run [`scan-hf-architecture.py`](../scripts/scan-hf-architecture.py) + [`port-arch-models.py`](../scripts/port-arch-models.py) first. +2. The registered `metadata.total_params` exceeds `MAX_VERIFY_PARAMS` (default 7.5B — a small buffer above "7B" models like Mistral-7B at 7.24B; blocks Llama-3-8B and anything larger). + +**Tunables:** `MAX_VERIFY_PARAMS` (default `7500000000`). Exported from the session environment. + +## gate-reviewer-writes-file.sh + +Enforces the "every review must be persisted to a file" rule from [`agents/reviewer.md`](../agents/reviewer.md) § Feedback Format. Turns a prompt-level convention into a runtime stop so reviewers can't silently return verbal approvals while leaving `.adapter-workspace/reviews/` empty. + +Fires on `SubagentStop`. Takes action only when the stopped subagent is `agent_type == "reviewer"`: + +1. Reads the reviewer's matching `SubagentStart` timestamp from `.adapter-workspace/timeline.jsonl` (by `agent_id`). +2. Checks whether any file in `.adapter-workspace/reviews/` or the single `.adapter-workspace/completion-report.md` has an `mtime` at or after that `SubagentStart` timestamp. +3. **If yes** → allow. The reviewer wrote a file; normal processing continues. +4. **If no** → blocks with a `{continue: false, decision: "block", reason: …}` response. The reason string instructs the orchestrator to treat the reviewer's result as void, resume the reviewer by `agent_id` with a message prefixed `Resuming you —` insisting on a file, and re-run the Review File Integrity Check after the next `SubagentStop`. +5. In the drift case, also appends a `ReviewerFileDrift` event to `timeline.jsonl` for visibility in `status` / `logs` / post-hoc inspection. + +Non-reviewer `SubagentStop` events pass through unchanged (the hook exits at the `agent_type` check). + +Complemented by the orchestrator-level Review File Integrity Check section in the protocol block (see `agents/signals.sh generate_protocol_block`), which tells the orchestrator to verify file existence manually without waiting for the hook. + +## gate-lint-checks.sh + +Refuses to let the session end until `mypy` and `make check-format` both pass — but only *after* verification has passed, so it doesn't block legitimate mid-build stops. + +Flow when the `Stop` event fires: + +1. If `.adapter-progress.json` doesn't exist → allow stop (not an adapter build) +2. If `verification_passed != true` → allow stop (agent is stopping legitimately mid-build) +3. If `final_review_passed == true` → allow stop (already gated this session, don't re-run the checks) +4. Run `uv run mypy .` — if it fails, block with the last 30 lines of output and a reminder that `# type: ignore` is not an acceptable fix +5. Run `make check-format` — if it fails, block with the last 30 lines of output +6. Both pass → mark `final_review_passed: true` in the progress file and allow stop + +The mark ensures subsequent `Stop` events on the same session skip straight to allow, so the expensive checks run exactly once per completed build. + +## notify-on-completion.sh + +Fires the Slack notification on `SessionEnd` when the progress file shows `verification_passed: true`. Uses a `.adapter-workspace/notification-sent` sentinel file to avoid double-sending on resumed sessions. + +Never blocks — this is a side-effect hook. Finds `scripts/notify.sh` via `TL_ADAPTER_BUILDER_ROOT` (exported by the launch wrapper) or by searching upward from the worktree as a fallback. + +## Hook settings registration + +[`agents/launch.sh`](../agents/launch.sh) writes `.claude/settings.json` in each worktree with this structure: + +```json +{ + "hooks": { + "SessionStart": [{ "hooks": [{ "command": "<timeline>" }] }], + "SessionEnd": [{ "hooks": [{ "command": "<timeline>" }, { "command": "<notify>" }] }], + "SubagentStart": [{ "hooks": [{ "command": "<timeline>" }] }], + "SubagentStop": [{ "hooks": [{ "command": "<timeline>" }, { "command": "<gate-reviewer-writes-file>" }] }], + "PreToolUse": [ + { "matcher": "Edit|Write|MultiEdit|NotebookEdit", + "hooks": [{ "command": "<guard-ht>" }, { "command": "<guard-review-rounds>" }] }, + { "matcher": "Bash", + "hooks": [{ "command": "<guard-git>" }, { "command": "<guard-verify-models>" }] }, + { "matcher": "", + "hooks": [{ "command": "<timeline>" }] } + ], + "PostToolUse": [ + { "matcher": "", + "hooks": [{ "command": "<timeline>" }] } + ], + "Stop": [{ "hooks": [{ "command": "<gate-lint>" }] }] + } +} +``` + +Hook commands are stored as absolute paths, single-quoted to preserve the space in "TL Adapter Builder". Nothing is copied from this repo into the worktree except this `.claude/settings.json` and `.claude/agents/{programmer,reviewer}.md`. diff --git a/devtools/adapter_builder/docs/memory-lock.md b/devtools/adapter_builder/docs/memory-lock.md new file mode 100644 index 0000000000..93512b2ad5 --- /dev/null +++ b/devtools/adapter_builder/docs/memory-lock.md @@ -0,0 +1,58 @@ +# Memory Lock + +A flock-based lock at `/tmp/tl-adapter-builder.lock` prevents concurrent +memory-intensive operations across all agent pairs. + +## When to use the lock + +**REQUIRES the lock:** +- Running `verify_models` or `run_benchmark_suite` +- Loading a full HuggingFace model (e.g., `AutoModelForCausalLM.from_pretrained`) + +**Does NOT require the lock (everything else):** +- Reading/writing/editing source files, git, unit tests, scripts, installs, etc. + +## Usage — ALWAYS use the `run` subcommand + +The `run` subcommand acquires the lock, executes the wrapped command while +holding the lock, and releases the lock when the wrapped command exits — all +in **one** shell process. This is the ONLY pattern that works from the Bash +tool: + +```bash +"$TL_ADAPTER_BUILDER_ROOT/agents/overlord-request.sh" run "verify_models: <model_id>" -- \ + uv run python -m transformer_lens.tools.model_registry.verify_models \ + --model <model_id> \ + --max-memory $MAX_MEMORY_GB \ + --device cpu --dtype float32 +``` + +The `--` separator terminates lock arguments; everything after it is the +wrapped command. Exit code of the wrapped command propagates out. + +## Why NOT `source … && overlord_acquire … ; <cmd> ; overlord_release` + +That pattern looks reasonable but is **broken** in this environment. Each +Bash tool call runs in a fresh shell — functions sourced in one call are +not defined in the next, and the lock fd opened by `overlord_acquire` is +held by the current shell, so it's released the instant that shell exits. +If you split acquire / work / release across separate Bash tool calls, +each call acquires and immediately releases the lock, providing **zero** +protection. + +```bash +# DO NOT do this — each line runs in its own shell, the lock is gone by +# the time you reach the work command: +source "$TL_ADAPTER_BUILDER_ROOT/agents/overlord-request.sh" # call 1 (lock acquired + released) +overlord_acquire "verify_models" # call 2 (overlord_acquire not defined) +uv run python -m transformer_lens.tools.model_registry.verify_models ... # call 3 (unprotected) +overlord_release # call 4 (no-op) +``` + +**Always use `overlord-request.sh run` to wrap the command. Do NOT source.** + +## Check lock status + +```bash +"$TL_ADAPTER_BUILDER_ROOT/agents/overlord-request.sh" status +``` diff --git a/devtools/adapter_builder/docs/review-specification.md b/devtools/adapter_builder/docs/review-specification.md new file mode 100644 index 0000000000..0c8ba07923 --- /dev/null +++ b/devtools/adapter_builder/docs/review-specification.md @@ -0,0 +1,152 @@ +# Plan & Code Review Specification + +## Execution Model + +### Findings Accumulator + +Maintain a running findings list across all phases. Each phase appends to it +using `[P0]`, `[P1]`, etc. tags. The final output is produced FROM this list, +not from memory. This prevents forgetting earlier findings and eliminates +redundant re-verification. + +### Phase 0: HF Source Reference (do this once) + +Before any evaluation, read the HuggingFace Transformers source and produce a +structured reference block. All subsequent phases reference this block instead +of re-reading the source independently. + +Record: module paths, config fields, forward pass flow, bias presence, +normalization details, attention type, MLP type. + +**Exit criteria:** Every architectural property has a recorded module path, +config field, or source line number. + +--- + +## Phase 1: Factual Verification + +Every concrete claim in the plan or code must be verified against the Phase 0 +reference and source code. Never trust that a description of existing code is +accurate. + +### For architecture/model claims: +- Cross-reference every module path against the P0 reference +- Verify every config attribute name matches what the code actually uses +- Verify parameter shapes, bias presence, and dimensional relationships +- Verify the forward pass flow matches what the plan/code describes + +### For claims about existing codebase infrastructure: +- Read the referenced files at the cited line numbers +- Verify function signatures, class hierarchies, and calling conventions +- Verify that claimed capabilities actually exist (e.g., "JointQKVAttentionBridge accepts split_qkv_matrix" — read the __init__ to confirm) +- Verify inheritance chains — does the class actually inherit the mixin/method the plan assumes? + +### For test claims: +- Verify test files exist, check skip/xfail markers, count actual test methods +- Verify claimed tolerances match actual code +- Verify claimed CI behavior by reading workflow files + +**Exit criteria:** Every concrete claim has a verified or refuted annotation. + +--- + +## Phase 2: Numerical/Semantic Correctness + +When a plan reimplements a computation or defines weight conversions, diff +against the Phase 0 reference line by line: + +- **Operation order**: Does split happen before or after layernorm? Before or after reshape? +- **Dimension handling**: Are transpose/reshape dimensions correct? Does `view(-1, ...)` infer the right dimension? +- **Dtype behavior**: Does HF upcast softmax to fp32? Does the bridge match? +- **Conditional logic**: Does HF only pad under certain conditions (flash attention)? Does the bridge match those conditions? +- **Config attribute access**: Is the bridge reading from the right object (HF config vs HF module instance vs bridge config)? +- **Return values**: Does the bridge return the same tuple structure as HF? + +**Exit criteria:** Every weight conversion key and rearrange pattern has been +checked against the P0 forward pass and dimension records. + +--- + +## Phase 3: Design Evaluation + +### Over-engineering detection: +- Count concrete implementations vs abstractions. If an abstraction has only 1-2 implementations today, it's probably premature. Prefer the simplest approach that works now. +- Watch for protocol/interface designs justified by "future architectures" that don't exist yet. +- Watch for configuration flexibility that no current use case exercises. + +### Under-specification detection: +- Any fix described as "ensure X" or "handle Y correctly" without specifying *how* is underspecified. There should be a concrete approach or at minimum enumerated options with tradeoffs. +- Any claim about behavior ("hooks fire correctly") without specifying the mechanism that makes it happen. + +### Dependency and ordering analysis: +- Can the proposed order actually work? Does step N depend on outputs of step N+2? +- Are investigative tasks (unknown scope) mixed with mechanical tasks (known scope) in the same work unit? If so, the mechanical tasks may be blocked unnecessarily. +- Do multiple steps touch the same files? If so, should they be merged or sequenced explicitly? + +**Exit criteria:** Every design decision that deviates from existing patterns +has been evaluated and annotated. + +--- + +## Phase 4: Completeness & Edge Cases + +### What happens when it fails? +- If a user accesses an unsupported property, do they get a clear error or a confusing crash? +- If a config variant isn't supported, does the adapter raise NotImplementedError or silently produce wrong results? +- If a submodule doesn't exist on a particular layer, does setup crash or skip gracefully? + +### What's missing? +- Are all files that need modification listed? +- Are all new exports/registrations covered? +- Is there a verification strategy that tests what the plan claims to fix? +- Does the test strategy use models that are actually available (CI-cached, programmatic, or small enough to download)? + +### Inventory accuracy: +- When the plan says "N files" or "N tests", count the actual files. Plans frequently undercount. + +**Exit criteria:** Every required file, registration, and edge case has been +checked and annotated. + +--- + +## Phase 5: Differential Review (for completed code) + +Phase 5 is NOT a re-run of Phase 1. Its scope is strictly: + +### Findings resolution: +- Walk the entire findings accumulator from P0-P4. Confirm each finding is + either resolved in the code or documented as accepted. +- Flag any finding that was neither fixed nor acknowledged. + +### Plan-to-code match: +- Every task in the plan should have corresponding code changes. Flag anything + planned but not implemented, or implemented but not planned. + +### Cross-phase issues: +- Flag any new discrepancies that only emerge when seeing plan + code + HF + source together — things no single phase would catch in isolation. + +### Test quality: +- Are assertions meaningful (testing actual behavior) or tautological (testing mock setup)? +- Are tolerances justified by observed values, not picked arbitrarily? +- Do tests exercise both the happy path and the edge cases the plan identified? +- Are test fixtures appropriately scoped (session for model loading, function for mutations)? + +**Exit criteria:** Every P0-P4 finding has a resolution status, every plan +item has a corresponding code change, and test quality has been assessed. + +--- + +## Meta-principles + +1. **Verify, don't trust.** Read the actual code for every claim. Plans written from memory are frequently wrong about attribute names, parameter counts, and conditional logic. + +2. **Severity triage.** Not every issue is equal. A wrong attribute name is CRITICAL (crashes at runtime). An over-designed abstraction is MEDIUM (works but adds maintenance cost). A documentation gap is LOW. Label accordingly. + +3. **One issue per finding.** Don't bundle "the LN name is wrong and also the tolerance is loose" — these are separate findings with separate severities. + +4. **Distinguish "wrong" from "could be better."** A plan that works but isn't optimal is fine. A plan that will crash or produce silently wrong results is not. Focus review energy on correctness first, then design. + +5. **Track across iterations.** On re-reviews, explicitly confirm each previous finding was addressed before looking for new issues. Don't re-raise resolved items. + +6. **Don't invent requirements.** Review against what the plan says it will do, not what you think it should do. If the plan explicitly defers something, that's a valid choice — flag only if the deferral creates a silent failure path. diff --git a/devtools/adapter_builder/docs/scripts-reference.md b/devtools/adapter_builder/docs/scripts-reference.md new file mode 100644 index 0000000000..ef37d6c7ed --- /dev/null +++ b/devtools/adapter_builder/docs/scripts-reference.md @@ -0,0 +1,152 @@ +# Scripts Reference + +Every standalone tool under [`scripts/`](../scripts/). The agent prompts invoke several of these directly via `$TL_ADAPTER_BUILDER_ROOT/scripts/…`; the rest are for human use. Shell scripts are bash; Python scripts expect to run via `uv run python` from inside the target worktree when they touch the TransformerLens registry. + +## Adapter analysis and scaffolding + +### [`analyze-hf-model.py`](../scripts/analyze-hf-model.py) + +Analyzes a HuggingFace model to extract adapter-relevant information. This is the Programmer's main tool during Step 1a.2 (generate a scaffold adapter). + +```bash +# Config-only analysis (no model download) +python scripts/analyze-hf-model.py meta-llama/Llama-3.1-8B + +# Generate a pre-filled adapter file (meta device, no weights downloaded) +python scripts/analyze-hf-model.py Salesforce/codegen-350M-mono --scaffold +python scripts/analyze-hf-model.py Salesforce/codegen-350M-mono --scaffold --scaffold-out adapter.py + +# Full model inspection (downloads model) +python scripts/analyze-hf-model.py meta-llama/Llama-3.1-8B --modules --state-dict + +# JSON output for programmatic use +python scripts/analyze-hf-model.py meta-llama/Llama-3.1-8B --json +``` + +The `--scaffold` flag is the key feature: it instantiates the model on a meta device (no weight download), inspects the module tree and state-dict keys, and emits a ready-to-refine adapter `.py` file with config attributes pre-set, component-mapping paths detected from real module names, and weight-conversion stubs. The Programmer uses this as the starting point — not the generic [`docs/adapter-template.py`](adapter-template.py). + +### [`scan-hf-architecture.py`](../scripts/scan-hf-architecture.py) + +Exhaustively scans HuggingFace for every model of a given architecture class and writes the results to `transformer_lens/tools/model_registry/data/supported_models_<arch_short>.json`. Invoked by the Programmer during Step 1a.2. + +```bash +# Run from the worktree root +uv run python "$TL_ADAPTER_BUILDER_ROOT/scripts/scan-hf-architecture.py" \ + Qwen3MoeForCausalLM --arch-short qwen3_moe +``` + +Produces an authoritative per-architecture artifact that stays in place for human review and later gets merged into the main registry via `port-arch-models.py`. Replaces the ~70-line inline Python that used to live in `programmer.md`. + +Tuning knobs: `--limit N` caps the HF listing pagination (default 10000), `--output-dir DIR` overrides the registry path (default matches the worktree layout). + +### [`validate-architecture.py`](../scripts/validate-architecture.py) + +Pre-flight check used by [`launch.sh`](../agents/launch.sh) to validate that an architecture class name actually exists before creating a worktree. Catches typos like `CohoreForCausalLM` before they burn agent time. + +```bash +# Run from any cwd that has uv + the TransformerLens project env available +uv run --project "$DEFAULT_TARGET_REPO" \ + python "$TL_ADAPTER_BUILDER_ROOT/scripts/validate-architecture.py" CohereForCausalLM +``` + +Two checks in order: + +1. **`transformers` package** — `getattr(transformers, <arch>)` to exercise the lazy-loading machinery. Fast, local, ~1–2s including cold import. +2. **HuggingFace Hub** — bounded scan of the top 500 models by download count (configurable via `--hf-limit`), breaking on first match. Only runs when the transformers check misses. ~1–3s on top of the transformers import. + +**Flags:** + +- `--hf-limit N` — cap on the HF Hub scan (default 500) +- `--skip-hf` — only check the transformers package; don't fall back to HF Hub +- `--quiet` — suppress the "OK: …" message on success (errors still print) + +**Exit codes:** + +- `0` — found in transformers or on HF Hub +- `1` — definitively not found (both checks ran, neither found a match) +- `2` — could not verify (missing deps, network error); caller decides + +The launcher treats `0` as proceed, `1` as abort, and `2` as warn+proceed. See [cli-reference.md § Pre-flight: Architecture Existence Check](cli-reference.md#pre-flight-architecture-existence-check) for details and the `--skip-arch-check` bypass. + +### [`port-arch-models.py`](../scripts/port-arch-models.py) + +Merges the per-architecture model list produced by `scan-hf-architecture.py` into `supported_models.json` so that `verify_models` can pick the models up. Invoked by the Programmer during Step 3.0 (before verification). + +```bash +uv run python "$TL_ADAPTER_BUILDER_ROOT/scripts/port-arch-models.py" \ + --arch-short qwen3_moe +``` + +Idempotent — models already present in the registry are skipped. The per-arch file is not modified. If this step is skipped, `verify_models` silently no-ops (the `guard-verify-models.sh` hook catches this case and blocks the call with an error message pointing here). + +## Adapter validation + +### [`validate-adapter.sh`](../scripts/validate-adapter.sh) + +Validates an adapter's structure and optionally its semantics against a real model. + +```bash +# Structural validation only (no model download) +./scripts/validate-adapter.sh llama + +# Deep validation — cross-references against a real model (config-only, meta device) +./scripts/validate-adapter.sh llama --model meta-llama/Llama-3.2-1B +``` + +**Structural checks:** adapter file exists, class extends `ArchitectureAdapter`, `component_mapping` is set, `weight_processing_conversions` is set, registered in `supported_architectures/__init__.py` and the factory, config attributes set, import test passes. + +**Deep checks (with `--model`):** loads the model on a meta device, verifies component-mapping paths resolve to real HF modules, weight conversion keys match state-dict parameters, no unmapped projection weights. Catches mismatches before `verify_models` downloads the full model. + +### [`validate-adapter-deep.py`](../scripts/validate-adapter-deep.py) + +The Python backend for `--model` deep validation. Usually invoked via `validate-adapter.sh` but can be run standalone for scripting. + +### [`compare-adapters.sh`](../scripts/compare-adapters.sh) + +Structured diff between two existing adapters. The Programmer uses this in Step 1a.3 to study differences between candidate reference adapters. + +```bash +./scripts/compare-adapters.sh llama qwen2 +./scripts/compare-adapters.sh gemma1 gemma2 +``` + +Outputs: config attribute differences, bridge-component differences, component-mapping path differences, weight-conversion differences, optional override differences, and a truncated code diff. + +## Operational tooling + +### [`format-timeline.py`](../scripts/format-timeline.py) + +Formats `.adapter-workspace/timeline.jsonl` entries for human-readable display. Reads JSONL from stdin, writes one formatted line per entry to stdout. + +Used internally by `ops.sh`: + +```bash +# From status (single latest event) +tail -1 timeline.jsonl | python3 scripts/format-timeline.py + +# From logs (live stream) +tail -f timeline.jsonl | python3 scripts/format-timeline.py +``` + +Each line renders as `HH:MM:SS EventName label`, where `label` is the agent-authored `description`, a truncated Bash command, a file path, or the raw event name in that order. Malformed lines are annotated rather than dropped. + +### [`notify.sh`](../scripts/notify.sh) + +Sends completion notifications via a fallback chain: + +1. **Slack/Discord webhook** — if `NOTIFICATION_WEBHOOK_URL` is set (auto-detects payload format) +2. **iMessage** — if `NOTIFICATION_NUMBER` is set and on macOS +3. **macOS Notification Center** — last resort on macOS +4. **stdout** — always works as a fallback + +Invoked automatically by `agents/hooks/notify-on-completion.sh` when a session ends with `verification_passed: true`. The orchestrator also calls it directly for the "stuck after 3 review rounds" and "verification skipped — all models too large" escalations. + +### [`dry-run-test.sh`](../scripts/dry-run-test.sh) + +Self-test suite that validates this project's integrity. Safe to run anytime — it doesn't touch the TransformerLens repo. + +```bash +./scripts/dry-run-test.sh +``` + +Auto-discovers and syntax-checks every `.sh` and `.py` file under `agents/`, `scripts/`, and `docs/`. Validates that domain docs exist, agent definitions have required sections, signals are consistent across `signals.sh` and the agent prompts, `validate-adapter.sh` runs against an existing adapter, and the TransformerLens repo is accessible. diff --git a/devtools/adapter_builder/scripts/analyze-hf-model.py b/devtools/adapter_builder/scripts/analyze-hf-model.py new file mode 100755 index 0000000000..dbbb8dfcfa --- /dev/null +++ b/devtools/adapter_builder/scripts/analyze-hf-model.py @@ -0,0 +1,621 @@ +#!/usr/bin/env python3 +"""Analyze a HuggingFace model to extract information needed for an Architecture Adapter. + +Usage: + python scripts/analyze-hf-model.py <model_name_or_path> + +Example: + python scripts/analyze-hf-model.py meta-llama/Llama-3.1-8B + python scripts/analyze-hf-model.py google/gemma-2-2b + +Outputs: + - Architecture class name + - Config field mappings (HF -> TL) + - Normalization type + - Positional embedding type + - Attention type (MHA/GQA/MQA) + - MLP type (gated/standard) + - Module path listing + - Suggested closest existing adapter +""" + +import argparse +import json +import re +import sys +from pathlib import Path + + +def analyze_config(config) -> dict: + """Extract adapter-relevant fields from a HuggingFace config.""" + info = {} + + # Architecture class + info["architecture_class"] = getattr(config, "architectures", ["Unknown"])[0] + info["model_type"] = getattr(config, "model_type", "unknown") + + # Dimensions + info["d_model"] = getattr(config, "hidden_size", None) + info["n_heads"] = getattr(config, "num_attention_heads", None) + info["n_key_value_heads"] = getattr(config, "num_key_value_heads", None) + info["d_mlp"] = getattr(config, "intermediate_size", None) + info["n_layers"] = getattr(config, "num_hidden_layers", None) + info["d_vocab"] = getattr(config, "vocab_size", None) + info["n_ctx"] = getattr(config, "max_position_embeddings", None) + + # Derived — guard against None or zero + if info["d_model"] and info["n_heads"] and info["n_heads"] > 0: + info["d_head"] = info["d_model"] // info["n_heads"] + else: + info["d_head"] = None + + # Normalization + if hasattr(config, "rms_norm_eps"): + info["normalization_type"] = "RMS" + info["eps"] = config.rms_norm_eps + info["eps_attr_candidates"] = ["variance_epsilon", "rms_norm_eps"] + elif hasattr(config, "layer_norm_eps"): + info["normalization_type"] = "LN" + info["eps"] = config.layer_norm_eps + info["eps_attr_candidates"] = ["layer_norm_eps", "eps"] + elif hasattr(config, "layer_norm_epsilon"): + info["normalization_type"] = "LN" + info["eps"] = config.layer_norm_epsilon + info["eps_attr_candidates"] = ["layer_norm_epsilon", "eps"] + else: + info["normalization_type"] = "unknown" + info["eps_attr_candidates"] = [] + + # Attention type + n_heads = info["n_heads"] + n_kv_heads = info["n_key_value_heads"] + if n_kv_heads is None or n_kv_heads == n_heads: + info["attention_type"] = "MHA" + elif n_kv_heads == 1: + info["attention_type"] = "MQA" + else: + info["attention_type"] = "GQA" + + # Positional embedding type + rope_type = getattr(config, "rope_scaling", None) + rope_theta = getattr(config, "rope_theta", None) + if rope_theta is not None or rope_type is not None: + info["positional_embedding_type"] = "rotary" + elif hasattr(config, "position_embedding_type"): + info["positional_embedding_type"] = config.position_embedding_type + else: + # Check for max_position_embeddings without rope indicators + info["positional_embedding_type"] = "standard (inferred — verify manually)" + + # MLP type (heuristic: if intermediate_size != 4 * hidden_size, likely gated) + if hasattr(config, "mlp_bias"): + info["mlp_has_bias"] = config.mlp_bias + if hasattr(config, "hidden_act"): + info["activation"] = config.hidden_act + if config.hidden_act in ("silu", "swiglu"): + info["mlp_type"] = "gated (SwiGLU)" + elif config.hidden_act in ("gelu", "gelu_new", "relu"): + info["mlp_type"] = "standard (inferred from activation — verify manually)" + else: + info["mlp_type"] = f"unknown (activation: {config.hidden_act})" + + # Biases + if hasattr(config, "attention_bias"): + info["attention_has_bias"] = config.attention_bias + if hasattr(config, "bias"): + info["global_bias"] = config.bias + + return info + + +def suggest_closest_adapter(info: dict) -> str: + """Suggest the closest existing adapter to use as a reference.""" + norm = info.get("normalization_type", "") + pos = info.get("positional_embedding_type", "") + mlp = info.get("mlp_type", "") + attn = info.get("attention_type", "") + + if "rotary" in pos and "RMS" in norm and "gated" in mlp: + if attn == "GQA": + return "llama.py (Llama-like with GQA)" + return "llama.py (Llama-like)" + elif "rotary" in pos and "RMS" in norm: + return "qwen2.py (RoPE + RMSNorm — uses MLPBridge instead of GatedMLPBridge)" + elif "standard" in pos and "LN" in norm: + return "gpt2.py (GPT-2-like)" + else: + return "llama.py (default starting point — verify manually)" + + +def analyze_modules(model) -> list[str]: + """List all named modules in the model.""" + modules = [] + for name, module in model.named_modules(): + modules.append(f"{name}: {type(module).__name__}") + return modules + + +def detect_module_paths(model) -> dict: + """Detect HF module paths for component mapping by inspecting the model tree.""" + paths = {} + module_map = {name: type(mod).__name__ for name, mod in model.named_modules()} + + # Find embedding + for name, cls in module_map.items(): + if cls == "Embedding" and "embed" in name.lower() and "pos" not in name.lower(): + paths["embed"] = name + break + + # Find positional embedding + for name, cls in module_map.items(): + if cls == "Embedding" and ("pos" in name.lower() or "wpe" in name.lower()): + paths["pos_embed"] = name + break + + # Find rotary embedding + for name, cls in module_map.items(): + if "RotaryEmbedding" in cls and "layers" not in name: + paths["rotary_emb"] = name + break + + # Find layer list (blocks) + for name, cls in module_map.items(): + if cls == "ModuleList" and any(k in name for k in ["layers", ".h", "blocks"]): + paths["blocks"] = name + break + + # Find final layer norm + for name, cls in module_map.items(): + if ("Norm" in cls or "LayerNorm" in cls) and name.count(".") <= 1: + if "layers" not in name and "block" not in name and "h." not in name: + if "final" in name or name.endswith("norm") or name.endswith("ln_f"): + paths["ln_final"] = name + break + + # Find lm_head / unembed + for name, cls in module_map.items(): + if cls == "Linear" and ("lm_head" in name or "embed_out" in name): + paths["unembed"] = name + break + + # Find block submodule paths (from layer 0) + blocks_prefix = paths.get("blocks", "") + if blocks_prefix: + layer0_prefix = f"{blocks_prefix}.0." + layer0_modules = { + name[len(layer0_prefix) :]: cls + for name, cls in module_map.items() + if name.startswith(layer0_prefix) and name.count(".") - layer0_prefix.count(".") <= 1 + } + + # Attention module + for name, cls in layer0_modules.items(): + if "attn" in name.lower() or "attention" in name.lower(): + paths["attn"] = name + # Find Q/K/V/O projections within attention + attn_prefix = f"{layer0_prefix}{name}." + attn_children = { + n[len(attn_prefix) :]: c + for n, c in module_map.items() + if n.startswith(attn_prefix) and n.count(".") == attn_prefix.count(".") + } + for proj_name, proj_cls in attn_children.items(): + if proj_cls == "Linear": + if "q" in proj_name.lower() and "k" not in proj_name.lower(): + paths["attn_q"] = proj_name + elif "k" in proj_name.lower() and "q" not in proj_name.lower(): + paths["attn_k"] = proj_name + elif "v" in proj_name.lower(): + paths["attn_v"] = proj_name + elif ( + "o" in proj_name.lower() + or "out" in proj_name.lower() + or proj_name == "dense" + ): + paths["attn_o"] = proj_name + elif "c_attn" in proj_name or "qkv" in proj_name.lower(): + paths["attn_qkv"] = proj_name + elif "c_proj" in proj_name: + paths["attn_o"] = proj_name + break + + # MLP module + for name, cls in layer0_modules.items(): + if "mlp" in name.lower() or "feed_forward" in name.lower(): + paths["mlp"] = name + mlp_prefix = f"{layer0_prefix}{name}." + mlp_children = { + n[len(mlp_prefix) :]: c + for n, c in module_map.items() + if n.startswith(mlp_prefix) and n.count(".") == mlp_prefix.count(".") + } + for proj_name, proj_cls in mlp_children.items(): + if proj_cls in ("Linear", "Conv1D"): + pn = proj_name.lower() + if "gate" in pn or pn == "w1": + paths["mlp_gate"] = proj_name + elif "up" in pn or pn == "w3" or "c_fc" in pn or "fc1" in pn: + paths["mlp_in"] = proj_name + elif "down" in pn or pn == "w2" or "fc2" in pn: + paths["mlp_out"] = proj_name + elif "c_proj" in pn: + paths["mlp_out"] = proj_name + break + + # Layer norms + for name, cls in layer0_modules.items(): + if "Norm" in cls or "LayerNorm" in cls: + if "input" in name or "ln_1" in name or name == "ln1": + paths["ln1"] = name + elif "post" in name or "ln_2" in name or name == "ln2": + paths["ln2"] = name + + return paths + + +def generate_scaffold(info: dict, paths: dict, state_dict_keys: list) -> str: + """Generate a pre-filled adapter Python file.""" + arch_class = info["architecture_class"] + # Derive class name: strip For*LM suffix, add ArchitectureAdapter + base_name = re.sub(r"(For(Causal|Conditional|Masked).*|LMHead.*)", "", arch_class) + adapter_class = f"{base_name}ArchitectureAdapter" + module_name = base_name.lower() + + norm_type = info.get("normalization_type", "unknown") + pos_type = info.get("positional_embedding_type", "unknown") + is_rope = "rotary" in pos_type + is_rms = norm_type == "RMS" + is_gated = "gated" in info.get("mlp_type", "").lower() + has_combined_qkv = "attn_qkv" in paths + has_gqa = info.get("attention_type") in ("GQA", "MQA") + + # Build imports + norm_bridge = "RMSNormalizationBridge" if is_rms else "NormalizationBridge" + attn_bridge = ( + "JointQKVAttentionBridge" if has_combined_qkv else "PositionEmbeddingsAttentionBridge" + ) + mlp_bridge = "GatedMLPBridge" if is_gated else "MLPBridge" + pos_bridge = "RotaryEmbeddingBridge" if is_rope else "PosEmbedBridge" + + imports = [ + "BlockBridge", + "EmbeddingBridge", + mlp_bridge, + "LinearBridge", + attn_bridge, + norm_bridge, + pos_bridge, + "UnembeddingBridge", + ] + + # Determine eps_attr + eps_attr = "variance_epsilon" + candidates = info.get("eps_attr_candidates", []) + if candidates: + eps_attr = candidates[0] + + # Detect bias presence from state_dict keys + has_attn_bias = any( + k.endswith(".q_proj.bias") or k.endswith(".c_attn.bias") for k in state_dict_keys + ) + has_mlp_bias = any( + k.endswith(".up_proj.bias") or k.endswith(".c_fc.bias") or k.endswith(".fc1.bias") + for k in state_dict_keys + ) + + # Build optional params docstring + optional_params = [] + if not has_attn_bias: + optional_params.extend( + [ + "- blocks.{i}.attn.b_Q - No bias on query projection", + "- blocks.{i}.attn.b_K - No bias on key projection", + "- blocks.{i}.attn.b_V - No bias on value projection", + "- blocks.{i}.attn.b_O - No bias on output projection", + ] + ) + if not has_mlp_bias: + gate_line = ( + "- blocks.{i}.mlp.b_gate - No bias on MLP gate projection\n " if is_gated else "" + ) + optional_params.extend( + [ + "- blocks.{i}.mlp.b_in - No bias on MLP input", + f"{gate_line}- blocks.{{i}}.mlp.b_out - No bias on MLP output", + ] + ) + if is_rms: + optional_params.extend( + [ + "- blocks.{i}.ln1.b - RMSNorm has no bias", + "- blocks.{i}.ln2.b - RMSNorm has no bias", + "- ln_final.b - RMSNorm has no bias", + ] + ) + + optional_block = ( + "\n ".join(optional_params) if optional_params else "None identified — verify manually" + ) + + # Build component mapping + embed_path = paths.get("embed", "model.embed_tokens # TODO: verify") + blocks_path = paths.get("blocks", "model.layers # TODO: verify") + ln1_path = paths.get("ln1", "input_layernorm # TODO: verify") + ln2_path = paths.get("ln2", "post_attention_layernorm # TODO: verify") + attn_path = paths.get("attn", "self_attn # TODO: verify") + mlp_path = paths.get("mlp", "mlp # TODO: verify") + ln_final_path = paths.get("ln_final", "model.norm # TODO: verify") + unembed_path = paths.get("unembed", "lm_head # TODO: verify") + + # Attention submodules + if has_combined_qkv: + attn_submodules = f""" "qkv": LinearBridge(name="{paths.get('attn_qkv', 'c_attn')}"), + "o": LinearBridge(name="{paths.get('attn_o', 'c_proj')}"),""" + else: + attn_submodules = f""" "q": LinearBridge(name="{paths.get('attn_q', 'q_proj')}"), + "k": LinearBridge(name="{paths.get('attn_k', 'k_proj')}"), + "v": LinearBridge(name="{paths.get('attn_v', 'v_proj')}"), + "o": LinearBridge(name="{paths.get('attn_o', 'o_proj')}"),""" + + # MLP submodules + if is_gated: + mlp_submodules = f""" "gate": LinearBridge(name="{paths.get('mlp_gate', 'gate_proj')}"), + "in": LinearBridge(name="{paths.get('mlp_in', 'up_proj')}"), + "out": LinearBridge(name="{paths.get('mlp_out', 'down_proj')}"),""" + else: + mlp_submodules = f""" "in": LinearBridge(name="{paths.get('mlp_in', 'c_fc')}"), + "out": LinearBridge(name="{paths.get('mlp_out', 'c_proj')}"),""" + + # Positional embedding component + if is_rope: + pos_component = f""" "rotary_emb": RotaryEmbeddingBridge(name="{paths.get('rotary_emb', 'model.rotary_emb # TODO: verify')}"),""" + attn_kwargs = """ + requires_attention_mask=True, + requires_position_embeddings=True,""" + else: + pos_component = f""" "pos_embed": PosEmbedBridge(name="{paths.get('pos_embed', 'transformer.wpe # TODO: verify')}"),""" + attn_kwargs = "" + + # GQA handling + gqa_block = "" + if has_gqa: + gqa_block = """ + # GQA support + if hasattr(cfg, "n_key_value_heads") and cfg.n_key_value_heads is not None: + self.cfg.n_key_value_heads = cfg.n_key_value_heads +""" + + # Setup component testing (for RoPE models) + setup_testing = "" + if is_rope: + rotary_path = paths.get("rotary_emb", "model.rotary_emb") + # Derive the Python access path from the dot-separated path + rotary_access = "hf_model." + rotary_path + setup_testing = f""" + def setup_component_testing(self, hf_model: Any, bridge_model: Any = None) -> None: + \"\"\"Set up rotary embedding references for component testing.\"\"\" + rotary_emb = {rotary_access} + + if bridge_model is not None and hasattr(bridge_model, "blocks"): + for block in bridge_model.blocks: + if hasattr(block, "attn"): + block.attn.set_rotary_emb(rotary_emb) + + attn_bridge = self.get_generalized_component("blocks.0.attn") + attn_bridge.set_rotary_emb(rotary_emb) +""" + + # Weight conversions + weight_conv = "**self._qkvo_weight_conversions()," + if has_combined_qkv: + weight_conv = """# TODO: Combined QKV requires custom weight conversions. + # See gpt2.py QKVSplitRearrangeConversion for reference. + # Standard _qkvo_weight_conversions() does NOT work for combined QKV.""" + + scaffold = f'''"""{base_name} architecture adapter. + +Generated by analyze-hf-model.py --scaffold from {info.get("_model_id", "unknown model")}. +Review all TODO comments and verify paths against the Transformers source. +""" + +from typing import Any + +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.generalized_components import ( + {(",{chr(10)} ").join(sorted(set(imports)))} +) + + +class {adapter_class}(ArchitectureAdapter): + """{base_name} architecture adapter. + + Optional Parameters (may not exist in state_dict): + ------------------------------------------------- + {optional_block} + """ + + def __init__(self, cfg: Any) -> None: + """Initialize the {base_name} architecture adapter.""" + super().__init__(cfg) + + # Config attributes (detected from model config) + self.cfg.normalization_type = "{norm_type}" + self.cfg.positional_embedding_type = "{"rotary" if is_rope else "standard"}" + self.cfg.final_rms = {is_rms} + self.cfg.gated_mlp = {is_gated} + self.cfg.attn_only = False + self.cfg.uses_rms_norm = {is_rms} + self.cfg.eps_attr = "{eps_attr}" +{gqa_block} + # Weight processing conversions + self.weight_processing_conversions = {{ + {weight_conv} + }} + + # Component mapping (paths detected from model module tree) + self.component_mapping = {{ + "embed": EmbeddingBridge(name="{embed_path}"), + {pos_component} + "blocks": BlockBridge( + name="{blocks_path}", + submodules={{ + "ln1": {norm_bridge}(name="{ln1_path}", config=self.cfg), + "ln2": {norm_bridge}(name="{ln2_path}", config=self.cfg), + "attn": {attn_bridge}( + name="{attn_path}", + config=self.cfg, + submodules={{ +{attn_submodules} + }},{attn_kwargs} + ), + "mlp": {mlp_bridge}( + name="{mlp_path}", + config=self.cfg, + submodules={{ +{mlp_submodules} + }}, + ), + }}, + ), + "ln_final": {norm_bridge}(name="{ln_final_path}", config=self.cfg), + "unembed": UnembeddingBridge(name="{unembed_path}", config=self.cfg), + }} +{setup_testing}''' + + return scaffold + + +def main(): + parser = argparse.ArgumentParser(description="Analyze HF model for adapter creation") + parser.add_argument("model", help="HuggingFace model name or path") + parser.add_argument( + "--modules", + action="store_true", + help="Also print model module hierarchy (requires downloading model)", + ) + parser.add_argument( + "--state-dict", + action="store_true", + help="Also print state dict keys and shapes (requires downloading model)", + ) + parser.add_argument( + "--scaffold", + action="store_true", + help="Generate a pre-filled adapter .py file (meta device, no weights)", + ) + parser.add_argument( + "--scaffold-out", default=None, help="Write scaffold to this file path (default: stdout)" + ) + parser.add_argument("--json", action="store_true", help="Output in JSON format") + args = parser.parse_args() + + try: + from transformers import AutoConfig + except ImportError: + print( + "Error: transformers package not installed. Run: pip install transformers", + file=sys.stderr, + ) + sys.exit(1) + + # Load config (doesn't download the model) + print(f"Loading config for: {args.model}", file=sys.stderr) + config = AutoConfig.from_pretrained(args.model, trust_remote_code=True) + + info = analyze_config(config) + info["suggested_adapter"] = suggest_closest_adapter(info) + + if args.scaffold: + import torch + from transformers import AutoModelForCausalLM + + print("Creating model on meta device (no weights downloaded)...", file=sys.stderr) + with torch.device("meta"): + model = AutoModelForCausalLM.from_config(config, trust_remote_code=True) + + state_dict_keys = list(model.state_dict().keys()) + paths = detect_module_paths(model) + info["_model_id"] = args.model + + print( + f"Detected {len(paths)} module paths, {len(state_dict_keys)} state_dict keys", + file=sys.stderr, + ) + + scaffold_code = generate_scaffold(info, paths, state_dict_keys) + + if args.scaffold_out: + Path(args.scaffold_out).write_text(scaffold_code) + print(f"Scaffold written to: {args.scaffold_out}", file=sys.stderr) + else: + print(scaffold_code) + sys.exit(0) + + if args.modules or args.state_dict: + import torch + from transformers import AutoModelForCausalLM + + print("Loading model (this may take a while)...", file=sys.stderr) + model = AutoModelForCausalLM.from_pretrained( + args.model, + torch_dtype=torch.float16, + trust_remote_code=True, + device_map="cpu", + ) + + if args.modules: + info["modules"] = analyze_modules(model) + + if args.state_dict: + info["state_dict_keys"] = [ + {"key": k, "shape": list(v.shape), "dtype": str(v.dtype)} + for k, v in model.state_dict().items() + ] + + if args.json: + print(json.dumps(info, indent=2, default=str)) + else: + print("\n" + "=" * 60) + print(f" Architecture Adapter Analysis: {args.model}") + print("=" * 60) + print(f"\n Architecture class : {info['architecture_class']}") + print(f" Model type : {info['model_type']}") + print(f"\n --- Dimensions ---") + print(f" d_model : {info.get('d_model')}") + print(f" n_heads : {info.get('n_heads')}") + print(f" n_key_value_heads : {info.get('n_key_value_heads')}") + print(f" d_head : {info.get('d_head')}") + print(f" d_mlp : {info.get('d_mlp')}") + print(f" n_layers : {info.get('n_layers')}") + print(f" d_vocab : {info.get('d_vocab')}") + print(f" n_ctx : {info.get('n_ctx')}") + print(f"\n --- Architecture ---") + print(f" Normalization : {info.get('normalization_type')}") + print(f" Epsilon attr : {info.get('eps_attr_candidates')}") + print(f" Pos embeddings : {info.get('positional_embedding_type')}") + print(f" Attention type : {info.get('attention_type')}") + print(f" MLP type : {info.get('mlp_type', 'unknown')}") + print(f" Activation : {info.get('activation', 'unknown')}") + print(f" Attention bias : {info.get('attention_has_bias', 'unknown')}") + print(f"\n --- Recommendation ---") + print(f" Closest adapter : {info['suggested_adapter']}") + + if "modules" in info: + print(f"\n --- Module Hierarchy ---") + for m in info["modules"][:100]: + print(f" {m}") + if len(info["modules"]) > 100: + print(f" ... ({len(info['modules']) - 100} more)") + + if "state_dict_keys" in info: + print(f"\n --- State Dict Keys ({len(info['state_dict_keys'])} params) ---") + for entry in info["state_dict_keys"][:50]: + print(f" {entry['key']}: {entry['shape']}") + if len(info["state_dict_keys"]) > 50: + print(f" ... ({len(info['state_dict_keys']) - 50} more)") + + print() + + +if __name__ == "__main__": + main() diff --git a/devtools/adapter_builder/scripts/compare-adapters.sh b/devtools/adapter_builder/scripts/compare-adapters.sh new file mode 100755 index 0000000000..1933a3ea60 --- /dev/null +++ b/devtools/adapter_builder/scripts/compare-adapters.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +# ============================================================================= +# compare-adapters.sh — Structured diff between two existing adapters +# +# Shows config attribute differences, component mapping differences, and +# weight conversion differences between two adapters. Designed to support +# the "find nearest reference adapter and adapt" workflow. +# +# Usage: +# ./scripts/compare-adapters.sh <adapter_a> <adapter_b> [--repo <path>] +# +# Example: +# ./scripts/compare-adapters.sh gemma1 gemma2 +# ./scripts/compare-adapters.sh llama qwen2 +# ./scripts/compare-adapters.sh gpt2 gptj --repo /path/to/TransformerLens +# ============================================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="${TL_REPO:-$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel 2>/dev/null || { cd "$SCRIPT_DIR/../../.." && pwd; })}" +ADAPTER_A="" +ADAPTER_B="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --repo) REPO_ROOT="$2"; shift 2 ;; + -h|--help) + echo "Usage: $0 <adapter_a> <adapter_b> [--repo <path>]" + echo "Example: $0 gemma1 gemma2" + exit 0 + ;; + *) + if [[ -z "$ADAPTER_A" ]]; then + ADAPTER_A="$1" + elif [[ -z "$ADAPTER_B" ]]; then + ADAPTER_B="$1" + fi + shift + ;; + esac +done + +[[ -z "$ADAPTER_A" || -z "$ADAPTER_B" ]] && { echo "Error: two adapter names required"; exit 1; } + +ADAPTERS_DIR="$REPO_ROOT/transformer_lens/model_bridge/supported_architectures" +FILE_A="$ADAPTERS_DIR/${ADAPTER_A}.py" +FILE_B="$ADAPTERS_DIR/${ADAPTER_B}.py" + +[[ -f "$FILE_A" ]] || { echo "Error: $FILE_A not found"; exit 1; } +[[ -f "$FILE_B" ]] || { echo "Error: $FILE_B not found"; exit 1; } + +echo "" +echo "================================================================" +echo " Adapter Comparison: ${ADAPTER_A} vs ${ADAPTER_B}" +echo "================================================================" + +# --- Config Attributes --- +echo "" +echo -e "\033[1;36m--- Config Attributes ---\033[0m" +echo "" + +CONFIG_ATTRS=( + normalization_type + positional_embedding_type + final_rms + gated_mlp + attn_only + uses_rms_norm + eps_attr + default_prepend_bos + uses_split_attention + split_attention_weights +) + +printf " %-30s %-25s %-25s\n" "Attribute" "$ADAPTER_A" "$ADAPTER_B" +printf " %-30s %-25s %-25s\n" "------------------------------" "-------------------------" "-------------------------" + +for attr in "${CONFIG_ATTRS[@]}"; do + val_a=$(grep "self\.cfg\.$attr" "$FILE_A" 2>/dev/null | head -1 | sed 's/.*=\s*//' | tr -d ' "'"'"' +' || echo "—") + val_b=$(grep "self\.cfg\.$attr" "$FILE_B" 2>/dev/null | head -1 | sed 's/.*=\s*//' | tr -d ' "'"'"' +' || echo "—") + [[ -z "$val_a" ]] && val_a="—" + [[ -z "$val_b" ]] && val_b="—" + if [[ "$val_a" == "$val_b" ]]; then + printf " %-30s %-25s %-25s\n" "$attr" "$val_a" "(same)" + else + printf " %-30s \033[33m%-25s\033[0m \033[33m%-25s\033[0m\n" "$attr" "$val_a" "$val_b" + fi +done + +# --- Imports (Bridge Components Used) --- +echo "" +echo -e "\033[1;36m--- Bridge Components Used ---\033[0m" +echo "" + +imports_a=$(grep -oE '[A-Z][A-Za-z]*Bridge' "$FILE_A" 2>/dev/null | sort -u) +imports_b=$(grep -oE '[A-Z][A-Za-z]*Bridge' "$FILE_B" 2>/dev/null | sort -u) + +# Find differences +only_a=$(comm -23 <(echo "$imports_a") <(echo "$imports_b")) +only_b=$(comm -13 <(echo "$imports_a") <(echo "$imports_b")) +both=$(comm -12 <(echo "$imports_a") <(echo "$imports_b")) + +for bridge in $both; do + echo " [both] $bridge" +done +for bridge in $only_a; do + echo -e " \033[33m[${ADAPTER_A} only]\033[0m $bridge" +done +for bridge in $only_b; do + echo -e " \033[33m[${ADAPTER_B} only]\033[0m $bridge" +done + +# --- Component Mapping Paths --- +echo "" +echo -e "\033[1;36m--- Component Mapping (HF paths) ---\033[0m" +echo "" + +# Extract name="..." from component mapping +extract_paths() { + local file="$1" + grep -oE 'name="[^"]*"' "$file" 2>/dev/null | sed 's/name="//;s/"//' | sort +} + +paths_a=$(extract_paths "$FILE_A") +paths_b=$(extract_paths "$FILE_B") + +only_pa=$(comm -23 <(echo "$paths_a") <(echo "$paths_b")) +only_pb=$(comm -13 <(echo "$paths_a") <(echo "$paths_b")) +both_p=$(comm -12 <(echo "$paths_a") <(echo "$paths_b")) + +for p in $both_p; do + echo " [both] $p" +done +for p in $only_pa; do + echo -e " \033[33m[${ADAPTER_A} only]\033[0m $p" +done +for p in $only_pb; do + echo -e " \033[33m[${ADAPTER_B} only]\033[0m $p" +done + +# --- Weight Conversions --- +echo "" +echo -e "\033[1;36m--- Weight Processing Conversions ---\033[0m" +echo "" + +extract_conversions() { + local file="$1" + grep -oE '"blocks\.\{i\}\.[^"]*"|"[a-z_]+\.[a-z_]+"' "$file" 2>/dev/null | tr -d '"' | sort -u +} + +conv_a=$(extract_conversions "$FILE_A") +conv_b=$(extract_conversions "$FILE_B") + +only_ca=$(comm -23 <(echo "$conv_a") <(echo "$conv_b")) +only_cb=$(comm -13 <(echo "$conv_a") <(echo "$conv_b")) +both_c=$(comm -12 <(echo "$conv_a") <(echo "$conv_b")) + +for c in $both_c; do + echo " [both] $c" +done +for c in $only_ca; do + echo -e " \033[33m[${ADAPTER_A} only]\033[0m $c" +done +for c in $only_cb; do + echo -e " \033[33m[${ADAPTER_B} only]\033[0m $c" +done + +# --- Optional Overrides --- +echo "" +echo -e "\033[1;36m--- Optional Overrides ---\033[0m" +echo "" + +OVERRIDES=( + "setup_component_testing" + "preprocess_weights" + "prepare_loading" + "prepare_model" +) + +for override in "${OVERRIDES[@]}"; do + label_a=$(grep -q "def $override" "$FILE_A" 2>/dev/null && echo "yes" || echo "—") + label_b=$(grep -q "def $override" "$FILE_B" 2>/dev/null && echo "yes" || echo "—") + if [[ "$label_a" == "$label_b" ]]; then + printf " %-30s %-10s %-10s\n" "$override" "$label_a" "(same)" + else + printf " %-30s \033[33m%-10s\033[0m \033[33m%-10s\033[0m\n" "$override" "$label_a" "$label_b" + fi +done + +# --- Line Count --- +echo "" +echo -e "\033[1;36m--- Size ---\033[0m" +echo "" +lines_a=$(wc -l < "$FILE_A" | tr -d ' ') +lines_b=$(wc -l < "$FILE_B" | tr -d ' ') +echo " ${ADAPTER_A}: ${lines_a} lines" +echo " ${ADAPTER_B}: ${lines_b} lines" + +# --- Full Diff (collapsed) --- +echo "" +echo -e "\033[1;36m--- Full Diff ---\033[0m" +echo "" +diff --color=always -u "$FILE_A" "$FILE_B" 2>/dev/null | head -80 || true +TOTAL_DIFF=$(diff -u "$FILE_A" "$FILE_B" 2>/dev/null | wc -l | tr -d ' \n') +if (( TOTAL_DIFF > 80 )); then + echo "" + echo " ... ($((TOTAL_DIFF - 80)) more lines — run 'diff -u $FILE_A $FILE_B' for full diff)" +fi + +echo "" +echo "================================================================" +echo "" diff --git a/devtools/adapter_builder/scripts/dry-run-test.sh b/devtools/adapter_builder/scripts/dry-run-test.sh new file mode 100755 index 0000000000..08b3e43ca1 --- /dev/null +++ b/devtools/adapter_builder/scripts/dry-run-test.sh @@ -0,0 +1,404 @@ +#!/usr/bin/env bash +# ============================================================================= +# dry-run-test.sh +# +# Tests the adapter builder system to verify all components are correctly +# structured without actually launching agents. +# +# Checks: +# 1. All .sh scripts have valid bash syntax +# 2. All .py scripts have valid Python syntax +# 3. Domain knowledge docs are present and non-empty +# 4. Agent definitions have required sections +# 5. Signal consistency across files +# 6. validate-adapter.sh passes for an existing adapter +# 7. Project files exist +# 8. TransformerLens repo is accessible +# +# Usage: +# ./scripts/dry-run-test.sh [--repo /path/to/TransformerLens] +# ============================================================================= + +set -euo pipefail + +log() { echo -e "\033[1;34m[test]\033[0m $*"; } +ok() { echo -e "\033[1;32m[ ok ]\033[0m $*"; } +fail() { echo -e "\033[1;31m[FAIL]\033[0m $*"; FAILURES=$((FAILURES + 1)); } + +FAILURES=0 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +TL_REPO="$(git -C "$PROJECT_ROOT" rev-parse --show-toplevel 2>/dev/null || { cd "$PROJECT_ROOT/../.." && pwd; })" + +# Parse --repo flag +while [[ $# -gt 0 ]]; do + case "$1" in + --repo) TL_REPO="$2"; shift 2 ;; + *) shift ;; + esac +done + +echo "" +echo "==========================================" +echo " Dry-Run Test Suite" +echo " TL Repo: $TL_REPO" +echo "==========================================" +echo "" + +# --------------------------------------------------------------------------- # +# 1. Bash syntax check — all .sh files in agents/ and scripts/ +# --------------------------------------------------------------------------- # +log "Checking bash syntax (all .sh files)..." +while IFS= read -r -d '' shfile; do + relpath="${shfile#"$PROJECT_ROOT/"}" + if bash -n "$shfile" 2>/dev/null; then + ok "$relpath" + else + fail "$relpath has syntax errors" + fi +done < <(find "$PROJECT_ROOT/agents" "$PROJECT_ROOT/scripts" -name '*.sh' -print0 2>/dev/null) + +# --------------------------------------------------------------------------- # +# 2. Python syntax check — all .py files in scripts/ +# --------------------------------------------------------------------------- # +log "Checking Python syntax (all .py files)..." +while IFS= read -r -d '' pyfile; do + relpath="${pyfile#"$PROJECT_ROOT/"}" + if python3 -c "import ast; ast.parse(open('$pyfile').read())" 2>/dev/null; then + ok "$relpath" + else + fail "$relpath has syntax errors" + fi +done < <(find "$PROJECT_ROOT/scripts" "$PROJECT_ROOT/docs" -name '*.py' -print0 2>/dev/null) + +# --------------------------------------------------------------------------- # +# 3. Domain knowledge docs +# --------------------------------------------------------------------------- # +log "Checking domain knowledge docs..." +for doc in docs/adapter-specification.md docs/adapter-template.py docs/hf-model-analysis-guide.md docs/review-specification.md; do + if [[ -s "$PROJECT_ROOT/$doc" ]]; then + ok "$doc exists and non-empty" + else + fail "$doc missing or empty" + fi +done + +# --------------------------------------------------------------------------- # +# 4. Agent definitions have required sections +# --------------------------------------------------------------------------- # +log "Checking agent definitions..." +for agent_file in "$PROJECT_ROOT"/agents/*.md; do + [[ -f "$agent_file" ]] || continue + name=$(basename "$agent_file") + if head -1 "$agent_file" | grep -q '^---'; then + ok "$name has YAML frontmatter" + else + # orchestrator.md doesn't need frontmatter (it's a template, not an agent def) + if [[ "$name" == "orchestrator.md" ]]; then + ok "$name is a template (no frontmatter expected)" + else + fail "$name missing YAML frontmatter" + fi + fi +done + +# Check programmer.md has adapter-specific content +if grep -q "Architecture Adapter" "$PROJECT_ROOT/agents/programmer.md" 2>/dev/null; then + ok "programmer.md has adapter domain knowledge" +else + fail "programmer.md missing adapter domain knowledge" +fi + +# Check the five-phase review process: reviewer.md references the spec, and +# the spec itself contains the phases (content lives in the doc, not the prompt) +if grep -q "review-specification.md" "$PROJECT_ROOT/agents/reviewer.md" 2>/dev/null \ + && grep -q "Phase 1: Factual Verification" "$PROJECT_ROOT/docs/review-specification.md" 2>/dev/null; then + ok "reviewer.md references five-phase review spec (docs/review-specification.md)" +else + fail "reviewer.md missing reference to five-phase review spec" +fi + +# Check reviewer.md has brief review mode +if grep -q "BRIEF APPROVED" "$PROJECT_ROOT/agents/reviewer.md" 2>/dev/null; then + ok "reviewer.md has brief review mode" +else + fail "reviewer.md missing brief review mode" +fi + +# --------------------------------------------------------------------------- # +# 5. Signal consistency +# --------------------------------------------------------------------------- # +log "Checking signal consistency..." +source "$PROJECT_ROOT/agents/signals.sh" + +# Programmer signals in programmer.md +for sig in "$SIG_BRIEF_READY" "$SIG_PLAN_READY" "$SIG_PHASE_READY" "$SIG_VERIFICATION_COMPLETE" "$SIG_VERIFICATION_FAILED" "$SIG_FINAL_READY"; do + if grep -qF "$sig" "$PROJECT_ROOT/agents/programmer.md" 2>/dev/null; then + ok "programmer.md: $sig" + else + fail "programmer.md missing signal: $sig" + fi +done + +# Reviewer signals in reviewer.md +for sig in "$SIG_BRIEF_APPROVED" "$SIG_BRIEF_CHANGES" "$SIG_PLAN_APPROVED" "$SIG_PLAN_CHANGES" "$SIG_CODE_APPROVED" "$SIG_CODE_CHANGES"; do + if grep -qF "$sig" "$PROJECT_ROOT/agents/reviewer.md" 2>/dev/null; then + ok "reviewer.md: $sig" + else + fail "reviewer.md missing signal: $sig" + fi +done + +# --------------------------------------------------------------------------- # +# 6. validate-adapter.sh against existing Llama adapter +# --------------------------------------------------------------------------- # +log "Running validate-adapter.sh against llama..." +if "$PROJECT_ROOT/scripts/validate-adapter.sh" llama --repo "$TL_REPO" >/dev/null 2>&1; then + ok "validate-adapter.sh passes for llama" +else + fail "validate-adapter.sh failed for llama" +fi + +# --------------------------------------------------------------------------- # +# 7. Project files exist +# --------------------------------------------------------------------------- # +log "Checking project files..." +for f in CLAUDE.md .gitignore .env.example; do + if [[ -s "$PROJECT_ROOT/$f" ]]; then + ok "$f exists" + else + fail "$f missing or empty" + fi +done + +# --------------------------------------------------------------------------- # +# 8. TransformerLens repo is accessible +# --------------------------------------------------------------------------- # +log "Checking TransformerLens repo..." +if [[ -d "$TL_REPO/transformer_lens/model_bridge/supported_architectures" ]]; then + ok "TransformerLens repo accessible at $TL_REPO" + ADAPTER_COUNT=$(find "$TL_REPO/transformer_lens/model_bridge/supported_architectures" -name '*.py' -not -name '__init__*' | wc -l | tr -d ' ') + ok "Found $ADAPTER_COUNT existing adapters" +else + fail "TransformerLens repo not found or missing expected directories" +fi + +# --------------------------------------------------------------------------- # +# State machine validation +# --------------------------------------------------------------------------- # +log "Validating routing rules in signals.sh..." + +# Source signals.sh to get the protocol block, then validate: +# - Every programmer signal has at least one routing rule that handles it +# - Every reviewer signal has at least one routing rule that handles it +# - No signal variable in signals.sh is absent from the routing rules +source "$PROJECT_ROOT/agents/signals.sh" 2>/dev/null + +PROTOCOL=$(generate_protocol_block 2>/dev/null || echo "") +if [[ -z "$PROTOCOL" ]]; then + fail "generate_protocol_block() returned empty output" +else + # Check that every signal defined at the top of signals.sh appears in routing + for sig_var in SIG_BRIEF_READY SIG_PLAN_READY SIG_PHASE_READY SIG_VERIFICATION_COMPLETE \ + SIG_VERIFICATION_FAILED SIG_VERIFICATION_SKIPPED SIG_FINAL_READY \ + SIG_BRIEF_APPROVED SIG_BRIEF_CHANGES SIG_PLAN_APPROVED SIG_PLAN_CHANGES \ + SIG_CODE_APPROVED SIG_CODE_CHANGES; do + # Extract the key phrase from the signal value (e.g. "BRIEF READY FOR REVIEW") + sig_value="${!sig_var}" + # Strip the "PROGRAMMER: " or "REVIEWER: " prefix to get the routing-rule keyword + keyword=$(echo "$sig_value" | sed 's/^PROGRAMMER: //; s/^REVIEWER: //') + if echo "$PROTOCOL" | grep -qF "$keyword"; then + ok "Signal $sig_var ('$keyword') has a routing rule" + else + fail "Signal $sig_var ('$keyword') has NO routing rule in the protocol block" + fi + done + + # Check that routing rules reference only defined signals (no typos) + rule_signals=$(echo "$PROTOCOL" | grep -oE '`[A-Z][A-Z ]+`' | tr -d '`' | sort -u) + while IFS= read -r rule_sig; do + [[ -z "$rule_sig" ]] && continue + # Check if any SIG_* variable contains this phrase + found=false + for sig_var in SIG_BRIEF_READY SIG_PLAN_READY SIG_PHASE_READY SIG_VERIFICATION_COMPLETE \ + SIG_VERIFICATION_FAILED SIG_VERIFICATION_SKIPPED SIG_FINAL_READY \ + SIG_BRIEF_APPROVED SIG_BRIEF_CHANGES SIG_PLAN_APPROVED SIG_PLAN_CHANGES \ + SIG_CODE_APPROVED SIG_CODE_CHANGES; do + if [[ "${!sig_var}" == *"$rule_sig"* ]]; then + found=true + break + fi + done + if [[ "$found" == false ]]; then + # Might be a partial match or a combined signal — only fail if it looks + # like a full signal (starts with a capital letter, >10 chars) + if [[ ${#rule_sig} -gt 10 ]]; then + fail "Routing rule references '$rule_sig' which matches no defined signal" + fi + fi + done <<< "$rule_signals" +fi + +# --------------------------------------------------------------------------- # +# Solo-mode coordinator: routing table + signal consumption +# --------------------------------------------------------------------------- # +log "Validating solo-coordinator routing table..." + +COORDINATOR="$PROJECT_ROOT/agents/solo-coordinator.sh" +_route_expect() { + local sig="$1" expected="$2" got + got=$("$COORDINATOR" --route "$sig" 2>/dev/null | cut -f1) + if [[ "$got" == "$expected" ]]; then + ok "route: $sig → $expected" + else + fail "route: $sig → '$got' (expected '$expected')" + fi +} + +_route_expect brief-ready reviewer +_route_expect plan-ready reviewer +_route_expect phase-A-ready reviewer +_route_expect phase-A+B-ready reviewer +_route_expect verification-complete reviewer +_route_expect verification-failed reviewer +_route_expect brief-approved programmer +_route_expect brief-changes-2 programmer +_route_expect plan-approved programmer +_route_expect plan-changes-1 programmer +_route_expect phase-A-approved programmer +_route_expect phase-A+B-changes-3 programmer +_route_expect final-approved programmer +_route_expect verification-skipped notify-human +_route_expect stuck notify-human +_route_expect done complete +_route_expect no-such-signal unknown + +log "Validating solo-coordinator signal consumption (--once)..." + +TEST_WT=$(mktemp -d) +mkdir -p "$TEST_WT/.adapter-workspace/signals" +touch "$TEST_WT/.adapter-workspace/signals/brief-ready" +sleep 1 # distinct mtimes so causal (oldest-first) ordering is observable +touch "$TEST_WT/.adapter-workspace/signals/plan-ready" + +ONCE_OUT=$("$COORDINATOR" --once "$TEST_WT" 2>/dev/null || true) + +# Both signals routed, oldest first +if [[ $(echo "$ONCE_OUT" | wc -l | tr -d ' ') == "2" ]] \ + && [[ $(echo "$ONCE_OUT" | sed -n 1p | cut -f1,2) == $'brief-ready\treviewer' ]] \ + && [[ $(echo "$ONCE_OUT" | sed -n 2p | cut -f1,2) == $'plan-ready\treviewer' ]]; then + ok "--once routes pending signals oldest-first" +else + fail "--once output unexpected: $ONCE_OUT" +fi + +# Signals consumed (dir empty) and archived +if ls "$TEST_WT/.adapter-workspace/signals" | grep -q .; then + fail "--once left unconsumed signals behind" +else + ok "--once consumed all pending signals" +fi +archived=$(ls "$TEST_WT/.adapter-workspace/signals/.archive" 2>/dev/null | wc -l | tr -d ' ') +if [[ "$archived" == "2" ]]; then + ok "--once archived both signals" +else + fail "--once archive has $archived entries (expected 2)" +fi + +# Re-running is a no-op: consumption means no stale re-delivery +ONCE_AGAIN=$("$COORDINATOR" --once "$TEST_WT" 2>/dev/null || true) +if [[ -z "$ONCE_AGAIN" ]]; then + ok "re-scan after consumption routes nothing (no stale signals)" +else + fail "re-scan re-delivered consumed signals: $ONCE_AGAIN" +fi + +# check-signals.sh reflects the archive +CHECK_OUT=$(cd "$TEST_WT" && "$PROJECT_ROOT/agents/check-signals.sh" 2>/dev/null || true) +if echo "$CHECK_OUT" | grep -q "brief-ready" && echo "$CHECK_OUT" | grep -q "plan-ready"; then + ok "check-signals.sh reports processed signals" +else + fail "check-signals.sh missing processed signals: $CHECK_OUT" +fi + +rm -rf "$TEST_WT" + +# Solo prompts must not contain blocking waits +log "Checking solo prompts are non-blocking..." +for f in solo-programmer.md solo-reviewer.md; do + if grep -q "wait-for-signal" "$PROJECT_ROOT/agents/$f"; then + fail "$f still references wait-for-signal (blocking waits are forbidden)" + else + ok "$f has no blocking wait references" + fi + if grep -q "end your turn" "$PROJECT_ROOT/agents/$f"; then + ok "$f instructs signal-then-end-turn" + else + fail "$f missing signal-then-end-turn instruction" + fi +done + +# Launcher must start the coordinator, not the teams watcher +if grep -q "solo-coordinator.sh" "$PROJECT_ROOT/agents/launch-solo-pair.sh"; then + ok "launch-solo-pair.sh starts solo-coordinator.sh" +else + fail "launch-solo-pair.sh does not start solo-coordinator.sh" +fi + +# --------------------------------------------------------------------------- # +# Launcher pre-flight: the already-supported check must read the BASE BRANCH +# --------------------------------------------------------------------------- # +log "Checking launcher early-exit reads the base branch (--dry-run)..." + +# An adapter present on the default base branch must early-exit... +EARLY_OUT=$("$PROJECT_ROOT/agents/launch-solo-pair.sh" \ + --architecture LlamaForCausalLM --dry-run 2>&1 || true) +if echo "$EARLY_OUT" | grep -q "already has an adapter"; then + ok "existing adapter early-exits against the base branch" +else + fail "expected early-exit for LlamaForCausalLM: $EARLY_OUT" +fi + +# ...and both launchers must consult git show <branch>:factory, not the +# working tree (golden-master test branches strip the adapter on a branch). +for launcher in launch.sh launch-solo-pair.sh; do + if grep -q 'git -C "$REPO_ROOT" show "${BASE_BRANCH}:${FACTORY_REL}"' "$PROJECT_ROOT/agents/$launcher"; then + ok "$launcher factory check is branch-aware" + else + fail "$launcher factory check does not read the base branch" + fi +done + +# Per-agent model env overrides must be honored (dry-run prints resolved models) +ENV_MODEL_OUT=$(PROGRAMMER_MODEL=env-test-prog REVIEWER_MODEL=env-test-rev \ + "$PROJECT_ROOT/agents/launch-solo-pair.sh" \ + --architecture FakeEnvTestForCausalLM --skip-arch-check --dry-run 2>&1 || true) +if echo "$ENV_MODEL_OUT" | grep -q "env-test-prog" && echo "$ENV_MODEL_OUT" | grep -q "env-test-rev"; then + ok "PROGRAMMER_MODEL / REVIEWER_MODEL env overrides resolve in dry-run" +else + fail "model env overrides not honored: $ENV_MODEL_OUT" +fi + +# ...and CLI flags must beat env vars (flag > env > frontmatter) +FLAG_MODEL_OUT=$(PROGRAMMER_MODEL=env-test-prog \ + "$PROJECT_ROOT/agents/launch-solo-pair.sh" \ + --architecture FakeEnvTestForCausalLM --skip-arch-check --dry-run \ + --programmer-model flag-test-prog 2>&1 || true) +if echo "$FLAG_MODEL_OUT" | grep -q "flag-test-prog" && ! echo "$FLAG_MODEL_OUT" | grep -q "env-test-prog"; then + ok "--programmer-model flag beats PROGRAMMER_MODEL env" +else + fail "flag precedence broken: $FLAG_MODEL_OUT" +fi + +# Summary +echo "" +echo "==========================================" +if [[ $FAILURES -eq 0 ]]; then + echo -e " \033[1;32mAll dry-run tests passed!\033[0m" +else + echo -e " \033[1;31m${FAILURES} test(s) failed\033[0m" +fi +echo "==========================================" +echo "" + +exit $FAILURES diff --git a/devtools/adapter_builder/scripts/format-timeline.py b/devtools/adapter_builder/scripts/format-timeline.py new file mode 100755 index 0000000000..9ffca0308b --- /dev/null +++ b/devtools/adapter_builder/scripts/format-timeline.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Format .adapter-workspace/timeline.jsonl entries for human-readable display. + +Reads JSONL entries from stdin and writes one formatted line per entry to +stdout. Malformed lines are annotated rather than dropped. + +Typical usage: + tail -1 timeline.jsonl | format-timeline.py # last event (status) + tail -f timeline.jsonl | format-timeline.py # live stream (logs) + +Each entry is expected to look like the structure written by +agents/hooks/timeline-capture.sh: + {"ts": "...", "event": "...", "tool": "...", "tool_input": {...}, ...} + +Kept as a standalone script (rather than embedded inside bash -c) so the +Python/bash boundary is debuggable and unit-testable. +""" +from __future__ import annotations + +import json +import sys +from typing import Any + + +def format_entry(entry: dict[str, Any]) -> str: + """Return a one-line human-readable summary of a single timeline entry.""" + ts = entry.get("ts", "?") + # Extract HH:MM:SS from ISO 8601 timestamp; fall back to whatever we have. + ts_short = ts[11:19] if isinstance(ts, str) and len(ts) >= 19 else str(ts) + + event = entry.get("event", "?") or "?" + tool = entry.get("tool") or "" + tool_input = entry.get("tool_input") or {} + + # Prefer the agent-authored description when present; it's already + # human-readable ("Run mypy", "Read file X", etc.). + description = None + if isinstance(tool_input, dict): + description = tool_input.get("description") + + if description: + label = str(description) + elif tool: + if tool == "Bash" and isinstance(tool_input, dict): + cmd = (tool_input.get("command") or "").strip().split("\n")[0] + label = f"{tool}: {cmd[:80]}" + elif isinstance(tool_input, dict) and tool_input.get("file_path"): + label = f"{tool}: {tool_input['file_path']}" + else: + label = tool + else: + label = event # Fall back to the raw event name (SessionStart, etc.) + + if len(label) > 100: + label = label[:97] + "..." + + return f"{ts_short} {event:14} {label}" + + +def main() -> int: + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + print(format_entry(json.loads(line)), flush=True) + except Exception: + print(f"(malformed) {line}", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/devtools/adapter_builder/scripts/notify.sh b/devtools/adapter_builder/scripts/notify.sh new file mode 100755 index 0000000000..5dc8591c4e --- /dev/null +++ b/devtools/adapter_builder/scripts/notify.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# ============================================================================= +# notify.sh — Send a completion notification +# +# Tries in order: +# 1. Webhook (Slack/Discord/custom) — if NOTIFICATION_WEBHOOK_URL is set +# 2. iMessage (macOS) — if NOTIFICATION_NUMBER is set and on macOS +# 3. macOS Notification Center — last resort on macOS +# 4. Stdout — always works +# +# Usage: +# ./scripts/notify.sh "Adapter completed for CohereForCausalLM" +# ============================================================================= + +set -euo pipefail + +MESSAGE="${1:-Adapter build complete}" +WEBHOOK_URL="${NOTIFICATION_WEBHOOK_URL:-}" +PHONE="${NOTIFICATION_NUMBER:-}" + +log() { echo -e "\033[1;35m[notify]\033[0m $*"; } + +# --------------------------------------------------------------------------- # +# 1. Webhook (Slack, Discord, or custom) +# --------------------------------------------------------------------------- # +if [[ -n "$WEBHOOK_URL" ]]; then + # Detect service from URL and format payload accordingly + if [[ "$WEBHOOK_URL" == *"slack"* ]]; then + PAYLOAD=$(printf '{"text": "%s"}' "$MESSAGE") + elif [[ "$WEBHOOK_URL" == *"discord"* ]]; then + PAYLOAD=$(printf '{"content": "%s"}' "$MESSAGE") + else + # Generic — send both common field names + PAYLOAD=$(printf '{"text": "%s", "content": "%s", "message": "%s"}' "$MESSAGE" "$MESSAGE" "$MESSAGE") + fi + + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \ + -X POST -H "Content-Type: application/json" \ + -d "$PAYLOAD" \ + "$WEBHOOK_URL" 2>/dev/null) || HTTP_CODE="000" + + if [[ "$HTTP_CODE" =~ ^2 ]]; then + log "Webhook notification sent (HTTP $HTTP_CODE)" + exit 0 + else + log "Webhook failed (HTTP $HTTP_CODE) — trying fallback" + fi +fi + +# --------------------------------------------------------------------------- # +# 2. iMessage (macOS only) +# --------------------------------------------------------------------------- # +if [[ -n "$PHONE" && "$(uname)" == "Darwin" ]] && command -v osascript &>/dev/null; then + osascript -e " + tell application \"Messages\" + set targetService to 1st account whose service type = iMessage + set targetBuddy to participant \"${PHONE}\" of targetService + send \"${MESSAGE}\" to targetBuddy + end tell + " 2>/dev/null && { + log "iMessage sent to $PHONE" + exit 0 + } || { + log "iMessage failed — trying fallback" + } +fi + +# --------------------------------------------------------------------------- # +# 3. macOS Notification Center +# --------------------------------------------------------------------------- # +if [[ "$(uname)" == "Darwin" ]]; then + osascript -e "display notification \"${MESSAGE}\" with title \"TL Adapter Builder\"" 2>/dev/null && { + log "macOS notification sent" + exit 0 + } || true +fi + +# --------------------------------------------------------------------------- # +# 4. Stdout (always works) +# --------------------------------------------------------------------------- # +log "$MESSAGE" +log "(no notification service configured — set NOTIFICATION_WEBHOOK_URL in .env)" diff --git a/devtools/adapter_builder/scripts/port-arch-models.py b/devtools/adapter_builder/scripts/port-arch-models.py new file mode 100755 index 0000000000..a329834bdb --- /dev/null +++ b/devtools/adapter_builder/scripts/port-arch-models.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Merge the architecture-specific model list produced by +scan-hf-architecture.py into `supported_models.json` so that `verify_models` +can pick the models up. + +Reads: transformer_lens/tools/model_registry/data/supported_models_<arch_short>.json +Writes: transformer_lens/tools/model_registry/data/supported_models.json + +This is an idempotent merge — models already present in the registry are +skipped, not duplicated. The per-arch file stays in place untouched. + +Run from inside the TransformerLens worktree: + + uv run python "$TL_ADAPTER_BUILDER_ROOT/scripts/port-arch-models.py" \\ + --arch-short qwen3_moe + +Skipping this step is a common failure mode that wastes hours — if the +target architecture's models aren't in `supported_models.json`, verify_models +silently skips them and you get no signal. +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--arch-short", + required=True, + help="Short lowercase name matching the scan output, e.g. qwen3_moe", + ) + parser.add_argument( + "--data-dir", + default="transformer_lens/tools/model_registry/data", + help="Registry data directory (resolved relative to cwd)", + ) + args = parser.parse_args() + + data_dir = Path(args.data_dir) + arch_file = data_dir / f"supported_models_{args.arch_short}.json" + registry_file = data_dir / "supported_models.json" + + if not arch_file.exists(): + print(f"ERROR: arch file not found: {arch_file}", file=sys.stderr) + print("Run scan-hf-architecture.py first.", file=sys.stderr) + return 1 + + if not registry_file.exists(): + print(f"ERROR: registry file not found: {registry_file}", file=sys.stderr) + return 1 + + arch_data = json.loads(arch_file.read_text()) + registry = json.loads(registry_file.read_text()) + + existing_ids = {m["model_id"] for m in registry["models"]} + arch_models = arch_data["models"] + new_entries = [m for m in arch_models if m["model_id"] not in existing_ids] + + registry["models"].extend(new_entries) + arch_ids = {m["architecture_id"] for m in registry["models"]} + registry["total_architectures"] = len(arch_ids) + registry["total_models"] = len(registry["models"]) + + registry_file.write_text(json.dumps(registry, indent=2) + "\n") + + skipped = len(arch_models) - len(new_entries) + print(f"Ported {len(new_entries)} new models to {registry_file.name}") + print(f"(Skipped {skipped} already present)") + print( + f"Registry now has {registry['total_models']} total models across " + f"{registry['total_architectures']} architectures" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/devtools/adapter_builder/scripts/scan-hf-architecture.py b/devtools/adapter_builder/scripts/scan-hf-architecture.py new file mode 100755 index 0000000000..fb019b0401 --- /dev/null +++ b/devtools/adapter_builder/scripts/scan-hf-architecture.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Scan HuggingFace for every model of a given architecture class and write +the results to `transformer_lens/tools/model_registry/data/supported_models_<arch_short>.json`. + +This produces the authoritative per-architecture model list used by the +adapter-builder workflow. It stays in place as an artifact for human review, +and later gets merged into `supported_models.json` via port-arch-models.py +so that `verify_models` can pick the models up. + +Run from inside the TransformerLens worktree: + + uv run python "$TL_ADAPTER_BUILDER_ROOT/scripts/scan-hf-architecture.py" \\ + Qwen3MoeForCausalLM --arch-short qwen3_moe + +The `architecture_gaps.json` file is useful for reference metadata but its +`sample_models` list is capped at 10 — this scan is exhaustive, which is +what the adapter workflow needs. +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def scan(arch_class: str, limit: int) -> tuple[list[dict], int]: + """Return (matches, scanned_count) from a paginated HF listing.""" + # Deferred import so --help works even when huggingface_hub isn't installed + # in the current env (the script is meant to run via `uv run` from the + # worktree, which has the project's deps available). + try: + from huggingface_hub import HfApi + except ImportError: + print( + "huggingface_hub is not installed. Run this with `uv run` from " + "the TransformerLens worktree so the project's env is available.", + file=sys.stderr, + ) + sys.exit(1) + + api = HfApi() + matches: list[dict] = [] + scanned = 0 + + for m in api.list_models( + pipeline_tag="text-generation", + sort="downloads", + expand=["config", "safetensors", "downloads"], + limit=limit, + ): + scanned += 1 + config = getattr(m, "config", None) or {} + archs = config.get("architectures") or [] + if arch_class not in archs: + continue + + safetensors = getattr(m, "safetensors", None) + total_params = None + if safetensors and isinstance(safetensors, dict): + total_params = safetensors.get("total") + + matches.append( + { + "architecture_id": arch_class, + "model_id": m.id, + "status": 0, + "verified_date": None, + "metadata": { + "downloads": m.downloads or 0, + "total_params": total_params, + }, + "note": None, + "phase1_score": None, + "phase2_score": None, + "phase3_score": None, + "phase4_score": None, + "phase7_score": None, + "phase8_score": None, + } + ) + + return matches, scanned + + +def main() -> int: + """Parse args, run the scan, and write the result JSON.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("arch_class", help="HF architecture class, e.g. Qwen3MoeForCausalLM") + parser.add_argument( + "--arch-short", + required=True, + help="Short lowercase name for the output file, e.g. qwen3_moe", + ) + parser.add_argument( + "--limit", + type=int, + default=10000, + help="Max models to scan from the HF listing (default: 10000)", + ) + parser.add_argument( + "--output-dir", + default="transformer_lens/tools/model_registry/data", + help="Output directory (resolved relative to cwd; default matches the registry layout)", + ) + args = parser.parse_args() + + output_path = Path(args.output_dir) / f"supported_models_{args.arch_short}.json" + output_path.parent.mkdir(parents=True, exist_ok=True) + + matches, scanned = scan(args.arch_class, args.limit) + + output_path.write_text( + json.dumps( + { + "architecture_id": args.arch_class, + "total_models": len(matches), + "scanned": scanned, + "models": matches, + }, + indent=2, + ) + + "\n" + ) + + print(f"Scanned {scanned} models, found {len(matches)} matches for {args.arch_class}") + print(f"Wrote: {output_path}") + print() + print("Top 20 by downloads:") + top = sorted(matches, key=lambda x: -(x["metadata"]["downloads"] or 0))[:20] + for m in top: + params = m["metadata"]["total_params"] + params_str = f"{params/1e9:.2f}B" if params else "?" + downloads = m["metadata"]["downloads"] + print(f" {m['model_id']:60s} downloads={downloads:>10} params={params_str}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/devtools/adapter_builder/scripts/strip-adapter.py b/devtools/adapter_builder/scripts/strip-adapter.py new file mode 100644 index 0000000000..9799a94d6a --- /dev/null +++ b/devtools/adapter_builder/scripts/strip-adapter.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Strip an existing adapter from a checkout for golden-master rebuild tests. + +Removes everything the adapter builder would have to recreate for an +architecture — the adapter module, its registrations, its unit tests, and +its model-registry entries — so the build system can be pointed at the +architecture as if it were unsupported, and the result diffed against the +original (`git diff <base-branch> -- <paths>`). + +Run from inside the checkout (worktree) to strip. Leaves shared +infrastructure alone (HT-side files, model_type maps, scan configs, +comments) — those are legitimately part of the environment a fresh +adapter author would see. + +Usage: + python3 strip-adapter.py --module neox --adapter-class NeoxArchitectureAdapter \\ + --arch-class GPTNeoXForCausalLM --arch-class NeoXForCausalLM + +Example (round 2): + python3 strip-adapter.py --module codegen --adapter-class CodeGenArchitectureAdapter \\ + --arch-class CodeGenForCausalLM +""" + +import argparse +import json +import subprocess +import sys +from pathlib import Path + + +def repo_root() -> Path: + out = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], capture_output=True, text=True, check=True + ) + return Path(out.stdout.strip()) + + +def remove_lines(path: Path, predicate, label: str) -> int: + lines = path.read_text().splitlines(keepends=True) + kept = [ln for ln in lines if not predicate(ln)] + removed = len(lines) - len(kept) + if removed: + path.write_text("".join(kept)) + print(f" {path.name}: removed {removed} line(s) ({label})") + return removed + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--module", required=True, help="adapter module name, e.g. neox") + parser.add_argument( + "--adapter-class", required=True, help="adapter class name, e.g. NeoxArchitectureAdapter" + ) + parser.add_argument( + "--arch-class", + action="append", + required=True, + help="HF architecture class(es) registered to this adapter (repeatable)", + ) + args = parser.parse_args() + + root = repo_root() + arch_dir = root / "transformer_lens" / "model_bridge" / "supported_architectures" + factory = root / "transformer_lens" / "factories" / "architecture_adapter_factory.py" + data_dir = root / "transformer_lens" / "tools" / "model_registry" / "data" + + print(f"Stripping adapter '{args.module}' ({args.adapter_class}) from {root}") + + # 1. Adapter module + adapter_file = arch_dir / f"{args.module}.py" + if adapter_file.exists(): + adapter_file.unlink() + print(f" deleted {adapter_file.relative_to(root)}") + else: + print(f" WARNING: {adapter_file.relative_to(root)} not found") + + # 2. Package exports + remove_lines( + arch_dir / "__init__.py", + lambda ln: f".{args.module} import" in ln or f'"{args.adapter_class}"' in ln, + "import + __all__", + ) + + # 3. Factory registration (import line + one dict entry per arch class) + remove_lines( + factory, + lambda ln: args.adapter_class in ln, + "import + SUPPORTED_ARCHITECTURES entries", + ) + + # 4. Unit tests + test_dir = root / "tests" / "unit" / "model_bridge" / "supported_architectures" + for test_file in sorted(test_dir.glob(f"test_{args.module}*.py")): + test_file.unlink() + print(f" deleted {test_file.relative_to(root)}") + + # 5. Registry: supported_models.json (entries + top-level counters) + sm_path = data_dir / "supported_models.json" + sm = json.loads(sm_path.read_text()) + before = len(sm["models"]) + removed_models = [m for m in sm["models"] if m["architecture_id"] in args.arch_class] + sm["models"] = [m for m in sm["models"] if m["architecture_id"] not in args.arch_class] + removed_archs = {m["architecture_id"] for m in removed_models} + sm["total_models"] = sm.get("total_models", before) - len(removed_models) + sm["total_verified"] = sm.get("total_verified", 0) - sum( + 1 for m in removed_models if m["status"] == 1 + ) + sm["total_architectures"] = sm.get("total_architectures", 0) - len(removed_archs) + sm_path.write_text(json.dumps(sm, indent=2) + "\n") + print(f" supported_models.json: removed {len(removed_models)} model entries") + + # 6. Registry: verification_history.json + vh_path = data_dir / "verification_history.json" + vh = json.loads(vh_path.read_text()) + n_rec = len(vh["records"]) + vh["records"] = [r for r in vh["records"] if r.get("architecture_id") not in args.arch_class] + vh_path.write_text(json.dumps(vh, indent=2) + "\n") + print(f" verification_history.json: removed {n_rec - len(vh['records'])} records") + + # 7. Sanity: the adapter class must be gone from the package + leftovers = subprocess.run( + ["grep", "-rl", args.adapter_class, str(root / "transformer_lens")], + capture_output=True, + text=True, + ).stdout.strip() + if leftovers: + print(f"FAIL: '{args.adapter_class}' still referenced in:\n{leftovers}") + return 1 + print(f"OK: no remaining references to {args.adapter_class} in the package") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/devtools/adapter_builder/scripts/validate-adapter-deep.py b/devtools/adapter_builder/scripts/validate-adapter-deep.py new file mode 100644 index 0000000000..6e3e27f13a --- /dev/null +++ b/devtools/adapter_builder/scripts/validate-adapter-deep.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +"""Deep adapter validation — cross-references adapter against a real HF model. + +Loads a model's config and state_dict keys (no weights downloaded) and checks: +1. Component mapping paths resolve to real HF modules +2. Weight conversion keys match actual state_dict parameters +3. No missing or extra entries in weight_processing_conversions + +Usage: + uv run python scripts/validate-adapter-deep.py <architecture_class> <hf_model_id> [--repo <path>] + +Example: + uv run python scripts/validate-adapter-deep.py LlamaForCausalLM meta-llama/Llama-3.2-1B + uv run python scripts/validate-adapter-deep.py CodeGenForCausalLM Salesforce/codegen-350M-mono +""" + +import argparse +import json +import sys +from pathlib import Path + + +def validate_component_paths(adapter, model): + """Check that component mapping paths resolve to real HF modules.""" + issues: list[str] = [] + ok: list[str] = [] + + if adapter.component_mapping is None: + issues.append("component_mapping is None") + return ok, issues + + # Get all named modules from the model + module_names = {name for name, _ in model.named_modules()} + + for tl_name, component in adapter.component_mapping.items(): + hf_path = component.name + if hf_path is None: + continue + + # Top-level components: check the full path + if hf_path in module_names: + ok.append(f"{tl_name} -> {hf_path}") + else: + # For block components, check with index 0 + test_path = hf_path.replace("{i}", "0") if "{i}" in hf_path else hf_path + if test_path in module_names: + ok.append(f"{tl_name} -> {hf_path}") + else: + issues.append(f"{tl_name} -> '{hf_path}' does not resolve to a module in the model") + + # Check submodules + if hasattr(component, "submodules") and component.submodules: + for sub_name, sub_component in component.submodules.items(): + sub_hf_path = sub_component.name + if sub_hf_path is None: + continue + + # Submodule paths are relative to the parent + if component.is_list_item if hasattr(component, "is_list_item") else False: + full_path = f"{hf_path}.0.{sub_hf_path}" + else: + full_path = f"{hf_path}.{sub_hf_path}" + + if full_path in module_names: + ok.append(f" {tl_name}.{sub_name} -> {sub_hf_path}") + else: + issues.append( + f" {tl_name}.{sub_name} -> '{sub_hf_path}' " + f"(full: '{full_path}') does not resolve" + ) + + # Check nested submodules (e.g., attn.q, mlp.gate) + if hasattr(sub_component, "submodules") and sub_component.submodules: + for nested_name, nested_comp in sub_component.submodules.items(): + nested_hf = nested_comp.name + if nested_hf is None: + continue + nested_full = f"{hf_path}.0.{sub_hf_path}.{nested_hf}" + if nested_full in module_names: + ok.append(f" {tl_name}.{sub_name}.{nested_name} -> {nested_hf}") + else: + issues.append( + f" {tl_name}.{sub_name}.{nested_name} -> '{nested_hf}' " + f"(full: '{nested_full}') does not resolve" + ) + + return ok, issues + + +def validate_weight_conversions(adapter, state_dict_keys, n_layers): + """Check that weight conversion keys match actual state_dict parameters.""" + issues: list[str] = [] + ok: list[str] = [] + + if adapter.weight_processing_conversions is None: + issues.append("weight_processing_conversions is None") + return ok, issues + + for conv_key, conversion in adapter.weight_processing_conversions.items(): + # Expand {i} to check against actual keys + if "{i}" in conv_key: + # Check layer 0 as representative + expanded = conv_key.replace("{i}", "0") + else: + expanded = conv_key + + # Convert TL-style key to what we'd expect in the HF state dict + # The conversion key is in TL format (blocks.{i}.attn.q.weight) + # We need to check that the corresponding HF key exists + # This is a heuristic — the adapter's convert_hf_key_to_tl_key does the mapping + found = False + for sd_key in state_dict_keys: + try: + tl_key = adapter.convert_hf_key_to_tl_key(sd_key) + if "{i}" in conv_key: + # Normalize layer index for comparison + import re + + pattern = conv_key.replace("{i}", r"\d+") + if re.match(pattern.replace(".", r"\."), tl_key): + found = True + break + elif tl_key == expanded: + found = True + break + except Exception: + continue + + if found: + ok.append(f"{conv_key}") + else: + # Check if the source_key (if any) maps to something + source = getattr(conversion, "source_key", None) + if source: + issues.append( + f"{conv_key} (source: {source}) — no matching HF state_dict key found" + ) + else: + issues.append(f"{conv_key} — no matching HF state_dict key found") + + return ok, issues + + +def check_unmapped_weights(adapter, state_dict_keys, n_layers): + """Find state_dict keys that have no corresponding weight conversion.""" + unmapped: list[str] = [] + + if adapter.weight_processing_conversions is None: + return unmapped + + # Get all conversion patterns + conv_patterns = set() + for conv_key in adapter.weight_processing_conversions: + if "{i}" in conv_key: + for i in range(n_layers): + conv_patterns.add(conv_key.replace("{i}", str(i))) + else: + conv_patterns.add(conv_key) + + # Check which attention/projection weights lack conversions + # (embeddings, norms, etc. typically don't need conversions) + attention_mlp_keys = [ + k + for k in state_dict_keys + if any( + proj in k + for proj in [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "query", + "key", + "value", + "c_attn", + "c_proj", + "gate_proj", + "up_proj", + "down_proj", + ] + ) + ] + + for sd_key in attention_mlp_keys: + try: + tl_key = adapter.convert_hf_key_to_tl_key(sd_key) + if tl_key not in conv_patterns: + # Check with {i} pattern + import re + + matched = False + for conv_key in adapter.weight_processing_conversions: + if "{i}" in conv_key: + pattern = conv_key.replace("{i}", r"\d+").replace(".", r"\.") + if re.match(pattern, tl_key): + matched = True + break + if not matched: + unmapped.append(f"{sd_key} (TL: {tl_key})") + except Exception: + pass + + return unmapped + + +def main(): + parser = argparse.ArgumentParser(description="Deep adapter validation against a real HF model") + parser.add_argument("architecture", help="HF architecture class name (e.g., LlamaForCausalLM)") + parser.add_argument("model_id", help="HuggingFace model ID (e.g., meta-llama/Llama-3.2-1B)") + parser.add_argument("--repo", default=None, help="Path to TransformerLens repo") + parser.add_argument("--json", action="store_true", help="Output results as JSON") + args = parser.parse_args() + + # Add TL repo to path — the builder lives in devtools/adapter_builder + # inside the repo, so the containing repo is the default. + repo = args.repo or str(Path(__file__).resolve().parents[3]) + sys.path.insert(0, repo) + + try: + import torch + from transformers import AutoConfig, AutoModelForCausalLM + except ImportError: + print( + "Error: transformers and torch required. Run: pip install transformers torch", + file=sys.stderr, + ) + sys.exit(1) + + try: + from transformer_lens.factories.architecture_adapter_factory import ( + SUPPORTED_ARCHITECTURES, + ) + from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_config_from_hf, + ) + except ImportError as e: + print(f"Error importing TransformerLens: {e}", file=sys.stderr) + print("Make sure the --repo path is correct.", file=sys.stderr) + sys.exit(1) + + # Check architecture is registered + if args.architecture not in SUPPORTED_ARCHITECTURES: + print(f"Error: {args.architecture} not found in SUPPORTED_ARCHITECTURES", file=sys.stderr) + print(f"Available: {', '.join(sorted(SUPPORTED_ARCHITECTURES.keys()))}", file=sys.stderr) + sys.exit(1) + + print(f"Loading config for {args.model_id}...", file=sys.stderr) + hf_config = AutoConfig.from_pretrained(args.model_id, trust_remote_code=True) + + # Get state_dict keys without downloading weights + print(f"Getting state_dict keys (no weight download)...", file=sys.stderr) + with torch.device("meta"): + model = AutoModelForCausalLM.from_config(hf_config, trust_remote_code=True) + state_dict_keys = list(model.state_dict().keys()) + n_layers = getattr(hf_config, "num_hidden_layers", 1) + + # Create adapter instance + print(f"Creating {args.architecture} adapter...", file=sys.stderr) + try: + bridge_cfg = build_bridge_config_from_hf( + hf_config, args.architecture, args.model_id, torch.float32 + ) + adapter_class = SUPPORTED_ARCHITECTURES[args.architecture] + adapter = adapter_class(bridge_cfg) + except Exception as e: + print(f"Error creating adapter: {e}", file=sys.stderr) + sys.exit(1) + + # Run validations + print( + f"Validating against {len(state_dict_keys)} state_dict keys, {n_layers} layers...", + file=sys.stderr, + ) + + comp_ok, comp_issues = validate_component_paths(adapter, model) + weight_ok, weight_issues = validate_weight_conversions(adapter, state_dict_keys, n_layers) + unmapped = check_unmapped_weights(adapter, state_dict_keys, n_layers) + + results = { + "architecture": args.architecture, + "model_id": args.model_id, + "state_dict_keys": len(state_dict_keys), + "n_layers": n_layers, + "component_mapping": {"ok": comp_ok, "issues": comp_issues}, + "weight_conversions": {"ok": weight_ok, "issues": weight_issues}, + "unmapped_projection_weights": unmapped, + } + + if args.json: + print(json.dumps(results, indent=2)) + else: + total_issues = len(comp_issues) + len(weight_issues) + len(unmapped) + + print(f"\n{'='*60}") + print(f" Deep Validation: {args.architecture}") + print(f" Model: {args.model_id}") + print(f" State dict keys: {len(state_dict_keys)}, Layers: {n_layers}") + print(f"{'='*60}\n") + + print(" --- Component Mapping ---") + for item in comp_ok: + print(f" \033[32m[ok]\033[0m {item}") + for item in comp_issues: + print(f" \033[31m[!!]\033[0m {item}") + + print(f"\n --- Weight Conversions ---") + for item in weight_ok: + print(f" \033[32m[ok]\033[0m {item}") + for item in weight_issues: + print(f" \033[31m[!!]\033[0m {item}") + + if unmapped: + print(f"\n --- Unmapped Projection Weights ({len(unmapped)}) ---") + for item in unmapped[:20]: + print(f" \033[33m[??]\033[0m {item}") + if len(unmapped) > 20: + print(f" ... and {len(unmapped) - 20} more") + + print(f"\n{'='*60}") + if total_issues == 0: + print(f" \033[32mAll deep checks passed!\033[0m") + else: + print(f" \033[31m{total_issues} issue(s) found\033[0m") + print(f"{'='*60}\n") + + sys.exit(1 if total_issues > 0 else 0) + + +if __name__ == "__main__": + main() diff --git a/devtools/adapter_builder/scripts/validate-adapter.sh b/devtools/adapter_builder/scripts/validate-adapter.sh new file mode 100755 index 0000000000..098c702511 --- /dev/null +++ b/devtools/adapter_builder/scripts/validate-adapter.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash +# ============================================================================= +# validate-adapter.sh +# +# Validates that a generated Architecture Adapter is correctly structured and +# registered in the TransformerLens codebase. +# +# Usage: +# ./scripts/validate-adapter.sh <adapter_module_name> [--repo <path>] [--model <hf_model_id>] +# +# Example: +# ./scripts/validate-adapter.sh llama # structural only +# ./scripts/validate-adapter.sh llama --model meta-llama/Llama-3.2-1B # + deep validation +# ./scripts/validate-adapter.sh cohere --repo /path/to/TransformerLens +# ============================================================================= + +set -euo pipefail + +log() { echo -e "\033[1;34m[check]\033[0m $*"; } +ok() { echo -e "\033[1;32m[ ok ]\033[0m $*"; } +fail() { echo -e "\033[1;31m[ FAIL]\033[0m $*"; FAILURES=$((FAILURES + 1)); } + +FAILURES=0 +ADAPTER_NAME="" +HF_MODEL_ID="" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="${TL_REPO:-$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel 2>/dev/null || { cd "$SCRIPT_DIR/../../.." && pwd; })}" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case "$1" in + --repo) REPO_ROOT="$2"; shift 2 ;; + --model) HF_MODEL_ID="$2"; shift 2 ;; + -h|--help) + echo "Usage: $0 <adapter_module_name> [--repo <path>] [--model <hf_model_id>]" + echo "Example: $0 llama --model meta-llama/Llama-3.2-1B" + exit 0 + ;; + *) ADAPTER_NAME="$1"; shift ;; + esac +done + +[[ -z "$ADAPTER_NAME" ]] && { echo "Error: adapter module name required"; exit 1; } + +ADAPTERS_DIR="$REPO_ROOT/transformer_lens/model_bridge/supported_architectures" +ADAPTER_FILE="$ADAPTERS_DIR/${ADAPTER_NAME}.py" +INIT_FILE="$ADAPTERS_DIR/__init__.py" +FACTORY_FILE="$REPO_ROOT/transformer_lens/factories/architecture_adapter_factory.py" + +echo "" +echo "==========================================" +echo " Validating adapter: ${ADAPTER_NAME}" +echo "==========================================" +echo "" + +# 1. Check adapter file exists +log "Checking adapter file exists..." +if [[ -f "$ADAPTER_FILE" ]]; then + ok "Found: $ADAPTER_FILE" +else + fail "Adapter file not found: $ADAPTER_FILE" +fi + +# 2. Check class definition +log "Checking class definition..." +CLASS_PATTERN="class.*ArchitectureAdapter" +if grep -qE "$CLASS_PATTERN" "$ADAPTER_FILE" 2>/dev/null; then + CLASS_NAME=$(grep -oE "class \w+ArchitectureAdapter" "$ADAPTER_FILE" | head -1 | sed 's/class //') + ok "Found class: $CLASS_NAME" +else + fail "No ArchitectureAdapter class found in $ADAPTER_FILE" + CLASS_NAME="" +fi + +# 3. Check extends ArchitectureAdapter +log "Checking inheritance..." +if grep -qE "class.*\(ArchitectureAdapter\)" "$ADAPTER_FILE" 2>/dev/null; then + ok "Extends ArchitectureAdapter" +else + fail "Class does not extend ArchitectureAdapter" +fi + +# 4. Check component_mapping is set +log "Checking component_mapping..." +if grep -q "self.component_mapping" "$ADAPTER_FILE" 2>/dev/null; then + ok "component_mapping is set" +else + fail "component_mapping not found in adapter" +fi + +# 5. Check weight_processing_conversions is set +log "Checking weight_processing_conversions..." +if grep -q "self.weight_processing_conversions" "$ADAPTER_FILE" 2>/dev/null; then + ok "weight_processing_conversions is set" +else + fail "weight_processing_conversions not found in adapter" +fi + +# 6. Check registered in __init__.py +log "Checking __init__.py registration..." +if [[ -n "$CLASS_NAME" ]]; then + if grep -q "$CLASS_NAME" "$INIT_FILE" 2>/dev/null; then + ok "Found in __init__.py" + else + fail "$CLASS_NAME not found in $INIT_FILE" + fi + + if grep -q "$CLASS_NAME" "$INIT_FILE" 2>/dev/null && grep -q "__all__" "$INIT_FILE" 2>/dev/null; then + if grep -A 200 "__all__" "$INIT_FILE" | grep -q "\"$CLASS_NAME\""; then + ok "Found in __all__ list" + else + fail "$CLASS_NAME not found in __all__ list" + fi + fi +fi + +# 7. Check registered in factory +log "Checking factory registration..." +if [[ -n "$CLASS_NAME" ]]; then + if grep -q "$CLASS_NAME" "$FACTORY_FILE" 2>/dev/null; then + ok "Found in architecture_adapter_factory.py" + else + fail "$CLASS_NAME not found in $FACTORY_FILE" + fi +fi + +# 8. Check config attributes are set — either explicitly (exotic archs) or via +# a base-class defaults helper like _set_rms_rotary_defaults() (the common case) +log "Checking config attributes..." +if grep -qE "self\.cfg\.(normalization_type|positional_embedding_type)|self\._set_[a-z_]*defaults\(" "$ADAPTER_FILE" 2>/dev/null; then + ok "Sets normalization/positional config (defaults helper or explicit)" +else + fail "Does not set normalization/positional config (no _set_*_defaults() call or explicit self.cfg assignment)" +fi + +# 9. Run adapter-related tests (if pytest is available) +log "Running adapter tests..." +if command -v uv &>/dev/null && [[ -f "$REPO_ROOT/pyproject.toml" ]]; then + # Run a quick import test to verify the adapter can be loaded + TEST_OUTPUT=$(cd "$REPO_ROOT" && uv run python -c " +from transformer_lens.model_bridge.supported_architectures import ${CLASS_NAME} +print('Import OK: ${CLASS_NAME}') +" 2>&1) && { + ok "Import test passed: $CLASS_NAME" + } || { + fail "Import test failed: $TEST_OUTPUT" + } + + # Run any existing tests matching the adapter name + ADAPTER_TEST_FILE="$REPO_ROOT/tests/unit/test_${ADAPTER_NAME}_adapter.py" + ADAPTER_TEST_FILE2="$REPO_ROOT/tests/unit/architecture_adapters/test_${ADAPTER_NAME}.py" + if [[ -f "$ADAPTER_TEST_FILE" ]]; then + log "Found test file: $ADAPTER_TEST_FILE" + TEST_RESULT=$(cd "$REPO_ROOT" && uv run pytest "$ADAPTER_TEST_FILE" -x -q 2>&1) && { + ok "Adapter tests passed" + } || { + fail "Adapter tests failed:\n$TEST_RESULT" + } + elif [[ -f "$ADAPTER_TEST_FILE2" ]]; then + log "Found test file: $ADAPTER_TEST_FILE2" + TEST_RESULT=$(cd "$REPO_ROOT" && uv run pytest "$ADAPTER_TEST_FILE2" -x -q 2>&1) && { + ok "Adapter tests passed" + } || { + fail "Adapter tests failed:\n$TEST_RESULT" + } + else + log "No adapter-specific test file found (checked test_${ADAPTER_NAME}_adapter.py and test_${ADAPTER_NAME}.py) — skipping." + fi +else + log "uv not available or no pyproject.toml — skipping test execution." +fi + +# 10. Deep validation against a real model (if --model provided) +if [[ -n "$HF_MODEL_ID" && -n "$CLASS_NAME" ]]; then + log "Running deep validation against model: $HF_MODEL_ID" + + # Find the architecture class name from the factory + ARCH_CLASS=$(grep -B1 "$CLASS_NAME" "$FACTORY_FILE" 2>/dev/null | grep -oE '"[^"]+ForCausalLM"|"[^"]+ForConditionalGeneration"|"[^"]+ForMaskedLM"|"[^"]+LMHeadModel"' | head -1 | tr -d '"') + + if [[ -z "$ARCH_CLASS" ]]; then + log "Could not determine architecture class from factory — using class name heuristic." + # Try common patterns from the factory + ARCH_CLASS=$(grep "$CLASS_NAME" "$FACTORY_FILE" 2>/dev/null | grep -oE '"[^"]+"' | head -1 | tr -d '"') + fi + + if [[ -n "$ARCH_CLASS" ]]; then + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + DEEP_RESULT=$(cd "$REPO_ROOT" && uv run python "$SCRIPT_DIR/validate-adapter-deep.py" "$ARCH_CLASS" "$HF_MODEL_ID" --repo "$REPO_ROOT" 2>&1) && { + ok "Deep validation passed" + echo "$DEEP_RESULT" | tail -20 + } || { + fail "Deep validation found issues" + echo "$DEEP_RESULT" | tail -30 + } + else + log "Could not determine architecture class — skipping deep validation." + fi +elif [[ -n "$HF_MODEL_ID" ]]; then + log "Skipping deep validation — no class name available." +fi + +# Summary +echo "" +echo "==========================================" +if [[ $FAILURES -eq 0 ]]; then + echo -e " \033[1;32mAll checks passed!\033[0m" +else + echo -e " \033[1;31m${FAILURES} check(s) failed\033[0m" +fi +echo "==========================================" +echo "" + +exit $FAILURES diff --git a/devtools/adapter_builder/scripts/validate-architecture.py b/devtools/adapter_builder/scripts/validate-architecture.py new file mode 100755 index 0000000000..af0ef0afda --- /dev/null +++ b/devtools/adapter_builder/scripts/validate-architecture.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Validate that a HuggingFace architecture class name exists in either the +installed `transformers` package or on the HuggingFace Hub. + +Used by agents/launch.sh as a pre-flight check so typo'd architecture names +fail fast (before worktree creation, agent launch, or any wasted work) +rather than grinding through 3 review rounds and hitting the stuck-report +escalation. + +The two checks are independent and run in order: + + 1. `transformers` package (fast, local, ~1s first import). + Uses `getattr(transformers, ARCH)` — this exercises the lazy-loading + machinery and raises AttributeError for unknown names. + + 2. HuggingFace Hub scan (slower, network, ~5-10s for --hf-limit=500). + Pages through the top models by download count, expanding the `config` + field, and breaks as soon as one model's `config.architectures` + contains ARCH. We don't need exhaustive coverage — just *any* match. + +Exit codes: + 0 — validated (found via transformers OR HF Hub) + 1 — definitively not found (both checks completed, neither found a match) + 2 — could not verify (missing deps or network error); caller should decide + whether to abort or proceed + +Usage: + python3 scripts/validate-architecture.py CohereForCausalLM + python3 scripts/validate-architecture.py FakeArch --skip-hf # tf only + python3 scripts/validate-architecture.py Qwen3MoeForCausalLM --hf-limit 1000 +""" +from __future__ import annotations + +import argparse +import sys +from typing import Optional + + +def check_transformers(arch: str) -> Optional[bool]: + """Return True if `arch` is importable from the transformers package, + False if transformers is importable but `arch` is not a known attribute, + None if transformers itself is unavailable.""" + try: + import transformers + except ImportError: + return None + + # transformers uses _LazyModule: known names trigger lazy import, unknown + # names raise AttributeError. Both the AttributeError path and a None + # return count as "not found". + try: + cls = getattr(transformers, arch) + except AttributeError: + return False + return cls is not None + + +def check_huggingface(arch: str, limit: int) -> Optional[bool]: + """Return True if at least one HuggingFace Hub model has `arch` in its + `config.architectures`, False if the bounded scan completes without a + match, None on error (missing dep, network failure, API change).""" + try: + from huggingface_hub import HfApi + except ImportError: + return None + + api = HfApi() + try: + for m in api.list_models( + expand=["config"], + sort="downloads", + limit=limit, + ): + config = getattr(m, "config", None) or {} + archs = config.get("architectures") or [] + if arch in archs: + return True + return False + except Exception: + return None + + +def main() -> int: + """Parse args, run the two checks in order, and return an exit code.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("arch", help="HF architecture class, e.g. CohereForCausalLM") + parser.add_argument( + "--hf-limit", + type=int, + default=500, + help="Max HF models to scan before giving up (default: 500)", + ) + parser.add_argument( + "--skip-hf", + action="store_true", + help="Only check the transformers package; skip the HF Hub scan", + ) + parser.add_argument( + "--quiet", + action="store_true", + help="Suppress non-error output on success", + ) + args = parser.parse_args() + + # --- Check 1: transformers package (fast, local) ----------------------- + tf = check_transformers(args.arch) + if tf is True: + if not args.quiet: + print(f"OK: '{args.arch}' found in transformers package") + return 0 + + if args.skip_hf: + if tf is False: + print( + f"NOT FOUND: '{args.arch}' is not in the installed transformers package " + f"(HF Hub check skipped).", + file=sys.stderr, + ) + return 1 + # tf is None + print( + "ERROR: transformers package is not importable in this environment. " + "Install it or skip this check with --skip-arch-check on the launcher.", + file=sys.stderr, + ) + return 2 + + # --- Check 2: HuggingFace Hub scan (bounded, network) ------------------ + if tf is False: + print( + f"'{args.arch}' not in transformers package; " + f"scanning HuggingFace Hub (top {args.hf_limit} models)...", + file=sys.stderr, + ) + else: + print( + f"transformers package not importable; " + f"scanning HuggingFace Hub (top {args.hf_limit} models) for '{args.arch}'...", + file=sys.stderr, + ) + + hf = check_huggingface(args.arch, args.hf_limit) + if hf is True: + if not args.quiet: + print(f"OK: '{args.arch}' found on HuggingFace Hub") + return 0 + + # --- Both checks ran; neither found it --------------------------------- + if tf is False and hf is False: + print( + f"NOT FOUND: '{args.arch}' is not in the installed transformers package " + f"and does not appear in the top {args.hf_limit} HuggingFace Hub models " + f"by download count. Check for typos (case-sensitive), or if this is a " + f"genuinely new/rare architecture, launch with --skip-arch-check.", + file=sys.stderr, + ) + return 1 + + # --- At least one check was inconclusive ------------------------------- + if tf is False: + print( + f"INCONCLUSIVE: '{args.arch}' not in transformers, " + "and HuggingFace Hub scan failed (network error or missing huggingface_hub).", + file=sys.stderr, + ) + elif hf is False: + print( + "INCONCLUSIVE: transformers package not importable, " + f"and HuggingFace Hub scan did not find '{args.arch}' " + f"in the top {args.hf_limit} models.", + file=sys.stderr, + ) + else: + print( + "INCONCLUSIVE: could not verify against either transformers or HuggingFace Hub. " + "Check your Python environment and network connectivity.", + file=sys.stderr, + ) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/source/_static/model_properties_table.jsonl b/docs/source/_static/model_properties_table.jsonl index d3e820a333..1998d357c3 100644 --- a/docs/source/_static/model_properties_table.jsonl +++ b/docs/source/_static/model_properties_table.jsonl @@ -1,246 +1,247 @@ -{"name.default_alias":"yi-34b","name.huggingface":"01-ai\/Yi-34B","name.aliases":"yi-34b, Yi-34B","model_type":"yi","name.from_cfg":"Yi-34B","n_params.as_str":"39B","n_params.as_int":38755368960,"n_params.from_name":"34b","cfg.n_params":38755368960,"cfg.n_layers":60,"cfg.n_heads":56,"cfg.d_model":7168,"cfg.d_vocab":64000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":7168,"d_head":128,"n_layers":60,"n_ctx":4096,"n_heads":56,"d_mlp":20480,"d_vocab":64000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Yi-34B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"01-ai\/Yi-34B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0094491118,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":64000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":38755368960,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 7168\nd_head: 128\nn_layers: 60\nn_ctx: 4096\nn_heads: 56\nd_mlp: 20480\nd_vocab: 64000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Yi-34B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: 01-ai\/Yi-34B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.00944911182523068\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 64000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 38755368960\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"01-ai\/Yi-34B","tokenizer.vocab_size":64000.0,"tokenizer.max_len":4096.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"VBBPi7l7j0Xrv93YNq1tizlalWw=","tensor_shapes.state_dict":"embed:\n W_E: (64000, 7168)\nblocks:\n '[0-59]':\n ln1:\n w: (7168,)\n ln2:\n w: (7168,)\n attn:\n W_Q: (56, 7168, 128)\n W_O: (56, 128, 7168)\n b_Q: (56, 128)\n b_O: (7168,)\n '[_W_K, _W_V]': (8, 7168, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n '[W_in, W_gate]': (7168, 20480)\n W_out: (20480, 7168)\n b_in: (20480,)\n b_out: (7168,)\nln_final:\n w: (7168,)\nunembed:\n W_U: (7168, 64000)\n b_U: (64000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(64000, 7168)"},"blocks":{"[0-59]":{"ln1":{"w":"(7168,)"},"ln2":{"w":"(7168,)"},"attn":{"W_Q":"(56, 7168, 128)","W_O":"(56, 128, 7168)","b_Q":"(56, 128)","b_O":"(7168,)","[_W_K, _W_V]":"(8, 7168, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"[W_in, W_gate]":"(7168, 20480)","W_out":"(20480, 7168)","b_in":"(20480,)","b_out":"(7168,)"}}},"ln_final":{"w":"(7168,)"},"unembed":{"W_U":"(7168, 64000)","b_U":"(64000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-59]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 56, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 56, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 20480)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 7168)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\nunembed:\n hook_in: (batch, seq_len, 7168)\n hook_out: (batch, seq_len, 64000)\nhook_embed: (batch, seq_len, 7168)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-59]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 56, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 56, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 20480)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 7168)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"unembed":{"hook_in":"(batch, seq_len, 7168)","hook_out":"(batch, seq_len, 64000)"},"hook_embed":"(batch, seq_len, 7168)"}} -{"name.default_alias":"yi-34b-chat","name.huggingface":"01-ai\/Yi-34B-Chat","name.aliases":"yi-34b-chat, Yi-34B-Chat","model_type":"yi","name.from_cfg":"Yi-34B-Chat","n_params.as_str":"39B","n_params.as_int":38755368960,"n_params.from_name":"34b","cfg.n_params":38755368960,"cfg.n_layers":60,"cfg.n_heads":56,"cfg.d_model":7168,"cfg.d_vocab":64000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":7168,"d_head":128,"n_layers":60,"n_ctx":4096,"n_heads":56,"d_mlp":20480,"d_vocab":64000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Yi-34B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"01-ai\/Yi-34B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0094491118,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":64000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":38755368960,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 7168\nd_head: 128\nn_layers: 60\nn_ctx: 4096\nn_heads: 56\nd_mlp: 20480\nd_vocab: 64000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Yi-34B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: 01-ai\/Yi-34B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.00944911182523068\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 64000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 38755368960\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"01-ai\/Yi-34B-Chat","tokenizer.vocab_size":63992.0,"tokenizer.max_len":4096.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"VGXAFrTzytwGdUlX6AWH0NacncM=","tensor_shapes.state_dict":"embed:\n W_E: (64000, 7168)\nblocks:\n '[0-59]':\n ln1:\n w: (7168,)\n ln2:\n w: (7168,)\n attn:\n W_Q: (56, 7168, 128)\n W_O: (56, 128, 7168)\n b_Q: (56, 128)\n b_O: (7168,)\n '[_W_K, _W_V]': (8, 7168, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n '[W_in, W_gate]': (7168, 20480)\n W_out: (20480, 7168)\n b_in: (20480,)\n b_out: (7168,)\nln_final:\n w: (7168,)\nunembed:\n W_U: (7168, 64000)\n b_U: (64000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(64000, 7168)"},"blocks":{"[0-59]":{"ln1":{"w":"(7168,)"},"ln2":{"w":"(7168,)"},"attn":{"W_Q":"(56, 7168, 128)","W_O":"(56, 128, 7168)","b_Q":"(56, 128)","b_O":"(7168,)","[_W_K, _W_V]":"(8, 7168, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"[W_in, W_gate]":"(7168, 20480)","W_out":"(20480, 7168)","b_in":"(20480,)","b_out":"(7168,)"}}},"ln_final":{"w":"(7168,)"},"unembed":{"W_U":"(7168, 64000)","b_U":"(64000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-59]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 56, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 56, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 20480)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 7168)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\nunembed:\n hook_in: (batch, seq_len, 7168)\n hook_out: (batch, seq_len, 64000)\nhook_embed: (batch, seq_len, 7168)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-59]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 56, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 56, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 20480)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 7168)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"unembed":{"hook_in":"(batch, seq_len, 7168)","hook_out":"(batch, seq_len, 64000)"},"hook_embed":"(batch, seq_len, 7168)"}} -{"name.default_alias":"yi-6b","name.huggingface":"01-ai\/Yi-6B","name.aliases":"yi-6b, Yi-6B","model_type":"yi","name.from_cfg":"Yi-6B","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"6b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":64000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":64000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Yi-6B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"01-ai\/Yi-6B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":64000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 64000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Yi-6B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: 01-ai\/Yi-6B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 64000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"01-ai\/Yi-6B","tokenizer.vocab_size":63992.0,"tokenizer.max_len":4096.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"VGXAFrTzytwGdUlX6AWH0NacncM=","tensor_shapes.state_dict":"embed:\n W_E: (64000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (4, 4096, 128)\n '[_b_K, _b_V]': (4, 128)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 64000)\n b_U: (64000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(64000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(4, 4096, 128)","[_b_K, _b_V]":"(4, 128)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 64000)","b_U":"(64000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 64000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 64000)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"yi-6b-chat","name.huggingface":"01-ai\/Yi-6B-Chat","name.aliases":"yi-6b-chat, Yi-6B-Chat","model_type":"yi","name.from_cfg":"Yi-6B-Chat","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"6b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":64000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":64000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Yi-6B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"01-ai\/Yi-6B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":64000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 64000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Yi-6B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: 01-ai\/Yi-6B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 64000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"01-ai\/Yi-6B-Chat","tokenizer.vocab_size":63992.0,"tokenizer.max_len":4096.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"VGXAFrTzytwGdUlX6AWH0NacncM=","tensor_shapes.state_dict":"embed:\n W_E: (64000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (4, 4096, 128)\n '[_b_K, _b_V]': (4, 128)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 64000)\n b_U: (64000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(64000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(4, 4096, 128)","[_b_K, _b_V]":"(4, 128)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 64000)","b_U":"(64000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 64000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 64000)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"mGPT","name.huggingface":null,"name.aliases":"","model_type":null,"name.from_cfg":"mGPT","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":null,"cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":100000,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":100000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"mGPT","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"ai-forever\/mGPT","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100000,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 100000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: mGPT\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: ai-forever\/mGPT\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100000\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"ai-forever\/mGPT","tokenizer.vocab_size":100000.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"8j6CU_p3zgyeEBZ1Z3lu358tiy0=","tensor_shapes.state_dict":"embed:\n W_E: (100000, 2048)\npos_embed:\n W_pos: (2048, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 100000)\n b_U: (100000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100000, 2048)"},"pos_embed":{"W_pos":"(2048, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 100000)","b_U":"(100000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 100000)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 100000)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 2048)"}} -{"name.default_alias":"olmo-1b","name.huggingface":"allenai\/OLMo-1B-hf","name.aliases":"olmo-1b","model_type":null,"name.from_cfg":"OLMo-1B-hf","n_params.as_str":"1.1B","n_params.as_int":1073741824,"n_params.from_name":"1b","cfg.n_params":1073741824,"cfg.n_layers":16,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OlmoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":16,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"OLMo-1B-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OlmoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/OLMo-1B-hf","window_size":null,"attn_types":["global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1073741824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 16\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: OLMo-1B-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OlmoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/OLMo-1B-hf\nwindow_size: null\nattn_types:\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1073741824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/OLMo-1B-hf","tokenizer.vocab_size":50280.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"Zon1p_mdHoNTi0EKxz2EJK3tpZg=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (2048, 8192)\n W_out: (8192, 2048)\n b_in: (8192,)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(2048, 8192)","W_out":"(8192, 2048)","b_in":"(8192,)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"olmo-2-1b","name.huggingface":"allenai\/OLMo-2-0425-1B","name.aliases":"olmo-2-1b","model_type":null,"name.from_cfg":"OLMo-2-0425-1B","n_params.as_str":"1.1B","n_params.as_int":1073741824,"n_params.from_name":"1b","cfg.n_params":1073741824,"cfg.n_layers":16,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":100352,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":16,"n_ctx":4096,"n_heads":16,"d_mlp":8192,"d_vocab":100352,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"OLMo-2-0425-1B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/OLMo-2-0425-1B","window_size":null,"attn_types":["global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100352,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1073741824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 16\nn_ctx: 4096\nn_heads: 16\nd_mlp: 8192\nd_vocab: 100352\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: OLMo-2-0425-1B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/OLMo-2-0425-1B\nwindow_size: null\nattn_types:\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100352\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1073741824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/OLMo-2-0425-1B","tokenizer.vocab_size":100278.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"VuUSAx0rE_5hHVJSa7DTIutKX9s=","tensor_shapes.state_dict":"embed:\n W_E: (100352, 2048)\nblocks:\n '[0-15]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n q_norm:\n w: (2048,)\n k_norm:\n w: (2048,)\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n '[W_in, W_gate]': (2048, 8192)\n W_out: (8192, 2048)\n b_in: (8192,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 100352)\n b_U: (100352,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100352, 2048)"},"blocks":{"[0-15]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"q_norm":{"w":"(2048,)"},"k_norm":{"w":"(2048,)"},"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"[W_in, W_gate]":"(2048, 8192)","W_out":"(8192, 2048)","b_in":"(8192,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 100352)","b_U":"(100352,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8192)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 100352)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8192)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 100352)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"olmo-2-7b","name.huggingface":"allenai\/OLMo-2-1124-7B","name.aliases":"olmo-2-7b","model_type":null,"name.from_cfg":"OLMo-2-1124-7B","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":100352,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":100352,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"OLMo-2-1124-7B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/OLMo-2-1124-7B","window_size":null,"attn_types":["global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100352,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 100352\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: OLMo-2-1124-7B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/OLMo-2-1124-7B\nwindow_size: null\nattn_types:\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100352\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/OLMo-2-1124-7B","tokenizer.vocab_size":100278.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"VuUSAx0rE_5hHVJSa7DTIutKX9s=","tensor_shapes.state_dict":"embed:\n W_E: (100352, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n q_norm:\n w: (4096,)\n k_norm:\n w: (4096,)\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 100352)\n b_U: (100352,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100352, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"q_norm":{"w":"(4096,)"},"k_norm":{"w":"(4096,)"},"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 100352)","b_U":"(100352,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 100352)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 100352)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"olmo-3-32b-think","name.huggingface":"allenai\/Olmo-3-32B-Think","name.aliases":"olmo-3-32b-think","model_type":null,"name.from_cfg":"Olmo-3-32B-Think","n_params.as_str":"34B","n_params.as_int":33889976320,"n_params.from_name":"32b","cfg.n_params":33889976320,"cfg.n_layers":64,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":100278,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":64,"n_ctx":65536,"n_heads":40,"d_mlp":27648,"d_vocab":100278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Olmo-3-32B-Think","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/Olmo-3-32B-Think","window_size":null,"attn_types":["local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100278,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":33889976320,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":true,"yarn_factor":8.0,"yarn_attention_factor":1.2079441542,"yarn_beta_fast":32,"yarn_beta_slow":1,"yarn_original_max_position_embeddings":8192,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 64\nn_ctx: 65536\nn_heads: 40\nd_mlp: 27648\nd_vocab: 100278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Olmo-3-32B-Think\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/Olmo-3-32B-Think\nwindow_size: null\nattn_types:\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100278\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 33889976320\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: true\nyarn_factor: 8.0\nyarn_attention_factor: 1.2079441541679836\nyarn_beta_fast: 32\nyarn_beta_slow: 1\nyarn_original_max_position_embeddings: 8192\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/Olmo-3-32B-Think","tokenizer.vocab_size":100278.0,"tokenizer.max_len":65536.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"VuUSAx0rE_5hHVJSa7DTIutKX9s=","tensor_shapes.state_dict":"embed:\n W_E: (100278, 5120)\nblocks:\n '[0-63]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n q_norm:\n w: (5120,)\n k_norm:\n w: (1024,)\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (65536, 65536)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (65536, 128)\n mlp:\n '[W_in, W_gate]': (5120, 27648)\n W_out: (27648, 5120)\n b_in: (27648,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 100278)\n b_U: (100278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100278, 5120)"},"blocks":{"[0-63]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"q_norm":{"w":"(5120,)"},"k_norm":{"w":"(1024,)"},"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(65536, 65536)","IGNORE":"()","[rotary_sin, rotary_cos]":"(65536, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 27648)","W_out":"(27648, 5120)","b_in":"(27648,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 100278)","b_U":"(100278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-63]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 27648)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 100278)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-63]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 27648)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 100278)"},"hook_embed":"(batch, seq_len, 5120)"}} -{"name.default_alias":"olmo-3-7b-instruct","name.huggingface":"allenai\/Olmo-3-7B-Instruct","name.aliases":"olmo-3-7b-instruct","model_type":null,"name.from_cfg":"Olmo-3-7B-Instruct","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":100278,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":65536,"n_heads":32,"d_mlp":11008,"d_vocab":100278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":32,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Olmo-3-7B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/Olmo-3-7B-Instruct","window_size":null,"attn_types":["local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100278,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":true,"yarn_factor":8.0,"yarn_attention_factor":1.2079441542,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":8192,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 65536\nn_heads: 32\nd_mlp: 11008\nd_vocab: 100278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 32\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Olmo-3-7B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/Olmo-3-7B-Instruct\nwindow_size: null\nattn_types:\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100278\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: true\nyarn_factor: 8.0\nyarn_attention_factor: 1.2079441541679836\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 8192\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/Olmo-3-7B-Instruct","tokenizer.vocab_size":100278.0,"tokenizer.max_len":65536.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"RNowY96fjQ_x29qA8x5WywVkiVk=","tensor_shapes.state_dict":"embed:\n W_E: (100278, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n q_norm:\n w: (4096,)\n k_norm:\n w: (4096,)\n '[W_Q, _W_K, _W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, _b_K, _b_V]': (32, 128)\n b_O: (4096,)\n mask: (65536, 65536)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (65536, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 100278)\n b_U: (100278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100278, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"q_norm":{"w":"(4096,)"},"k_norm":{"w":"(4096,)"},"[W_Q, _W_K, _W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, _b_K, _b_V]":"(32, 128)","b_O":"(4096,)","mask":"(65536, 65536)","IGNORE":"()","[rotary_sin, rotary_cos]":"(65536, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 100278)","b_U":"(100278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 100278)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 100278)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"olmo-3-7b-think","name.huggingface":"allenai\/Olmo-3-7B-Think","name.aliases":"olmo-3-7b-think","model_type":null,"name.from_cfg":"Olmo-3-7B-Think","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":100278,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":65536,"n_heads":32,"d_mlp":11008,"d_vocab":100278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":32,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Olmo-3-7B-Think","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/Olmo-3-7B-Think","window_size":null,"attn_types":["local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100278,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":true,"yarn_factor":8.0,"yarn_attention_factor":1.2079441542,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":8192,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 65536\nn_heads: 32\nd_mlp: 11008\nd_vocab: 100278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 32\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Olmo-3-7B-Think\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/Olmo-3-7B-Think\nwindow_size: null\nattn_types:\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100278\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: true\nyarn_factor: 8.0\nyarn_attention_factor: 1.2079441541679836\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 8192\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/Olmo-3-7B-Think","tokenizer.vocab_size":100278.0,"tokenizer.max_len":65536.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"VuUSAx0rE_5hHVJSa7DTIutKX9s=","tensor_shapes.state_dict":"embed:\n W_E: (100278, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n q_norm:\n w: (4096,)\n k_norm:\n w: (4096,)\n '[W_Q, _W_K, _W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, _b_K, _b_V]': (32, 128)\n b_O: (4096,)\n mask: (65536, 65536)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (65536, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 100278)\n b_U: (100278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100278, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"q_norm":{"w":"(4096,)"},"k_norm":{"w":"(4096,)"},"[W_Q, _W_K, _W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, _b_K, _b_V]":"(32, 128)","b_O":"(4096,)","mask":"(65536, 65536)","IGNORE":"()","[rotary_sin, rotary_cos]":"(65536, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 100278)","b_U":"(100278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 100278)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 100278)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"olmo-3.1-32b-instruct","name.huggingface":"allenai\/Olmo-3.1-32B-Instruct","name.aliases":"olmo-3.1-32b-instruct","model_type":null,"name.from_cfg":"Olmo-3.1-32B-Instruct","n_params.as_str":"34B","n_params.as_int":33889976320,"n_params.from_name":"32b","cfg.n_params":33889976320,"cfg.n_layers":64,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":100278,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":64,"n_ctx":65536,"n_heads":40,"d_mlp":27648,"d_vocab":100278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Olmo-3.1-32B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/Olmo-3.1-32B-Instruct","window_size":null,"attn_types":["local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100278,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":33889976320,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":true,"yarn_factor":8.0,"yarn_attention_factor":1.2079441542,"yarn_beta_fast":32,"yarn_beta_slow":1,"yarn_original_max_position_embeddings":8192,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 64\nn_ctx: 65536\nn_heads: 40\nd_mlp: 27648\nd_vocab: 100278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Olmo-3.1-32B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/Olmo-3.1-32B-Instruct\nwindow_size: null\nattn_types:\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100278\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 33889976320\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: true\nyarn_factor: 8.0\nyarn_attention_factor: 1.2079441541679836\nyarn_beta_fast: 32\nyarn_beta_slow: 1\nyarn_original_max_position_embeddings: 8192\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/Olmo-3.1-32B-Instruct","tokenizer.vocab_size":100278.0,"tokenizer.max_len":65536.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"RNowY96fjQ_x29qA8x5WywVkiVk=","tensor_shapes.state_dict":"embed:\n W_E: (100278, 5120)\nblocks:\n '[0-63]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n q_norm:\n w: (5120,)\n k_norm:\n w: (1024,)\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (65536, 65536)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (65536, 128)\n mlp:\n '[W_in, W_gate]': (5120, 27648)\n W_out: (27648, 5120)\n b_in: (27648,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 100278)\n b_U: (100278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100278, 5120)"},"blocks":{"[0-63]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"q_norm":{"w":"(5120,)"},"k_norm":{"w":"(1024,)"},"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(65536, 65536)","IGNORE":"()","[rotary_sin, rotary_cos]":"(65536, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 27648)","W_out":"(27648, 5120)","b_in":"(27648,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 100278)","b_U":"(100278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-63]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 27648)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 100278)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-63]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 27648)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 100278)"},"hook_embed":"(batch, seq_len, 5120)"}} -{"name.default_alias":"olmo-3.1-32b-think","name.huggingface":"allenai\/Olmo-3.1-32B-Think","name.aliases":"olmo-3.1-32b-think","model_type":null,"name.from_cfg":"Olmo-3.1-32B-Think","n_params.as_str":"34B","n_params.as_int":33889976320,"n_params.from_name":"32b","cfg.n_params":33889976320,"cfg.n_layers":64,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":100278,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":64,"n_ctx":65536,"n_heads":40,"d_mlp":27648,"d_vocab":100278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Olmo-3.1-32B-Think","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/Olmo-3.1-32B-Think","window_size":null,"attn_types":["local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100278,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":33889976320,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":true,"yarn_factor":8.0,"yarn_attention_factor":1.2079441542,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":8192,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 64\nn_ctx: 65536\nn_heads: 40\nd_mlp: 27648\nd_vocab: 100278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Olmo-3.1-32B-Think\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/Olmo-3.1-32B-Think\nwindow_size: null\nattn_types:\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100278\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 33889976320\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: true\nyarn_factor: 8.0\nyarn_attention_factor: 1.2079441541679836\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 8192\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/Olmo-3.1-32B-Think","tokenizer.vocab_size":100278.0,"tokenizer.max_len":65536.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"VuUSAx0rE_5hHVJSa7DTIutKX9s=","tensor_shapes.state_dict":"embed:\n W_E: (100278, 5120)\nblocks:\n '[0-63]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n q_norm:\n w: (5120,)\n k_norm:\n w: (1024,)\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (65536, 65536)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (65536, 128)\n mlp:\n '[W_in, W_gate]': (5120, 27648)\n W_out: (27648, 5120)\n b_in: (27648,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 100278)\n b_U: (100278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100278, 5120)"},"blocks":{"[0-63]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"q_norm":{"w":"(5120,)"},"k_norm":{"w":"(1024,)"},"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(65536, 65536)","IGNORE":"()","[rotary_sin, rotary_cos]":"(65536, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 27648)","W_out":"(27648, 5120)","b_in":"(27648,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 100278)","b_U":"(100278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-63]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 27648)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 100278)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-63]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 27648)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 100278)"},"hook_embed":"(batch, seq_len, 5120)"}} -{"name.default_alias":"olmo-7b","name.huggingface":"allenai\/OLMo-7B-hf","name.aliases":"olmo-7b","model_type":null,"name.from_cfg":"OLMo-7B-hf","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50304,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OlmoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":11008,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"OLMo-7B-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OlmoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/OLMo-7B-hf","window_size":null,"attn_types":["global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 11008\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: OLMo-7B-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OlmoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/OLMo-7B-hf\nwindow_size: null\nattn_types:\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/OLMo-7B-hf","tokenizer.vocab_size":50280.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"Zon1p_mdHoNTi0EKxz2EJK3tpZg=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 4096)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 4096)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"olmoe","name.huggingface":"allenai\/OLMoE-1B-7B-0924","name.aliases":"olmoe","model_type":null,"name.from_cfg":"OLMoE-1B-7B-0924","n_params.as_str":"6.7B","n_params.as_int":6712983552,"n_params.from_name":null,"cfg.n_params":6712983552,"cfg.n_layers":16,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OlmoeForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":16,"n_ctx":4096,"n_heads":16,"d_mlp":1024,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":64,"experts_per_token":8,"final_rms":false,"dtype":"torch.float32","model_name":"OLMoE-1B-7B-0924","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OlmoeForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/OLMoE-1B-7B-0924","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6712983552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 16\nn_ctx: 4096\nn_heads: 16\nd_mlp: 1024\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: 64\nexperts_per_token: 8\nfinal_rms: false\ndtype: torch.float32\nmodel_name: OLMoE-1B-7B-0924\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OlmoeForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/OLMoE-1B-7B-0924\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6712983552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/OLMoE-1B-7B-0924","tokenizer.vocab_size":50280.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"Zon1p_mdHoNTi0EKxz2EJK3tpZg=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-15]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n q_norm:\n w: (2048,)\n k_norm:\n w: (2048,)\n '[W_Q, _W_K, _W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, _b_K, _b_V]': (16, 128)\n b_O: (2048,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n experts:\n '[0-63]':\n W_in:\n weight: (1024, 2048)\n W_out:\n weight: (2048, 1024)\n W_gate:\n weight: (1024, 2048)\n W_gate:\n weight: (64, 2048)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-15]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"q_norm":{"w":"(2048,)"},"k_norm":{"w":"(2048,)"},"[W_Q, _W_K, _W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, _b_K, _b_V]":"(16, 128)","b_O":"(2048,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"experts":{"[0-63]":{"W_in":{"weight":"(1024, 2048)"},"W_out":{"weight":"(2048, 1024)"},"W_gate":{"weight":"(1024, 2048)"}}},"W_gate":{"weight":"(64, 2048)"}}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"redwood_attn_2l","name.huggingface":"ArthurConmy\/redwood_attn_2l","name.aliases":"redwood_attn_2l","model_type":null,"name.from_cfg":"redwood_attn_2l","n_params.as_str":"524K","n_params.as_int":524288,"n_params.from_name":null,"cfg.n_params":524288,"cfg.n_layers":2,"cfg.n_heads":8,"cfg.d_model":256,"cfg.d_vocab":50259,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"shortformer","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":256,"d_head":32,"n_layers":2,"n_ctx":2048,"n_heads":8,"d_mlp":-1,"d_vocab":50259,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"shortformer","n_key_value_heads":null,"attn_only":true,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"redwood_attn_2l","use_attn_scale":true,"attn_scale":5.6568542495,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"ArthurConmy\/redwood_tokenizer","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.05,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50259,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":524288,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 256\nd_head: 32\nn_layers: 2\nn_ctx: 2048\nn_heads: 8\nd_mlp: -1\nd_vocab: 50259\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: shortformer\nn_key_value_heads: null\nattn_only: true\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: redwood_attn_2l\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gFkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: ArthurConmy\/redwood_tokenizer\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.05\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50259\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 524288\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"othello-gpt","name.huggingface":"Baidicoot\/Othello-GPT-Transformer-Lens","name.aliases":"othello-gpt","model_type":null,"name.from_cfg":"Othello-GPT-Transformer-Lens","n_params.as_str":"25M","n_params.as_int":25165824,"n_params.from_name":null,"cfg.n_params":25165824,"cfg.n_layers":8,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":61,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"mingpt","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":8,"n_ctx":59,"n_heads":8,"d_mlp":2048,"d_vocab":61,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Othello-GPT-Transformer-Lens","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"mingpt","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":null,"window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":61,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":25165824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 8\nn_ctx: 59\nn_heads: 8\nd_mlp: 2048\nd_vocab: 61\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Othello-GPT-Transformer-Lens\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: mingpt\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: null\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 61\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 25165824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":"embed:\n W_E: (61, 512)\npos_embed:\n W_pos: (59, 512)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (59, 59)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 61)\n b_U: (61,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(61, 512)"},"pos_embed":{"W_pos":"(59, 512)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(59, 59)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 61)","b_U":"(61,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 61)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 61)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"bloom-1b1","name.huggingface":"bigscience\/bloom-1b1","name.aliases":"bloom-1b1","model_type":"bloom","name.from_cfg":"bloom-1b1","n_params.as_str":"679M","n_params.as_int":679477248,"n_params.from_name":null,"cfg.n_params":679477248,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1536,"cfg.d_vocab":250880,"cfg.act_fn":"gelu_fast","cfg.positional_embedding_type":"alibi","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BloomForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1536,"d_head":96,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":6144,"d_vocab":250880,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"alibi","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_fast","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bloom-1b1","use_attn_scale":true,"attn_scale":9.7979589711,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BloomForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"bigscience\/bloom-1b1","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0204124145,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":250880,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":679477248,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":true,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1536\nd_head: 96\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 6144\nd_vocab: 250880\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: alibi\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_fast\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bloom-1b1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n LiEJFI6YI0A=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BloomForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: bigscience\/bloom-1b1\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.020412414523193152\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 250880\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 679477248\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: true\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"bigscience\/bloom-1b1","tokenizer.vocab_size":250680.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"OO9NZoesMCpWsijo1O2DAbq9GqI=","tensor_shapes.state_dict":"embed:\n ln:\n '[w, b]': (1536,)\n W_E: (250880, 1536)\npos_embed:\n W_pos: (2048, 1536)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1536,)\n ln2:\n '[w, b]': (1536,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1536, 96)\n W_O: (16, 96, 1536)\n '[b_Q, b_K, b_V]': (16, 96)\n b_O: (1536,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (1536, 6144)\n b_in: (6144,)\n W_out: (6144, 1536)\n b_out: (1536,)\nln_final:\n '[w, b]': (1536,)\nunembed:\n W_U: (1536, 250880)\n b_U: (250880,)\n","tensor_shapes.state_dict.raw__":{"embed":{"ln":{"[w, b]":"(1536,)"},"W_E":"(250880, 1536)"},"pos_embed":{"W_pos":"(2048, 1536)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1536,)"},"ln2":{"[w, b]":"(1536,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1536, 96)","W_O":"(16, 96, 1536)","[b_Q, b_K, b_V]":"(16, 96)","b_O":"(1536,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(1536, 6144)","b_in":"(6144,)","W_out":"(6144, 1536)","b_out":"(1536,)"}}},"ln_final":{"[w, b]":"(1536,)"},"unembed":{"W_U":"(1536, 250880)","b_U":"(250880,)"}},"tensor_shapes.activation_cache":"embed:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\nblocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 96)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 6144)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1536)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\nunembed:\n hook_in: (batch, seq_len, 1536)\n hook_out: (batch, seq_len, 250880)\nhook_embed: (batch, seq_len, 1536)\n","tensor_shapes.activation_cache.raw__":{"embed":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"}},"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 96)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 6144)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1536)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"unembed":{"hook_in":"(batch, seq_len, 1536)","hook_out":"(batch, seq_len, 250880)"},"hook_embed":"(batch, seq_len, 1536)"}} -{"name.default_alias":"bloom-1b7","name.huggingface":"bigscience\/bloom-1b7","name.aliases":"bloom-1b7","model_type":"bloom","name.from_cfg":"bloom-1b7","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":null,"cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":250880,"cfg.act_fn":"gelu_fast","cfg.positional_embedding_type":"alibi","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BloomForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":250880,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"alibi","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_fast","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bloom-1b7","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BloomForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"bigscience\/bloom-1b7","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":250880,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":true,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 250880\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: alibi\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_fast\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bloom-1b7\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BloomForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: bigscience\/bloom-1b7\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 250880\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: true\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"bigscience\/bloom-1b7","tokenizer.vocab_size":250680.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"OO9NZoesMCpWsijo1O2DAbq9GqI=","tensor_shapes.state_dict":"embed:\n ln:\n '[w, b]': (2048,)\n W_E: (250880, 2048)\npos_embed:\n W_pos: (2048, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 250880)\n b_U: (250880,)\n","tensor_shapes.state_dict.raw__":{"embed":{"ln":{"[w, b]":"(2048,)"},"W_E":"(250880, 2048)"},"pos_embed":{"W_pos":"(2048, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 250880)","b_U":"(250880,)"}},"tensor_shapes.activation_cache":"embed:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nblocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 250880)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"embed":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"}},"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 250880)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"bloom-3b","name.huggingface":"bigscience\/bloom-3b","name.aliases":"bloom-3b","model_type":"bloom","name.from_cfg":"bloom-3b","n_params.as_str":"2.4B","n_params.as_int":2359296000,"n_params.from_name":"3b","cfg.n_params":2359296000,"cfg.n_layers":30,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":250880,"cfg.act_fn":"gelu_fast","cfg.positional_embedding_type":"alibi","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BloomForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":80,"n_layers":30,"n_ctx":2048,"n_heads":32,"d_mlp":10240,"d_vocab":250880,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"alibi","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_fast","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bloom-3b","use_attn_scale":true,"attn_scale":8.94427191,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BloomForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"bigscience\/bloom-3b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0158113883,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":250880,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":2359296000,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":true,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 80\nn_layers: 30\nn_ctx: 2048\nn_heads: 32\nd_mlp: 10240\nd_vocab: 250880\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: alibi\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_fast\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bloom-3b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n qPSXm3fjIUA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BloomForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: bigscience\/bloom-3b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.015811388300841896\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 250880\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 2359296000\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: true\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"bigscience\/bloom-3b","tokenizer.vocab_size":250680.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"OO9NZoesMCpWsijo1O2DAbq9GqI=","tensor_shapes.state_dict":"embed:\n ln:\n '[w, b]': (2560,)\n W_E: (250880, 2560)\npos_embed:\n W_pos: (2048, 2560)\nblocks:\n '[0-29]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2560, 80)\n W_O: (32, 80, 2560)\n '[b_Q, b_K, b_V]': (32, 80)\n b_O: (2560,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 250880)\n b_U: (250880,)\n","tensor_shapes.state_dict.raw__":{"embed":{"ln":{"[w, b]":"(2560,)"},"W_E":"(250880, 2560)"},"pos_embed":{"W_pos":"(2048, 2560)"},"blocks":{"[0-29]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2560, 80)","W_O":"(32, 80, 2560)","[b_Q, b_K, b_V]":"(32, 80)","b_O":"(2560,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 250880)","b_U":"(250880,)"}},"tensor_shapes.activation_cache":"embed:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nblocks:\n '[0-29]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 32, 80)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 250880)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"embed":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"}},"blocks":{"[0-29]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 32, 80)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 250880)"},"hook_embed":"(batch, seq_len, 2560)"}} -{"name.default_alias":"bloom-560m","name.huggingface":"bigscience\/bloom-560m","name.aliases":"bloom-560m","model_type":"bloom","name.from_cfg":"bloom-560m","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":"560m","cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":250880,"cfg.act_fn":"gelu_fast","cfg.positional_embedding_type":"alibi","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BloomForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":4096,"d_vocab":250880,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"alibi","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_fast","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bloom-560m","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BloomForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"bigscience\/bloom-560m","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":250880,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":true,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 4096\nd_vocab: 250880\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: alibi\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_fast\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bloom-560m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BloomForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: bigscience\/bloom-560m\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 250880\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: true\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"bigscience\/bloom-560m","tokenizer.vocab_size":250680.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"OO9NZoesMCpWsijo1O2DAbq9GqI=","tensor_shapes.state_dict":"embed:\n ln:\n '[w, b]': (1024,)\n W_E: (250880, 1024)\npos_embed:\n W_pos: (2048, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 250880)\n b_U: (250880,)\n","tensor_shapes.state_dict.raw__":{"embed":{"ln":{"[w, b]":"(1024,)"},"W_E":"(250880, 1024)"},"pos_embed":{"W_pos":"(2048, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 250880)","b_U":"(250880,)"}},"tensor_shapes.activation_cache":"embed:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nblocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 250880)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"embed":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"}},"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 250880)"},"hook_embed":"(batch, seq_len, 1024)"}} -{"name.default_alias":"bloom-7b1","name.huggingface":"bigscience\/bloom-7b1","name.aliases":"bloom-7b1","model_type":"bloom","name.from_cfg":"bloom-7b1","n_params.as_str":"6.0B","n_params.as_int":6039797760,"n_params.from_name":null,"cfg.n_params":6039797760,"cfg.n_layers":30,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":250880,"cfg.act_fn":"gelu_fast","cfg.positional_embedding_type":"alibi","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BloomForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":30,"n_ctx":2048,"n_heads":32,"d_mlp":16384,"d_vocab":250880,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"alibi","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_fast","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bloom-7b1","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BloomForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"bigscience\/bloom-7b1","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":250880,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":6039797760,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":true,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 30\nn_ctx: 2048\nn_heads: 32\nd_mlp: 16384\nd_vocab: 250880\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: alibi\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_fast\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bloom-7b1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BloomForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: bigscience\/bloom-7b1\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 250880\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 6039797760\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: true\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"bigscience\/bloom-7b1","tokenizer.vocab_size":250680.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"OO9NZoesMCpWsijo1O2DAbq9GqI=","tensor_shapes.state_dict":"embed:\n ln:\n '[w, b]': (4096,)\n W_E: (250880, 4096)\npos_embed:\n W_pos: (2048, 4096)\nblocks:\n '[0-29]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 250880)\n b_U: (250880,)\n","tensor_shapes.state_dict.raw__":{"embed":{"ln":{"[w, b]":"(4096,)"},"W_E":"(250880, 4096)"},"pos_embed":{"W_pos":"(2048, 4096)"},"blocks":{"[0-29]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 250880)","b_U":"(250880,)"}},"tensor_shapes.activation_cache":"embed:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nblocks:\n '[0-29]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 250880)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"embed":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"}},"blocks":{"[0-29]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 250880)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"CodeLlamallama-2-7b","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"CodeLlama-7b-hf","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32016,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":32016,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"CodeLlama-7b-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"codellama\/CodeLlama-7b-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32016,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 32016\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: CodeLlama-7b-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: codellama\/CodeLlama-7b-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32016\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"codellama\/CodeLlama-7b-hf","tokenizer.vocab_size":32016.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"Tq7bUWJcm1X5kj9R-2uR1o7lSq8=","tensor_shapes.state_dict":"embed:\n W_E: (32016, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32016)\n b_U: (32016,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32016, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32016)","b_U":"(32016,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32016)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32016)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"CodeLlama-7b-instruct","name.huggingface":null,"name.aliases":"","model_type":"CodeLlama","name.from_cfg":"CodeLlama-7b-Instruct-hf","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32016,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":32016,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"CodeLlama-7b-Instruct-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"codellama\/CodeLlama-7b-Instruct-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32016,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 32016\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: CodeLlama-7b-Instruct-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: codellama\/CodeLlama-7b-Instruct-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32016\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"codellama\/CodeLlama-7b-Instruct-hf","tokenizer.vocab_size":32016.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"Tq7bUWJcm1X5kj9R-2uR1o7lSq8=","tensor_shapes.state_dict":"embed:\n W_E: (32016, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32016)\n b_U: (32016,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32016, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32016)","b_U":"(32016,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32016)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32016)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"CodeLlama-7b-python","name.huggingface":null,"name.aliases":"","model_type":"CodeLlama","name.from_cfg":"CodeLlama-7b-Python-hf","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"CodeLlama-7b-Python-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"codellama\/CodeLlama-7b-Python-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: CodeLlama-7b-Python-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: codellama\/CodeLlama-7b-Python-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"codellama\/CodeLlama-7b-Python-hf","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"distillgpt2","name.huggingface":"distilgpt2","name.aliases":"distillgpt2, distill-gpt2, distil-gpt2, gpt2-xs","model_type":"gpt2","name.from_cfg":"distilgpt2","n_params.as_str":"42M","n_params.as_int":42467328,"n_params.from_name":null,"cfg.n_params":42467328,"cfg.n_layers":6,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":6,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"distilgpt2","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"distilgpt2","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":42467328,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 6\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: distilgpt2\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: distilgpt2\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 42467328\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"distilgpt2","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} -{"name.default_alias":"gpt-j-6B","name.huggingface":null,"name.aliases":"","model_type":"gpt-j","name.from_cfg":"gpt-j-6B","n_params.as_str":"5.6B","n_params.as_int":5637144576,"n_params.from_name":"6B","cfg.n_params":5637144576,"cfg.n_layers":28,"cfg.n_heads":16,"cfg.d_model":4096,"cfg.d_vocab":50400,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTJForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":256,"n_layers":28,"n_ctx":2048,"n_heads":16,"d_mlp":16384,"d_vocab":50400,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt-j-6B","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTJForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-j-6B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50400,"parallel_attn_mlp":true,"rotary_dim":64,"n_params":5637144576,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":true,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 256\nn_layers: 28\nn_ctx: 2048\nn_heads: 16\nd_mlp: 16384\nd_vocab: 50400\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt-j-6B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTJForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-j-6B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50400\nparallel_attn_mlp: true\nrotary_dim: 64\nn_params: 5637144576\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: true\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-j-6B","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"aKfp-BCA9d3W27qknxFiS0DGC5s=","tensor_shapes.state_dict":"embed:\n W_E: (50400, 4096)\nblocks:\n '[0-27]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (16, 4096, 256)\n W_O: (16, 256, 4096)\n '[b_Q, b_K, b_V]': (16, 256)\n b_O: (4096,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50400)\n b_U: (50400,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50400, 4096)"},"blocks":{"[0-27]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 4096, 256)","W_O":"(16, 256, 4096)","[b_Q, b_K, b_V]":"(16, 256)","b_O":"(4096,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50400)","b_U":"(50400,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50400)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 256)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50400)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"gpt-neo-1.3B","name.huggingface":null,"name.aliases":"","model_type":"gpt-neo","name.from_cfg":"gpt-neo-1.3B","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":"1.3B","cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt-neo-1.3B","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neo-1.3B","window_size":256,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt-neo-1.3B\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neo-1.3B\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neo-1.3B","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 2048)\npos_embed:\n W_pos: (2048, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 2048)"},"pos_embed":{"W_pos":"(2048, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 2048)"}} -{"name.default_alias":"gpt-neo-125M","name.huggingface":null,"name.aliases":"","model_type":"gpt-neo","name.from_cfg":"gpt-neo-125M","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"125M","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt-neo-125M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neo-125M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt-neo-125M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neo-125M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neo-125M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (2048, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(2048, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} -{"name.default_alias":"gpt-neo-2.7B","name.huggingface":null,"name.aliases":"","model_type":"gpt-neo","name.from_cfg":"gpt-neo-2.7B","n_params.as_str":"2.5B","n_params.as_int":2516582400,"n_params.from_name":"2.7B","cfg.n_params":2516582400,"cfg.n_layers":32,"cfg.n_heads":20,"cfg.d_model":2560,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":20,"d_mlp":10240,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt-neo-2.7B","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neo-2.7B","window_size":256,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0158113883,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":2516582400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 20\nd_mlp: 10240\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt-neo-2.7B\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neo-2.7B\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.015811388300841896\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 2516582400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neo-2.7B","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 2560)\npos_embed:\n W_pos: (2048, 2560)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (20, 2560, 128)\n W_O: (20, 128, 2560)\n '[b_Q, b_K, b_V]': (20, 128)\n b_O: (2560,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 2560)"},"pos_embed":{"W_pos":"(2048, 2560)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(20, 2560, 128)","W_O":"(20, 128, 2560)","[b_Q, b_K, b_V]":"(20, 128)","b_O":"(2560,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 20, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 20, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 20, 128)","[hook_attn_scores, hook_pattern]":"(batch, 20, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 2560)"}} -{"name.default_alias":"gpt-neox-20b","name.huggingface":"EleutherAI\/gpt-neox-20b","name.aliases":"gpt-neox-20b, gpt-neox, neox","model_type":"gpt-neo","name.from_cfg":"gpt-neox-20b","n_params.as_str":"20B","n_params.as_int":19931332608,"n_params.from_name":"20b","cfg.n_params":19931332608,"cfg.n_layers":44,"cfg.n_heads":64,"cfg.d_model":6144,"cfg.d_vocab":50432,"cfg.act_fn":"gelu_fast","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":6144,"d_head":96,"n_layers":44,"n_ctx":2048,"n_heads":64,"d_mlp":24576,"d_vocab":50432,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_fast","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt-neox-20b","use_attn_scale":true,"attn_scale":9.7979589711,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0102062073,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50432,"parallel_attn_mlp":true,"rotary_dim":24,"n_params":19931332608,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 6144\nd_head: 96\nn_layers: 44\nn_ctx: 2048\nn_heads: 64\nd_mlp: 24576\nd_vocab: 50432\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_fast\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt-neox-20b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n LiEJFI6YI0A=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.010206207261596576\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50432\nparallel_attn_mlp: true\nrotary_dim: 24\nn_params: 19931332608\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50432, 6144)\nblocks:\n '[0-43]':\n ln1:\n '[w, b]': (6144,)\n ln2:\n '[w, b]': (6144,)\n attn:\n '[W_Q, W_K, W_V]': (64, 6144, 96)\n W_O: (64, 96, 6144)\n '[b_Q, b_K, b_V]': (64, 96)\n b_O: (6144,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 24)\n mlp:\n W_in: (6144, 24576)\n b_in: (24576,)\n W_out: (24576, 6144)\n b_out: (6144,)\nln_final:\n '[w, b]': (6144,)\nunembed:\n W_U: (6144, 50432)\n b_U: (50432,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50432, 6144)"},"blocks":{"[0-43]":{"ln1":{"[w, b]":"(6144,)"},"ln2":{"[w, b]":"(6144,)"},"attn":{"[W_Q, W_K, W_V]":"(64, 6144, 96)","W_O":"(64, 96, 6144)","[b_Q, b_K, b_V]":"(64, 96)","b_O":"(6144,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 24)"},"mlp":{"W_in":"(6144, 24576)","b_in":"(24576,)","W_out":"(24576, 6144)","b_out":"(6144,)"}}},"ln_final":{"[w, b]":"(6144,)"},"unembed":{"W_U":"(6144, 50432)","b_U":"(50432,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-43]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 64, 96)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 24576)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 6144)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\nunembed:\n hook_in: (batch, seq_len, 6144)\n hook_out: (batch, seq_len, 50432)\nhook_embed: (batch, seq_len, 6144)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-43]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 64, 96)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 24576)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 6144)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"unembed":{"hook_in":"(batch, seq_len, 6144)","hook_out":"(batch, seq_len, 50432)"},"hook_embed":"(batch, seq_len, 6144)"}} -{"name.default_alias":"pythia-1.4b","name.huggingface":"EleutherAI\/pythia-1.4b","name.aliases":"pythia-1.4b, EleutherAI\/pythia-1.3b, pythia-1.3b","model_type":"pythia","name.from_cfg":"pythia-1.4b","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":"1.4b","cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1.4b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1.4b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1.4b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1.4b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1.4b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"pythia-1.4b-deduped","name.huggingface":"EleutherAI\/pythia-1.4b-deduped","name.aliases":"pythia-1.4b-deduped, EleutherAI\/pythia-1.3b-deduped, pythia-1.3b-deduped","model_type":"pythia","name.from_cfg":"pythia-1.4b-deduped","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":"1.4b","cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1.4b-deduped","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1.4b-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1.4b-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1.4b-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1.4b-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"pythia-1.4b-deduped-v0","name.huggingface":"EleutherAI\/pythia-1.4b-deduped-v0","name.aliases":"pythia-1.4b-deduped-v0, EleutherAI\/pythia-1.3b-deduped-v0, pythia-1.3b-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-1.4b-deduped-v0","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":"1.4b","cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1.4b-deduped-v0","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1.4b-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1.4b-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1.4b-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1.4b-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"pythia-1.4b-v0","name.huggingface":"EleutherAI\/pythia-1.4b-v0","name.aliases":"pythia-1.4b-v0, EleutherAI\/pythia-1.3b-v0, pythia-1.3b-v0","model_type":"pythia","name.from_cfg":"pythia-1.4b-v0","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":"1.4b","cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1.4b-v0","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1.4b-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1.4b-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1.4b-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1.4b-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"pythia-12b","name.huggingface":"EleutherAI\/pythia-12b","name.aliases":"pythia-12b, EleutherAI\/pythia-13b, pythia-13b","model_type":"pythia","name.from_cfg":"pythia-12b","n_params.as_str":"11B","n_params.as_int":11324620800,"n_params.from_name":"12b","cfg.n_params":11324620800,"cfg.n_layers":36,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":50688,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":5120,"d_head":128,"n_layers":36,"n_ctx":2048,"n_heads":40,"d_mlp":20480,"d_vocab":50688,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-12b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-12b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50688,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":11324620800,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 36\nn_ctx: 2048\nn_heads: 40\nd_mlp: 20480\nd_vocab: 50688\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-12b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-12b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50688\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 11324620800\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-12b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50688, 5120)\nblocks:\n '[0-35]':\n ln1:\n '[w, b]': (5120,)\n ln2:\n '[w, b]': (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (5120, 20480)\n b_in: (20480,)\n W_out: (20480, 5120)\n b_out: (5120,)\nln_final:\n '[w, b]': (5120,)\nunembed:\n W_U: (5120, 50688)\n b_U: (50688,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50688, 5120)"},"blocks":{"[0-35]":{"ln1":{"[w, b]":"(5120,)"},"ln2":{"[w, b]":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(5120, 20480)","b_in":"(20480,)","W_out":"(20480, 5120)","b_out":"(5120,)"}}},"ln_final":{"[w, b]":"(5120,)"},"unembed":{"W_U":"(5120, 50688)","b_U":"(50688,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 20480)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 50688)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 20480)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 50688)"},"hook_embed":"(batch, seq_len, 5120)"}} -{"name.default_alias":"pythia-12b-deduped","name.huggingface":"EleutherAI\/pythia-12b-deduped","name.aliases":"pythia-12b-deduped, EleutherAI\/pythia-13b-deduped, pythia-13b-deduped","model_type":"pythia","name.from_cfg":"pythia-12b-deduped","n_params.as_str":"11B","n_params.as_int":11324620800,"n_params.from_name":"12b","cfg.n_params":11324620800,"cfg.n_layers":36,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":50688,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":5120,"d_head":128,"n_layers":36,"n_ctx":2048,"n_heads":40,"d_mlp":20480,"d_vocab":50688,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-12b-deduped","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-12b-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50688,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":11324620800,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 36\nn_ctx: 2048\nn_heads: 40\nd_mlp: 20480\nd_vocab: 50688\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-12b-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-12b-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50688\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 11324620800\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-12b-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50688, 5120)\nblocks:\n '[0-35]':\n ln1:\n '[w, b]': (5120,)\n ln2:\n '[w, b]': (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (5120, 20480)\n b_in: (20480,)\n W_out: (20480, 5120)\n b_out: (5120,)\nln_final:\n '[w, b]': (5120,)\nunembed:\n W_U: (5120, 50688)\n b_U: (50688,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50688, 5120)"},"blocks":{"[0-35]":{"ln1":{"[w, b]":"(5120,)"},"ln2":{"[w, b]":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(5120, 20480)","b_in":"(20480,)","W_out":"(20480, 5120)","b_out":"(5120,)"}}},"ln_final":{"[w, b]":"(5120,)"},"unembed":{"W_U":"(5120, 50688)","b_U":"(50688,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 20480)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 50688)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 20480)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 50688)"},"hook_embed":"(batch, seq_len, 5120)"}} -{"name.default_alias":"pythia-12b-deduped-v0","name.huggingface":"EleutherAI\/pythia-12b-deduped-v0","name.aliases":"pythia-12b-deduped-v0, EleutherAI\/pythia-13b-deduped-v0, pythia-13b-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-12b-deduped-v0","n_params.as_str":"11B","n_params.as_int":11324620800,"n_params.from_name":"12b","cfg.n_params":11324620800,"cfg.n_layers":36,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":50688,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":5120,"d_head":128,"n_layers":36,"n_ctx":2048,"n_heads":40,"d_mlp":20480,"d_vocab":50688,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-12b-deduped-v0","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-12b-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50688,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":11324620800,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 36\nn_ctx: 2048\nn_heads: 40\nd_mlp: 20480\nd_vocab: 50688\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-12b-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-12b-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50688\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 11324620800\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-12b-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50688, 5120)\nblocks:\n '[0-35]':\n ln1:\n '[w, b]': (5120,)\n ln2:\n '[w, b]': (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (5120, 20480)\n b_in: (20480,)\n W_out: (20480, 5120)\n b_out: (5120,)\nln_final:\n '[w, b]': (5120,)\nunembed:\n W_U: (5120, 50688)\n b_U: (50688,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50688, 5120)"},"blocks":{"[0-35]":{"ln1":{"[w, b]":"(5120,)"},"ln2":{"[w, b]":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(5120, 20480)","b_in":"(20480,)","W_out":"(20480, 5120)","b_out":"(5120,)"}}},"ln_final":{"[w, b]":"(5120,)"},"unembed":{"W_U":"(5120, 50688)","b_U":"(50688,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 20480)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 50688)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 20480)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 50688)"},"hook_embed":"(batch, seq_len, 5120)"}} -{"name.default_alias":"pythia-12b-v0","name.huggingface":"EleutherAI\/pythia-12b-v0","name.aliases":"pythia-12b-v0, EleutherAI\/pythia-13b-v0, pythia-13b-v0","model_type":"pythia","name.from_cfg":"pythia-12b-v0","n_params.as_str":"11B","n_params.as_int":11324620800,"n_params.from_name":"12b","cfg.n_params":11324620800,"cfg.n_layers":36,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":50688,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":5120,"d_head":128,"n_layers":36,"n_ctx":2048,"n_heads":40,"d_mlp":20480,"d_vocab":50688,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-12b-v0","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-12b-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50688,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":11324620800,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 36\nn_ctx: 2048\nn_heads: 40\nd_mlp: 20480\nd_vocab: 50688\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-12b-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-12b-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50688\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 11324620800\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-12b-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50688, 5120)\nblocks:\n '[0-35]':\n ln1:\n '[w, b]': (5120,)\n ln2:\n '[w, b]': (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (5120, 20480)\n b_in: (20480,)\n W_out: (20480, 5120)\n b_out: (5120,)\nln_final:\n '[w, b]': (5120,)\nunembed:\n W_U: (5120, 50688)\n b_U: (50688,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50688, 5120)"},"blocks":{"[0-35]":{"ln1":{"[w, b]":"(5120,)"},"ln2":{"[w, b]":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(5120, 20480)","b_in":"(20480,)","W_out":"(20480, 5120)","b_out":"(5120,)"}}},"ln_final":{"[w, b]":"(5120,)"},"unembed":{"W_U":"(5120, 50688)","b_U":"(50688,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 20480)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 50688)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 20480)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 50688)"},"hook_embed":"(batch, seq_len, 5120)"}} -{"name.default_alias":"pythia-14m","name.huggingface":"EleutherAI\/pythia-14m","name.aliases":"pythia-14m","model_type":"pythia","name.from_cfg":"pythia-14m","n_params.as_str":"1.2M","n_params.as_int":1179648,"n_params.from_name":"14m","cfg.n_params":1179648,"cfg.n_layers":6,"cfg.n_heads":4,"cfg.d_model":128,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":128,"d_head":32,"n_layers":6,"n_ctx":2048,"n_heads":4,"d_mlp":512,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-14m","use_attn_scale":true,"attn_scale":5.6568542495,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-14m","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0707106781,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":8,"n_params":1179648,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 128\nd_head: 32\nn_layers: 6\nn_ctx: 2048\nn_heads: 4\nd_mlp: 512\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-14m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gFkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-14m\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.07071067811865475\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 8\nn_params: 1179648\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-14m","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 128)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (128,)\n ln2:\n '[w, b]': (128,)\n attn:\n '[W_Q, W_K, W_V]': (4, 128, 32)\n W_O: (4, 32, 128)\n '[b_Q, b_K, b_V]': (4, 32)\n b_O: (128,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 8)\n mlp:\n W_in: (128, 512)\n b_in: (512,)\n W_out: (512, 128)\n b_out: (128,)\nln_final:\n '[w, b]': (128,)\nunembed:\n W_U: (128, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 128)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(128,)"},"ln2":{"[w, b]":"(128,)"},"attn":{"[W_Q, W_K, W_V]":"(4, 128, 32)","W_O":"(4, 32, 128)","[b_Q, b_K, b_V]":"(4, 32)","b_O":"(128,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 8)"},"mlp":{"W_in":"(128, 512)","b_in":"(512,)","W_out":"(512, 128)","b_out":"(128,)"}}},"ln_final":{"[w, b]":"(128,)"},"unembed":{"W_U":"(128, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 4, 32)\n '[hook_attn_scores, hook_pattern]': (batch, 4, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 512)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 128)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\nunembed:\n hook_in: (batch, seq_len, 128)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 128)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 4, 32)","[hook_attn_scores, hook_pattern]":"(batch, 4, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 512)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 128)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"unembed":{"hook_in":"(batch, seq_len, 128)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 128)"}} -{"name.default_alias":"pythia-160m","name.huggingface":"EleutherAI\/pythia-160m","name.aliases":"pythia-160m, EleutherAI\/pythia-125m, pythia-125m","model_type":"pythia","name.from_cfg":"pythia-160m","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"160m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-160m","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-160m","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-160m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-160m\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-160m","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 768)"}} -{"name.default_alias":"pythia-160m-deduped","name.huggingface":"EleutherAI\/pythia-160m-deduped","name.aliases":"pythia-160m-deduped, EleutherAI\/pythia-125m-deduped, pythia-125m-deduped","model_type":"pythia","name.from_cfg":"pythia-160m-deduped","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"160m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-160m-deduped","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-160m-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-160m-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-160m-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-160m-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 768)"}} -{"name.default_alias":"pythia-160m-deduped-v0","name.huggingface":"EleutherAI\/pythia-160m-deduped-v0","name.aliases":"pythia-160m-deduped-v0, EleutherAI\/pythia-125m-deduped-v0, pythia-125m-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-160m-deduped-v0","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"160m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-160m-deduped-v0","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-160m-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-160m-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-160m-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-160m-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 768)"}} -{"name.default_alias":"pythia-160m-seed1","name.huggingface":"EleutherAI\/pythia-160m-seed1","name.aliases":"pythia-160m-seed1, EleutherAI\/pythia-125m-seed1, pythia-125m-seed1","model_type":"pythia","name.from_cfg":"pythia-160m-seed1","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"160m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-160m-seed1","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-160m-seed1","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-160m-seed1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-160m-seed1\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-160m-seed1","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 768)"}} -{"name.default_alias":"pythia-160m-seed2","name.huggingface":"EleutherAI\/pythia-160m-seed2","name.aliases":"pythia-160m-seed2, EleutherAI\/pythia-125m-seed2, pythia-125m-seed2","model_type":"pythia","name.from_cfg":"pythia-160m-seed2","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"160m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-160m-seed2","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-160m-seed2","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-160m-seed2\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-160m-seed2\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-160m-seed2","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 768)"}} -{"name.default_alias":"pythia-160m-seed3","name.huggingface":"EleutherAI\/pythia-160m-seed3","name.aliases":"pythia-160m-seed3, EleutherAI\/pythia-125m-seed3, pythia-125m-seed3","model_type":"pythia","name.from_cfg":"pythia-160m-seed3","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"160m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-160m-seed3","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-160m-seed3","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-160m-seed3\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-160m-seed3\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-160m-seed3","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 768)"}} -{"name.default_alias":"pythia-160m-v0","name.huggingface":"EleutherAI\/pythia-160m-v0","name.aliases":"pythia-160m-v0, EleutherAI\/pythia-125m-v0, pythia-125m-v0","model_type":"pythia","name.from_cfg":"pythia-160m-v0","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"160m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-160m-v0","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-160m-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-160m-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-160m-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-160m-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 768)"}} -{"name.default_alias":"pythia-1b","name.huggingface":"EleutherAI\/pythia-1b","name.aliases":"pythia-1b, EleutherAI\/pythia-800m, pythia-800m","model_type":"pythia","name.from_cfg":"pythia-1b","n_params.as_str":"805M","n_params.as_int":805306368,"n_params.from_name":"1b","cfg.n_params":805306368,"cfg.n_layers":16,"cfg.n_heads":8,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":256,"n_layers":16,"n_ctx":2048,"n_heads":8,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1b","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":64,"n_params":805306368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 256\nn_layers: 16\nn_ctx: 2048\nn_heads: 8\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 64\nn_params: 805306368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (8, 2048, 256)\n W_O: (8, 256, 2048)\n '[b_Q, b_K, b_V]': (8, 256)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 2048, 256)","W_O":"(8, 256, 2048)","[b_Q, b_K, b_V]":"(8, 256)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"pythia-1b-deduped","name.huggingface":"EleutherAI\/pythia-1b-deduped","name.aliases":"pythia-1b-deduped, EleutherAI\/pythia-800m-deduped, pythia-800m-deduped","model_type":"pythia","name.from_cfg":"pythia-1b-deduped","n_params.as_str":"805M","n_params.as_int":805306368,"n_params.from_name":"1b","cfg.n_params":805306368,"cfg.n_layers":16,"cfg.n_heads":8,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":256,"n_layers":16,"n_ctx":2048,"n_heads":8,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1b-deduped","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1b-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":64,"n_params":805306368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 256\nn_layers: 16\nn_ctx: 2048\nn_heads: 8\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1b-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1b-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 64\nn_params: 805306368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1b-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (8, 2048, 256)\n W_O: (8, 256, 2048)\n '[b_Q, b_K, b_V]': (8, 256)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 2048, 256)","W_O":"(8, 256, 2048)","[b_Q, b_K, b_V]":"(8, 256)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"pythia-1b-deduped-v0","name.huggingface":"EleutherAI\/pythia-1b-deduped-v0","name.aliases":"pythia-1b-deduped-v0, EleutherAI\/pythia-800m-deduped-v0, pythia-800m-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-1b-deduped-v0","n_params.as_str":"805M","n_params.as_int":805306368,"n_params.from_name":"1b","cfg.n_params":805306368,"cfg.n_layers":16,"cfg.n_heads":8,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":256,"n_layers":16,"n_ctx":2048,"n_heads":8,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1b-deduped-v0","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1b-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":64,"n_params":805306368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 256\nn_layers: 16\nn_ctx: 2048\nn_heads: 8\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1b-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1b-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 64\nn_params: 805306368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1b-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (8, 2048, 256)\n W_O: (8, 256, 2048)\n '[b_Q, b_K, b_V]': (8, 256)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 2048, 256)","W_O":"(8, 256, 2048)","[b_Q, b_K, b_V]":"(8, 256)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"pythia-1b-v0","name.huggingface":"EleutherAI\/pythia-1b-v0","name.aliases":"pythia-1b-v0, EleutherAI\/pythia-800m-v0, pythia-800m-v0","model_type":"pythia","name.from_cfg":"pythia-1b-v0","n_params.as_str":"805M","n_params.as_int":805306368,"n_params.from_name":"1b","cfg.n_params":805306368,"cfg.n_layers":16,"cfg.n_heads":8,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":256,"n_layers":16,"n_ctx":2048,"n_heads":8,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1b-v0","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1b-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":64,"n_params":805306368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 256\nn_layers: 16\nn_ctx: 2048\nn_heads: 8\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1b-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1b-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 64\nn_params: 805306368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1b-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (8, 2048, 256)\n W_O: (8, 256, 2048)\n '[b_Q, b_K, b_V]': (8, 256)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 2048, 256)","W_O":"(8, 256, 2048)","[b_Q, b_K, b_V]":"(8, 256)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"pythia-2.8b","name.huggingface":"EleutherAI\/pythia-2.8b","name.aliases":"pythia-2.8b, EleutherAI\/pythia-2.7b, pythia-2.7b","model_type":"pythia","name.from_cfg":"pythia-2.8b","n_params.as_str":"2.5B","n_params.as_int":2516582400,"n_params.from_name":"2.8b","cfg.n_params":2516582400,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":80,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":10240,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-2.8b","use_attn_scale":true,"attn_scale":8.94427191,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-2.8b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0158113883,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":20,"n_params":2516582400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 80\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 10240\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-2.8b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n qPSXm3fjIUA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-2.8b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.015811388300841896\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 20\nn_params: 2516582400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-2.8b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2560)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2560, 80)\n W_O: (32, 80, 2560)\n '[b_Q, b_K, b_V]': (32, 80)\n b_O: (2560,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 20)\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2560)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2560, 80)","W_O":"(32, 80, 2560)","[b_Q, b_K, b_V]":"(32, 80)","b_O":"(2560,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 20)"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 80)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 80)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2560)"}} -{"name.default_alias":"pythia-2.8b-deduped","name.huggingface":"EleutherAI\/pythia-2.8b-deduped","name.aliases":"pythia-2.8b-deduped, EleutherAI\/pythia-2.7b-deduped, pythia-2.7b-deduped","model_type":"pythia","name.from_cfg":"pythia-2.8b-deduped","n_params.as_str":"2.5B","n_params.as_int":2516582400,"n_params.from_name":"2.8b","cfg.n_params":2516582400,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":80,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":10240,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-2.8b-deduped","use_attn_scale":true,"attn_scale":8.94427191,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-2.8b-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0158113883,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":20,"n_params":2516582400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 80\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 10240\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-2.8b-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n qPSXm3fjIUA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-2.8b-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.015811388300841896\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 20\nn_params: 2516582400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-2.8b-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2560)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2560, 80)\n W_O: (32, 80, 2560)\n '[b_Q, b_K, b_V]': (32, 80)\n b_O: (2560,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 20)\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2560)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2560, 80)","W_O":"(32, 80, 2560)","[b_Q, b_K, b_V]":"(32, 80)","b_O":"(2560,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 20)"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 80)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 80)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2560)"}} -{"name.default_alias":"pythia-2.8b-deduped-v0","name.huggingface":"EleutherAI\/pythia-2.8b-deduped-v0","name.aliases":"pythia-2.8b-deduped-v0, EleutherAI\/pythia-2.7b-deduped-v0, pythia-2.7b-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-2.8b-deduped-v0","n_params.as_str":"2.5B","n_params.as_int":2516582400,"n_params.from_name":"2.8b","cfg.n_params":2516582400,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":80,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":10240,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-2.8b-deduped-v0","use_attn_scale":true,"attn_scale":8.94427191,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-2.8b-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0158113883,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":20,"n_params":2516582400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 80\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 10240\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-2.8b-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n qPSXm3fjIUA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-2.8b-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.015811388300841896\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 20\nn_params: 2516582400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-2.8b-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2560)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2560, 80)\n W_O: (32, 80, 2560)\n '[b_Q, b_K, b_V]': (32, 80)\n b_O: (2560,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 20)\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2560)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2560, 80)","W_O":"(32, 80, 2560)","[b_Q, b_K, b_V]":"(32, 80)","b_O":"(2560,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 20)"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 80)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 80)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2560)"}} -{"name.default_alias":"pythia-2.8b-v0","name.huggingface":"EleutherAI\/pythia-2.8b-v0","name.aliases":"pythia-2.8b-v0, EleutherAI\/pythia-2.7b-v0, pythia-2.7b-v0","model_type":"pythia","name.from_cfg":"pythia-2.8b-v0","n_params.as_str":"2.5B","n_params.as_int":2516582400,"n_params.from_name":"2.8b","cfg.n_params":2516582400,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":80,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":10240,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-2.8b-v0","use_attn_scale":true,"attn_scale":8.94427191,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-2.8b-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0158113883,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":20,"n_params":2516582400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 80\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 10240\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-2.8b-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n qPSXm3fjIUA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-2.8b-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.015811388300841896\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 20\nn_params: 2516582400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-2.8b-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2560)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2560, 80)\n W_O: (32, 80, 2560)\n '[b_Q, b_K, b_V]': (32, 80)\n b_O: (2560,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 20)\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2560)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2560, 80)","W_O":"(32, 80, 2560)","[b_Q, b_K, b_V]":"(32, 80)","b_O":"(2560,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 20)"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 80)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 80)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2560)"}} -{"name.default_alias":"pythia-31m","name.huggingface":"EleutherAI\/pythia-31m","name.aliases":"pythia-31m","model_type":"pythia","name.from_cfg":"pythia-31m","n_params.as_str":"4.7M","n_params.as_int":4718592,"n_params.from_name":"31m","cfg.n_params":4718592,"cfg.n_layers":6,"cfg.n_heads":8,"cfg.d_model":256,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":256,"d_head":32,"n_layers":6,"n_ctx":2048,"n_heads":8,"d_mlp":1024,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-31m","use_attn_scale":true,"attn_scale":5.6568542495,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-31m","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.05,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":8,"n_params":4718592,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 256\nd_head: 32\nn_layers: 6\nn_ctx: 2048\nn_heads: 8\nd_mlp: 1024\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-31m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gFkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-31m\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.05\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 8\nn_params: 4718592\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-31m","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 256)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (256,)\n ln2:\n '[w, b]': (256,)\n attn:\n '[W_Q, W_K, W_V]': (8, 256, 32)\n W_O: (8, 32, 256)\n '[b_Q, b_K, b_V]': (8, 32)\n b_O: (256,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 8)\n mlp:\n W_in: (256, 1024)\n b_in: (1024,)\n W_out: (1024, 256)\n b_out: (256,)\nln_final:\n '[w, b]': (256,)\nunembed:\n W_U: (256, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 256)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(256,)"},"ln2":{"[w, b]":"(256,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 256, 32)","W_O":"(8, 32, 256)","[b_Q, b_K, b_V]":"(8, 32)","b_O":"(256,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 8)"},"mlp":{"W_in":"(256, 1024)","b_in":"(1024,)","W_out":"(1024, 256)","b_out":"(256,)"}}},"ln_final":{"[w, b]":"(256,)"},"unembed":{"W_U":"(256, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 32)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 1024)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 256)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\nunembed:\n hook_in: (batch, seq_len, 256)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 256)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 32)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 1024)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 256)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"unembed":{"hook_in":"(batch, seq_len, 256)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 256)"}} -{"name.default_alias":"pythia-410m","name.huggingface":"EleutherAI\/pythia-410m","name.aliases":"pythia-410m, EleutherAI\/pythia-350m, pythia-350m","model_type":"pythia","name.from_cfg":"pythia-410m","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":"410m","cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":4096,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-410m","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-410m","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-410m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-410m\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-410m","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 1024)"}} -{"name.default_alias":"pythia-410m-deduped","name.huggingface":"EleutherAI\/pythia-410m-deduped","name.aliases":"pythia-410m-deduped, EleutherAI\/pythia-350m-deduped, pythia-350m-deduped","model_type":"pythia","name.from_cfg":"pythia-410m-deduped","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":"410m","cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":4096,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-410m-deduped","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-410m-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-410m-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-410m-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-410m-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 1024)"}} -{"name.default_alias":"pythia-410m-deduped-v0","name.huggingface":"EleutherAI\/pythia-410m-deduped-v0","name.aliases":"pythia-410m-deduped-v0, EleutherAI\/pythia-350m-deduped-v0, pythia-350m-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-410m-deduped-v0","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":"410m","cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":4096,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-410m-deduped-v0","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-410m-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-410m-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-410m-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-410m-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 1024)"}} -{"name.default_alias":"pythia-410m-v0","name.huggingface":"EleutherAI\/pythia-410m-v0","name.aliases":"pythia-410m-v0, EleutherAI\/pythia-350m-v0, pythia-350m-v0","model_type":"pythia","name.from_cfg":"pythia-410m-v0","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":"410m","cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":4096,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-410m-v0","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-410m-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-410m-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-410m-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-410m-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 1024)"}} -{"name.default_alias":"pythia-6.9b","name.huggingface":"EleutherAI\/pythia-6.9b","name.aliases":"pythia-6.9b, EleutherAI\/pythia-6.7b, pythia-6.7b","model_type":"pythia","name.from_cfg":"pythia-6.9b","n_params.as_str":"6.4B","n_params.as_int":6442450944,"n_params.from_name":"6.9b","cfg.n_params":6442450944,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50432,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":16384,"d_vocab":50432,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-6.9b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-6.9b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50432,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":6442450944,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 16384\nd_vocab: 50432\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-6.9b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-6.9b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50432\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 6442450944\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-6.9b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50432, 4096)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50432)\n b_U: (50432,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50432, 4096)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50432)","b_U":"(50432,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50432)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50432)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"pythia-6.9b-deduped","name.huggingface":"EleutherAI\/pythia-6.9b-deduped","name.aliases":"pythia-6.9b-deduped, EleutherAI\/pythia-6.7b-deduped, pythia-6.7b-deduped","model_type":"pythia","name.from_cfg":"pythia-6.9b-deduped","n_params.as_str":"6.4B","n_params.as_int":6442450944,"n_params.from_name":"6.9b","cfg.n_params":6442450944,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50432,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":16384,"d_vocab":50432,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-6.9b-deduped","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-6.9b-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50432,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":6442450944,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 16384\nd_vocab: 50432\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-6.9b-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-6.9b-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50432\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 6442450944\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-6.9b-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50432, 4096)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50432)\n b_U: (50432,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50432, 4096)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50432)","b_U":"(50432,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50432)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50432)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"pythia-6.9b-deduped-v0","name.huggingface":"EleutherAI\/pythia-6.9b-deduped-v0","name.aliases":"pythia-6.9b-deduped-v0, EleutherAI\/pythia-6.7b-deduped-v0, pythia-6.7b-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-6.9b-deduped-v0","n_params.as_str":"6.4B","n_params.as_int":6442450944,"n_params.from_name":"6.9b","cfg.n_params":6442450944,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50432,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":16384,"d_vocab":50432,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-6.9b-deduped-v0","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-6.9b-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50432,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":6442450944,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 16384\nd_vocab: 50432\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-6.9b-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-6.9b-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50432\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 6442450944\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-6.9b-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50432, 4096)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50432)\n b_U: (50432,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50432, 4096)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50432)","b_U":"(50432,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50432)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50432)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"pythia-6.9b-v0","name.huggingface":"EleutherAI\/pythia-6.9b-v0","name.aliases":"pythia-6.9b-v0, EleutherAI\/pythia-6.7b-v0, pythia-6.7b-v0","model_type":"pythia","name.from_cfg":"pythia-6.9b-v0","n_params.as_str":"6.4B","n_params.as_int":6442450944,"n_params.from_name":"6.9b","cfg.n_params":6442450944,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50432,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":16384,"d_vocab":50432,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-6.9b-v0","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-6.9b-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50432,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":6442450944,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 16384\nd_vocab: 50432\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-6.9b-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-6.9b-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50432\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 6442450944\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-6.9b-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50432, 4096)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50432)\n b_U: (50432,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50432, 4096)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50432)","b_U":"(50432,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50432)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50432)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"pythia-70m","name.huggingface":"EleutherAI\/pythia-70m","name.aliases":"pythia-70m, pythia, EleutherAI\/pythia-19m, pythia-19m","model_type":"pythia","name.from_cfg":"pythia-70m","n_params.as_str":"19M","n_params.as_int":18874368,"n_params.from_name":"70m","cfg.n_params":18874368,"cfg.n_layers":6,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":6,"n_ctx":2048,"n_heads":8,"d_mlp":2048,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-70m","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-70m","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":18874368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 6\nn_ctx: 2048\nn_heads: 8\nd_mlp: 2048\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-70m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-70m\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 18874368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-70m","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 512)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 512)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 512)"}} -{"name.default_alias":"pythia-70m-deduped","name.huggingface":"EleutherAI\/pythia-70m-deduped","name.aliases":"pythia-70m-deduped, EleutherAI\/pythia-19m-deduped, pythia-19m-deduped","model_type":"pythia","name.from_cfg":"pythia-70m-deduped","n_params.as_str":"19M","n_params.as_int":18874368,"n_params.from_name":"70m","cfg.n_params":18874368,"cfg.n_layers":6,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":6,"n_ctx":2048,"n_heads":8,"d_mlp":2048,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-70m-deduped","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-70m-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":18874368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 6\nn_ctx: 2048\nn_heads: 8\nd_mlp: 2048\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-70m-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-70m-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 18874368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-70m-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 512)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 512)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 512)"}} -{"name.default_alias":"pythia-70m-deduped-v0","name.huggingface":"EleutherAI\/pythia-70m-deduped-v0","name.aliases":"pythia-70m-deduped-v0, EleutherAI\/pythia-19m-deduped-v0, pythia-19m-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-70m-deduped-v0","n_params.as_str":"19M","n_params.as_int":18874368,"n_params.from_name":"70m","cfg.n_params":18874368,"cfg.n_layers":6,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":6,"n_ctx":2048,"n_heads":8,"d_mlp":2048,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-70m-deduped-v0","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-70m-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":18874368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 6\nn_ctx: 2048\nn_heads: 8\nd_mlp: 2048\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-70m-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-70m-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 18874368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-70m-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 512)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 512)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 512)"}} -{"name.default_alias":"pythia-70m-v0","name.huggingface":"EleutherAI\/pythia-70m-v0","name.aliases":"pythia-70m-v0, pythia-v0, EleutherAI\/pythia-19m-v0, pythia-19m-v0","model_type":"pythia","name.from_cfg":"pythia-70m-v0","n_params.as_str":"19M","n_params.as_int":18874368,"n_params.from_name":"70m","cfg.n_params":18874368,"cfg.n_layers":6,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":6,"n_ctx":2048,"n_heads":8,"d_mlp":2048,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-70m-v0","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-70m-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":18874368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 6\nn_ctx: 2048\nn_heads: 8\nd_mlp: 2048\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-70m-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-70m-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 18874368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-70m-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 512)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 512)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 512)"}} -{"name.default_alias":"hubert-base-ls960","name.huggingface":"facebook\/hubert-base-ls960","name.aliases":"hubert-base-ls960","model_type":"bert","name.from_cfg":"hubert-base-ls960","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":-1,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"HubertModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":8192,"n_heads":12,"d_mlp":3072,"d_vocab":-1,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"hubert-base-ls960","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"HubertModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/hubert-base-ls960","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":-1,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 8192\nn_heads: 12\nd_mlp: 3072\nd_vocab: -1\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: hubert-base-ls960\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: HubertModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/hubert-base-ls960\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: -1\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"opt-1.3b","name.huggingface":"facebook\/opt-1.3b","name.aliases":"opt-1.3b, opt-medium","model_type":"opt","name.from_cfg":"opt-1.3b","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":"1.3b","cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":32,"cfg.d_model":2048,"cfg.d_vocab":50272,"cfg.act_fn":"relu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OPTForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":32,"d_mlp":8192,"d_vocab":50272,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"opt-1.3b","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OPTForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/opt-1.3b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50272,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 32\nd_mlp: 8192\nd_vocab: 50272\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: opt-1.3b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OPTForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/opt-1.3b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50272\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/opt-1.3b","tokenizer.vocab_size":50265.0,"tokenizer.max_len":null,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"f1FIzqnRiMYzke1CU0hp8TDxq7k=","tensor_shapes.state_dict":"embed:\n W_E: (50272, 2048)\npos_embed:\n W_pos: (2048, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2048, 64)\n W_O: (32, 64, 2048)\n '[b_Q, b_K, b_V]': (32, 64)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50272)\n b_U: (50272,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50272, 2048)"},"pos_embed":{"W_pos":"(2048, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2048, 64)","W_O":"(32, 64, 2048)","[b_Q, b_K, b_V]":"(32, 64)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50272)","b_U":"(50272,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 32, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50272)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 32, 64)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50272)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 2048)"}} -{"name.default_alias":"opt-125m","name.huggingface":"facebook\/opt-125m","name.aliases":"opt-125m, opt-small, opt","model_type":"opt","name.from_cfg":"opt-125m","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"125m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50272,"cfg.act_fn":"relu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OPTForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50272,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"opt-125m","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OPTForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/opt-125m","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50272,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50272\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: opt-125m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OPTForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/opt-125m\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50272\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/opt-125m","tokenizer.vocab_size":50265.0,"tokenizer.max_len":null,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"f1FIzqnRiMYzke1CU0hp8TDxq7k=","tensor_shapes.state_dict":"embed:\n W_E: (50272, 768)\npos_embed:\n W_pos: (2048, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50272)\n b_U: (50272,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50272, 768)"},"pos_embed":{"W_pos":"(2048, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50272)","b_U":"(50272,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50272)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50272)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} -{"name.default_alias":"opt-13b","name.huggingface":"facebook\/opt-13b","name.aliases":"opt-13b, opt-xxl","model_type":"opt","name.from_cfg":"opt-13b","n_params.as_str":"13B","n_params.as_int":12582912000,"n_params.from_name":"13b","cfg.n_params":12582912000,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":50272,"cfg.act_fn":"relu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OPTForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":40,"d_mlp":20480,"d_vocab":50272,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"opt-13b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OPTForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/opt-13b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50272,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912000,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 40\nd_mlp: 20480\nd_vocab: 50272\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: opt-13b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OPTForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/opt-13b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50272\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912000\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/opt-13b","tokenizer.vocab_size":50265.0,"tokenizer.max_len":null,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"f1FIzqnRiMYzke1CU0hp8TDxq7k=","tensor_shapes.state_dict":"embed:\n W_E: (50272, 5120)\npos_embed:\n W_pos: (2048, 5120)\nblocks:\n '[0-39]':\n ln1:\n '[w, b]': (5120,)\n ln2:\n '[w, b]': (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (5120, 20480)\n b_in: (20480,)\n W_out: (20480, 5120)\n b_out: (5120,)\nln_final:\n '[w, b]': (5120,)\nunembed:\n W_U: (5120, 50272)\n b_U: (50272,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50272, 5120)"},"pos_embed":{"W_pos":"(2048, 5120)"},"blocks":{"[0-39]":{"ln1":{"[w, b]":"(5120,)"},"ln2":{"[w, b]":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(5120, 20480)","b_in":"(20480,)","W_out":"(20480, 5120)","b_out":"(5120,)"}}},"ln_final":{"[w, b]":"(5120,)"},"unembed":{"W_U":"(5120, 50272)","b_U":"(50272,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 20480)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 50272)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 20480)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 50272)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 5120)"}} -{"name.default_alias":"opt-2.7b","name.huggingface":"facebook\/opt-2.7b","name.aliases":"opt-2.7b, opt-large","model_type":"opt","name.from_cfg":"opt-2.7b","n_params.as_str":"2.5B","n_params.as_int":2516582400,"n_params.from_name":"2.7b","cfg.n_params":2516582400,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":50272,"cfg.act_fn":"relu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OPTForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":80,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":10240,"d_vocab":50272,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"opt-2.7b","use_attn_scale":true,"attn_scale":8.94427191,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OPTForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/opt-2.7b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0158113883,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50272,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":2516582400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 80\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 10240\nd_vocab: 50272\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: opt-2.7b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n qPSXm3fjIUA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OPTForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/opt-2.7b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.015811388300841896\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50272\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 2516582400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/opt-2.7b","tokenizer.vocab_size":50265.0,"tokenizer.max_len":null,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"f1FIzqnRiMYzke1CU0hp8TDxq7k=","tensor_shapes.state_dict":"embed:\n W_E: (50272, 2560)\npos_embed:\n W_pos: (2048, 2560)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2560, 80)\n W_O: (32, 80, 2560)\n '[b_Q, b_K, b_V]': (32, 80)\n b_O: (2560,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 50272)\n b_U: (50272,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50272, 2560)"},"pos_embed":{"W_pos":"(2048, 2560)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2560, 80)","W_O":"(32, 80, 2560)","[b_Q, b_K, b_V]":"(32, 80)","b_O":"(2560,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 50272)","b_U":"(50272,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 32, 80)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 50272)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 32, 80)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 50272)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 2560)"}} -{"name.default_alias":"opt-30b","name.huggingface":"facebook\/opt-30b","name.aliases":"opt-30b, opt-xxxl","model_type":"opt","name.from_cfg":"opt-30b","n_params.as_str":"30B","n_params.as_int":29595009024,"n_params.from_name":"30b","cfg.n_params":29595009024,"cfg.n_layers":48,"cfg.n_heads":56,"cfg.d_model":7168,"cfg.d_vocab":50272,"cfg.act_fn":"relu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OPTForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":7168,"d_head":128,"n_layers":48,"n_ctx":2048,"n_heads":56,"d_mlp":28672,"d_vocab":50272,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"opt-30b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OPTForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/opt-30b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0094491118,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50272,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":29595009024,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 7168\nd_head: 128\nn_layers: 48\nn_ctx: 2048\nn_heads: 56\nd_mlp: 28672\nd_vocab: 50272\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: opt-30b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OPTForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/opt-30b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.00944911182523068\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50272\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 29595009024\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/opt-30b","tokenizer.vocab_size":50265.0,"tokenizer.max_len":null,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"f1FIzqnRiMYzke1CU0hp8TDxq7k=","tensor_shapes.state_dict":"embed:\n W_E: (50272, 7168)\npos_embed:\n W_pos: (2048, 7168)\nblocks:\n '[0-47]':\n ln1:\n '[w, b]': (7168,)\n ln2:\n '[w, b]': (7168,)\n attn:\n '[W_Q, W_K, W_V]': (56, 7168, 128)\n W_O: (56, 128, 7168)\n '[b_Q, b_K, b_V]': (56, 128)\n b_O: (7168,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (7168, 28672)\n b_in: (28672,)\n W_out: (28672, 7168)\n b_out: (7168,)\nln_final:\n '[w, b]': (7168,)\nunembed:\n W_U: (7168, 50272)\n b_U: (50272,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50272, 7168)"},"pos_embed":{"W_pos":"(2048, 7168)"},"blocks":{"[0-47]":{"ln1":{"[w, b]":"(7168,)"},"ln2":{"[w, b]":"(7168,)"},"attn":{"[W_Q, W_K, W_V]":"(56, 7168, 128)","W_O":"(56, 128, 7168)","[b_Q, b_K, b_V]":"(56, 128)","b_O":"(7168,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(7168, 28672)","b_in":"(28672,)","W_out":"(28672, 7168)","b_out":"(7168,)"}}},"ln_final":{"[w, b]":"(7168,)"},"unembed":{"W_U":"(7168, 50272)","b_U":"(50272,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-47]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 56, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 56, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 28672)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 7168)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\nunembed:\n hook_in: (batch, seq_len, 7168)\n hook_out: (batch, seq_len, 50272)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 7168)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-47]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 56, 128)","[hook_attn_scores, hook_pattern]":"(batch, 56, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 28672)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 7168)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"unembed":{"hook_in":"(batch, seq_len, 7168)","hook_out":"(batch, seq_len, 50272)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 7168)"}} -{"name.default_alias":"opt-6.7b","name.huggingface":"facebook\/opt-6.7b","name.aliases":"opt-6.7b, opt-xl","model_type":"opt","name.from_cfg":"opt-6.7b","n_params.as_str":"6.4B","n_params.as_int":6442450944,"n_params.from_name":"6.7b","cfg.n_params":6442450944,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50272,"cfg.act_fn":"relu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OPTForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":16384,"d_vocab":50272,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"opt-6.7b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OPTForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/opt-6.7b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50272,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":6442450944,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 16384\nd_vocab: 50272\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: opt-6.7b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OPTForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/opt-6.7b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50272\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 6442450944\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/opt-6.7b","tokenizer.vocab_size":50265.0,"tokenizer.max_len":null,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"f1FIzqnRiMYzke1CU0hp8TDxq7k=","tensor_shapes.state_dict":"embed:\n W_E: (50272, 4096)\npos_embed:\n W_pos: (2048, 4096)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50272)\n b_U: (50272,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50272, 4096)"},"pos_embed":{"W_pos":"(2048, 4096)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50272)","b_U":"(50272,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50272)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50272)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 4096)"}} -{"name.default_alias":"opt-66b","name.huggingface":"facebook\/opt-66b","name.aliases":"opt-66b, opt-xxxxl","model_type":"opt","name.from_cfg":"opt-66b","n_params.as_str":"65B","n_params.as_int":65229815808,"n_params.from_name":"66b","cfg.n_params":65229815808,"cfg.n_layers":64,"cfg.n_heads":72,"cfg.d_model":9216,"cfg.d_vocab":50272,"cfg.act_fn":"relu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OPTForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":9216,"d_head":128,"n_layers":64,"n_ctx":2048,"n_heads":72,"d_mlp":36864,"d_vocab":50272,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"opt-66b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OPTForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/opt-66b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0083333333,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50272,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":65229815808,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 9216\nd_head: 128\nn_layers: 64\nn_ctx: 2048\nn_heads: 72\nd_mlp: 36864\nd_vocab: 50272\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: opt-66b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OPTForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/opt-66b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008333333333333333\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50272\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 65229815808\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/opt-66b","tokenizer.vocab_size":50265.0,"tokenizer.max_len":null,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"f1FIzqnRiMYzke1CU0hp8TDxq7k=","tensor_shapes.state_dict":"embed:\n W_E: (50272, 9216)\npos_embed:\n W_pos: (2048, 9216)\nblocks:\n '[0-63]':\n ln1:\n '[w, b]': (9216,)\n ln2:\n '[w, b]': (9216,)\n attn:\n '[W_Q, W_K, W_V]': (72, 9216, 128)\n W_O: (72, 128, 9216)\n '[b_Q, b_K, b_V]': (72, 128)\n b_O: (9216,)\n mask: (2048, 2048)\n IGNORE: ()\n mlp:\n W_in: (9216, 36864)\n b_in: (36864,)\n W_out: (36864, 9216)\n b_out: (9216,)\nln_final:\n '[w, b]': (9216,)\nunembed:\n W_U: (9216, 50272)\n b_U: (50272,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50272, 9216)"},"pos_embed":{"W_pos":"(2048, 9216)"},"blocks":{"[0-63]":{"ln1":{"[w, b]":"(9216,)"},"ln2":{"[w, b]":"(9216,)"},"attn":{"[W_Q, W_K, W_V]":"(72, 9216, 128)","W_O":"(72, 128, 9216)","[b_Q, b_K, b_V]":"(72, 128)","b_O":"(9216,)","mask":"(2048, 2048)","IGNORE":"()"},"mlp":{"W_in":"(9216, 36864)","b_in":"(36864,)","W_out":"(36864, 9216)","b_out":"(9216,)"}}},"ln_final":{"[w, b]":"(9216,)"},"unembed":{"W_U":"(9216, 50272)","b_U":"(50272,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-63]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 9216)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 72, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 72, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 9216)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 36864)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 9216)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 9216)\nunembed:\n hook_in: (batch, seq_len, 9216)\n hook_out: (batch, seq_len, 50272)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 9216)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-63]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 9216)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 72, 128)","[hook_attn_scores, hook_pattern]":"(batch, 72, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 9216)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 36864)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 9216)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 9216)"},"unembed":{"hook_in":"(batch, seq_len, 9216)","hook_out":"(batch, seq_len, 50272)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 9216)"}} -{"name.default_alias":"wav2vec2-base","name.huggingface":"facebook\/wav2vec2-base","name.aliases":"wav2vec2-base, w2v2-base","model_type":null,"name.from_cfg":"wav2vec2-base","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":-1,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Wav2Vec2ForPreTraining","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":8192,"n_heads":12,"d_mlp":3072,"d_vocab":-1,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"wav2vec2-base","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Wav2Vec2ForPreTraining","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/wav2vec2-base","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":-1,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 8192\nn_heads: 12\nd_mlp: 3072\nd_vocab: -1\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: wav2vec2-base\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Wav2Vec2ForPreTraining\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/wav2vec2-base\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: -1\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/wav2vec2-base","tokenizer.vocab_size":32.0,"tokenizer.max_len":null,"tokenizer.class":"Wav2Vec2CTCTokenizer","tokenizer.vocab_hash":"tJ24E7IRYtfON5KH_y7lgJl6WV4=","tensor_shapes.state_dict":"embed:\n W_E: (32, 768)\npos_embed:\n W_pos: (8192, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (8192, 8192)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 32)\n b_U: (32,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32, 768)"},"pos_embed":{"W_pos":"(8192, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(8192, 8192)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 32)","b_U":"(32,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 32)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 32)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} -{"name.default_alias":"wav2vec2-large","name.huggingface":"facebook\/wav2vec2-large","name.aliases":"wav2vec2-large, w2v2-large","model_type":null,"name.from_cfg":"wav2vec2-large","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":-1,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Wav2Vec2ForPreTraining","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":8192,"n_heads":16,"d_mlp":4096,"d_vocab":-1,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"wav2vec2-large","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Wav2Vec2ForPreTraining","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/wav2vec2-large","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":-1,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 8192\nn_heads: 16\nd_mlp: 4096\nd_vocab: -1\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: wav2vec2-large\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Wav2Vec2ForPreTraining\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/wav2vec2-large\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: -1\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"bert-base-cased","name.huggingface":"google-bert\/bert-base-cased","name.aliases":"bert-base-cased","model_type":"bert","name.from_cfg":"bert-base-cased","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":28996,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BertForMaskedLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":512,"n_heads":12,"d_mlp":3072,"d_vocab":28996,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.0,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bert-base-cased","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BertForMaskedLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google-bert\/bert-base-cased","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":28996,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 512\nn_heads: 12\nd_mlp: 3072\nd_vocab: 28996\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-12\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bert-base-cased\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BertForMaskedLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google-bert\/bert-base-cased\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 28996\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"bert-base-uncased","name.huggingface":"google-bert\/bert-base-uncased","name.aliases":"bert-base-uncased","model_type":"bert","name.from_cfg":"bert-base-uncased","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":30522,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BertForMaskedLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":512,"n_heads":12,"d_mlp":3072,"d_vocab":30522,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.0,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bert-base-uncased","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BertForMaskedLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google-bert\/bert-base-uncased","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":30522,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 512\nn_heads: 12\nd_mlp: 3072\nd_vocab: 30522\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-12\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bert-base-uncased\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BertForMaskedLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google-bert\/bert-base-uncased\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 30522\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"bert-large-cased","name.huggingface":"google-bert\/bert-large-cased","name.aliases":"bert-large-cased","model_type":"bert","name.from_cfg":"bert-large-cased","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":28996,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BertForMaskedLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":512,"n_heads":16,"d_mlp":4096,"d_vocab":28996,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.0,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bert-large-cased","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BertForMaskedLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google-bert\/bert-large-cased","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":28996,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 512\nn_heads: 16\nd_mlp: 4096\nd_vocab: 28996\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-12\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bert-large-cased\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BertForMaskedLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google-bert\/bert-large-cased\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 28996\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"bert-large-uncased","name.huggingface":"google-bert\/bert-large-uncased","name.aliases":"bert-large-uncased","model_type":"bert","name.from_cfg":"bert-large-uncased","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":30522,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BertForMaskedLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":512,"n_heads":16,"d_mlp":4096,"d_vocab":30522,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.0,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bert-large-uncased","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BertForMaskedLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google-bert\/bert-large-uncased","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":30522,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 512\nn_heads: 16\nd_mlp: 4096\nd_vocab: 30522\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-12\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bert-large-uncased\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BertForMaskedLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google-bert\/bert-large-uncased\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 30522\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"t5-base","name.huggingface":"google-t5\/t5-base","name.aliases":"t5-base","model_type":"t5","name.from_cfg":"t5-base","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":32128,"cfg.act_fn":"relu","cfg.positional_embedding_type":"relative_positional_bias","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"T5ForConditionalGeneration","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":512,"n_heads":12,"d_mlp":3072,"d_vocab":32128,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"relative_positional_bias","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"t5-base","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"T5ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google-t5\/t5-base","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32128,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":128,"relative_attention_num_buckets":32,"decoder_start_token_id":0,"tie_word_embeddings":true,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 512\nn_heads: 12\nd_mlp: 3072\nd_vocab: 32128\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: relative_positional_bias\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: t5-base\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: T5ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google-t5\/t5-base\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32128\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: 128\nrelative_attention_num_buckets: 32\ndecoder_start_token_id: 0\ntie_word_embeddings: true\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"t5-large","name.huggingface":"google-t5\/t5-large","name.aliases":"t5-large","model_type":"t5","name.from_cfg":"t5-large","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":32128,"cfg.act_fn":"relu","cfg.positional_embedding_type":"relative_positional_bias","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"T5ForConditionalGeneration","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":512,"n_heads":16,"d_mlp":4096,"d_vocab":32128,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"relative_positional_bias","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"t5-large","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"T5ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google-t5\/t5-large","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32128,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":128,"relative_attention_num_buckets":32,"decoder_start_token_id":0,"tie_word_embeddings":true,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 512\nn_heads: 16\nd_mlp: 4096\nd_vocab: 32128\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: relative_positional_bias\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: t5-large\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: T5ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google-t5\/t5-large\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32128\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: 128\nrelative_attention_num_buckets: 32\ndecoder_start_token_id: 0\ntie_word_embeddings: true\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"t5-small","name.huggingface":"google-t5\/t5-small","name.aliases":"t5-small","model_type":"t5","name.from_cfg":"t5-small","n_params.as_str":"19M","n_params.as_int":18874368,"n_params.from_name":null,"cfg.n_params":18874368,"cfg.n_layers":6,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":32128,"cfg.act_fn":"relu","cfg.positional_embedding_type":"relative_positional_bias","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"T5ForConditionalGeneration","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":6,"n_ctx":512,"n_heads":8,"d_mlp":2048,"d_vocab":32128,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"relative_positional_bias","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"t5-small","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"T5ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google-t5\/t5-small","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32128,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":18874368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":128,"relative_attention_num_buckets":32,"decoder_start_token_id":0,"tie_word_embeddings":true,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 6\nn_ctx: 512\nn_heads: 8\nd_mlp: 2048\nd_vocab: 32128\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: relative_positional_bias\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: t5-small\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: T5ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google-t5\/t5-small\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32128\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 18874368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: 128\nrelative_attention_num_buckets: 32\ndecoder_start_token_id: 0\ntie_word_embeddings: true\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"gemma-2-27b","name.huggingface":"google\/gemma-2-27b","name.aliases":"gemma-2-27b","model_type":"gemma","name.from_cfg":"gemma-2-27b","n_params.as_str":"27B","n_params.as_int":26914848768,"n_params.from_name":"27b","cfg.n_params":26914848768,"cfg.n_layers":46,"cfg.n_heads":32,"cfg.d_model":4608,"cfg.d_vocab":256000,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4608,"d_head":128,"n_layers":46,"n_ctx":8192,"n_heads":32,"d_mlp":36864,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2-27b","use_attn_scale":true,"attn_scale":12.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2-27b","window_size":4096,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":26914848768,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":50.0,"output_logits_soft_cap":30.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4608\nd_head: 128\nn_layers: 46\nn_ctx: 8192\nn_heads: 32\nd_mlp: 36864\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2-27b\nuse_attn_scale: true\nattn_scale: 12.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2-27b\nwindow_size: 4096\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 26914848768\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: 50.0\noutput_logits_soft_cap: 30.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2-27b","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 4608)\nblocks:\n '[0-45]':\n ln1:\n w: (4608,)\n ln1_post:\n w: (4608,)\n ln2:\n w: (4608,)\n ln2_post:\n w: (4608,)\n attn:\n W_Q: (32, 4608, 128)\n W_O: (32, 128, 4608)\n b_Q: (32, 128)\n b_O: (4608,)\n '[_W_K, _W_V]': (16, 4608, 128)\n '[_b_K, _b_V]': (16, 128)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 128)\n mlp:\n '[W_in, W_gate]': (4608, 36864)\n W_out: (36864, 4608)\n b_in: (36864,)\n b_out: (4608,)\nln_final:\n w: (4608,)\nunembed:\n W_U: (4608, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 4608)"},"blocks":{"[0-45]":{"ln1":{"w":"(4608,)"},"ln1_post":{"w":"(4608,)"},"ln2":{"w":"(4608,)"},"ln2_post":{"w":"(4608,)"},"attn":{"W_Q":"(32, 4608, 128)","W_O":"(32, 128, 4608)","b_Q":"(32, 128)","b_O":"(4608,)","[_W_K, _W_V]":"(16, 4608, 128)","[_b_K, _b_V]":"(16, 128)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 128)"},"mlp":{"[W_in, W_gate]":"(4608, 36864)","W_out":"(36864, 4608)","b_in":"(36864,)","b_out":"(4608,)"}}},"ln_final":{"w":"(4608,)"},"unembed":{"W_U":"(4608, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-45]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 36864)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4608)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\nunembed:\n hook_in: (batch, seq_len, 4608)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 4608)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-45]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 36864)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4608)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"unembed":{"hook_in":"(batch, seq_len, 4608)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 4608)"}} -{"name.default_alias":"gemma-2-27b-it","name.huggingface":"google\/gemma-2-27b-it","name.aliases":"gemma-2-27b-it","model_type":"gemma","name.from_cfg":"gemma-2-27b-it","n_params.as_str":"27B","n_params.as_int":26914848768,"n_params.from_name":"27b","cfg.n_params":26914848768,"cfg.n_layers":46,"cfg.n_heads":32,"cfg.d_model":4608,"cfg.d_vocab":256000,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4608,"d_head":128,"n_layers":46,"n_ctx":8192,"n_heads":32,"d_mlp":36864,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2-27b-it","use_attn_scale":true,"attn_scale":12.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2-27b-it","window_size":4096,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":26914848768,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":50.0,"output_logits_soft_cap":30.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4608\nd_head: 128\nn_layers: 46\nn_ctx: 8192\nn_heads: 32\nd_mlp: 36864\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2-27b-it\nuse_attn_scale: true\nattn_scale: 12.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2-27b-it\nwindow_size: 4096\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 26914848768\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: 50.0\noutput_logits_soft_cap: 30.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2-27b-it","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 4608)\nblocks:\n '[0-45]':\n ln1:\n w: (4608,)\n ln1_post:\n w: (4608,)\n ln2:\n w: (4608,)\n ln2_post:\n w: (4608,)\n attn:\n W_Q: (32, 4608, 128)\n W_O: (32, 128, 4608)\n b_Q: (32, 128)\n b_O: (4608,)\n '[_W_K, _W_V]': (16, 4608, 128)\n '[_b_K, _b_V]': (16, 128)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 128)\n mlp:\n '[W_in, W_gate]': (4608, 36864)\n W_out: (36864, 4608)\n b_in: (36864,)\n b_out: (4608,)\nln_final:\n w: (4608,)\nunembed:\n W_U: (4608, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 4608)"},"blocks":{"[0-45]":{"ln1":{"w":"(4608,)"},"ln1_post":{"w":"(4608,)"},"ln2":{"w":"(4608,)"},"ln2_post":{"w":"(4608,)"},"attn":{"W_Q":"(32, 4608, 128)","W_O":"(32, 128, 4608)","b_Q":"(32, 128)","b_O":"(4608,)","[_W_K, _W_V]":"(16, 4608, 128)","[_b_K, _b_V]":"(16, 128)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 128)"},"mlp":{"[W_in, W_gate]":"(4608, 36864)","W_out":"(36864, 4608)","b_in":"(36864,)","b_out":"(4608,)"}}},"ln_final":{"w":"(4608,)"},"unembed":{"W_U":"(4608, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-45]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 36864)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4608)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\nunembed:\n hook_in: (batch, seq_len, 4608)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 4608)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-45]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 36864)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4608)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"unembed":{"hook_in":"(batch, seq_len, 4608)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 4608)"}} -{"name.default_alias":"gemma-2-2b","name.huggingface":"google\/gemma-2-2b","name.aliases":"gemma-2-2b","model_type":"gemma","name.from_cfg":"gemma-2-2b","n_params.as_str":"2.1B","n_params.as_int":2146959360,"n_params.from_name":"2b","cfg.n_params":2146959360,"cfg.n_layers":26,"cfg.n_heads":8,"cfg.d_model":2304,"cfg.d_vocab":256000,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2304,"d_head":256,"n_layers":26,"n_ctx":8192,"n_heads":8,"d_mlp":9216,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2-2b","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2-2b","window_size":4096,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":2146959360,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":50.0,"output_logits_soft_cap":30.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2304\nd_head: 256\nn_layers: 26\nn_ctx: 8192\nn_heads: 8\nd_mlp: 9216\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2-2b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2-2b\nwindow_size: 4096\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 2146959360\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: 50.0\noutput_logits_soft_cap: 30.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2-2b","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 2304)\nblocks:\n '[0-25]':\n ln1:\n w: (2304,)\n ln1_post:\n w: (2304,)\n ln2:\n w: (2304,)\n ln2_post:\n w: (2304,)\n attn:\n W_Q: (8, 2304, 256)\n W_O: (8, 256, 2304)\n b_Q: (8, 256)\n b_O: (2304,)\n '[_W_K, _W_V]': (4, 2304, 256)\n '[_b_K, _b_V]': (4, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (2304, 9216)\n W_out: (9216, 2304)\n b_in: (9216,)\n b_out: (2304,)\nln_final:\n w: (2304,)\nunembed:\n W_U: (2304, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 2304)"},"blocks":{"[0-25]":{"ln1":{"w":"(2304,)"},"ln1_post":{"w":"(2304,)"},"ln2":{"w":"(2304,)"},"ln2_post":{"w":"(2304,)"},"attn":{"W_Q":"(8, 2304, 256)","W_O":"(8, 256, 2304)","b_Q":"(8, 256)","b_O":"(2304,)","[_W_K, _W_V]":"(4, 2304, 256)","[_b_K, _b_V]":"(4, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(2304, 9216)","W_out":"(9216, 2304)","b_in":"(9216,)","b_out":"(2304,)"}}},"ln_final":{"w":"(2304,)"},"unembed":{"W_U":"(2304, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-25]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 8, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 9216)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2304)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\nunembed:\n hook_in: (batch, seq_len, 2304)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 2304)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-25]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 8, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 9216)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2304)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"unembed":{"hook_in":"(batch, seq_len, 2304)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 2304)"}} -{"name.default_alias":"gemma-2-2b-it","name.huggingface":"google\/gemma-2-2b-it","name.aliases":"gemma-2-2b-it","model_type":"gemma","name.from_cfg":"gemma-2-2b-it","n_params.as_str":"2.1B","n_params.as_int":2146959360,"n_params.from_name":"2b","cfg.n_params":2146959360,"cfg.n_layers":26,"cfg.n_heads":8,"cfg.d_model":2304,"cfg.d_vocab":256000,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2304,"d_head":256,"n_layers":26,"n_ctx":8192,"n_heads":8,"d_mlp":9216,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2-2b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2-2b-it","window_size":4096,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":2146959360,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":50.0,"output_logits_soft_cap":30.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2304\nd_head: 256\nn_layers: 26\nn_ctx: 8192\nn_heads: 8\nd_mlp: 9216\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2-2b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2-2b-it\nwindow_size: 4096\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 2146959360\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: 50.0\noutput_logits_soft_cap: 30.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2-2b-it","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 2304)\nblocks:\n '[0-25]':\n ln1:\n w: (2304,)\n ln1_post:\n w: (2304,)\n ln2:\n w: (2304,)\n ln2_post:\n w: (2304,)\n attn:\n W_Q: (8, 2304, 256)\n W_O: (8, 256, 2304)\n b_Q: (8, 256)\n b_O: (2304,)\n '[_W_K, _W_V]': (4, 2304, 256)\n '[_b_K, _b_V]': (4, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (2304, 9216)\n W_out: (9216, 2304)\n b_in: (9216,)\n b_out: (2304,)\nln_final:\n w: (2304,)\nunembed:\n W_U: (2304, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 2304)"},"blocks":{"[0-25]":{"ln1":{"w":"(2304,)"},"ln1_post":{"w":"(2304,)"},"ln2":{"w":"(2304,)"},"ln2_post":{"w":"(2304,)"},"attn":{"W_Q":"(8, 2304, 256)","W_O":"(8, 256, 2304)","b_Q":"(8, 256)","b_O":"(2304,)","[_W_K, _W_V]":"(4, 2304, 256)","[_b_K, _b_V]":"(4, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(2304, 9216)","W_out":"(9216, 2304)","b_in":"(9216,)","b_out":"(2304,)"}}},"ln_final":{"w":"(2304,)"},"unembed":{"W_U":"(2304, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-25]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 8, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 9216)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2304)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\nunembed:\n hook_in: (batch, seq_len, 2304)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 2304)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-25]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 8, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 9216)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2304)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"unembed":{"hook_in":"(batch, seq_len, 2304)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 2304)"}} -{"name.default_alias":"gemma-2-9b","name.huggingface":"google\/gemma-2-9b","name.aliases":"gemma-2-9b","model_type":"gemma","name.from_cfg":"gemma-2-9b","n_params.as_str":"8.9B","n_params.as_int":8940158976,"n_params.from_name":"9b","cfg.n_params":8940158976,"cfg.n_layers":42,"cfg.n_heads":16,"cfg.d_model":3584,"cfg.d_vocab":256000,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3584,"d_head":256,"n_layers":42,"n_ctx":8192,"n_heads":16,"d_mlp":14336,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2-9b","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2-9b","window_size":4096,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":8940158976,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":50.0,"output_logits_soft_cap":30.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 3584\nd_head: 256\nn_layers: 42\nn_ctx: 8192\nn_heads: 16\nd_mlp: 14336\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2-9b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2-9b\nwindow_size: 4096\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 8940158976\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: 50.0\noutput_logits_soft_cap: 30.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2-9b","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 3584)\nblocks:\n '[0-41]':\n ln1:\n w: (3584,)\n ln1_post:\n w: (3584,)\n ln2:\n w: (3584,)\n ln2_post:\n w: (3584,)\n attn:\n W_Q: (16, 3584, 256)\n W_O: (16, 256, 3584)\n b_Q: (16, 256)\n b_O: (3584,)\n '[_W_K, _W_V]': (8, 3584, 256)\n '[_b_K, _b_V]': (8, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (3584, 14336)\n W_out: (14336, 3584)\n b_in: (14336,)\n b_out: (3584,)\nln_final:\n w: (3584,)\nunembed:\n W_U: (3584, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 3584)"},"blocks":{"[0-41]":{"ln1":{"w":"(3584,)"},"ln1_post":{"w":"(3584,)"},"ln2":{"w":"(3584,)"},"ln2_post":{"w":"(3584,)"},"attn":{"W_Q":"(16, 3584, 256)","W_O":"(16, 256, 3584)","b_Q":"(16, 256)","b_O":"(3584,)","[_W_K, _W_V]":"(8, 3584, 256)","[_b_K, _b_V]":"(8, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(3584, 14336)","W_out":"(14336, 3584)","b_in":"(14336,)","b_out":"(3584,)"}}},"ln_final":{"w":"(3584,)"},"unembed":{"W_U":"(3584, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-41]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3584)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\nunembed:\n hook_in: (batch, seq_len, 3584)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 3584)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-41]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3584)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"unembed":{"hook_in":"(batch, seq_len, 3584)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 3584)"}} -{"name.default_alias":"gemma-2-9b-it","name.huggingface":"google\/gemma-2-9b-it","name.aliases":"gemma-2-9b-it","model_type":"gemma","name.from_cfg":"gemma-2-9b-it","n_params.as_str":"8.9B","n_params.as_int":8940158976,"n_params.from_name":"9b","cfg.n_params":8940158976,"cfg.n_layers":42,"cfg.n_heads":16,"cfg.d_model":3584,"cfg.d_vocab":256000,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3584,"d_head":256,"n_layers":42,"n_ctx":8192,"n_heads":16,"d_mlp":14336,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2-9b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2-9b-it","window_size":4096,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":8940158976,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":50.0,"output_logits_soft_cap":30.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 3584\nd_head: 256\nn_layers: 42\nn_ctx: 8192\nn_heads: 16\nd_mlp: 14336\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2-9b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2-9b-it\nwindow_size: 4096\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 8940158976\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: 50.0\noutput_logits_soft_cap: 30.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2-9b-it","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 3584)\nblocks:\n '[0-41]':\n ln1:\n w: (3584,)\n ln1_post:\n w: (3584,)\n ln2:\n w: (3584,)\n ln2_post:\n w: (3584,)\n attn:\n W_Q: (16, 3584, 256)\n W_O: (16, 256, 3584)\n b_Q: (16, 256)\n b_O: (3584,)\n '[_W_K, _W_V]': (8, 3584, 256)\n '[_b_K, _b_V]': (8, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (3584, 14336)\n W_out: (14336, 3584)\n b_in: (14336,)\n b_out: (3584,)\nln_final:\n w: (3584,)\nunembed:\n W_U: (3584, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 3584)"},"blocks":{"[0-41]":{"ln1":{"w":"(3584,)"},"ln1_post":{"w":"(3584,)"},"ln2":{"w":"(3584,)"},"ln2_post":{"w":"(3584,)"},"attn":{"W_Q":"(16, 3584, 256)","W_O":"(16, 256, 3584)","b_Q":"(16, 256)","b_O":"(3584,)","[_W_K, _W_V]":"(8, 3584, 256)","[_b_K, _b_V]":"(8, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(3584, 14336)","W_out":"(14336, 3584)","b_in":"(14336,)","b_out":"(3584,)"}}},"ln_final":{"w":"(3584,)"},"unembed":{"W_U":"(3584, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-41]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3584)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\nunembed:\n hook_in: (batch, seq_len, 3584)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 3584)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-41]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3584)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"unembed":{"hook_in":"(batch, seq_len, 3584)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 3584)"}} -{"name.default_alias":"gemma-2b","name.huggingface":"google\/gemma-2b","name.aliases":"gemma-2b","model_type":"gemma","name.from_cfg":"gemma-2b","n_params.as_str":"2.1B","n_params.as_int":2113929216,"n_params.from_name":"2b","cfg.n_params":2113929216,"cfg.n_layers":18,"cfg.n_heads":8,"cfg.d_model":2048,"cfg.d_vocab":256000,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GemmaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":256,"n_layers":18,"n_ctx":8192,"n_heads":8,"d_mlp":16384,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":1,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2b","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GemmaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":2113929216,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 256\nn_layers: 18\nn_ctx: 8192\nn_heads: 8\nd_mlp: 16384\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 1\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GemmaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 2113929216\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2b","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 2048)\nblocks:\n '[0-17]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n W_Q: (8, 2048, 256)\n W_O: (8, 256, 2048)\n b_Q: (8, 256)\n b_O: (2048,)\n '[_W_K, _W_V]': (1, 2048, 256)\n '[_b_K, _b_V]': (1, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (2048, 16384)\n W_out: (16384, 2048)\n b_in: (16384,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 2048)"},"blocks":{"[0-17]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"W_Q":"(8, 2048, 256)","W_O":"(8, 256, 2048)","b_Q":"(8, 256)","b_O":"(2048,)","[_W_K, _W_V]":"(1, 2048, 256)","[_b_K, _b_V]":"(1, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(2048, 16384)","W_out":"(16384, 2048)","b_in":"(16384,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-17]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 8, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 1, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-17]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 8, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 1, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"gemma-2b-it","name.huggingface":"google\/gemma-2b-it","name.aliases":"gemma-2b-it","model_type":"gemma","name.from_cfg":"gemma-2b-it","n_params.as_str":"2.1B","n_params.as_int":2113929216,"n_params.from_name":"2b","cfg.n_params":2113929216,"cfg.n_layers":18,"cfg.n_heads":8,"cfg.d_model":2048,"cfg.d_vocab":256000,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GemmaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":256,"n_layers":18,"n_ctx":8192,"n_heads":8,"d_mlp":16384,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":1,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GemmaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2b-it","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":2113929216,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 256\nn_layers: 18\nn_ctx: 8192\nn_heads: 8\nd_mlp: 16384\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 1\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GemmaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2b-it\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 2113929216\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2b-it","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 2048)\nblocks:\n '[0-17]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n W_Q: (8, 2048, 256)\n W_O: (8, 256, 2048)\n b_Q: (8, 256)\n b_O: (2048,)\n '[_W_K, _W_V]': (1, 2048, 256)\n '[_b_K, _b_V]': (1, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (2048, 16384)\n W_out: (16384, 2048)\n b_in: (16384,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 2048)"},"blocks":{"[0-17]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"W_Q":"(8, 2048, 256)","W_O":"(8, 256, 2048)","b_Q":"(8, 256)","b_O":"(2048,)","[_W_K, _W_V]":"(1, 2048, 256)","[_b_K, _b_V]":"(1, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(2048, 16384)","W_out":"(16384, 2048)","b_in":"(16384,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-17]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 8, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 1, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-17]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 8, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 1, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"gemma-3-12b-it","name.huggingface":"google\/gemma-3-12b-it","name.aliases":"gemma-3-12b-it","model_type":"gemma","name.from_cfg":"gemma-3-12b-it","n_params.as_str":"12B","n_params.as_int":11513364480,"n_params.from_name":"12b","cfg.n_params":11513364480,"cfg.n_layers":48,"cfg.n_heads":16,"cfg.d_model":3840,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":3840,"d_head":256,"n_layers":48,"n_ctx":8192,"n_heads":16,"d_mlp":15360,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-12b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-12b-it","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":11513364480,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 3840\nd_head: 256\nn_layers: 48\nn_ctx: 8192\nn_heads: 16\nd_mlp: 15360\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-12b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-12b-it\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 11513364480\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-12b-it","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262208, 3840)\nblocks:\n '[0-47]':\n ln1:\n w: (3840,)\n ln1_post:\n w: (3840,)\n ln2:\n w: (3840,)\n ln2_post:\n w: (3840,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (16, 3840, 256)\n W_O: (16, 256, 3840)\n b_Q: (16, 256)\n b_O: (3840,)\n '[_W_K, _W_V]': (8, 3840, 256)\n '[_b_K, _b_V]': (8, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (3840, 15360)\n W_out: (15360, 3840)\n b_in: (15360,)\n b_out: (3840,)\nln_final:\n w: (3840,)\nunembed:\n W_U: (3840, 262208)\n b_U: (262208,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262208, 3840)"},"blocks":{"[0-47]":{"ln1":{"w":"(3840,)"},"ln1_post":{"w":"(3840,)"},"ln2":{"w":"(3840,)"},"ln2_post":{"w":"(3840,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(16, 3840, 256)","W_O":"(16, 256, 3840)","b_Q":"(16, 256)","b_O":"(3840,)","[_W_K, _W_V]":"(8, 3840, 256)","[_b_K, _b_V]":"(8, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(3840, 15360)","W_out":"(15360, 3840)","b_in":"(15360,)","b_out":"(3840,)"}}},"ln_final":{"w":"(3840,)"},"unembed":{"W_U":"(3840, 262208)","b_U":"(262208,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-47]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n attn:\n q_norm:\n hook_scale: (110923120, 1)\n hook_normalized: (110923120, 256)\n k_norm:\n hook_scale: (55461560, 1)\n hook_normalized: (55461560, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 15360)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3840)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\nunembed:\n hook_in: (batch, seq_len, 3840)\n hook_out: (batch, seq_len, 262208)\nhook_embed: (batch, seq_len, 3840)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-47]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"attn":{"q_norm":{"hook_scale":"(110923120, 1)","hook_normalized":"(110923120, 256)"},"k_norm":{"hook_scale":"(55461560, 1)","hook_normalized":"(55461560, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 15360)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3840)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"unembed":{"hook_in":"(batch, seq_len, 3840)","hook_out":"(batch, seq_len, 262208)"},"hook_embed":"(batch, seq_len, 3840)"}} -{"name.default_alias":"gemma-3-12b-pt","name.huggingface":"google\/gemma-3-12b-pt","name.aliases":"gemma-3-12b-pt","model_type":"gemma","name.from_cfg":"gemma-3-12b-pt","n_params.as_str":"12B","n_params.as_int":11513364480,"n_params.from_name":"12b","cfg.n_params":11513364480,"cfg.n_layers":48,"cfg.n_heads":16,"cfg.d_model":3840,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":3840,"d_head":256,"n_layers":48,"n_ctx":8192,"n_heads":16,"d_mlp":15360,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-12b-pt","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-12b-pt","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":11513364480,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 3840\nd_head: 256\nn_layers: 48\nn_ctx: 8192\nn_heads: 16\nd_mlp: 15360\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-12b-pt\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-12b-pt\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 11513364480\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-12b-pt","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262208, 3840)\nblocks:\n '[0-47]':\n ln1:\n w: (3840,)\n ln1_post:\n w: (3840,)\n ln2:\n w: (3840,)\n ln2_post:\n w: (3840,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (16, 3840, 256)\n W_O: (16, 256, 3840)\n b_Q: (16, 256)\n b_O: (3840,)\n '[_W_K, _W_V]': (8, 3840, 256)\n '[_b_K, _b_V]': (8, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (3840, 15360)\n W_out: (15360, 3840)\n b_in: (15360,)\n b_out: (3840,)\nln_final:\n w: (3840,)\nunembed:\n W_U: (3840, 262208)\n b_U: (262208,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262208, 3840)"},"blocks":{"[0-47]":{"ln1":{"w":"(3840,)"},"ln1_post":{"w":"(3840,)"},"ln2":{"w":"(3840,)"},"ln2_post":{"w":"(3840,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(16, 3840, 256)","W_O":"(16, 256, 3840)","b_Q":"(16, 256)","b_O":"(3840,)","[_W_K, _W_V]":"(8, 3840, 256)","[_b_K, _b_V]":"(8, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(3840, 15360)","W_out":"(15360, 3840)","b_in":"(15360,)","b_out":"(3840,)"}}},"ln_final":{"w":"(3840,)"},"unembed":{"W_U":"(3840, 262208)","b_U":"(262208,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-47]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n attn:\n q_norm:\n hook_scale: (110923120, 1)\n hook_normalized: (110923120, 256)\n k_norm:\n hook_scale: (55461560, 1)\n hook_normalized: (55461560, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 15360)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3840)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\nunembed:\n hook_in: (batch, seq_len, 3840)\n hook_out: (batch, seq_len, 262208)\nhook_embed: (batch, seq_len, 3840)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-47]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"attn":{"q_norm":{"hook_scale":"(110923120, 1)","hook_normalized":"(110923120, 256)"},"k_norm":{"hook_scale":"(55461560, 1)","hook_normalized":"(55461560, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 15360)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3840)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"unembed":{"hook_in":"(batch, seq_len, 3840)","hook_out":"(batch, seq_len, 262208)"},"hook_embed":"(batch, seq_len, 3840)"}} -{"name.default_alias":"gemma-3-1b-it","name.huggingface":"google\/gemma-3-1b-it","name.aliases":"gemma-3-1b-it","model_type":"gemma","name.from_cfg":"gemma-3-1b-it","n_params.as_str":"744M","n_params.as_int":743768064,"n_params.from_name":"1b","cfg.n_params":743768064,"cfg.n_layers":26,"cfg.n_heads":4,"cfg.d_model":1152,"cfg.d_vocab":262144,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1152,"d_head":256,"n_layers":26,"n_ctx":8192,"n_heads":4,"d_mlp":6912,"d_vocab":262144,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":1,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-1b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-1b-it","window_size":512,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262144,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":743768064,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1152\nd_head: 256\nn_layers: 26\nn_ctx: 8192\nn_heads: 4\nd_mlp: 6912\nd_vocab: 262144\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 1\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-1b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-1b-it\nwindow_size: 512\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262144\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 743768064\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-1b-it","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262144, 1152)\nblocks:\n '[0-25]':\n ln1:\n w: (1152,)\n ln1_post:\n w: (1152,)\n ln2:\n w: (1152,)\n ln2_post:\n w: (1152,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (4, 1152, 256)\n W_O: (4, 256, 1152)\n b_Q: (4, 256)\n b_O: (1152,)\n '[_W_K, _W_V]': (1, 1152, 256)\n '[_b_K, _b_V]': (1, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (1152, 6912)\n W_out: (6912, 1152)\n b_in: (6912,)\n b_out: (1152,)\nln_final:\n w: (1152,)\nunembed:\n W_U: (1152, 262144)\n b_U: (262144,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262144, 1152)"},"blocks":{"[0-25]":{"ln1":{"w":"(1152,)"},"ln1_post":{"w":"(1152,)"},"ln2":{"w":"(1152,)"},"ln2_post":{"w":"(1152,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(4, 1152, 256)","W_O":"(4, 256, 1152)","b_Q":"(4, 256)","b_O":"(1152,)","[_W_K, _W_V]":"(1, 1152, 256)","[_b_K, _b_V]":"(1, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(1152, 6912)","W_out":"(6912, 1152)","b_in":"(6912,)","b_out":"(1152,)"}}},"ln_final":{"w":"(1152,)"},"unembed":{"W_U":"(1152, 262144)","b_U":"(262144,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-25]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n attn:\n q_norm:\n hook_scale: (27730780, 1)\n hook_normalized: (27730780, 256)\n k_norm:\n hook_scale: (6932695, 1)\n hook_normalized: (6932695, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 4, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 1, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 4, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 6912)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1152)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\nunembed:\n hook_in: (batch, seq_len, 1152)\n hook_out: (batch, seq_len, 262144)\nhook_embed: (batch, seq_len, 1152)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-25]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"attn":{"q_norm":{"hook_scale":"(27730780, 1)","hook_normalized":"(27730780, 256)"},"k_norm":{"hook_scale":"(6932695, 1)","hook_normalized":"(6932695, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 4, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 1, 256)","[hook_attn_scores, hook_pattern]":"(batch, 4, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 6912)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1152)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"unembed":{"hook_in":"(batch, seq_len, 1152)","hook_out":"(batch, seq_len, 262144)"},"hook_embed":"(batch, seq_len, 1152)"}} -{"name.default_alias":"gemma-3-1b-pt","name.huggingface":"google\/gemma-3-1b-pt","name.aliases":"gemma-3-1b-pt","model_type":"gemma","name.from_cfg":"gemma-3-1b-pt","n_params.as_str":"744M","n_params.as_int":743768064,"n_params.from_name":"1b","cfg.n_params":743768064,"cfg.n_layers":26,"cfg.n_heads":4,"cfg.d_model":1152,"cfg.d_vocab":262144,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1152,"d_head":256,"n_layers":26,"n_ctx":8192,"n_heads":4,"d_mlp":6912,"d_vocab":262144,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":1,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-1b-pt","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-1b-pt","window_size":512,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262144,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":743768064,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1152\nd_head: 256\nn_layers: 26\nn_ctx: 8192\nn_heads: 4\nd_mlp: 6912\nd_vocab: 262144\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 1\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-1b-pt\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-1b-pt\nwindow_size: 512\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262144\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 743768064\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-1b-pt","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262144, 1152)\nblocks:\n '[0-25]':\n ln1:\n w: (1152,)\n ln1_post:\n w: (1152,)\n ln2:\n w: (1152,)\n ln2_post:\n w: (1152,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (4, 1152, 256)\n W_O: (4, 256, 1152)\n b_Q: (4, 256)\n b_O: (1152,)\n '[_W_K, _W_V]': (1, 1152, 256)\n '[_b_K, _b_V]': (1, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (1152, 6912)\n W_out: (6912, 1152)\n b_in: (6912,)\n b_out: (1152,)\nln_final:\n w: (1152,)\nunembed:\n W_U: (1152, 262144)\n b_U: (262144,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262144, 1152)"},"blocks":{"[0-25]":{"ln1":{"w":"(1152,)"},"ln1_post":{"w":"(1152,)"},"ln2":{"w":"(1152,)"},"ln2_post":{"w":"(1152,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(4, 1152, 256)","W_O":"(4, 256, 1152)","b_Q":"(4, 256)","b_O":"(1152,)","[_W_K, _W_V]":"(1, 1152, 256)","[_b_K, _b_V]":"(1, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(1152, 6912)","W_out":"(6912, 1152)","b_in":"(6912,)","b_out":"(1152,)"}}},"ln_final":{"w":"(1152,)"},"unembed":{"W_U":"(1152, 262144)","b_U":"(262144,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-25]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n attn:\n q_norm:\n hook_scale: (27730780, 1)\n hook_normalized: (27730780, 256)\n k_norm:\n hook_scale: (6932695, 1)\n hook_normalized: (6932695, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 4, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 1, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 4, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 6912)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1152)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\nunembed:\n hook_in: (batch, seq_len, 1152)\n hook_out: (batch, seq_len, 262144)\nhook_embed: (batch, seq_len, 1152)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-25]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"attn":{"q_norm":{"hook_scale":"(27730780, 1)","hook_normalized":"(27730780, 256)"},"k_norm":{"hook_scale":"(6932695, 1)","hook_normalized":"(6932695, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 4, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 1, 256)","[hook_attn_scores, hook_pattern]":"(batch, 4, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 6912)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1152)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"unembed":{"hook_in":"(batch, seq_len, 1152)","hook_out":"(batch, seq_len, 262144)"},"hook_embed":"(batch, seq_len, 1152)"}} -{"name.default_alias":"gemma-3-270m","name.huggingface":"google\/gemma-3-270m","name.aliases":"gemma-3-270m","model_type":"gemma","name.from_cfg":"gemma-3-270m","n_params.as_str":"118M","n_params.as_int":117964800,"n_params.from_name":"270m","cfg.n_params":117964800,"cfg.n_layers":18,"cfg.n_heads":4,"cfg.d_model":640,"cfg.d_vocab":262144,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":640,"d_head":256,"n_layers":18,"n_ctx":8192,"n_heads":4,"d_mlp":2048,"d_vocab":262144,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":1,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-270m","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-270m","window_size":512,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262144,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":117964800,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 640\nd_head: 256\nn_layers: 18\nn_ctx: 8192\nn_heads: 4\nd_mlp: 2048\nd_vocab: 262144\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 1\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-270m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-270m\nwindow_size: 512\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262144\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 117964800\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-270m","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262144, 640)\nblocks:\n '[0-17]':\n ln1:\n w: (640,)\n ln1_post:\n w: (640,)\n ln2:\n w: (640,)\n ln2_post:\n w: (640,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (4, 640, 256)\n W_O: (4, 256, 640)\n b_Q: (4, 256)\n b_O: (640,)\n '[_W_K, _W_V]': (1, 640, 256)\n '[_b_K, _b_V]': (1, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (640, 2048)\n W_out: (2048, 640)\n b_in: (2048,)\n b_out: (640,)\nln_final:\n w: (640,)\nunembed:\n W_U: (640, 262144)\n b_U: (262144,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262144, 640)"},"blocks":{"[0-17]":{"ln1":{"w":"(640,)"},"ln1_post":{"w":"(640,)"},"ln2":{"w":"(640,)"},"ln2_post":{"w":"(640,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(4, 640, 256)","W_O":"(4, 256, 640)","b_Q":"(4, 256)","b_O":"(640,)","[_W_K, _W_V]":"(1, 640, 256)","[_b_K, _b_V]":"(1, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(640, 2048)","W_out":"(2048, 640)","b_in":"(2048,)","b_out":"(640,)"}}},"ln_final":{"w":"(640,)"},"unembed":{"W_U":"(640, 262144)","b_U":"(262144,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-17]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n attn:\n q_norm:\n hook_scale: (27730780, 1)\n hook_normalized: (27730780, 256)\n k_norm:\n hook_scale: (6932695, 1)\n hook_normalized: (6932695, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 4, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 1, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 4, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 2048)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 640)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\nunembed:\n hook_in: (batch, seq_len, 640)\n hook_out: (batch, seq_len, 262144)\nhook_embed: (batch, seq_len, 640)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-17]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"attn":{"q_norm":{"hook_scale":"(27730780, 1)","hook_normalized":"(27730780, 256)"},"k_norm":{"hook_scale":"(6932695, 1)","hook_normalized":"(6932695, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 4, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 1, 256)","[hook_attn_scores, hook_pattern]":"(batch, 4, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 2048)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 640)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"unembed":{"hook_in":"(batch, seq_len, 640)","hook_out":"(batch, seq_len, 262144)"},"hook_embed":"(batch, seq_len, 640)"}} -{"name.default_alias":"gemma-3-270m-it","name.huggingface":"google\/gemma-3-270m-it","name.aliases":"gemma-3-270m-it","model_type":"gemma","name.from_cfg":"gemma-3-270m-it","n_params.as_str":"118M","n_params.as_int":117964800,"n_params.from_name":"270m","cfg.n_params":117964800,"cfg.n_layers":18,"cfg.n_heads":4,"cfg.d_model":640,"cfg.d_vocab":262144,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":640,"d_head":256,"n_layers":18,"n_ctx":8192,"n_heads":4,"d_mlp":2048,"d_vocab":262144,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":1,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-270m-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-270m-it","window_size":512,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262144,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":117964800,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 640\nd_head: 256\nn_layers: 18\nn_ctx: 8192\nn_heads: 4\nd_mlp: 2048\nd_vocab: 262144\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 1\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-270m-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-270m-it\nwindow_size: 512\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262144\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 117964800\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-270m-it","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262144, 640)\nblocks:\n '[0-17]':\n ln1:\n w: (640,)\n ln1_post:\n w: (640,)\n ln2:\n w: (640,)\n ln2_post:\n w: (640,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (4, 640, 256)\n W_O: (4, 256, 640)\n b_Q: (4, 256)\n b_O: (640,)\n '[_W_K, _W_V]': (1, 640, 256)\n '[_b_K, _b_V]': (1, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (640, 2048)\n W_out: (2048, 640)\n b_in: (2048,)\n b_out: (640,)\nln_final:\n w: (640,)\nunembed:\n W_U: (640, 262144)\n b_U: (262144,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262144, 640)"},"blocks":{"[0-17]":{"ln1":{"w":"(640,)"},"ln1_post":{"w":"(640,)"},"ln2":{"w":"(640,)"},"ln2_post":{"w":"(640,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(4, 640, 256)","W_O":"(4, 256, 640)","b_Q":"(4, 256)","b_O":"(640,)","[_W_K, _W_V]":"(1, 640, 256)","[_b_K, _b_V]":"(1, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(640, 2048)","W_out":"(2048, 640)","b_in":"(2048,)","b_out":"(640,)"}}},"ln_final":{"w":"(640,)"},"unembed":{"W_U":"(640, 262144)","b_U":"(262144,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-17]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n attn:\n q_norm:\n hook_scale: (27730780, 1)\n hook_normalized: (27730780, 256)\n k_norm:\n hook_scale: (6932695, 1)\n hook_normalized: (6932695, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 4, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 1, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 4, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 2048)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 640)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\nunembed:\n hook_in: (batch, seq_len, 640)\n hook_out: (batch, seq_len, 262144)\nhook_embed: (batch, seq_len, 640)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-17]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"attn":{"q_norm":{"hook_scale":"(27730780, 1)","hook_normalized":"(27730780, 256)"},"k_norm":{"hook_scale":"(6932695, 1)","hook_normalized":"(6932695, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 4, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 1, 256)","[hook_attn_scores, hook_pattern]":"(batch, 4, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 2048)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 640)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"unembed":{"hook_in":"(batch, seq_len, 640)","hook_out":"(batch, seq_len, 262144)"},"hook_embed":"(batch, seq_len, 640)"}} -{"name.default_alias":"gemma-3-27b-it","name.huggingface":"google\/gemma-3-27b-it","name.aliases":"gemma-3-27b-it","model_type":"gemma","name.from_cfg":"gemma-3-27b-it","n_params.as_str":"27B","n_params.as_int":26963607552,"n_params.from_name":"27b","cfg.n_params":26963607552,"cfg.n_layers":62,"cfg.n_heads":32,"cfg.d_model":5376,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":5376,"d_head":128,"n_layers":62,"n_ctx":8192,"n_heads":32,"d_mlp":21504,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-27b-it","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-27b-it","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":26963607552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5376\nd_head: 128\nn_layers: 62\nn_ctx: 8192\nn_heads: 32\nd_mlp: 21504\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-27b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-27b-it\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 26963607552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-27b-it","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262208, 5376)\nblocks:\n '[0-61]':\n ln1:\n w: (5376,)\n ln1_post:\n w: (5376,)\n ln2:\n w: (5376,)\n ln2_post:\n w: (5376,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (32, 5376, 128)\n W_O: (32, 128, 5376)\n b_Q: (32, 128)\n b_O: (5376,)\n '[_W_K, _W_V]': (16, 5376, 128)\n '[_b_K, _b_V]': (16, 128)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 128)\n mlp:\n '[W_in, W_gate]': (5376, 21504)\n W_out: (21504, 5376)\n b_in: (21504,)\n b_out: (5376,)\nln_final:\n w: (5376,)\nunembed:\n W_U: (5376, 262208)\n b_U: (262208,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262208, 5376)"},"blocks":{"[0-61]":{"ln1":{"w":"(5376,)"},"ln1_post":{"w":"(5376,)"},"ln2":{"w":"(5376,)"},"ln2_post":{"w":"(5376,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(32, 5376, 128)","W_O":"(32, 128, 5376)","b_Q":"(32, 128)","b_O":"(5376,)","[_W_K, _W_V]":"(16, 5376, 128)","[_b_K, _b_V]":"(16, 128)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 128)"},"mlp":{"[W_in, W_gate]":"(5376, 21504)","W_out":"(21504, 5376)","b_in":"(21504,)","b_out":"(5376,)"}}},"ln_final":{"w":"(5376,)"},"unembed":{"W_U":"(5376, 262208)","b_U":"(262208,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-61]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n attn:\n q_norm:\n hook_scale: (221846240, 1)\n hook_normalized: (221846240, 128)\n k_norm:\n hook_scale: (110923120, 1)\n hook_normalized: (110923120, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 21504)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5376)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\nunembed:\n hook_in: (batch, seq_len, 5376)\n hook_out: (batch, seq_len, 262208)\nhook_embed: (batch, seq_len, 5376)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-61]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"attn":{"q_norm":{"hook_scale":"(221846240, 1)","hook_normalized":"(221846240, 128)"},"k_norm":{"hook_scale":"(110923120, 1)","hook_normalized":"(110923120, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 21504)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5376)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"unembed":{"hook_in":"(batch, seq_len, 5376)","hook_out":"(batch, seq_len, 262208)"},"hook_embed":"(batch, seq_len, 5376)"}} -{"name.default_alias":"gemma-3-27b-pt","name.huggingface":"google\/gemma-3-27b-pt","name.aliases":"gemma-3-27b-pt","model_type":"gemma","name.from_cfg":"gemma-3-27b-pt","n_params.as_str":"27B","n_params.as_int":26963607552,"n_params.from_name":"27b","cfg.n_params":26963607552,"cfg.n_layers":62,"cfg.n_heads":32,"cfg.d_model":5376,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":5376,"d_head":128,"n_layers":62,"n_ctx":8192,"n_heads":32,"d_mlp":21504,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-27b-pt","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-27b-pt","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":26963607552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5376\nd_head: 128\nn_layers: 62\nn_ctx: 8192\nn_heads: 32\nd_mlp: 21504\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-27b-pt\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-27b-pt\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 26963607552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-27b-pt","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262208, 5376)\nblocks:\n '[0-61]':\n ln1:\n w: (5376,)\n ln1_post:\n w: (5376,)\n ln2:\n w: (5376,)\n ln2_post:\n w: (5376,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (32, 5376, 128)\n W_O: (32, 128, 5376)\n b_Q: (32, 128)\n b_O: (5376,)\n '[_W_K, _W_V]': (16, 5376, 128)\n '[_b_K, _b_V]': (16, 128)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 128)\n mlp:\n '[W_in, W_gate]': (5376, 21504)\n W_out: (21504, 5376)\n b_in: (21504,)\n b_out: (5376,)\nln_final:\n w: (5376,)\nunembed:\n W_U: (5376, 262208)\n b_U: (262208,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262208, 5376)"},"blocks":{"[0-61]":{"ln1":{"w":"(5376,)"},"ln1_post":{"w":"(5376,)"},"ln2":{"w":"(5376,)"},"ln2_post":{"w":"(5376,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(32, 5376, 128)","W_O":"(32, 128, 5376)","b_Q":"(32, 128)","b_O":"(5376,)","[_W_K, _W_V]":"(16, 5376, 128)","[_b_K, _b_V]":"(16, 128)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 128)"},"mlp":{"[W_in, W_gate]":"(5376, 21504)","W_out":"(21504, 5376)","b_in":"(21504,)","b_out":"(5376,)"}}},"ln_final":{"w":"(5376,)"},"unembed":{"W_U":"(5376, 262208)","b_U":"(262208,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-61]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n attn:\n q_norm:\n hook_scale: (221846240, 1)\n hook_normalized: (221846240, 128)\n k_norm:\n hook_scale: (110923120, 1)\n hook_normalized: (110923120, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 21504)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5376)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\nunembed:\n hook_in: (batch, seq_len, 5376)\n hook_out: (batch, seq_len, 262208)\nhook_embed: (batch, seq_len, 5376)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-61]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"attn":{"q_norm":{"hook_scale":"(221846240, 1)","hook_normalized":"(221846240, 128)"},"k_norm":{"hook_scale":"(110923120, 1)","hook_normalized":"(110923120, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 21504)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5376)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"unembed":{"hook_in":"(batch, seq_len, 5376)","hook_out":"(batch, seq_len, 262208)"},"hook_embed":"(batch, seq_len, 5376)"}} -{"name.default_alias":"gemma-3-4b-it","name.huggingface":"google\/gemma-3-4b-it","name.aliases":"gemma-3-4b-it","model_type":"gemma","name.from_cfg":"gemma-3-4b-it","n_params.as_str":"3.4B","n_params.as_int":3386900480,"n_params.from_name":"4b","cfg.n_params":3386900480,"cfg.n_layers":34,"cfg.n_heads":8,"cfg.d_model":2560,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":2560,"d_head":256,"n_layers":34,"n_ctx":8192,"n_heads":8,"d_mlp":10240,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-4b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-4b-it","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":3386900480,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 256\nn_layers: 34\nn_ctx: 8192\nn_heads: 8\nd_mlp: 10240\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-4b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-4b-it\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 3386900480\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-4b-it","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262208, 2560)\nblocks:\n '[0-33]':\n ln1:\n w: (2560,)\n ln1_post:\n w: (2560,)\n ln2:\n w: (2560,)\n ln2_post:\n w: (2560,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (8, 2560, 256)\n W_O: (8, 256, 2560)\n b_Q: (8, 256)\n b_O: (2560,)\n '[_W_K, _W_V]': (4, 2560, 256)\n '[_b_K, _b_V]': (4, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (2560, 10240)\n W_out: (10240, 2560)\n b_in: (10240,)\n b_out: (2560,)\nln_final:\n w: (2560,)\nunembed:\n W_U: (2560, 262208)\n b_U: (262208,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262208, 2560)"},"blocks":{"[0-33]":{"ln1":{"w":"(2560,)"},"ln1_post":{"w":"(2560,)"},"ln2":{"w":"(2560,)"},"ln2_post":{"w":"(2560,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(8, 2560, 256)","W_O":"(8, 256, 2560)","b_Q":"(8, 256)","b_O":"(2560,)","[_W_K, _W_V]":"(4, 2560, 256)","[_b_K, _b_V]":"(4, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(2560, 10240)","W_out":"(10240, 2560)","b_in":"(10240,)","b_out":"(2560,)"}}},"ln_final":{"w":"(2560,)"},"unembed":{"W_U":"(2560, 262208)","b_U":"(262208,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-33]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n q_norm:\n hook_scale: (55461560, 1)\n hook_normalized: (55461560, 256)\n k_norm:\n hook_scale: (27730780, 1)\n hook_normalized: (27730780, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 8, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 10240)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 262208)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-33]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"q_norm":{"hook_scale":"(55461560, 1)","hook_normalized":"(55461560, 256)"},"k_norm":{"hook_scale":"(27730780, 1)","hook_normalized":"(27730780, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 8, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 10240)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 262208)"},"hook_embed":"(batch, seq_len, 2560)"}} -{"name.default_alias":"gemma-3-4b-pt","name.huggingface":"google\/gemma-3-4b-pt","name.aliases":"gemma-3-4b-pt","model_type":"gemma","name.from_cfg":"gemma-3-4b-pt","n_params.as_str":"3.4B","n_params.as_int":3386900480,"n_params.from_name":"4b","cfg.n_params":3386900480,"cfg.n_layers":34,"cfg.n_heads":8,"cfg.d_model":2560,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":2560,"d_head":256,"n_layers":34,"n_ctx":8192,"n_heads":8,"d_mlp":10240,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-4b-pt","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-4b-pt","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":3386900480,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 256\nn_layers: 34\nn_ctx: 8192\nn_heads: 8\nd_mlp: 10240\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-4b-pt\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-4b-pt\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 3386900480\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-4b-pt","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262208, 2560)\nblocks:\n '[0-33]':\n ln1:\n w: (2560,)\n ln1_post:\n w: (2560,)\n ln2:\n w: (2560,)\n ln2_post:\n w: (2560,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (8, 2560, 256)\n W_O: (8, 256, 2560)\n b_Q: (8, 256)\n b_O: (2560,)\n '[_W_K, _W_V]': (4, 2560, 256)\n '[_b_K, _b_V]': (4, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (2560, 10240)\n W_out: (10240, 2560)\n b_in: (10240,)\n b_out: (2560,)\nln_final:\n w: (2560,)\nunembed:\n W_U: (2560, 262208)\n b_U: (262208,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262208, 2560)"},"blocks":{"[0-33]":{"ln1":{"w":"(2560,)"},"ln1_post":{"w":"(2560,)"},"ln2":{"w":"(2560,)"},"ln2_post":{"w":"(2560,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(8, 2560, 256)","W_O":"(8, 256, 2560)","b_Q":"(8, 256)","b_O":"(2560,)","[_W_K, _W_V]":"(4, 2560, 256)","[_b_K, _b_V]":"(4, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(2560, 10240)","W_out":"(10240, 2560)","b_in":"(10240,)","b_out":"(2560,)"}}},"ln_final":{"w":"(2560,)"},"unembed":{"W_U":"(2560, 262208)","b_U":"(262208,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-33]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n q_norm:\n hook_scale: (55461560, 1)\n hook_normalized: (55461560, 256)\n k_norm:\n hook_scale: (27730780, 1)\n hook_normalized: (27730780, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 8, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 10240)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 262208)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-33]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"q_norm":{"hook_scale":"(55461560, 1)","hook_normalized":"(55461560, 256)"},"k_norm":{"hook_scale":"(27730780, 1)","hook_normalized":"(27730780, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 8, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 10240)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 262208)"},"hook_embed":"(batch, seq_len, 2560)"}} -{"name.default_alias":"gemma-7b","name.huggingface":"google\/gemma-7b","name.aliases":"gemma-7b","model_type":"gemma","name.from_cfg":"gemma-7b","n_params.as_str":"7.8B","n_params.as_int":7751073792,"n_params.from_name":"7b","cfg.n_params":7751073792,"cfg.n_layers":28,"cfg.n_heads":16,"cfg.d_model":3072,"cfg.d_vocab":256000,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GemmaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3072,"d_head":256,"n_layers":28,"n_ctx":8192,"n_heads":16,"d_mlp":24576,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-7b","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GemmaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-7b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":7751073792,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 3072\nd_head: 256\nn_layers: 28\nn_ctx: 8192\nn_heads: 16\nd_mlp: 24576\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-7b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GemmaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-7b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 7751073792\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-7b","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 3072)\nblocks:\n '[0-27]':\n ln1:\n w: (3072,)\n ln2:\n w: (3072,)\n attn:\n '[W_Q, _W_K, _W_V]': (16, 3072, 256)\n W_O: (16, 256, 3072)\n '[b_Q, _b_K, _b_V]': (16, 256)\n b_O: (3072,)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (3072, 24576)\n W_out: (24576, 3072)\n b_in: (24576,)\n b_out: (3072,)\nln_final:\n w: (3072,)\nunembed:\n W_U: (3072, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 3072)"},"blocks":{"[0-27]":{"ln1":{"w":"(3072,)"},"ln2":{"w":"(3072,)"},"attn":{"[W_Q, _W_K, _W_V]":"(16, 3072, 256)","W_O":"(16, 256, 3072)","[b_Q, _b_K, _b_V]":"(16, 256)","b_O":"(3072,)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(3072, 24576)","W_out":"(24576, 3072)","b_in":"(24576,)","b_out":"(3072,)"}}},"ln_final":{"w":"(3072,)"},"unembed":{"W_U":"(3072, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 24576)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3072)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\nunembed:\n hook_in: (batch, seq_len, 3072)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 3072)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 256)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 24576)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3072)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"unembed":{"hook_in":"(batch, seq_len, 3072)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 3072)"}} -{"name.default_alias":"gemma-7b-it","name.huggingface":"google\/gemma-7b-it","name.aliases":"gemma-7b-it","model_type":"gemma","name.from_cfg":"gemma-7b-it","n_params.as_str":"7.8B","n_params.as_int":7751073792,"n_params.from_name":"7b","cfg.n_params":7751073792,"cfg.n_layers":28,"cfg.n_heads":16,"cfg.d_model":3072,"cfg.d_vocab":256000,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GemmaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3072,"d_head":256,"n_layers":28,"n_ctx":8192,"n_heads":16,"d_mlp":24576,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-7b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GemmaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-7b-it","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":7751073792,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 3072\nd_head: 256\nn_layers: 28\nn_ctx: 8192\nn_heads: 16\nd_mlp: 24576\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-7b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GemmaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-7b-it\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 7751073792\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-7b-it","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 3072)\nblocks:\n '[0-27]':\n ln1:\n w: (3072,)\n ln2:\n w: (3072,)\n attn:\n '[W_Q, _W_K, _W_V]': (16, 3072, 256)\n W_O: (16, 256, 3072)\n '[b_Q, _b_K, _b_V]': (16, 256)\n b_O: (3072,)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (3072, 24576)\n W_out: (24576, 3072)\n b_in: (24576,)\n b_out: (3072,)\nln_final:\n w: (3072,)\nunembed:\n W_U: (3072, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 3072)"},"blocks":{"[0-27]":{"ln1":{"w":"(3072,)"},"ln2":{"w":"(3072,)"},"attn":{"[W_Q, _W_K, _W_V]":"(16, 3072, 256)","W_O":"(16, 256, 3072)","[b_Q, _b_K, _b_V]":"(16, 256)","b_O":"(3072,)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(3072, 24576)","W_out":"(24576, 3072)","b_in":"(24576,)","b_out":"(3072,)"}}},"ln_final":{"w":"(3072,)"},"unembed":{"W_U":"(3072, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 24576)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3072)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\nunembed:\n hook_in: (batch, seq_len, 3072)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 3072)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 256)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 24576)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3072)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"unembed":{"hook_in":"(batch, seq_len, 3072)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 3072)"}} -{"name.default_alias":"medgemma-27b-it","name.huggingface":"google\/medgemma-27b-it","name.aliases":"medgemma-27b-it","model_type":"gemma","name.from_cfg":"medgemma-27b-it","n_params.as_str":"27B","n_params.as_int":26963607552,"n_params.from_name":"27b","cfg.n_params":26963607552,"cfg.n_layers":62,"cfg.n_heads":32,"cfg.d_model":5376,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":5376,"d_head":128,"n_layers":62,"n_ctx":8192,"n_heads":32,"d_mlp":21504,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"medgemma-27b-it","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/medgemma-27b-it","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":26963607552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5376\nd_head: 128\nn_layers: 62\nn_ctx: 8192\nn_heads: 32\nd_mlp: 21504\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: medgemma-27b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/medgemma-27b-it\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 26963607552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"medgemma-27b-text-it","name.huggingface":"google\/medgemma-27b-text-it","name.aliases":"medgemma-27b-text-it","model_type":"gemma","name.from_cfg":"medgemma-27b-text-it","n_params.as_str":"27B","n_params.as_int":26963607552,"n_params.from_name":"27b","cfg.n_params":26963607552,"cfg.n_layers":62,"cfg.n_heads":32,"cfg.d_model":5376,"cfg.d_vocab":262144,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5376,"d_head":128,"n_layers":62,"n_ctx":8192,"n_heads":32,"d_mlp":21504,"d_vocab":262144,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"medgemma-27b-text-it","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/medgemma-27b-text-it","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262144,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":26963607552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5376\nd_head: 128\nn_layers: 62\nn_ctx: 8192\nn_heads: 32\nd_mlp: 21504\nd_vocab: 262144\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: medgemma-27b-text-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/medgemma-27b-text-it\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262144\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 26963607552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"medgemma-4b-it","name.huggingface":"google\/medgemma-4b-it","name.aliases":"medgemma-4b-it","model_type":"gemma","name.from_cfg":"medgemma-4b-it","n_params.as_str":"3.4B","n_params.as_int":3386900480,"n_params.from_name":"4b","cfg.n_params":3386900480,"cfg.n_layers":34,"cfg.n_heads":8,"cfg.d_model":2560,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":2560,"d_head":256,"n_layers":34,"n_ctx":8192,"n_heads":8,"d_mlp":10240,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"medgemma-4b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/medgemma-4b-it","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":3386900480,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 256\nn_layers: 34\nn_ctx: 8192\nn_heads: 8\nd_mlp: 10240\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: medgemma-4b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/medgemma-4b-it\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 3386900480\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"medgemma-4b-pt","name.huggingface":"google\/medgemma-4b-pt","name.aliases":"medgemma-4b-pt","model_type":"gemma","name.from_cfg":"medgemma-4b-pt","n_params.as_str":"3.4B","n_params.as_int":3386900480,"n_params.from_name":"4b","cfg.n_params":3386900480,"cfg.n_layers":34,"cfg.n_heads":8,"cfg.d_model":2560,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":2560,"d_head":256,"n_layers":34,"n_ctx":8192,"n_heads":8,"d_mlp":10240,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"medgemma-4b-pt","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/medgemma-4b-pt","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":3386900480,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 256\nn_layers: 34\nn_ctx: 8192\nn_heads: 8\nd_mlp: 10240\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: medgemma-4b-pt\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/medgemma-4b-pt\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 3386900480\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"google\/medgemma-4b-pt","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262208, 2560)\nblocks:\n '[0-33]':\n ln1:\n w: (2560,)\n ln1_post:\n w: (2560,)\n ln2:\n w: (2560,)\n ln2_post:\n w: (2560,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (8, 2560, 256)\n W_O: (8, 256, 2560)\n b_Q: (8, 256)\n b_O: (2560,)\n '[_W_K, _W_V]': (4, 2560, 256)\n '[_b_K, _b_V]': (4, 256)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 256)\n mlp:\n '[W_in, W_gate]': (2560, 10240)\n W_out: (10240, 2560)\n b_in: (10240,)\n b_out: (2560,)\nln_final:\n w: (2560,)\nunembed:\n W_U: (2560, 262208)\n b_U: (262208,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262208, 2560)"},"blocks":{"[0-33]":{"ln1":{"w":"(2560,)"},"ln1_post":{"w":"(2560,)"},"ln2":{"w":"(2560,)"},"ln2_post":{"w":"(2560,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(8, 2560, 256)","W_O":"(8, 256, 2560)","b_Q":"(8, 256)","b_O":"(2560,)","[_W_K, _W_V]":"(4, 2560, 256)","[_b_K, _b_V]":"(4, 256)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 256)"},"mlp":{"[W_in, W_gate]":"(2560, 10240)","W_out":"(10240, 2560)","b_in":"(10240,)","b_out":"(2560,)"}}},"ln_final":{"w":"(2560,)"},"unembed":{"W_U":"(2560, 262208)","b_U":"(262208,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-33]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n q_norm:\n hook_scale: (55461560, 1)\n hook_normalized: (55461560, 256)\n k_norm:\n hook_scale: (27730780, 1)\n hook_normalized: (27730780, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 8, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 10240)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 262208)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-33]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"q_norm":{"hook_scale":"(55461560, 1)","hook_normalized":"(55461560, 256)"},"k_norm":{"hook_scale":"(27730780, 1)","hook_normalized":"(27730780, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 8, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 10240)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 262208)"},"hook_embed":"(batch, seq_len, 2560)"}} -{"name.default_alias":"gpt2-small","name.huggingface":"gpt2","name.aliases":"gpt2-small","model_type":"gpt2","name.from_cfg":"gpt2","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt2","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"gpt2","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt2\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: gpt2\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"gpt2","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} -{"name.default_alias":"gpt2-large","name.huggingface":"gpt2-large","name.aliases":"","model_type":"gpt2","name.from_cfg":"gpt2-large","n_params.as_str":"708M","n_params.as_int":707788800,"n_params.from_name":null,"cfg.n_params":707788800,"cfg.n_layers":36,"cfg.n_heads":20,"cfg.d_model":1280,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1280,"d_head":64,"n_layers":36,"n_ctx":1024,"n_heads":20,"d_mlp":5120,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt2-large","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"gpt2-large","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0223606798,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":707788800,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1280\nd_head: 64\nn_layers: 36\nn_ctx: 1024\nn_heads: 20\nd_mlp: 5120\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt2-large\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: gpt2-large\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.022360679774997897\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 707788800\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"gpt2-large","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1280)\npos_embed:\n W_pos: (1024, 1280)\nblocks:\n '[0-35]':\n ln1:\n '[w, b]': (1280,)\n ln2:\n '[w, b]': (1280,)\n attn:\n '[W_Q, W_K, W_V]': (20, 1280, 64)\n W_O: (20, 64, 1280)\n '[b_Q, b_K, b_V]': (20, 64)\n b_O: (1280,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (1280, 5120)\n b_in: (5120,)\n W_out: (5120, 1280)\n b_out: (1280,)\nln_final:\n '[w, b]': (1280,)\nunembed:\n W_U: (1280, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1280)"},"pos_embed":{"W_pos":"(1024, 1280)"},"blocks":{"[0-35]":{"ln1":{"[w, b]":"(1280,)"},"ln2":{"[w, b]":"(1280,)"},"attn":{"[W_Q, W_K, W_V]":"(20, 1280, 64)","W_O":"(20, 64, 1280)","[b_Q, b_K, b_V]":"(20, 64)","b_O":"(1280,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(1280, 5120)","b_in":"(5120,)","W_out":"(5120, 1280)","b_out":"(1280,)"}}},"ln_final":{"[w, b]":"(1280,)"},"unembed":{"W_U":"(1280, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 20, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 20, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 5120)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1280)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\nunembed:\n hook_in: (batch, seq_len, 1280)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1280)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 20, 64)","[hook_attn_scores, hook_pattern]":"(batch, 20, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 5120)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1280)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"unembed":{"hook_in":"(batch, seq_len, 1280)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1280)"}} -{"name.default_alias":"gpt2-medium","name.huggingface":"gpt2-medium","name.aliases":"","model_type":"gpt2","name.from_cfg":"gpt2-medium","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt2-medium","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"gpt2-medium","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt2-medium\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: gpt2-medium\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"gpt2-medium","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} -{"name.default_alias":"gpt2-xl","name.huggingface":"gpt2-xl","name.aliases":"","model_type":"gpt2","name.from_cfg":"gpt2-xl","n_params.as_str":"1.5B","n_params.as_int":1474560000,"n_params.from_name":null,"cfg.n_params":1474560000,"cfg.n_layers":48,"cfg.n_heads":25,"cfg.d_model":1600,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1600,"d_head":64,"n_layers":48,"n_ctx":1024,"n_heads":25,"d_mlp":6400,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt2-xl","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"gpt2-xl","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1474560000,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1600\nd_head: 64\nn_layers: 48\nn_ctx: 1024\nn_heads: 25\nd_mlp: 6400\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt2-xl\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: gpt2-xl\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1474560000\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"gpt2-xl","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1600)\npos_embed:\n W_pos: (1024, 1600)\nblocks:\n '[0-47]':\n ln1:\n '[w, b]': (1600,)\n ln2:\n '[w, b]': (1600,)\n attn:\n '[W_Q, W_K, W_V]': (25, 1600, 64)\n W_O: (25, 64, 1600)\n '[b_Q, b_K, b_V]': (25, 64)\n b_O: (1600,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (1600, 6400)\n b_in: (6400,)\n W_out: (6400, 1600)\n b_out: (1600,)\nln_final:\n '[w, b]': (1600,)\nunembed:\n W_U: (1600, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1600)"},"pos_embed":{"W_pos":"(1024, 1600)"},"blocks":{"[0-47]":{"ln1":{"[w, b]":"(1600,)"},"ln2":{"[w, b]":"(1600,)"},"attn":{"[W_Q, W_K, W_V]":"(25, 1600, 64)","W_O":"(25, 64, 1600)","[b_Q, b_K, b_V]":"(25, 64)","b_O":"(1600,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(1600, 6400)","b_in":"(6400,)","W_out":"(6400, 1600)","b_out":"(1600,)"}}},"ln_final":{"[w, b]":"(1600,)"},"unembed":{"W_U":"(1600, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-47]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1600)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 25, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 25, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1600)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 6400)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1600)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1600)\nunembed:\n hook_in: (batch, seq_len, 1600)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1600)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-47]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1600)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 25, 64)","[hook_attn_scores, hook_pattern]":"(batch, 25, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1600)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 6400)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1600)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1600)"},"unembed":{"hook_in":"(batch, seq_len, 1600)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1600)"}} -{"name.default_alias":"llama-13b","name.huggingface":"llama-13b-hf","name.aliases":"llama-13b","model_type":"llama","name.from_cfg":"llama-13b-hf","n_params.as_str":"13B","n_params.as_int":12687769600,"n_params.from_name":"13b","cfg.n_params":12687769600,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":40,"d_mlp":13824,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"llama-13b-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"llama-13b-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12687769600,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 40\nd_mlp: 13824\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: llama-13b-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: llama-13b-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12687769600\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"huggyllama\/llama-13b","tokenizer.vocab_size":32000.0,"tokenizer.max_len":2048.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 13824)\n W_out: (13824, 5120)\n b_in: (13824,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 13824)","W_out":"(13824, 5120)","b_in":"(13824,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 13824)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 13824)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 5120)"}} -{"name.default_alias":"llama-30b","name.huggingface":"llama-30b-hf","name.aliases":"llama-30b","model_type":"llama","name.from_cfg":"llama-30b-hf","n_params.as_str":"32B","n_params.as_int":32102154240,"n_params.from_name":"30b","cfg.n_params":32102154240,"cfg.n_layers":60,"cfg.n_heads":52,"cfg.d_model":6656,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":6656,"d_head":128,"n_layers":60,"n_ctx":2048,"n_heads":52,"d_mlp":17920,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"llama-30b-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"llama-30b-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0098058068,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":32102154240,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 6656\nd_head: 128\nn_layers: 60\nn_ctx: 2048\nn_heads: 52\nd_mlp: 17920\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: llama-30b-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: llama-30b-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.009805806756909202\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 32102154240\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"huggyllama\/llama-30b","tokenizer.vocab_size":32000.0,"tokenizer.max_len":2048.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 6656)\nblocks:\n '[0-59]':\n ln1:\n w: (6656,)\n ln2:\n w: (6656,)\n attn:\n '[W_Q, W_K, W_V]': (52, 6656, 128)\n W_O: (52, 128, 6656)\n '[b_Q, b_K, b_V]': (52, 128)\n b_O: (6656,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (6656, 17920)\n W_out: (17920, 6656)\n b_in: (17920,)\n b_out: (6656,)\nln_final:\n w: (6656,)\nunembed:\n W_U: (6656, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 6656)"},"blocks":{"[0-59]":{"ln1":{"w":"(6656,)"},"ln2":{"w":"(6656,)"},"attn":{"[W_Q, W_K, W_V]":"(52, 6656, 128)","W_O":"(52, 128, 6656)","[b_Q, b_K, b_V]":"(52, 128)","b_O":"(6656,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(6656, 17920)","W_out":"(17920, 6656)","b_in":"(17920,)","b_out":"(6656,)"}}},"ln_final":{"w":"(6656,)"},"unembed":{"W_U":"(6656, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-59]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6656)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 52, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 52, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6656)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 17920)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 6656)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6656)\nunembed:\n hook_in: (batch, seq_len, 6656)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 6656)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-59]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6656)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 52, 128)","[hook_attn_scores, hook_pattern]":"(batch, 52, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6656)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 17920)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 6656)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6656)"},"unembed":{"hook_in":"(batch, seq_len, 6656)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 6656)"}} -{"name.default_alias":"llama-65b","name.huggingface":"llama-65b-hf","name.aliases":"llama-65b","model_type":"llama","name.from_cfg":"llama-65b-hf","n_params.as_str":"65B","n_params.as_int":64760053760,"n_params.from_name":"65b","cfg.n_params":64760053760,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":2048,"n_heads":64,"d_mlp":22016,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"llama-65b-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"llama-65b-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0088388348,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":64760053760,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 2048\nn_heads: 64\nd_mlp: 22016\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: llama-65b-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: llama-65b-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008838834764831844\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 64760053760\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"huggyllama\/llama-65b","tokenizer.vocab_size":32000.0,"tokenizer.max_len":2048.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n '[W_Q, W_K, W_V]': (64, 8192, 128)\n W_O: (64, 128, 8192)\n '[b_Q, b_K, b_V]': (64, 128)\n b_O: (8192,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (8192, 22016)\n W_out: (22016, 8192)\n b_in: (22016,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"[W_Q, W_K, W_V]":"(64, 8192, 128)","W_O":"(64, 128, 8192)","[b_Q, b_K, b_V]":"(64, 128)","b_O":"(8192,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 22016)","W_out":"(22016, 8192)","b_in":"(22016,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 64, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 22016)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 64, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 22016)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 8192)"}} -{"name.default_alias":"llama-7b","name.huggingface":"llama-7b-hf","name.aliases":"llama-7b","model_type":"llama","name.from_cfg":"llama-7b-hf","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":11008,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"llama-7b-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"llama-7b-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 11008\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: llama-7b-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: llama-7b-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"huggyllama\/llama-7b","tokenizer.vocab_size":32000.0,"tokenizer.max_len":2048.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"Llama-2-13b-chat","name.huggingface":null,"name.aliases":"","model_type":"Llama-2","name.from_cfg":"Llama-2-13b-chat-hf","n_params.as_str":"13B","n_params.as_int":12687769600,"n_params.from_name":"13b","cfg.n_params":12687769600,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":4096,"n_heads":40,"d_mlp":13824,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-2-13b-chat-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-2-13b-chat-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12687769600,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 4096\nn_heads: 40\nd_mlp: 13824\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-2-13b-chat-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-2-13b-chat-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12687769600\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-2-13b-chat-hf","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n '[W_in, W_gate]': (5120, 13824)\n W_out: (13824, 5120)\n b_in: (13824,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 13824)","W_out":"(13824, 5120)","b_in":"(13824,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 13824)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 13824)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 5120)"}} -{"name.default_alias":"Llama-2-13b","name.huggingface":null,"name.aliases":"","model_type":"Llama-2","name.from_cfg":"Llama-2-13b-hf","n_params.as_str":"13B","n_params.as_int":12687769600,"n_params.from_name":"13b","cfg.n_params":12687769600,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":4096,"n_heads":40,"d_mlp":13824,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-2-13b-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-2-13b-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12687769600,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 4096\nn_heads: 40\nd_mlp: 13824\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-2-13b-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-2-13b-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12687769600\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-2-13b-hf","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n '[W_in, W_gate]': (5120, 13824)\n W_out: (13824, 5120)\n b_in: (13824,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 13824)","W_out":"(13824, 5120)","b_in":"(13824,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 13824)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 13824)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 5120)"}} -{"name.default_alias":"Llama-2-70b-chat","name.huggingface":null,"name.aliases":"","model_type":"Llama-2","name.from_cfg":"Llama-2-70b-chat-hf","n_params.as_str":"78B","n_params.as_int":77846282240,"n_params.from_name":"70b","cfg.n_params":77846282240,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":4096,"n_heads":64,"d_mlp":28672,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-2-70b-chat-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-2-70b-chat-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0088388348,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":77846282240,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 4096\nn_heads: 64\nd_mlp: 28672\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-2-70b-chat-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-2-70b-chat-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008838834764831844\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 77846282240\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-2-70b-chat-hf","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n W_Q: (64, 8192, 128)\n W_O: (64, 128, 8192)\n b_Q: (64, 128)\n b_O: (8192,)\n '[_W_K, _W_V]': (8, 8192, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n '[W_in, W_gate]': (8192, 28672)\n W_out: (28672, 8192)\n b_in: (28672,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"W_Q":"(64, 8192, 128)","W_O":"(64, 128, 8192)","b_Q":"(64, 128)","b_O":"(8192,)","[_W_K, _W_V]":"(8, 8192, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 28672)","W_out":"(28672, 8192)","b_in":"(28672,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 64, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 28672)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 64, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 28672)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 8192)"}} -{"name.default_alias":"Llama-2-7b-chat","name.huggingface":null,"name.aliases":"","model_type":"Llama-2","name.from_cfg":"Llama-2-7b-chat-hf","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-2-7b-chat-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-2-7b-chat-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-2-7b-chat-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-2-7b-chat-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-2-7b-chat-hf","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"Llama-2-7b","name.huggingface":null,"name.aliases":"","model_type":"Llama-2","name.from_cfg":"Llama-2-7b-hf","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-2-7b-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-2-7b-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-2-7b-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-2-7b-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-2-7b-hf","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"meta-llama\/Llama-3.1-70B","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.1-70B","n_params.as_str":"78B","n_params.as_int":77846282240,"n_params.from_name":"70B","cfg.n_params":77846282240,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":2048,"n_heads":64,"d_mlp":28672,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.1-70B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.1-70B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0088388348,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":77846282240,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 2048\nn_heads: 64\nd_mlp: 28672\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.1-70B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.1-70B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008838834764831844\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 77846282240\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.1-70B","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n W_Q: (64, 8192, 128)\n W_O: (64, 128, 8192)\n b_Q: (64, 128)\n b_O: (8192,)\n '[_W_K, _W_V]': (8, 8192, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (8192, 28672)\n W_out: (28672, 8192)\n b_in: (28672,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"W_Q":"(64, 8192, 128)","W_O":"(64, 128, 8192)","b_Q":"(64, 128)","b_O":"(8192,)","[_W_K, _W_V]":"(8, 8192, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 28672)","W_out":"(28672, 8192)","b_in":"(28672,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 64, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 28672)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 64, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 28672)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 8192)"}} -{"name.default_alias":"meta-llama\/Llama-3.1-70B-Instruct","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.1-70B-Instruct","n_params.as_str":"78B","n_params.as_int":77846282240,"n_params.from_name":"70B","cfg.n_params":77846282240,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":2048,"n_heads":64,"d_mlp":28672,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.1-70B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.1-70B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0088388348,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":77846282240,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 2048\nn_heads: 64\nd_mlp: 28672\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.1-70B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.1-70B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008838834764831844\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 77846282240\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.1-70B-Instruct","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n W_Q: (64, 8192, 128)\n W_O: (64, 128, 8192)\n b_Q: (64, 128)\n b_O: (8192,)\n '[_W_K, _W_V]': (8, 8192, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (8192, 28672)\n W_out: (28672, 8192)\n b_in: (28672,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"W_Q":"(64, 8192, 128)","W_O":"(64, 128, 8192)","b_Q":"(64, 128)","b_O":"(8192,)","[_W_K, _W_V]":"(8, 8192, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 28672)","W_out":"(28672, 8192)","b_in":"(28672,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 64, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 28672)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 64, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 28672)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 8192)"}} -{"name.default_alias":"meta-llama\/Llama-3.1-8B","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.1-8B","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"8B","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":14336,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.1-8B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.1-8B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 14336\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.1-8B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.1-8B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.1-8B","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 14336)\n W_out: (14336, 4096)\n b_in: (14336,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 14336)","W_out":"(14336, 4096)","b_in":"(14336,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"meta-llama\/Llama-3.1-8B-Instruct","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.1-8B-Instruct","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"8B","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":14336,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.1-8B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.1-8B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 14336\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.1-8B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.1-8B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.1-8B-Instruct","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 14336)\n W_out: (14336, 4096)\n b_in: (14336,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 14336)","W_out":"(14336, 4096)","b_in":"(14336,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"meta-llama\/Llama-3.2-1B","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.2-1B","n_params.as_str":"1.1B","n_params.as_int":1073741824,"n_params.from_name":"1B","cfg.n_params":1073741824,"cfg.n_layers":16,"cfg.n_heads":32,"cfg.d_model":2048,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":64,"n_layers":16,"n_ctx":2048,"n_heads":32,"d_mlp":8192,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.2-1B","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.2-1B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":1073741824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":32.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 64\nn_layers: 16\nn_ctx: 2048\nn_heads: 32\nd_mlp: 8192\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.2-1B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.2-1B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 1073741824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 32.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.2-1B","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 2048)\nblocks:\n '[0-15]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n W_Q: (32, 2048, 64)\n W_O: (32, 64, 2048)\n b_Q: (32, 64)\n b_O: (2048,)\n '[_W_K, _W_V]': (8, 2048, 64)\n '[_b_K, _b_V]': (8, 64)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n '[W_in, W_gate]': (2048, 8192)\n W_out: (8192, 2048)\n b_in: (8192,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 2048)"},"blocks":{"[0-15]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"W_Q":"(32, 2048, 64)","W_O":"(32, 64, 2048)","b_Q":"(32, 64)","b_O":"(2048,)","[_W_K, _W_V]":"(8, 2048, 64)","[_b_K, _b_V]":"(8, 64)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"[W_in, W_gate]":"(2048, 8192)","W_out":"(8192, 2048)","b_in":"(8192,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 64)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 64)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"meta-llama\/Llama-3.2-1B-Instruct","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.2-1B-Instruct","n_params.as_str":"1.1B","n_params.as_int":1073741824,"n_params.from_name":"1B","cfg.n_params":1073741824,"cfg.n_layers":16,"cfg.n_heads":32,"cfg.d_model":2048,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":64,"n_layers":16,"n_ctx":2048,"n_heads":32,"d_mlp":8192,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.2-1B-Instruct","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.2-1B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":1073741824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":32.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 64\nn_layers: 16\nn_ctx: 2048\nn_heads: 32\nd_mlp: 8192\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.2-1B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.2-1B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 1073741824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 32.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.2-1B-Instruct","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 2048)\nblocks:\n '[0-15]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n W_Q: (32, 2048, 64)\n W_O: (32, 64, 2048)\n b_Q: (32, 64)\n b_O: (2048,)\n '[_W_K, _W_V]': (8, 2048, 64)\n '[_b_K, _b_V]': (8, 64)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n '[W_in, W_gate]': (2048, 8192)\n W_out: (8192, 2048)\n b_in: (8192,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 2048)"},"blocks":{"[0-15]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"W_Q":"(32, 2048, 64)","W_O":"(32, 64, 2048)","b_Q":"(32, 64)","b_O":"(2048,)","[_W_K, _W_V]":"(8, 2048, 64)","[_b_K, _b_V]":"(8, 64)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"[W_in, W_gate]":"(2048, 8192)","W_out":"(8192, 2048)","b_in":"(8192,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 64)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 64)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"meta-llama\/Llama-3.2-3B","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.2-3B","n_params.as_str":"3.2B","n_params.as_int":3170893824,"n_params.from_name":"3B","cfg.n_params":3170893824,"cfg.n_layers":28,"cfg.n_heads":24,"cfg.d_model":3072,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3072,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":24,"d_mlp":8192,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.2-3B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.2-3B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0144337567,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":3170893824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":32.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 3072\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 24\nd_mlp: 8192\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.2-3B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.2-3B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.014433756729740645\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 3170893824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 32.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.2-3B","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 3072)\nblocks:\n '[0-27]':\n ln1:\n w: (3072,)\n ln2:\n w: (3072,)\n attn:\n W_Q: (24, 3072, 128)\n W_O: (24, 128, 3072)\n b_Q: (24, 128)\n b_O: (3072,)\n '[_W_K, _W_V]': (8, 3072, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (3072, 8192)\n W_out: (8192, 3072)\n b_in: (8192,)\n b_out: (3072,)\nln_final:\n w: (3072,)\nunembed:\n W_U: (3072, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 3072)"},"blocks":{"[0-27]":{"ln1":{"w":"(3072,)"},"ln2":{"w":"(3072,)"},"attn":{"W_Q":"(24, 3072, 128)","W_O":"(24, 128, 3072)","b_Q":"(24, 128)","b_O":"(3072,)","[_W_K, _W_V]":"(8, 3072, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(3072, 8192)","W_out":"(8192, 3072)","b_in":"(8192,)","b_out":"(3072,)"}}},"ln_final":{"w":"(3072,)"},"unembed":{"W_U":"(3072, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 24, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 24, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3072)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\nunembed:\n hook_in: (batch, seq_len, 3072)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 3072)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 24, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 24, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3072)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"unembed":{"hook_in":"(batch, seq_len, 3072)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 3072)"}} -{"name.default_alias":"meta-llama\/Llama-3.2-3B-Instruct","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.2-3B-Instruct","n_params.as_str":"3.2B","n_params.as_int":3170893824,"n_params.from_name":"3B","cfg.n_params":3170893824,"cfg.n_layers":28,"cfg.n_heads":24,"cfg.d_model":3072,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3072,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":24,"d_mlp":8192,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.2-3B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.2-3B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0144337567,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":3170893824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":32.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 3072\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 24\nd_mlp: 8192\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.2-3B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.2-3B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.014433756729740645\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 3170893824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 32.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.2-3B-Instruct","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 3072)\nblocks:\n '[0-27]':\n ln1:\n w: (3072,)\n ln2:\n w: (3072,)\n attn:\n W_Q: (24, 3072, 128)\n W_O: (24, 128, 3072)\n b_Q: (24, 128)\n b_O: (3072,)\n '[_W_K, _W_V]': (8, 3072, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (3072, 8192)\n W_out: (8192, 3072)\n b_in: (8192,)\n b_out: (3072,)\nln_final:\n w: (3072,)\nunembed:\n W_U: (3072, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 3072)"},"blocks":{"[0-27]":{"ln1":{"w":"(3072,)"},"ln2":{"w":"(3072,)"},"attn":{"W_Q":"(24, 3072, 128)","W_O":"(24, 128, 3072)","b_Q":"(24, 128)","b_O":"(3072,)","[_W_K, _W_V]":"(8, 3072, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(3072, 8192)","W_out":"(8192, 3072)","b_in":"(8192,)","b_out":"(3072,)"}}},"ln_final":{"w":"(3072,)"},"unembed":{"W_U":"(3072, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 24, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 24, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3072)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\nunembed:\n hook_in: (batch, seq_len, 3072)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 3072)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 24, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 24, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3072)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"unembed":{"hook_in":"(batch, seq_len, 3072)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 3072)"}} -{"name.default_alias":"meta-llama\/Llama-3.3-70B-Instruct","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.3-70B-Instruct","n_params.as_str":"78B","n_params.as_int":77846282240,"n_params.from_name":"70B","cfg.n_params":77846282240,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":2048,"n_heads":64,"d_mlp":28672,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.3-70B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.3-70B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0088388348,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":77846282240,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 2048\nn_heads: 64\nd_mlp: 28672\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.3-70B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.3-70B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008838834764831844\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 77846282240\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.3-70B-Instruct","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n W_Q: (64, 8192, 128)\n W_O: (64, 128, 8192)\n b_Q: (64, 128)\n b_O: (8192,)\n '[_W_K, _W_V]': (8, 8192, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (8192, 28672)\n W_out: (28672, 8192)\n b_in: (28672,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"W_Q":"(64, 8192, 128)","W_O":"(64, 128, 8192)","b_Q":"(64, 128)","b_O":"(8192,)","[_W_K, _W_V]":"(8, 8192, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 28672)","W_out":"(28672, 8192)","b_in":"(28672,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 64, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 28672)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 64, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 28672)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 8192)"}} -{"name.default_alias":"meta-llama\/Meta-Llama-3-70B","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Meta-Llama-3-70B","n_params.as_str":"78B","n_params.as_int":77846282240,"n_params.from_name":"70B","cfg.n_params":77846282240,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":8192,"n_heads":64,"d_mlp":28672,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Meta-Llama-3-70B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Meta-Llama-3-70B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0088388348,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":77846282240,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 8192\nn_heads: 64\nd_mlp: 28672\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Meta-Llama-3-70B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Meta-Llama-3-70B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008838834764831844\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 77846282240\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Meta-Llama-3-70B","tokenizer.vocab_size":128000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"RnzNv9w_ITBp6b2dcibKR7_l85I=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n W_Q: (64, 8192, 128)\n W_O: (64, 128, 8192)\n b_Q: (64, 128)\n b_O: (8192,)\n '[_W_K, _W_V]': (8, 8192, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 128)\n mlp:\n '[W_in, W_gate]': (8192, 28672)\n W_out: (28672, 8192)\n b_in: (28672,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"W_Q":"(64, 8192, 128)","W_O":"(64, 128, 8192)","b_Q":"(64, 128)","b_O":"(8192,)","[_W_K, _W_V]":"(8, 8192, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 28672)","W_out":"(28672, 8192)","b_in":"(28672,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 64, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 28672)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 64, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 28672)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 8192)"}} -{"name.default_alias":"meta-llama\/Meta-Llama-3-70B-Instruct","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Meta-Llama-3-70B-Instruct","n_params.as_str":"78B","n_params.as_int":77846282240,"n_params.from_name":"70B","cfg.n_params":77846282240,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":8192,"n_heads":64,"d_mlp":28672,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Meta-Llama-3-70B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Meta-Llama-3-70B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0088388348,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":77846282240,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 8192\nn_heads: 64\nd_mlp: 28672\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Meta-Llama-3-70B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Meta-Llama-3-70B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008838834764831844\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 77846282240\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Meta-Llama-3-70B-Instruct","tokenizer.vocab_size":128000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"RnzNv9w_ITBp6b2dcibKR7_l85I=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n W_Q: (64, 8192, 128)\n W_O: (64, 128, 8192)\n b_Q: (64, 128)\n b_O: (8192,)\n '[_W_K, _W_V]': (8, 8192, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 128)\n mlp:\n '[W_in, W_gate]': (8192, 28672)\n W_out: (28672, 8192)\n b_in: (28672,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"W_Q":"(64, 8192, 128)","W_O":"(64, 128, 8192)","b_Q":"(64, 128)","b_O":"(8192,)","[_W_K, _W_V]":"(8, 8192, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 28672)","W_out":"(28672, 8192)","b_in":"(28672,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 64, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 28672)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 64, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 28672)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 8192)"}} -{"name.default_alias":"meta-llama\/Meta-Llama-3-8B","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Meta-Llama-3-8B","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"8B","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":8192,"n_heads":32,"d_mlp":14336,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Meta-Llama-3-8B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Meta-Llama-3-8B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 8192\nn_heads: 32\nd_mlp: 14336\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Meta-Llama-3-8B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Meta-Llama-3-8B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Meta-Llama-3-8B","tokenizer.vocab_size":128000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"RnzNv9w_ITBp6b2dcibKR7_l85I=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 128)\n mlp:\n '[W_in, W_gate]': (4096, 14336)\n W_out: (14336, 4096)\n b_in: (14336,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 14336)","W_out":"(14336, 4096)","b_in":"(14336,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"meta-llama\/Meta-Llama-3-8B-Instruct","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Meta-Llama-3-8B-Instruct","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"8B","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":8192,"n_heads":32,"d_mlp":14336,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Meta-Llama-3-8B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Meta-Llama-3-8B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 8192\nn_heads: 32\nd_mlp: 14336\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Meta-Llama-3-8B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Meta-Llama-3-8B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Meta-Llama-3-8B-Instruct","tokenizer.vocab_size":128000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"RnzNv9w_ITBp6b2dcibKR7_l85I=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (8192, 8192)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (8192, 128)\n mlp:\n '[W_in, W_gate]': (4096, 14336)\n W_out: (14336, 4096)\n b_in: (14336,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(8192, 8192)","IGNORE":"()","[rotary_sin, rotary_cos]":"(8192, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 14336)","W_out":"(14336, 4096)","b_in":"(14336,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"phi-1","name.huggingface":"microsoft\/phi-1","name.aliases":"phi-1","model_type":"phi","name.from_cfg":"phi-1","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":null,"cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":32,"cfg.d_model":2048,"cfg.d_vocab":51200,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"PhiForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":32,"d_mlp":8192,"d_vocab":51200,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"phi-1","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"PhiForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"microsoft\/phi-1","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":51200,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 32\nd_mlp: 8192\nd_vocab: 51200\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: phi-1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: PhiForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: microsoft\/phi-1\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 51200\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"microsoft\/phi-1","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"TYk6J3OrqdU2F7JYiSfFXtd-vB4=","tensor_shapes.state_dict":"embed:\n W_E: (51200, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2048, 64)\n W_O: (32, 64, 2048)\n '[b_Q, b_K, b_V]': (32, 64)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 51200)\n b_U: (51200,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(51200, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2048, 64)","W_O":"(32, 64, 2048)","[b_Q, b_K, b_V]":"(32, 64)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 51200)","b_U":"(51200,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 51200)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 64)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 51200)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"phi-1_5","name.huggingface":"microsoft\/phi-1_5","name.aliases":"phi-1_5","model_type":"phi","name.from_cfg":"phi-1_5","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":null,"cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":32,"cfg.d_model":2048,"cfg.d_vocab":51200,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"PhiForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":32,"d_mlp":8192,"d_vocab":51200,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"phi-1_5","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"PhiForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"microsoft\/phi-1_5","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":51200,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 32\nd_mlp: 8192\nd_vocab: 51200\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: phi-1_5\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: PhiForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: microsoft\/phi-1_5\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 51200\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"microsoft\/phi-1_5","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"TYk6J3OrqdU2F7JYiSfFXtd-vB4=","tensor_shapes.state_dict":"embed:\n W_E: (51200, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2048, 64)\n W_O: (32, 64, 2048)\n '[b_Q, b_K, b_V]': (32, 64)\n b_O: (2048,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 51200)\n b_U: (51200,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(51200, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2048, 64)","W_O":"(32, 64, 2048)","[b_Q, b_K, b_V]":"(32, 64)","b_O":"(2048,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 51200)","b_U":"(51200,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 51200)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 64)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 51200)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"phi-2","name.huggingface":"microsoft\/phi-2","name.aliases":"phi-2","model_type":"phi","name.from_cfg":"phi-2","n_params.as_str":"2.5B","n_params.as_int":2516582400,"n_params.from_name":null,"cfg.n_params":2516582400,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":51200,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"PhiForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":80,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":10240,"d_vocab":51200,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"phi-2","use_attn_scale":true,"attn_scale":8.94427191,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"PhiForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"microsoft\/phi-2","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":51200,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":2516582400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 80\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 10240\nd_vocab: 51200\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: phi-2\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n qPSXm3fjIUA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: PhiForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: microsoft\/phi-2\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 51200\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 2516582400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"microsoft\/phi-2","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"TYk6J3OrqdU2F7JYiSfFXtd-vB4=","tensor_shapes.state_dict":"embed:\n W_E: (51200, 2560)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2560, 80)\n W_O: (32, 80, 2560)\n '[b_Q, b_K, b_V]': (32, 80)\n b_O: (2560,)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 51200)\n b_U: (51200,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(51200, 2560)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2560, 80)","W_O":"(32, 80, 2560)","[b_Q, b_K, b_V]":"(32, 80)","b_O":"(2560,)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 51200)","b_U":"(51200,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 80)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 51200)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 80)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 51200)"},"hook_embed":"(batch, seq_len, 2560)"}} -{"name.default_alias":"phi-3","name.huggingface":"microsoft\/Phi-3-mini-4k-instruct","name.aliases":"phi-3","model_type":"phi","name.from_cfg":"Phi-3-mini-4k-instruct","n_params.as_str":"3.6B","n_params.as_int":3623878656,"n_params.from_name":null,"cfg.n_params":3623878656,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":3072,"cfg.d_vocab":32064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Phi3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3072,"d_head":96,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":8192,"d_vocab":32064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Phi-3-mini-4k-instruct","use_attn_scale":true,"attn_scale":9.7979589711,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Phi3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"microsoft\/Phi-3-mini-4k-instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32064,"parallel_attn_mlp":false,"rotary_dim":96,"n_params":3623878656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 3072\nd_head: 96\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 8192\nd_vocab: 32064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Phi-3-mini-4k-instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n LiEJFI6YI0A=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Phi3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: microsoft\/Phi-3-mini-4k-instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32064\nparallel_attn_mlp: false\nrotary_dim: 96\nn_params: 3623878656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"microsoft\/Phi-3-mini-4k-instruct","tokenizer.vocab_size":32000.0,"tokenizer.max_len":4096.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"2BcGXsWoZjuOkMtb6uTbGL68fbc=","tensor_shapes.state_dict":"embed:\n W_E: (32064, 3072)\nblocks:\n '[0-31]':\n ln1:\n w: (3072,)\n ln2:\n w: (3072,)\n attn:\n '[W_Q, W_K, W_V]': (32, 3072, 96)\n W_O: (32, 96, 3072)\n '[b_Q, b_K, b_V]': (32, 96)\n b_O: (3072,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 96)\n mlp:\n '[W_in, W_gate]': (3072, 8192)\n W_out: (8192, 3072)\n b_in: (8192,)\n b_out: (3072,)\nln_final:\n w: (3072,)\nunembed:\n W_U: (3072, 32064)\n b_U: (32064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32064, 3072)"},"blocks":{"[0-31]":{"ln1":{"w":"(3072,)"},"ln2":{"w":"(3072,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 3072, 96)","W_O":"(32, 96, 3072)","[b_Q, b_K, b_V]":"(32, 96)","b_O":"(3072,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 96)"},"mlp":{"[W_in, W_gate]":"(3072, 8192)","W_out":"(8192, 3072)","b_in":"(8192,)","b_out":"(3072,)"}}},"ln_final":{"w":"(3072,)"},"unembed":{"W_U":"(3072, 32064)","b_U":"(32064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 96)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3072)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\nunembed:\n hook_in: (batch, seq_len, 3072)\n hook_out: (batch, seq_len, 32064)\nhook_embed: (batch, seq_len, 3072)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 96)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3072)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"unembed":{"hook_in":"(batch, seq_len, 3072)","hook_out":"(batch, seq_len, 32064)"},"hook_embed":"(batch, seq_len, 3072)"}} -{"name.default_alias":"phi-4","name.huggingface":"microsoft\/phi-4","name.aliases":"phi-4","model_type":"phi","name.from_cfg":"phi-4","n_params.as_str":"15B","n_params.as_int":15204352000,"n_params.from_name":null,"cfg.n_params":15204352000,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":100352,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Phi3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":16384,"n_heads":40,"d_mlp":17920,"d_vocab":100352,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":10,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"phi-4","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Phi3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"microsoft\/phi-4","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100352,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":15204352000,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":250000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 16384\nn_heads: 40\nd_mlp: 17920\nd_vocab: 100352\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 10\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: phi-4\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Phi3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: microsoft\/phi-4\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100352\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 15204352000\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 250000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"microsoft\/phi-4","tokenizer.vocab_size":100352.0,"tokenizer.max_len":16384.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"uJZqWk6gqn6tO_nlSJEZsP9MITQ=","tensor_shapes.state_dict":"embed:\n W_E: (100352, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (10, 5120, 128)\n '[_b_K, _b_V]': (10, 128)\n mask: (16384, 16384)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (16384, 128)\n mlp:\n '[W_in, W_gate]': (5120, 17920)\n W_out: (17920, 5120)\n b_in: (17920,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 100352)\n b_U: (100352,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100352, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(10, 5120, 128)","[_b_K, _b_V]":"(10, 128)","mask":"(16384, 16384)","IGNORE":"()","[rotary_sin, rotary_cos]":"(16384, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 17920)","W_out":"(17920, 5120)","b_in":"(17920,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 100352)","b_U":"(100352,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 10, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 17920)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 100352)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 10, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 17920)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 100352)"},"hook_embed":"(batch, seq_len, 5120)"}} -{"name.default_alias":"mistral-7b-instruct","name.huggingface":"mistralai\/Mistral-7B-Instruct-v0.1","name.aliases":"mistral-7b-instruct","model_type":"mistral","name.from_cfg":"Mistral-7B-Instruct-v0.1","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"7b","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"MistralForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":14336,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Mistral-7B-Instruct-v0.1","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"MistralForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"mistralai\/Mistral-7B-Instruct-v0.1","window_size":4096,"attn_types":["local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 14336\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Mistral-7B-Instruct-v0.1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: MistralForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: mistralai\/Mistral-7B-Instruct-v0.1\nwindow_size: 4096\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"mistralai\/Mistral-7B-Instruct-v0.1","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"kkCQxUk-PF9Ay_ZKDdKCh02YaGQ=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 14336)\n W_out: (14336, 4096)\n b_in: (14336,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 14336)","W_out":"(14336, 4096)","b_in":"(14336,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"mistral-7b","name.huggingface":"mistralai\/Mistral-7B-v0.1","name.aliases":"mistral-7b","model_type":"mistral","name.from_cfg":"Mistral-7B-v0.1","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"7b","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"MistralForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":14336,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Mistral-7B-v0.1","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"MistralForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"mistralai\/Mistral-7B-v0.1","window_size":4096,"attn_types":["local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 14336\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Mistral-7B-v0.1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: MistralForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: mistralai\/Mistral-7B-v0.1\nwindow_size: 4096\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"mistralai\/Mistral-7B-v0.1","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"kkCQxUk-PF9Ay_ZKDdKCh02YaGQ=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 14336)\n W_out: (14336, 4096)\n b_in: (14336,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 14336)","W_out":"(14336, 4096)","b_in":"(14336,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"mistral-nemo-base-2407","name.huggingface":"mistralai\/Mistral-Nemo-Base-2407","name.aliases":"mistral-nemo-base-2407","model_type":"mistral","name.from_cfg":"Mistral-Nemo-Base-2407","n_params.as_str":"12B","n_params.as_int":12163481600,"n_params.from_name":null,"cfg.n_params":12163481600,"cfg.n_layers":40,"cfg.n_heads":32,"cfg.d_model":5120,"cfg.d_vocab":131072,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"MistralForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":32,"d_mlp":14336,"d_vocab":131072,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Mistral-Nemo-Base-2407","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"MistralForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"mistralai\/Mistral-Nemo-Base-2407","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":131072,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12163481600,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 32\nd_mlp: 14336\nd_vocab: 131072\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Mistral-Nemo-Base-2407\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: MistralForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: mistralai\/Mistral-Nemo-Base-2407\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 131072\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12163481600\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"mistralai\/Mistral-Nemo-Base-2407","tokenizer.vocab_size":131072.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"0xs_eSvVgsyZGbcLSIpGUn4Gdms=","tensor_shapes.state_dict":"embed:\n W_E: (131072, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n W_Q: (32, 5120, 128)\n W_O: (32, 128, 5120)\n b_Q: (32, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 14336)\n W_out: (14336, 5120)\n b_in: (14336,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 131072)\n b_U: (131072,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(131072, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"W_Q":"(32, 5120, 128)","W_O":"(32, 128, 5120)","b_Q":"(32, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 14336)","W_out":"(14336, 5120)","b_in":"(14336,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 131072)","b_U":"(131072,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 131072)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 131072)"},"hook_embed":"(batch, seq_len, 5120)"}} -{"name.default_alias":"mistralai\/Mistral-Small-24B-Base-2501","name.huggingface":null,"name.aliases":"","model_type":"mistral","name.from_cfg":"Mistral-Small-24B-Base-2501","n_params.as_str":"23B","n_params.as_int":23488102400,"n_params.from_name":"24B","cfg.n_params":23488102400,"cfg.n_layers":40,"cfg.n_heads":32,"cfg.d_model":5120,"cfg.d_vocab":131072,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"MistralForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":32,"d_mlp":32768,"d_vocab":131072,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Mistral-Small-24B-Base-2501","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"MistralForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"mistralai\/Mistral-Small-24B-Base-2501","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":131072,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":23488102400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":100000000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 32\nd_mlp: 32768\nd_vocab: 131072\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Mistral-Small-24B-Base-2501\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: MistralForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: mistralai\/Mistral-Small-24B-Base-2501\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 131072\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 23488102400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 100000000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"mistralai\/Mistral-Small-24B-Base-2501","tokenizer.vocab_size":131072.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"GEwgZayWxpmhQxtMq-WrVFJEfqM=","tensor_shapes.state_dict":"embed:\n W_E: (131072, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n W_Q: (32, 5120, 128)\n W_O: (32, 128, 5120)\n b_Q: (32, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 32768)\n W_out: (32768, 5120)\n b_in: (32768,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 131072)\n b_U: (131072,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(131072, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"W_Q":"(32, 5120, 128)","W_O":"(32, 128, 5120)","b_Q":"(32, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 32768)","W_out":"(32768, 5120)","b_in":"(32768,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 131072)","b_U":"(131072,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 32768)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 131072)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 32768)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 131072)"},"hook_embed":"(batch, seq_len, 5120)"}} -{"name.default_alias":"mixtral-instruct","name.huggingface":"mistralai\/Mixtral-8x7B-Instruct-v0.1","name.aliases":"mixtral-instruct, mixtral-8x7b-instruct","model_type":"mixtral","name.from_cfg":"Mixtral-8x7B-Instruct-v0.1","n_params.as_str":"47B","n_params.as_int":47245688832,"n_params.from_name":null,"cfg.n_params":47245688832,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"MixtralForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":32768,"n_heads":32,"d_mlp":14336,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":8,"experts_per_token":2,"final_rms":false,"dtype":"torch.float32","model_name":"Mixtral-8x7B-Instruct-v0.1","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"MixtralForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"mistralai\/Mixtral-8x7B-Instruct-v0.1","window_size":null,"attn_types":["global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":47245688832,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 32768\nn_heads: 32\nd_mlp: 14336\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: 8\nexperts_per_token: 2\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Mixtral-8x7B-Instruct-v0.1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: MixtralForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: mistralai\/Mixtral-8x7B-Instruct-v0.1\nwindow_size: null\nattn_types:\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 47245688832\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"mistralai\/Mixtral-8x7B-Instruct-v0.1","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"kkCQxUk-PF9Ay_ZKDdKCh02YaGQ=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (32768, 32768)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (32768, 128)\n mlp:\n experts:\n '[0-7]':\n W_in:\n weight: (14336, 4096)\n W_out:\n weight: (4096, 14336)\n W_gate:\n weight: (14336, 4096)\n W_gate:\n weight: (8, 4096)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(32768, 32768)","IGNORE":"()","[rotary_sin, rotary_cos]":"(32768, 128)"},"mlp":{"experts":{"[0-7]":{"W_in":{"weight":"(14336, 4096)"},"W_out":{"weight":"(4096, 14336)"},"W_gate":{"weight":"(14336, 4096)"}}},"W_gate":{"weight":"(8, 4096)"}}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"mixtral","name.huggingface":"mistralai\/Mixtral-8x7B-v0.1","name.aliases":"mixtral, mixtral-8x7b","model_type":"mixtral","name.from_cfg":"Mixtral-8x7B-v0.1","n_params.as_str":"47B","n_params.as_int":47245688832,"n_params.from_name":null,"cfg.n_params":47245688832,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"MixtralForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":32768,"n_heads":32,"d_mlp":14336,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":8,"experts_per_token":2,"final_rms":false,"dtype":"torch.float32","model_name":"Mixtral-8x7B-v0.1","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"MixtralForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"mistralai\/Mixtral-8x7B-v0.1","window_size":null,"attn_types":["global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":47245688832,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 32768\nn_heads: 32\nd_mlp: 14336\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: 8\nexperts_per_token: 2\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Mixtral-8x7B-v0.1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: MixtralForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: mistralai\/Mixtral-8x7B-v0.1\nwindow_size: null\nattn_types:\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 47245688832\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"mistralai\/Mixtral-8x7B-v0.1","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"kkCQxUk-PF9Ay_ZKDdKCh02YaGQ=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (32768, 32768)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (32768, 128)\n mlp:\n experts:\n '[0-7]':\n W_in:\n weight: (14336, 4096)\n W_out:\n weight: (4096, 14336)\n W_gate:\n weight: (14336, 4096)\n W_gate:\n weight: (8, 4096)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(32768, 32768)","IGNORE":"()","[rotary_sin, rotary_cos]":"(32768, 128)"},"mlp":{"experts":{"[0-7]":{"W_in":{"weight":"(14336, 4096)"},"W_out":{"weight":"(4096, 14336)"},"W_gate":{"weight":"(14336, 4096)"}}},"W_gate":{"weight":"(8, 4096)"}}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"attn-only-2l-demo","name.huggingface":"NeelNanda\/Attn-Only-2L512W-Shortformer-6B-big-lr","name.aliases":"attn-only-2l-demo, attn-only-2l-shortformer-6b-big-lr, attn-only-2l-induction-demo, attn-only-demo","model_type":"attn-only","name.from_cfg":"Attn-Only-2L512W-Shortformer-6B-big-lr","n_params.as_str":"2.1M","n_params.as_int":2097152,"n_params.from_name":null,"cfg.n_params":2097152,"cfg.n_layers":2,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":50277,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"shortformer","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":null,"config.raw__":{"d_model":512,"d_head":64,"n_layers":2,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":50277,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"shortformer","n_key_value_heads":null,"attn_only":true,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":null,"num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Attn-Only-2L512W-Shortformer-6B-big-lr","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50277,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":2097152,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 2\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 50277\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: shortformer\nn_key_value_heads: null\nattn_only: true\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: null\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Attn-Only-2L512W-Shortformer-6B-big-lr\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50277\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 2097152\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50277, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-1]':\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\nunembed:\n W_U: (512, 50277)\n b_U: (50277,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50277, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-1]":{"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"}}},"unembed":{"W_U":"(512, 50277)","b_U":"(50277,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-1]':\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n '[hook_resid_pre, hook_attn_out, hook_resid_post]': (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50277)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-1]":{"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"[hook_resid_pre, hook_attn_out, hook_resid_post]":"(batch, seq_len, 512)"}},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50277)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"attn-only-1l","name.huggingface":"NeelNanda\/Attn_Only_1L512W_C4_Code","name.aliases":"attn-only-1l, attn-only-1l-new, attn-only-1l-c4-code","model_type":"attn-only","name.from_cfg":"Attn_Only_1L512W_C4_Code","n_params.as_str":"1.0M","n_params.as_int":1048576,"n_params.from_name":null,"cfg.n_params":1048576,"cfg.n_layers":1,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":1,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":true,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Attn_Only_1L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1048576,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 1\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: true\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Attn_Only_1L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1048576\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '0':\n ln1:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"0":{"ln1":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '0':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n '[hook_resid_pre, hook_attn_out, hook_resid_post]': (batch, seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"0":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"[hook_resid_pre, hook_attn_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"attn-only-2l","name.huggingface":"NeelNanda\/Attn_Only_2L512W_C4_Code","name.aliases":"attn-only-2l, attn-only-2l-new, attn-only-2l-c4-code","model_type":"attn-only","name.from_cfg":"Attn_Only_2L512W_C4_Code","n_params.as_str":"2.1M","n_params.as_int":2097152,"n_params.from_name":null,"cfg.n_params":2097152,"cfg.n_layers":2,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":2,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":true,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Attn_Only_2L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":2097152,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 2\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: true\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Attn_Only_2L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 2097152\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-1]':\n ln1:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-1]":{"ln1":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-1]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n '[hook_resid_pre, hook_attn_out, hook_resid_post]': (batch, seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-1]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"[hook_resid_pre, hook_attn_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"attn-only-3l","name.huggingface":"NeelNanda\/Attn_Only_3L512W_C4_Code","name.aliases":"attn-only-3l, attn-only-3l-new, attn-only-3l-c4-code","model_type":"attn-only","name.from_cfg":"Attn_Only_3L512W_C4_Code","n_params.as_str":"3.1M","n_params.as_int":3145728,"n_params.from_name":null,"cfg.n_params":3145728,"cfg.n_layers":3,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":3,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":true,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Attn_Only_3L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":3145728,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 3\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: true\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Attn_Only_3L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 3145728\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-2]':\n ln1:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-2]":{"ln1":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-2]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n '[hook_resid_pre, hook_attn_out, hook_resid_post]': (batch, seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-2]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"[hook_resid_pre, hook_attn_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"attn-only-4l","name.huggingface":"NeelNanda\/Attn_Only_4L512W_C4_Code","name.aliases":"attn-only-4l, attn-only-4l-new, attn-only-4l-c4-code","model_type":"attn-only","name.from_cfg":"Attn_Only_4L512W_C4_Code","n_params.as_str":"4.2M","n_params.as_int":4194304,"n_params.from_name":null,"cfg.n_params":4194304,"cfg.n_layers":4,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":4,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":true,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Attn_Only_4L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":4194304,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 4\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: true\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Attn_Only_4L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 4194304\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-3]':\n ln1:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-3]":{"ln1":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-3]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n '[hook_resid_pre, hook_attn_out, hook_resid_post]': (batch, seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-3]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"[hook_resid_pre, hook_attn_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"gelu-1l","name.huggingface":"NeelNanda\/GELU_1L512W_C4_Code","name.aliases":"gelu-1l, gelu-1l-new, gelu-1l-c4-code","model_type":"gelu","name.from_cfg":"GELU_1L512W_C4_Code","n_params.as_str":"3.1M","n_params.as_int":3145728,"n_params.from_name":null,"cfg.n_params":3145728,"cfg.n_layers":1,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":1,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"GELU_1L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":3145728,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 1\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: GELU_1L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 3145728\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '0':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"0":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '0':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"0":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"gelu-2l","name.huggingface":"NeelNanda\/GELU_2L512W_C4_Code","name.aliases":"gelu-2l, gelu-2l-new, gelu-2l-c4-code","model_type":"gelu","name.from_cfg":"GELU_2L512W_C4_Code","n_params.as_str":"6.3M","n_params.as_int":6291456,"n_params.from_name":null,"cfg.n_params":6291456,"cfg.n_layers":2,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":2,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"GELU_2L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":6291456,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 2\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: GELU_2L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 6291456\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-1]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-1]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-1]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-1]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"gelu-3l","name.huggingface":"NeelNanda\/GELU_3L512W_C4_Code","name.aliases":"gelu-3l, gelu-3l-new, gelu-3l-c4-code","model_type":"gelu","name.from_cfg":"GELU_3L512W_C4_Code","n_params.as_str":"9.4M","n_params.as_int":9437184,"n_params.from_name":null,"cfg.n_params":9437184,"cfg.n_layers":3,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":3,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"GELU_3L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":9437184,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 3\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: GELU_3L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 9437184\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-2]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-2]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-2]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-2]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"gelu-4l","name.huggingface":"NeelNanda\/GELU_4L512W_C4_Code","name.aliases":"gelu-4l, gelu-4l-new, gelu-4l-c4-code","model_type":"gelu","name.from_cfg":"GELU_4L512W_C4_Code","n_params.as_str":"13M","n_params.as_int":12582912,"n_params.from_name":null,"cfg.n_params":12582912,"cfg.n_layers":4,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":4,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"GELU_4L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 4\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: GELU_4L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-3]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-3]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-3]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-3]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"solu-10l","name.huggingface":"NeelNanda\/SoLU_10L1280W_C4_Code","name.aliases":"solu-10l, solu-10l-new, solu-10l-c4-code","model_type":"solu","name.from_cfg":"SoLU_10L1280W_C4_Code","n_params.as_str":"197M","n_params.as_int":196608000,"n_params.from_name":null,"cfg.n_params":196608000,"cfg.n_layers":10,"cfg.n_heads":20,"cfg.d_model":1280,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":1280,"d_head":64,"n_layers":10,"n_ctx":1024,"n_heads":20,"d_mlp":5120,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_10L1280W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0223606798,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":196608000,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1280\nd_head: 64\nn_layers: 10\nn_ctx: 1024\nn_heads: 20\nd_mlp: 5120\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_10L1280W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.022360679774997897\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 196608000\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 1280)\npos_embed:\n W_pos: (1024, 1280)\nblocks:\n '[0-9]':\n ln1:\n '[w, b]': (1280,)\n ln2:\n '[w, b]': (1280,)\n attn:\n '[W_Q, W_K, W_V]': (20, 1280, 64)\n W_O: (20, 64, 1280)\n '[b_Q, b_K, b_V]': (20, 64)\n b_O: (1280,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (5120,)\n W_in: (1280, 5120)\n b_in: (5120,)\n W_out: (5120, 1280)\n b_out: (1280,)\nln_final:\n '[w, b]': (1280,)\nunembed:\n W_U: (1280, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 1280)"},"pos_embed":{"W_pos":"(1024, 1280)"},"blocks":{"[0-9]":{"ln1":{"[w, b]":"(1280,)"},"ln2":{"[w, b]":"(1280,)"},"attn":{"[W_Q, W_K, W_V]":"(20, 1280, 64)","W_O":"(20, 64, 1280)","[b_Q, b_K, b_V]":"(20, 64)","b_O":"(1280,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(5120,)"},"W_in":"(1280, 5120)","b_in":"(5120,)","W_out":"(5120, 1280)","b_out":"(1280,)"}}},"ln_final":{"[w, b]":"(1280,)"},"unembed":{"W_U":"(1280, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-9]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 20, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 20, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 5120)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1280)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\nunembed:\n hook_in: (batch, seq_len, 1280)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1280)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-9]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 20, 64)","[hook_attn_scores, hook_pattern]":"(batch, 20, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 5120)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1280)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"unembed":{"hook_in":"(batch, seq_len, 1280)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1280)"}} -{"name.default_alias":"solu-10l-pile","name.huggingface":"NeelNanda\/SoLU_10L_v22_old","name.aliases":"solu-10l-pile, solu-10l-old","model_type":"solu","name.from_cfg":"SoLU_10L_v22_old","n_params.as_str":"197M","n_params.as_int":196608000,"n_params.from_name":null,"cfg.n_params":196608000,"cfg.n_layers":10,"cfg.n_heads":20,"cfg.d_model":1280,"cfg.d_vocab":50278,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel-solu-old","cfg.normalization_type":"LNPre","config.raw__":{"d_model":1280,"d_head":64,"n_layers":10,"n_ctx":1024,"n_heads":20,"d_mlp":5120,"d_vocab":50278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LNPre","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_10L_v22_old","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel-solu-old","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0223606798,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50278,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":196608000,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1280\nd_head: 64\nn_layers: 10\nn_ctx: 1024\nn_heads: 20\nd_mlp: 5120\nd_vocab: 50278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LNPre\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_10L_v22_old\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel-solu-old\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.022360679774997897\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50278\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 196608000\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50278, 1280)\npos_embed:\n W_pos: (1024, 1280)\nblocks:\n '[0-9]':\n attn:\n '[W_Q, W_K, W_V]': (20, 1280, 64)\n W_O: (20, 64, 1280)\n '[b_Q, b_K, b_V]': (20, 64)\n b_O: (1280,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (1280, 5120)\n b_in: (5120,)\n W_out: (5120, 1280)\n b_out: (1280,)\nunembed:\n W_U: (1280, 50278)\n b_U: (50278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50278, 1280)"},"pos_embed":{"W_pos":"(1024, 1280)"},"blocks":{"[0-9]":{"attn":{"[W_Q, W_K, W_V]":"(20, 1280, 64)","W_O":"(20, 64, 1280)","[b_Q, b_K, b_V]":"(20, 64)","b_O":"(1280,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(1280, 5120)","b_in":"(5120,)","W_out":"(5120, 1280)","b_out":"(1280,)"}}},"unembed":{"W_U":"(1280, 50278)","b_U":"(50278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-9]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 20, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 20, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 5120)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1280)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\nunembed:\n hook_in: (batch, seq_len, 1280)\n hook_out: (batch, seq_len, 50278)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1280)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-9]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 20, 64)","[hook_attn_scores, hook_pattern]":"(batch, 20, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 5120)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1280)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"unembed":{"hook_in":"(batch, seq_len, 1280)","hook_out":"(batch, seq_len, 50278)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1280)"}} -{"name.default_alias":"solu-12l","name.huggingface":"NeelNanda\/SoLU_12L1536W_C4_Code","name.aliases":"solu-12l, solu-12l-new, solu-12l-c4-code","model_type":"solu","name.from_cfg":"SoLU_12L1536W_C4_Code","n_params.as_str":"340M","n_params.as_int":339738624,"n_params.from_name":null,"cfg.n_params":339738624,"cfg.n_layers":12,"cfg.n_heads":24,"cfg.d_model":1536,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":1536,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":24,"d_mlp":6144,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_12L1536W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0204124145,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":339738624,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1536\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 24\nd_mlp: 6144\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_12L1536W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.020412414523193152\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 339738624\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 1536)\npos_embed:\n W_pos: (1024, 1536)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (1536,)\n ln2:\n '[w, b]': (1536,)\n attn:\n '[W_Q, W_K, W_V]': (24, 1536, 64)\n W_O: (24, 64, 1536)\n '[b_Q, b_K, b_V]': (24, 64)\n b_O: (1536,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (6144,)\n W_in: (1536, 6144)\n b_in: (6144,)\n W_out: (6144, 1536)\n b_out: (1536,)\nln_final:\n '[w, b]': (1536,)\nunembed:\n W_U: (1536, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 1536)"},"pos_embed":{"W_pos":"(1024, 1536)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(1536,)"},"ln2":{"[w, b]":"(1536,)"},"attn":{"[W_Q, W_K, W_V]":"(24, 1536, 64)","W_O":"(24, 64, 1536)","[b_Q, b_K, b_V]":"(24, 64)","b_O":"(1536,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(6144,)"},"W_in":"(1536, 6144)","b_in":"(6144,)","W_out":"(6144, 1536)","b_out":"(1536,)"}}},"ln_final":{"[w, b]":"(1536,)"},"unembed":{"W_U":"(1536, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 24, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 24, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 6144)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1536)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\nunembed:\n hook_in: (batch, seq_len, 1536)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1536)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 24, 64)","[hook_attn_scores, hook_pattern]":"(batch, 24, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 6144)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1536)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"unembed":{"hook_in":"(batch, seq_len, 1536)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1536)"}} -{"name.default_alias":"solu-12l-pile","name.huggingface":"NeelNanda\/SoLU_12L_v23_old","name.aliases":"solu-12l-pile, solu-12l-old","model_type":"solu","name.from_cfg":"SoLU_12L_v23_old","n_params.as_str":"340M","n_params.as_int":339738624,"n_params.from_name":null,"cfg.n_params":339738624,"cfg.n_layers":12,"cfg.n_heads":24,"cfg.d_model":1536,"cfg.d_vocab":50278,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel-solu-old","cfg.normalization_type":"LN","config.raw__":{"d_model":1536,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":24,"d_mlp":6144,"d_vocab":50278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_12L_v23_old","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel-solu-old","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0204124145,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50278,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":339738624,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1536\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 24\nd_mlp: 6144\nd_vocab: 50278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_12L_v23_old\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel-solu-old\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.020412414523193152\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50278\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 339738624\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50278, 1536)\npos_embed:\n W_pos: (1024, 1536)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (1536,)\n ln2:\n '[w, b]': (1536,)\n attn:\n '[W_Q, W_K, W_V]': (24, 1536, 64)\n W_O: (24, 64, 1536)\n '[b_Q, b_K, b_V]': (24, 64)\n b_O: (1536,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (6144,)\n W_in: (1536, 6144)\n b_in: (6144,)\n W_out: (6144, 1536)\n b_out: (1536,)\nln_final:\n '[w, b]': (1536,)\nunembed:\n W_U: (1536, 50278)\n b_U: (50278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50278, 1536)"},"pos_embed":{"W_pos":"(1024, 1536)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(1536,)"},"ln2":{"[w, b]":"(1536,)"},"attn":{"[W_Q, W_K, W_V]":"(24, 1536, 64)","W_O":"(24, 64, 1536)","[b_Q, b_K, b_V]":"(24, 64)","b_O":"(1536,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(6144,)"},"W_in":"(1536, 6144)","b_in":"(6144,)","W_out":"(6144, 1536)","b_out":"(1536,)"}}},"ln_final":{"[w, b]":"(1536,)"},"unembed":{"W_U":"(1536, 50278)","b_U":"(50278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 24, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 24, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 6144)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1536)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\nunembed:\n hook_in: (batch, seq_len, 1536)\n hook_out: (batch, seq_len, 50278)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1536)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 24, 64)","[hook_attn_scores, hook_pattern]":"(batch, 24, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 6144)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1536)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"unembed":{"hook_in":"(batch, seq_len, 1536)","hook_out":"(batch, seq_len, 50278)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1536)"}} -{"name.default_alias":"solu-1l","name.huggingface":"NeelNanda\/SoLU_1L512W_C4_Code","name.aliases":"solu-1l, solu-1l-new, solu-1l-c4-code","model_type":"solu","name.from_cfg":"SoLU_1L512W_C4_Code","n_params.as_str":"3.1M","n_params.as_int":3145728,"n_params.from_name":null,"cfg.n_params":3145728,"cfg.n_layers":1,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":1,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_1L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":3145728,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 1\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_1L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 3145728\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '0':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (2048,)\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"0":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(2048,)"},"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '0':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"0":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"solu-1l-wiki","name.huggingface":"NeelNanda\/SoLU_1L512W_Wiki_Finetune","name.aliases":"solu-1l-wiki, solu-1l-wiki-finetune, solu-1l-finetune","model_type":"solu","name.from_cfg":"SoLU_1L512W_Wiki_Finetune","n_params.as_str":"3.1M","n_params.as_int":3145728,"n_params.from_name":null,"cfg.n_params":3145728,"cfg.n_layers":1,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":1,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_1L512W_Wiki_Finetune","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":3145728,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 1\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_1L512W_Wiki_Finetune\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 3145728\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '0':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (2048,)\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"0":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(2048,)"},"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '0':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"0":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"solu-1l-pile","name.huggingface":"NeelNanda\/SoLU_1L_v9_old","name.aliases":"solu-1l-pile, solu-1l-old","model_type":"solu","name.from_cfg":"SoLU_1L_v9_old","n_params.as_str":"13M","n_params.as_int":12582912,"n_params.from_name":null,"cfg.n_params":12582912,"cfg.n_layers":1,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50278,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel-solu-old","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":1,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"SoLU_1L_v9_old","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel-solu-old","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50278,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 1\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: SoLU_1L_v9_old\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel-solu-old\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50278\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50278, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '0':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (4096,)\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n w: (1024,)\nunembed:\n W_U: (1024, 50278)\n b_U: (50278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50278, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"0":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(4096,)"},"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"w":"(1024,)"},"unembed":{"W_U":"(1024, 50278)","b_U":"(50278,)"}},"tensor_shapes.activation_cache":"blocks:\n '0':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50278)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"0":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50278)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} -{"name.default_alias":"solu-2l","name.huggingface":"NeelNanda\/SoLU_2L512W_C4_Code","name.aliases":"solu-2l, solu-2l-new, solu-2l-c4-code","model_type":"solu","name.from_cfg":"SoLU_2L512W_C4_Code","n_params.as_str":"6.3M","n_params.as_int":6291456,"n_params.from_name":null,"cfg.n_params":6291456,"cfg.n_layers":2,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":2,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_2L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":6291456,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 2\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_2L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 6291456\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-1]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (2048,)\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-1]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(2048,)"},"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-1]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-1]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"solu-2l-pile","name.huggingface":"NeelNanda\/SoLU_2L_v10_old","name.aliases":"solu-2l-pile, solu-2l-old","model_type":"solu","name.from_cfg":"SoLU_2L_v10_old","n_params.as_str":"13M","n_params.as_int":12812288,"n_params.from_name":null,"cfg.n_params":12812288,"cfg.n_layers":2,"cfg.n_heads":11,"cfg.d_model":736,"cfg.d_vocab":50278,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel-solu-old","cfg.normalization_type":"LNPre","config.raw__":{"d_model":736,"d_head":64,"n_layers":2,"n_ctx":1024,"n_heads":11,"d_mlp":2944,"d_vocab":50278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LNPre","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"SoLU_2L_v10_old","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel-solu-old","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0294883912,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50278,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12812288,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 736\nd_head: 64\nn_layers: 2\nn_ctx: 1024\nn_heads: 11\nd_mlp: 2944\nd_vocab: 50278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LNPre\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: SoLU_2L_v10_old\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel-solu-old\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02948839123097943\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50278\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12812288\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50278, 736)\npos_embed:\n W_pos: (1024, 736)\nblocks:\n '[0-1]':\n attn:\n '[W_Q, W_K, W_V]': (11, 736, 64)\n W_O: (11, 64, 736)\n '[b_Q, b_K, b_V]': (11, 64)\n b_O: (736,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (736, 2944)\n b_in: (2944,)\n W_out: (2944, 736)\n b_out: (736,)\nunembed:\n W_U: (736, 50278)\n b_U: (50278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50278, 736)"},"pos_embed":{"W_pos":"(1024, 736)"},"blocks":{"[0-1]":{"attn":{"[W_Q, W_K, W_V]":"(11, 736, 64)","W_O":"(11, 64, 736)","[b_Q, b_K, b_V]":"(11, 64)","b_O":"(736,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(736, 2944)","b_in":"(2944,)","W_out":"(2944, 736)","b_out":"(736,)"}}},"unembed":{"W_U":"(736, 50278)","b_U":"(50278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-1]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 736)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 11, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 11, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 736)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2944)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2944)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 736)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 736)\nunembed:\n hook_in: (batch, seq_len, 736)\n hook_out: (batch, seq_len, 50278)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 736)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-1]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 736)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 11, 64)","[hook_attn_scores, hook_pattern]":"(batch, 11, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 736)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2944)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2944)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 736)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 736)"},"unembed":{"hook_in":"(batch, seq_len, 736)","hook_out":"(batch, seq_len, 50278)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 736)"}} -{"name.default_alias":"solu-3l","name.huggingface":"NeelNanda\/SoLU_3L512W_C4_Code","name.aliases":"solu-3l, solu-3l-new, solu-3l-c4-code","model_type":"solu","name.from_cfg":"SoLU_3L512W_C4_Code","n_params.as_str":"9.4M","n_params.as_int":9437184,"n_params.from_name":null,"cfg.n_params":9437184,"cfg.n_layers":3,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":3,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_3L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":9437184,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 3\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_3L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 9437184\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-2]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (2048,)\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-2]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(2048,)"},"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-2]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-2]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"solu-4l","name.huggingface":"NeelNanda\/SoLU_4L512W_C4_Code","name.aliases":"solu-4l, solu-4l-new, solu-4l-c4-code","model_type":"solu","name.from_cfg":"SoLU_4L512W_C4_Code","n_params.as_str":"13M","n_params.as_int":12582912,"n_params.from_name":null,"cfg.n_params":12582912,"cfg.n_layers":4,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":4,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_4L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 4\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_4L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-3]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (2048,)\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-3]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(2048,)"},"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-3]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-3]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"solu-4l-wiki","name.huggingface":"NeelNanda\/SoLU_4L512W_Wiki_Finetune","name.aliases":"solu-4l-wiki, solu-4l-wiki-finetune, solu-4l-finetune","model_type":"solu","name.from_cfg":"SoLU_4L512W_Wiki_Finetune","n_params.as_str":"13M","n_params.as_int":12582912,"n_params.from_name":null,"cfg.n_params":12582912,"cfg.n_layers":4,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":4,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_4L512W_Wiki_Finetune","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 4\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_4L512W_Wiki_Finetune\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-3]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (2048,)\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-3]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(2048,)"},"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-3]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-3]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"solu-4l-pile","name.huggingface":"NeelNanda\/SoLU_4L_v11_old","name.aliases":"solu-4l-pile, solu-4l-old","model_type":"solu","name.from_cfg":"SoLU_4L_v11_old","n_params.as_str":"13M","n_params.as_int":12582912,"n_params.from_name":null,"cfg.n_params":12582912,"cfg.n_layers":4,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":50278,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel-solu-old","cfg.normalization_type":"LNPre","config.raw__":{"d_model":512,"d_head":64,"n_layers":4,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":50278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LNPre","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"SoLU_4L_v11_old","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel-solu-old","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50278,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 4\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 50278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LNPre\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: SoLU_4L_v11_old\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel-solu-old\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50278\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50278, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-3]':\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nunembed:\n W_U: (512, 50278)\n b_U: (50278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50278, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-3]":{"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"unembed":{"W_U":"(512, 50278)","b_U":"(50278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-3]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50278)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-3]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50278)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"solu-6l","name.huggingface":"NeelNanda\/SoLU_6L768W_C4_Code","name.aliases":"solu-6l, solu-6l-new, solu-6l-c4-code","model_type":"solu","name.from_cfg":"SoLU_6L768W_C4_Code","n_params.as_str":"42M","n_params.as_int":42467328,"n_params.from_name":null,"cfg.n_params":42467328,"cfg.n_layers":6,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":6,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_6L768W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":42467328,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 6\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_6L768W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 42467328\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (3072,)\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(3072,)"},"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} -{"name.default_alias":"solu-6l-pile","name.huggingface":"NeelNanda\/SoLU_6L_v13_old","name.aliases":"solu-6l-pile, solu-6l-old","model_type":"solu","name.from_cfg":"SoLU_6L_v13_old","n_params.as_str":"42M","n_params.as_int":42467328,"n_params.from_name":null,"cfg.n_params":42467328,"cfg.n_layers":6,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50278,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel-solu-old","cfg.normalization_type":"LNPre","config.raw__":{"d_model":768,"d_head":64,"n_layers":6,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LNPre","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"SoLU_6L_v13_old","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel-solu-old","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50278,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":42467328,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 6\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LNPre\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: SoLU_6L_v13_old\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel-solu-old\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50278\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 42467328\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50278, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-5]':\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nunembed:\n W_U: (768, 50278)\n b_U: (50278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50278, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-5]":{"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"unembed":{"W_U":"(768, 50278)","b_U":"(50278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50278)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50278)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} -{"name.default_alias":"solu-8l","name.huggingface":"NeelNanda\/SoLU_8L1024W_C4_Code","name.aliases":"solu-8l, solu-8l-new, solu-8l-c4-code","model_type":"solu","name.from_cfg":"SoLU_8L1024W_C4_Code","n_params.as_str":"101M","n_params.as_int":100663296,"n_params.from_name":null,"cfg.n_params":100663296,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":8,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_8L1024W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":100663296,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 8\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_8L1024W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 100663296\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (4096,)\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(4096,)"},"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} -{"name.default_alias":"solu-8l-pile","name.huggingface":"NeelNanda\/SoLU_8L_v21_old","name.aliases":"solu-8l-pile, solu-8l-old","model_type":"solu","name.from_cfg":"SoLU_8L_v21_old","n_params.as_str":"101M","n_params.as_int":100663296,"n_params.from_name":null,"cfg.n_params":100663296,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50278,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel-solu-old","cfg.normalization_type":"LNPre","config.raw__":{"d_model":1024,"d_head":64,"n_layers":8,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LNPre","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_8L_v21_old","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel-solu-old","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50278,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":100663296,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 8\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LNPre\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_8L_v21_old\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel-solu-old\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50278\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 100663296\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50278, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-7]':\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nunembed:\n W_U: (1024, 50278)\n b_U: (50278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50278, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-7]":{"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"unembed":{"W_U":"(1024, 50278)","b_U":"(50278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50278)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50278)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} -{"name.default_alias":"gpt-oss-20b","name.huggingface":"openai\/gpt-oss-20b","name.aliases":"gpt-oss-20b, gpt-oss","model_type":null,"name.from_cfg":"gpt-oss-20b","n_params.as_str":"20B","n_params.as_int":20244971520,"n_params.from_name":"20b","cfg.n_params":20244971520,"cfg.n_layers":24,"cfg.n_heads":64,"cfg.d_model":2880,"cfg.d_vocab":201088,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GptOssForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2880,"d_head":64,"n_layers":24,"n_ctx":131072,"n_heads":64,"d_mlp":2880,"d_vocab":201088,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":32,"experts_per_token":4,"final_rms":true,"dtype":"torch.float32","model_name":"gpt-oss-20b","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GptOssForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"openai\/gpt-oss-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0149071198,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":201088,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":20244971520,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":150000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2880\nd_head: 64\nn_layers: 24\nn_ctx: 131072\nn_heads: 64\nd_mlp: 2880\nd_vocab: 201088\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: 32\nexperts_per_token: 4\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gpt-oss-20b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GptOssForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: openai\/gpt-oss-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.014907119849998597\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 201088\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 20244971520\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 150000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"openai\/gpt-oss-20b","tokenizer.vocab_size":199998.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"c9AMJHU0SnHAXwla61AkYA78n-w=","tensor_shapes.state_dict":"embed:\n W_E: (201088, 2880)\nblocks:\n '[0-23]':\n ln1:\n w: (2880,)\n ln2:\n w: (2880,)\n attn:\n W_Q: (64, 2880, 64)\n W_O: (64, 64, 2880)\n b_Q: (64, 64)\n b_O: (2880,)\n '[_W_K, _W_V]': (8, 2880, 64)\n '[_b_K, _b_V]': (8, 64)\n mask: (131072, 131072)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (131072, 64)\n mlp:\n experts:\n '[0-31]':\n W_gate:\n weight: (2880, 2880)\n bias: (2880,)\n W_in:\n weight: (2880, 2880)\n bias: (2880,)\n W_out:\n weight: (2880, 2880)\n bias: (2880,)\n W_gate:\n weight: (32, 2880)\n bias: (32,)\nln_final:\n w: (2880,)\nunembed:\n W_U: (2880, 201088)\n b_U: (201088,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(201088, 2880)"},"blocks":{"[0-23]":{"ln1":{"w":"(2880,)"},"ln2":{"w":"(2880,)"},"attn":{"W_Q":"(64, 2880, 64)","W_O":"(64, 64, 2880)","b_Q":"(64, 64)","b_O":"(2880,)","[_W_K, _W_V]":"(8, 2880, 64)","[_b_K, _b_V]":"(8, 64)","mask":"(131072, 131072)","IGNORE":"()","[rotary_sin, rotary_cos]":"(131072, 64)"},"mlp":{"experts":{"[0-31]":{"W_gate":{"weight":"(2880, 2880)","bias":"(2880,)"},"W_in":{"weight":"(2880, 2880)","bias":"(2880,)"},"W_out":{"weight":"(2880, 2880)","bias":"(2880,)"}}},"W_gate":{"weight":"(32, 2880)","bias":"(32,)"}}}},"ln_final":{"w":"(2880,)"},"unembed":{"W_U":"(2880, 201088)","b_U":"(201088,)"}},"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen-14b","name.huggingface":"Qwen\/Qwen-14B","name.aliases":"qwen-14b","model_type":"qwen","name.from_cfg":"Qwen-14B","n_params.as_str":"13B","n_params.as_int":12609126400,"n_params.from_name":"14b","cfg.n_params":12609126400,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"QWenLMHeadModel","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":40,"d_mlp":13696,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen-14B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"QWenLMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen-14B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12609126400,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 40\nd_mlp: 13696\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen-14B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: QWenLMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen-14B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12609126400\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen-14b-chat","name.huggingface":"Qwen\/Qwen-14B-Chat","name.aliases":"qwen-14b-chat","model_type":"qwen","name.from_cfg":"Qwen-14B-Chat","n_params.as_str":"13B","n_params.as_int":12609126400,"n_params.from_name":"14b","cfg.n_params":12609126400,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"QWenLMHeadModel","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":40,"d_mlp":13696,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen-14B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"QWenLMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen-14B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12609126400,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 40\nd_mlp: 13696\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen-14B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: QWenLMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen-14B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12609126400\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen-1.8b","name.huggingface":"Qwen\/Qwen-1_8B","name.aliases":"qwen-1.8b","model_type":"qwen","name.from_cfg":"Qwen-1_8B","n_params.as_str":"1.2B","n_params.as_int":1214251008,"n_params.from_name":"1.8b","cfg.n_params":1214251008,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"QWenLMHeadModel","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":5504,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen-1_8B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"QWenLMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen-1_8B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1214251008,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 5504\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen-1_8B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: QWenLMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen-1_8B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1214251008\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen-1.8b-chat","name.huggingface":"Qwen\/Qwen-1_8B-Chat","name.aliases":"qwen-1.8b-chat","model_type":"qwen","name.from_cfg":"Qwen-1_8B-Chat","n_params.as_str":"1.2B","n_params.as_int":1214251008,"n_params.from_name":"1.8b","cfg.n_params":1214251008,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"QWenLMHeadModel","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":5504,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen-1_8B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"QWenLMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen-1_8B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1214251008,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 5504\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen-1_8B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: QWenLMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen-1_8B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1214251008\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen-7b","name.huggingface":"Qwen\/Qwen-7B","name.aliases":"qwen-7b","model_type":"qwen","name.from_cfg":"Qwen-7B","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"QWenLMHeadModel","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":11008,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen-7B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"QWenLMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen-7B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 11008\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen-7B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: QWenLMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen-7B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen-7b-chat","name.huggingface":"Qwen\/Qwen-7B-Chat","name.aliases":"qwen-7b-chat","model_type":"qwen","name.from_cfg":"Qwen-7B-Chat","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"QWenLMHeadModel","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":11008,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen-7B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"QWenLMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen-7B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 11008\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen-7B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: QWenLMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen-7B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen1.5-0.5b","name.huggingface":"Qwen\/Qwen1.5-0.5B","name.aliases":"qwen1.5-0.5b","model_type":"qwen","name.from_cfg":"Qwen1.5-0.5B","n_params.as_str":"308M","n_params.as_int":308281344,"n_params.from_name":"0.5b","cfg.n_params":308281344,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":2816,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-0.5B","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-0.5B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":308281344,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 2816\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-0.5B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-0.5B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 308281344\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen1.5-0.5b-chat","name.huggingface":"Qwen\/Qwen1.5-0.5B-Chat","name.aliases":"qwen1.5-0.5b-chat","model_type":"qwen","name.from_cfg":"Qwen1.5-0.5B-Chat","n_params.as_str":"308M","n_params.as_int":308281344,"n_params.from_name":"0.5b","cfg.n_params":308281344,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":2816,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-0.5B-Chat","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-0.5B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":308281344,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 2816\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-0.5B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-0.5B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 308281344\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen1.5-1.8b","name.huggingface":"Qwen\/Qwen1.5-1.8B","name.aliases":"qwen1.5-1.8b","model_type":"qwen","name.from_cfg":"Qwen1.5-1.8B","n_params.as_str":"1.2B","n_params.as_int":1214251008,"n_params.from_name":"1.8b","cfg.n_params":1214251008,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":5504,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-1.8B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-1.8B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1214251008,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 5504\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-1.8B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-1.8B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1214251008\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen1.5-1.8b-chat","name.huggingface":"Qwen\/Qwen1.5-1.8B-Chat","name.aliases":"qwen1.5-1.8b-chat","model_type":"qwen","name.from_cfg":"Qwen1.5-1.8B-Chat","n_params.as_str":"1.2B","n_params.as_int":1214251008,"n_params.from_name":"1.8b","cfg.n_params":1214251008,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":5504,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-1.8B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-1.8B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1214251008,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 5504\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-1.8B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-1.8B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1214251008\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen1.5-14b","name.huggingface":"Qwen\/Qwen1.5-14B","name.aliases":"qwen1.5-14b","model_type":"qwen","name.from_cfg":"Qwen1.5-14B","n_params.as_str":"13B","n_params.as_int":12609126400,"n_params.from_name":"14b","cfg.n_params":12609126400,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":40,"d_mlp":13696,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":40,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-14B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-14B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12609126400,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 40\nd_mlp: 13696\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 40\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-14B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-14B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12609126400\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen1.5-14b-chat","name.huggingface":"Qwen\/Qwen1.5-14B-Chat","name.aliases":"qwen1.5-14b-chat","model_type":"qwen","name.from_cfg":"Qwen1.5-14B-Chat","n_params.as_str":"13B","n_params.as_int":12609126400,"n_params.from_name":"14b","cfg.n_params":12609126400,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":40,"d_mlp":13696,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":40,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-14B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-14B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12609126400,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 40\nd_mlp: 13696\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 40\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-14B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-14B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12609126400\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen1.5-4b","name.huggingface":"Qwen\/Qwen1.5-4B","name.aliases":"qwen1.5-4b","model_type":"qwen","name.from_cfg":"Qwen1.5-4B","n_params.as_str":"3.2B","n_params.as_int":3171942400,"n_params.from_name":"4b","cfg.n_params":3171942400,"cfg.n_layers":40,"cfg.n_heads":20,"cfg.d_model":2560,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2560,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":20,"d_mlp":6912,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":20,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-4B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-4B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":3171942400,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":5000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 20\nd_mlp: 6912\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 20\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-4B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-4B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 3171942400\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 5000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen1.5-4b-chat","name.huggingface":"Qwen\/Qwen1.5-4B-Chat","name.aliases":"qwen1.5-4b-chat","model_type":"qwen","name.from_cfg":"Qwen1.5-4B-Chat","n_params.as_str":"3.2B","n_params.as_int":3171942400,"n_params.from_name":"4b","cfg.n_params":3171942400,"cfg.n_layers":40,"cfg.n_heads":20,"cfg.d_model":2560,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2560,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":20,"d_mlp":6912,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":20,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-4B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-4B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":3171942400,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":5000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 20\nd_mlp: 6912\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 20\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-4B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-4B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 3171942400\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 5000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen1.5-7b","name.huggingface":"Qwen\/Qwen1.5-7B","name.aliases":"qwen1.5-7b","model_type":"qwen","name.from_cfg":"Qwen1.5-7B","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":11008,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":32,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-7B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-7B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 11008\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 32\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-7B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-7B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen1.5-7b-chat","name.huggingface":"Qwen\/Qwen1.5-7B-Chat","name.aliases":"qwen1.5-7b-chat","model_type":"qwen","name.from_cfg":"Qwen1.5-7B-Chat","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":11008,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":32,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-7B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-7B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 11008\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 32\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-7B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-7B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2-0.5b","name.huggingface":"Qwen\/Qwen2-0.5B","name.aliases":"qwen2-0.5b","model_type":"qwen","name.from_cfg":"Qwen2-0.5B","n_params.as_str":"391M","n_params.as_int":390856704,"n_params.from_name":"0.5b","cfg.n_params":390856704,"cfg.n_layers":24,"cfg.n_heads":14,"cfg.d_model":896,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":896,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":14,"d_mlp":4864,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2-0.5B","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2-0.5B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":390856704,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 896\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 14\nd_mlp: 4864\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2-0.5B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2-0.5B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 390856704\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2-0.5b-instruct","name.huggingface":"Qwen\/Qwen2-0.5B-Instruct","name.aliases":"qwen2-0.5b-instruct","model_type":"qwen","name.from_cfg":"Qwen2-0.5B-Instruct","n_params.as_str":"391M","n_params.as_int":390856704,"n_params.from_name":"0.5b","cfg.n_params":390856704,"cfg.n_layers":24,"cfg.n_heads":14,"cfg.d_model":896,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":896,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":14,"d_mlp":4864,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2-0.5B-Instruct","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2-0.5B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":390856704,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 896\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 14\nd_mlp: 4864\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2-0.5B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2-0.5B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 390856704\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2-1.5b","name.huggingface":"Qwen\/Qwen2-1.5B","name.aliases":"qwen2-1.5b","model_type":"qwen","name.from_cfg":"Qwen2-1.5B","n_params.as_str":"1.4B","n_params.as_int":1420296192,"n_params.from_name":"1.5b","cfg.n_params":1420296192,"cfg.n_layers":28,"cfg.n_heads":12,"cfg.d_model":1536,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1536,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":12,"d_mlp":8960,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2-1.5B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2-1.5B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1420296192,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1536\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 12\nd_mlp: 8960\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2-1.5B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2-1.5B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1420296192\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2-1.5b-instruct","name.huggingface":"Qwen\/Qwen2-1.5B-Instruct","name.aliases":"qwen2-1.5b-instruct","model_type":"qwen","name.from_cfg":"Qwen2-1.5B-Instruct","n_params.as_str":"1.4B","n_params.as_int":1420296192,"n_params.from_name":"1.5b","cfg.n_params":1420296192,"cfg.n_layers":28,"cfg.n_heads":12,"cfg.d_model":1536,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1536,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":12,"d_mlp":8960,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2-1.5B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2-1.5B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1420296192,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1536\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 12\nd_mlp: 8960\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2-1.5B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2-1.5B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1420296192\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2-7b","name.huggingface":"Qwen\/Qwen2-7B","name.aliases":"qwen2-7b","model_type":"qwen","name.from_cfg":"Qwen2-7B","n_params.as_str":"7.1B","n_params.as_int":7141851136,"n_params.from_name":"7b","cfg.n_params":7141851136,"cfg.n_layers":28,"cfg.n_heads":28,"cfg.d_model":3584,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3584,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":28,"d_mlp":18944,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2-7B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2-7B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7141851136,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 3584\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 28\nd_mlp: 18944\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2-7B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2-7B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7141851136\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2-7b-instruct","name.huggingface":"Qwen\/Qwen2-7B-Instruct","name.aliases":"qwen2-7b-instruct","model_type":"qwen","name.from_cfg":"Qwen2-7B-Instruct","n_params.as_str":"7.1B","n_params.as_int":7141851136,"n_params.from_name":"7b","cfg.n_params":7141851136,"cfg.n_layers":28,"cfg.n_heads":28,"cfg.d_model":3584,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3584,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":28,"d_mlp":18944,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2-7B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2-7B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7141851136,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 3584\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 28\nd_mlp: 18944\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2-7B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2-7B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7141851136\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2.5-0.5b","name.huggingface":"Qwen\/Qwen2.5-0.5B","name.aliases":"qwen2.5-0.5b","model_type":"qwen","name.from_cfg":"Qwen2.5-0.5B","n_params.as_str":"391M","n_params.as_int":390856704,"n_params.from_name":"0.5b","cfg.n_params":390856704,"cfg.n_layers":24,"cfg.n_heads":14,"cfg.d_model":896,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":896,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":14,"d_mlp":4864,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-0.5B","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-0.5B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":390856704,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 896\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 14\nd_mlp: 4864\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-0.5B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-0.5B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 390856704\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2.5-0.5b-instruct","name.huggingface":"Qwen\/Qwen2.5-0.5B-Instruct","name.aliases":"qwen2.5-0.5b-instruct","model_type":"qwen","name.from_cfg":"Qwen2.5-0.5B-Instruct","n_params.as_str":"391M","n_params.as_int":390856704,"n_params.from_name":"0.5b","cfg.n_params":390856704,"cfg.n_layers":24,"cfg.n_heads":14,"cfg.d_model":896,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":896,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":14,"d_mlp":4864,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-0.5B-Instruct","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-0.5B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":390856704,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 896\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 14\nd_mlp: 4864\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-0.5B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-0.5B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 390856704\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2.5-1.5b","name.huggingface":"Qwen\/Qwen2.5-1.5B","name.aliases":"qwen2.5-1.5b","model_type":"qwen","name.from_cfg":"Qwen2.5-1.5B","n_params.as_str":"1.4B","n_params.as_int":1420296192,"n_params.from_name":"1.5b","cfg.n_params":1420296192,"cfg.n_layers":28,"cfg.n_heads":12,"cfg.d_model":1536,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1536,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":12,"d_mlp":8960,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-1.5B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-1.5B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1420296192,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1536\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 12\nd_mlp: 8960\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-1.5B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-1.5B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1420296192\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2.5-1.5b-instruct","name.huggingface":"Qwen\/Qwen2.5-1.5B-Instruct","name.aliases":"qwen2.5-1.5b-instruct","model_type":"qwen","name.from_cfg":"Qwen2.5-1.5B-Instruct","n_params.as_str":"1.4B","n_params.as_int":1420296192,"n_params.from_name":"1.5b","cfg.n_params":1420296192,"cfg.n_layers":28,"cfg.n_heads":12,"cfg.d_model":1536,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1536,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":12,"d_mlp":8960,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-1.5B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-1.5B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1420296192,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1536\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 12\nd_mlp: 8960\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-1.5B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-1.5B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1420296192\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2.5-14b","name.huggingface":"Qwen\/Qwen2.5-14B","name.aliases":"qwen2.5-14b","model_type":"qwen","name.from_cfg":"Qwen2.5-14B","n_params.as_str":"15B","n_params.as_int":15225323520,"n_params.from_name":"14b","cfg.n_params":15225323520,"cfg.n_layers":48,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":48,"n_ctx":2048,"n_heads":40,"d_mlp":13824,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-14B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-14B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":15225323520,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 48\nn_ctx: 2048\nn_heads: 40\nd_mlp: 13824\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-14B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-14B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 15225323520\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2.5-14b-instruct","name.huggingface":"Qwen\/Qwen2.5-14B-Instruct","name.aliases":"qwen2.5-14b-instruct","model_type":"qwen","name.from_cfg":"Qwen2.5-14B-Instruct","n_params.as_str":"15B","n_params.as_int":15225323520,"n_params.from_name":"14b","cfg.n_params":15225323520,"cfg.n_layers":48,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":48,"n_ctx":2048,"n_heads":40,"d_mlp":13824,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-14B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-14B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":15225323520,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 48\nn_ctx: 2048\nn_heads: 40\nd_mlp: 13824\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-14B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-14B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 15225323520\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2.5-32b","name.huggingface":"Qwen\/Qwen2.5-32B","name.aliases":"qwen2.5-32b","model_type":"qwen","name.from_cfg":"Qwen2.5-32B","n_params.as_str":"34B","n_params.as_int":33889976320,"n_params.from_name":"32b","cfg.n_params":33889976320,"cfg.n_layers":64,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":64,"n_ctx":2048,"n_heads":40,"d_mlp":27648,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-32B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-32B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":33889976320,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 64\nn_ctx: 2048\nn_heads: 40\nd_mlp: 27648\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-32B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-32B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 33889976320\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2.5-32b-instruct","name.huggingface":"Qwen\/Qwen2.5-32B-Instruct","name.aliases":"qwen2.5-32b-instruct","model_type":"qwen","name.from_cfg":"Qwen2.5-32B-Instruct","n_params.as_str":"34B","n_params.as_int":33889976320,"n_params.from_name":"32b","cfg.n_params":33889976320,"cfg.n_layers":64,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":64,"n_ctx":2048,"n_heads":40,"d_mlp":27648,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-32B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-32B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":33889976320,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 64\nn_ctx: 2048\nn_heads: 40\nd_mlp: 27648\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-32B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-32B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 33889976320\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2.5-3b","name.huggingface":"Qwen\/Qwen2.5-3B","name.aliases":"qwen2.5-3b","model_type":"qwen","name.from_cfg":"Qwen2.5-3B","n_params.as_str":"3.0B","n_params.as_int":3038773248,"n_params.from_name":"3b","cfg.n_params":3038773248,"cfg.n_layers":36,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":36,"n_ctx":2048,"n_heads":16,"d_mlp":11008,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-3B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-3B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":3038773248,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 36\nn_ctx: 2048\nn_heads: 16\nd_mlp: 11008\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-3B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-3B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 3038773248\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2.5-3b-instruct","name.huggingface":"Qwen\/Qwen2.5-3B-Instruct","name.aliases":"qwen2.5-3b-instruct","model_type":"qwen","name.from_cfg":"Qwen2.5-3B-Instruct","n_params.as_str":"3.0B","n_params.as_int":3038773248,"n_params.from_name":"3b","cfg.n_params":3038773248,"cfg.n_layers":36,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":36,"n_ctx":2048,"n_heads":16,"d_mlp":11008,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-3B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-3B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":3038773248,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 36\nn_ctx: 2048\nn_heads: 16\nd_mlp: 11008\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-3B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-3B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 3038773248\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2.5-72b","name.huggingface":"Qwen\/Qwen2.5-72B","name.aliases":"qwen2.5-72b","model_type":"qwen","name.from_cfg":"Qwen2.5-72B","n_params.as_str":"80B","n_params.as_int":79607889920,"n_params.from_name":"72b","cfg.n_params":79607889920,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":2048,"n_heads":64,"d_mlp":29568,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-72B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-72B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":79607889920,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 2048\nn_heads: 64\nd_mlp: 29568\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-72B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-72B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 79607889920\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2.5-72b-instruct","name.huggingface":"Qwen\/Qwen2.5-72B-Instruct","name.aliases":"qwen2.5-72b-instruct","model_type":"qwen","name.from_cfg":"Qwen2.5-72B-Instruct","n_params.as_str":"80B","n_params.as_int":79607889920,"n_params.from_name":"72b","cfg.n_params":79607889920,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":2048,"n_heads":64,"d_mlp":29568,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-72B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-72B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":79607889920,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 2048\nn_heads: 64\nd_mlp: 29568\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-72B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-72B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 79607889920\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2.5-7b","name.huggingface":"Qwen\/Qwen2.5-7B","name.aliases":"qwen2.5-7b","model_type":"qwen","name.from_cfg":"Qwen2.5-7B","n_params.as_str":"7.1B","n_params.as_int":7141851136,"n_params.from_name":"7b","cfg.n_params":7141851136,"cfg.n_layers":28,"cfg.n_heads":28,"cfg.d_model":3584,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3584,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":28,"d_mlp":18944,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-7B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-7B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7141851136,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 3584\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 28\nd_mlp: 18944\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-7B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-7B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7141851136\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen2.5-7b-instruct","name.huggingface":"Qwen\/Qwen2.5-7B-Instruct","name.aliases":"qwen2.5-7b-instruct","model_type":"qwen","name.from_cfg":"Qwen2.5-7B-Instruct","n_params.as_str":"7.1B","n_params.as_int":7141851136,"n_params.from_name":"7b","cfg.n_params":7141851136,"cfg.n_layers":28,"cfg.n_heads":28,"cfg.d_model":3584,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3584,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":28,"d_mlp":18944,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-7B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-7B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7141851136,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 3584\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 28\nd_mlp: 18944\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-7B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-7B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7141851136\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"qwen3-0.6b","name.huggingface":"Qwen\/Qwen3-0.6B","name.aliases":"qwen3-0.6b","model_type":"qwen","name.from_cfg":"Qwen3-0.6B","n_params.as_str":"499M","n_params.as_int":499122176,"n_params.from_name":"0.6b","cfg.n_params":499122176,"cfg.n_layers":28,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1024,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":16,"d_mlp":3072,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen3-0.6B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen3-0.6B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":499122176,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 16\nd_mlp: 3072\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen3-0.6B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen3-0.6B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 499122176\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen3-0.6B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"OfOK7SjmrN4KFUqlDylX-Up77SM=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 1024)\nblocks:\n '[0-27]':\n ln1:\n w: (1024,)\n ln2:\n w: (1024,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (16, 1024, 128)\n W_O: (16, 128, 1024)\n b_Q: (16, 128)\n b_O: (1024,)\n '[_W_K, _W_V]': (8, 1024, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (1024, 3072)\n W_out: (3072, 1024)\n b_in: (3072,)\n b_out: (1024,)\nln_final:\n w: (1024,)\nunembed:\n W_U: (1024, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 1024)"},"blocks":{"[0-27]":{"ln1":{"w":"(1024,)"},"ln2":{"w":"(1024,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(16, 1024, 128)","W_O":"(16, 128, 1024)","b_Q":"(16, 128)","b_O":"(1024,)","[_W_K, _W_V]":"(8, 1024, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(1024, 3072)","W_out":"(3072, 1024)","b_in":"(3072,)","b_out":"(1024,)"}}},"ln_final":{"w":"(1024,)"},"unembed":{"W_U":"(1024, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n q_norm:\n hook_scale: (27659632, 1)\n hook_normalized: (27659632, 128)\n k_norm:\n hook_scale: (13829816, 1)\n hook_normalized: (13829816, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"q_norm":{"hook_scale":"(27659632, 1)","hook_normalized":"(27659632, 128)"},"k_norm":{"hook_scale":"(13829816, 1)","hook_normalized":"(13829816, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 1024)"}} -{"name.default_alias":"qwen3-0.6b-base","name.huggingface":"Qwen\/Qwen3-0.6B-Base","name.aliases":"qwen3-0.6b-base","model_type":"qwen","name.from_cfg":"Qwen3-0.6B-Base","n_params.as_str":"499M","n_params.as_int":499122176,"n_params.from_name":"0.6b","cfg.n_params":499122176,"cfg.n_layers":28,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1024,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":16,"d_mlp":3072,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen3-0.6B-Base","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen3-0.6B-Base","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":499122176,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 16\nd_mlp: 3072\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen3-0.6B-Base\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen3-0.6B-Base\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 499122176\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen3-0.6B-Base","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"OfOK7SjmrN4KFUqlDylX-Up77SM=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 1024)\nblocks:\n '[0-27]':\n ln1:\n w: (1024,)\n ln2:\n w: (1024,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (16, 1024, 128)\n W_O: (16, 128, 1024)\n b_Q: (16, 128)\n b_O: (1024,)\n '[_W_K, _W_V]': (8, 1024, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (1024, 3072)\n W_out: (3072, 1024)\n b_in: (3072,)\n b_out: (1024,)\nln_final:\n w: (1024,)\nunembed:\n W_U: (1024, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 1024)"},"blocks":{"[0-27]":{"ln1":{"w":"(1024,)"},"ln2":{"w":"(1024,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(16, 1024, 128)","W_O":"(16, 128, 1024)","b_Q":"(16, 128)","b_O":"(1024,)","[_W_K, _W_V]":"(8, 1024, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(1024, 3072)","W_out":"(3072, 1024)","b_in":"(3072,)","b_out":"(1024,)"}}},"ln_final":{"w":"(1024,)"},"unembed":{"W_U":"(1024, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n q_norm:\n hook_scale: (27659632, 1)\n hook_normalized: (27659632, 128)\n k_norm:\n hook_scale: (13829816, 1)\n hook_normalized: (13829816, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"q_norm":{"hook_scale":"(27659632, 1)","hook_normalized":"(27659632, 128)"},"k_norm":{"hook_scale":"(13829816, 1)","hook_normalized":"(13829816, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 1024)"}} -{"name.default_alias":"qwen3-1.7b","name.huggingface":"Qwen\/Qwen3-1.7B","name.aliases":"qwen3-1.7b","model_type":"qwen","name.from_cfg":"Qwen3-1.7B","n_params.as_str":"1.5B","n_params.as_int":1526726656,"n_params.from_name":"1.7b","cfg.n_params":1526726656,"cfg.n_layers":28,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":16,"d_mlp":6144,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen3-1.7B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen3-1.7B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1526726656,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 16\nd_mlp: 6144\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen3-1.7B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen3-1.7B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1526726656\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen3-1.7B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"OfOK7SjmrN4KFUqlDylX-Up77SM=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 2048)\nblocks:\n '[0-27]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (16, 2048, 128)\n W_O: (16, 128, 2048)\n b_Q: (16, 128)\n b_O: (2048,)\n '[_W_K, _W_V]': (8, 2048, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (2048, 6144)\n W_out: (6144, 2048)\n b_in: (6144,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 2048)"},"blocks":{"[0-27]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(16, 2048, 128)","W_O":"(16, 128, 2048)","b_Q":"(16, 128)","b_O":"(2048,)","[_W_K, _W_V]":"(8, 2048, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(2048, 6144)","W_out":"(6144, 2048)","b_in":"(6144,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n q_norm:\n hook_scale: (27659632, 1)\n hook_normalized: (27659632, 128)\n k_norm:\n hook_scale: (13829816, 1)\n hook_normalized: (13829816, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 6144)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"q_norm":{"hook_scale":"(27659632, 1)","hook_normalized":"(27659632, 128)"},"k_norm":{"hook_scale":"(13829816, 1)","hook_normalized":"(13829816, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 6144)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 2048)"}} -{"name.default_alias":"qwen3-14b","name.huggingface":"Qwen\/Qwen3-14B","name.aliases":"qwen3-14b","model_type":"qwen","name.from_cfg":"Qwen3-14B","n_params.as_str":"15B","n_params.as_int":14889779200,"n_params.from_name":"14b","cfg.n_params":14889779200,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":40,"d_mlp":17408,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen3-14B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen3-14B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":14889779200,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 40\nd_mlp: 17408\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen3-14B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen3-14B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 14889779200\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen3-14B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"OfOK7SjmrN4KFUqlDylX-Up77SM=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 17408)\n W_out: (17408, 5120)\n b_in: (17408,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 17408)","W_out":"(17408, 5120)","b_in":"(17408,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n q_norm:\n hook_scale: (69149080, 1)\n hook_normalized: (69149080, 128)\n k_norm:\n hook_scale: (13829816, 1)\n hook_normalized: (13829816, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 17408)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"q_norm":{"hook_scale":"(69149080, 1)","hook_normalized":"(69149080, 128)"},"k_norm":{"hook_scale":"(13829816, 1)","hook_normalized":"(13829816, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 17408)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 5120)"}} -{"name.default_alias":"qwen3-4b","name.huggingface":"Qwen\/Qwen3-4B","name.aliases":"qwen3-4b","model_type":"qwen","name.from_cfg":"Qwen3-4B","n_params.as_str":"4.2B","n_params.as_int":4199546880,"n_params.from_name":"4b","cfg.n_params":4199546880,"cfg.n_layers":36,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2560,"d_head":128,"n_layers":36,"n_ctx":2048,"n_heads":32,"d_mlp":9728,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen3-4B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen3-4B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":4199546880,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 128\nn_layers: 36\nn_ctx: 2048\nn_heads: 32\nd_mlp: 9728\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen3-4B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen3-4B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 4199546880\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen3-4B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"OfOK7SjmrN4KFUqlDylX-Up77SM=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 2560)\nblocks:\n '[0-35]':\n ln1:\n w: (2560,)\n ln2:\n w: (2560,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (32, 2560, 128)\n W_O: (32, 128, 2560)\n b_Q: (32, 128)\n b_O: (2560,)\n '[_W_K, _W_V]': (8, 2560, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (2560, 9728)\n W_out: (9728, 2560)\n b_in: (9728,)\n b_out: (2560,)\nln_final:\n w: (2560,)\nunembed:\n W_U: (2560, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 2560)"},"blocks":{"[0-35]":{"ln1":{"w":"(2560,)"},"ln2":{"w":"(2560,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(32, 2560, 128)","W_O":"(32, 128, 2560)","b_Q":"(32, 128)","b_O":"(2560,)","[_W_K, _W_V]":"(8, 2560, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(2560, 9728)","W_out":"(9728, 2560)","b_in":"(9728,)","b_out":"(2560,)"}}},"ln_final":{"w":"(2560,)"},"unembed":{"W_U":"(2560, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n q_norm:\n hook_scale: (55319264, 1)\n hook_normalized: (55319264, 128)\n k_norm:\n hook_scale: (13829816, 1)\n hook_normalized: (13829816, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 9728)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"q_norm":{"hook_scale":"(55319264, 1)","hook_normalized":"(55319264, 128)"},"k_norm":{"hook_scale":"(13829816, 1)","hook_normalized":"(13829816, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 9728)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 2560)"}} -{"name.default_alias":"qwen3-8b","name.huggingface":"Qwen\/Qwen3-8B","name.aliases":"qwen3-8b","model_type":"qwen","name.from_cfg":"Qwen3-8B","n_params.as_str":"7.9B","n_params.as_int":7851737088,"n_params.from_name":"8b","cfg.n_params":7851737088,"cfg.n_layers":36,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":36,"n_ctx":2048,"n_heads":32,"d_mlp":12288,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen3-8B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen3-8B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7851737088,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 36\nn_ctx: 2048\nn_heads: 32\nd_mlp: 12288\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen3-8B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen3-8B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7851737088\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen3-8B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"OfOK7SjmrN4KFUqlDylX-Up77SM=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 4096)\nblocks:\n '[0-35]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (2048, 2048)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 12288)\n W_out: (12288, 4096)\n b_in: (12288,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 4096)"},"blocks":{"[0-35]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(2048, 2048)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 12288)","W_out":"(12288, 4096)","b_in":"(12288,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n q_norm:\n hook_scale: (55319264, 1)\n hook_normalized: (55319264, 128)\n k_norm:\n hook_scale: (13829816, 1)\n hook_normalized: (13829816, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 12288)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"q_norm":{"hook_scale":"(55319264, 1)","hook_normalized":"(55319264, 128)"},"k_norm":{"hook_scale":"(13829816, 1)","hook_normalized":"(13829816, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 12288)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"qwen-32b-preview","name.huggingface":"Qwen\/QwQ-32B-Preview","name.aliases":"qwen-32b-preview","model_type":"qwen","name.from_cfg":"QwQ-32B-Preview","n_params.as_str":"34B","n_params.as_int":33889976320,"n_params.from_name":"32b","cfg.n_params":33889976320,"cfg.n_layers":64,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":64,"n_ctx":2048,"n_heads":40,"d_mlp":27648,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"QwQ-32B-Preview","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/QwQ-32B-Preview","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":33889976320,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 64\nn_ctx: 2048\nn_heads: 40\nd_mlp: 27648\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: QwQ-32B-Preview\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/QwQ-32B-Preview\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 33889976320\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} -{"name.default_alias":"tiny-stories-1L-21M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-1Layer-21M","n_params.as_str":"13M","n_params.as_int":12582912,"n_params.from_name":"21M","cfg.n_params":12582912,"cfg.n_layers":1,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":1,"n_ctx":512,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-1Layer-21M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-1Layer-21M","window_size":256,"attn_types":["global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 1\nn_ctx: 512\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-1Layer-21M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-1Layer-21M\nwindow_size: 256\nattn_types:\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-1Layer-21M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (512, 1024)\nblocks:\n '0':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (512, 512)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(512, 1024)"},"blocks":{"0":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(512, 512)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '0':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"0":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} -{"name.default_alias":"tiny-stories-1M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-1M","n_params.as_str":"393K","n_params.as_int":393216,"n_params.from_name":"1M","cfg.n_params":393216,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":64,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":64,"d_head":4,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":256,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-1M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-1M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.1,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":393216,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 64\nd_head: 4\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 256\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-1M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-1M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.1\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 393216\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-1M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 64)\npos_embed:\n W_pos: (512, 64)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (64,)\n ln2:\n '[w, b]': (64,)\n attn:\n '[W_Q, W_K, W_V]': (16, 64, 4)\n W_O: (16, 4, 64)\n '[b_Q, b_K, b_V]': (16, 4)\n b_O: (64,)\n mask: (512, 512)\n IGNORE: ()\n mlp:\n W_in: (64, 256)\n b_in: (256,)\n W_out: (256, 64)\n b_out: (64,)\nln_final:\n '[w, b]': (64,)\nunembed:\n W_U: (64, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 64)"},"pos_embed":{"W_pos":"(512, 64)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(64,)"},"ln2":{"[w, b]":"(64,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 64, 4)","W_O":"(16, 4, 64)","[b_Q, b_K, b_V]":"(16, 4)","b_O":"(64,)","mask":"(512, 512)","IGNORE":"()"},"mlp":{"W_in":"(64, 256)","b_in":"(256,)","W_out":"(256, 64)","b_out":"(64,)"}}},"ln_final":{"[w, b]":"(64,)"},"unembed":{"W_U":"(64, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 64)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 4)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 64)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 256)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 64)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 64)\nunembed:\n hook_in: (batch, seq_len, 64)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 64)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 64)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 4)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 64)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 256)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 64)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 64)"},"unembed":{"hook_in":"(batch, seq_len, 64)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 64)"}} -{"name.default_alias":"tiny-stories-28M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-28M","n_params.as_str":"25M","n_params.as_int":25165824,"n_params.from_name":"28M","cfg.n_params":25165824,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":512,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":32,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":2048,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-28M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-28M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":25165824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 32\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 2048\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-28M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-28M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 25165824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-28M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 512)\npos_embed:\n W_pos: (512, 512)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (16, 512, 32)\n W_O: (16, 32, 512)\n '[b_Q, b_K, b_V]': (16, 32)\n b_O: (512,)\n mask: (512, 512)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 512)"},"pos_embed":{"W_pos":"(512, 512)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 512, 32)","W_O":"(16, 32, 512)","[b_Q, b_K, b_V]":"(16, 32)","b_O":"(512,)","mask":"(512, 512)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 32)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 32)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"tiny-stories-2L-33M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-2Layers-33M","n_params.as_str":"25M","n_params.as_int":25165824,"n_params.from_name":"33M","cfg.n_params":25165824,"cfg.n_layers":2,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":2,"n_ctx":512,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-2Layers-33M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-2Layers-33M","window_size":256,"attn_types":["global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":25165824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 2\nn_ctx: 512\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-2Layers-33M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-2Layers-33M\nwindow_size: 256\nattn_types:\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 25165824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-2Layers-33M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (512, 1024)\nblocks:\n '[0-1]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (512, 512)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(512, 1024)"},"blocks":{"[0-1]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(512, 512)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-1]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-1]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} -{"name.default_alias":"tiny-stories-33M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-33M","n_params.as_str":"28M","n_params.as_int":28311552,"n_params.from_name":"33M","cfg.n_params":28311552,"cfg.n_layers":4,"cfg.n_heads":16,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":48,"n_layers":4,"n_ctx":512,"n_heads":16,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-33M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-33M","window_size":256,"attn_types":["global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":28311552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 48\nn_layers: 4\nn_ctx: 512\nn_heads: 16\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-33M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-33M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 28311552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-33M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (512, 768)\nblocks:\n '[0-3]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (16, 768, 48)\n W_O: (16, 48, 768)\n '[b_Q, b_K, b_V]': (16, 48)\n b_O: (768,)\n mask: (512, 512)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(512, 768)"},"blocks":{"[0-3]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 768, 48)","W_O":"(16, 48, 768)","[b_Q, b_K, b_V]":"(16, 48)","b_O":"(768,)","mask":"(512, 512)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-3]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 48)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-3]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 48)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} -{"name.default_alias":"tiny-stories-3M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-3M","n_params.as_str":"1.6M","n_params.as_int":1572864,"n_params.from_name":"3M","cfg.n_params":1572864,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":128,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":128,"d_head":8,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":512,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-3M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-3M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0707106781,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1572864,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 128\nd_head: 8\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 512\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-3M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-3M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.07071067811865475\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1572864\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-3M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 128)\npos_embed:\n W_pos: (512, 128)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (128,)\n ln2:\n '[w, b]': (128,)\n attn:\n '[W_Q, W_K, W_V]': (16, 128, 8)\n W_O: (16, 8, 128)\n '[b_Q, b_K, b_V]': (16, 8)\n b_O: (128,)\n mask: (512, 512)\n IGNORE: ()\n mlp:\n W_in: (128, 512)\n b_in: (512,)\n W_out: (512, 128)\n b_out: (128,)\nln_final:\n '[w, b]': (128,)\nunembed:\n W_U: (128, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 128)"},"pos_embed":{"W_pos":"(512, 128)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(128,)"},"ln2":{"[w, b]":"(128,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 128, 8)","W_O":"(16, 8, 128)","[b_Q, b_K, b_V]":"(16, 8)","b_O":"(128,)","mask":"(512, 512)","IGNORE":"()"},"mlp":{"W_in":"(128, 512)","b_in":"(512,)","W_out":"(512, 128)","b_out":"(128,)"}}},"ln_final":{"[w, b]":"(128,)"},"unembed":{"W_U":"(128, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 8)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 512)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 128)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\nunembed:\n hook_in: (batch, seq_len, 128)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 128)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 8)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 512)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 128)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"unembed":{"hook_in":"(batch, seq_len, 128)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 128)"}} -{"name.default_alias":"tiny-stories-8M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-8M","n_params.as_str":"6.3M","n_params.as_int":6291456,"n_params.from_name":"8M","cfg.n_params":6291456,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":256,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":256,"d_head":16,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":1024,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-8M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-8M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.05,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":6291456,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 256\nd_head: 16\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 1024\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-8M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-8M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.05\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 6291456\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-8M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 256)\npos_embed:\n W_pos: (512, 256)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (256,)\n ln2:\n '[w, b]': (256,)\n attn:\n '[W_Q, W_K, W_V]': (16, 256, 16)\n W_O: (16, 16, 256)\n '[b_Q, b_K, b_V]': (16, 16)\n b_O: (256,)\n mask: (512, 512)\n IGNORE: ()\n mlp:\n W_in: (256, 1024)\n b_in: (1024,)\n W_out: (1024, 256)\n b_out: (256,)\nln_final:\n '[w, b]': (256,)\nunembed:\n W_U: (256, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 256)"},"pos_embed":{"W_pos":"(512, 256)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(256,)"},"ln2":{"[w, b]":"(256,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 256, 16)","W_O":"(16, 16, 256)","[b_Q, b_K, b_V]":"(16, 16)","b_O":"(256,)","mask":"(512, 512)","IGNORE":"()"},"mlp":{"W_in":"(256, 1024)","b_in":"(1024,)","W_out":"(1024, 256)","b_out":"(256,)"}}},"ln_final":{"[w, b]":"(256,)"},"unembed":{"W_U":"(256, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 16)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 1024)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 256)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\nunembed:\n hook_in: (batch, seq_len, 256)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 256)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 16)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 1024)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 256)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"unembed":{"hook_in":"(batch, seq_len, 256)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 256)"}} -{"name.default_alias":"tiny-stories-instruct-1M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-Instruct-1M","n_params.as_str":"393K","n_params.as_int":393216,"n_params.from_name":"1M","cfg.n_params":393216,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":64,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":64,"d_head":4,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":256,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-Instruct-1M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-Instruct-1M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.1,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":393216,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 64\nd_head: 4\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 256\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-Instruct-1M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-Instruct-1M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.1\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 393216\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-Instruct-1M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 64)\npos_embed:\n W_pos: (512, 64)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (64,)\n ln2:\n '[w, b]': (64,)\n attn:\n '[W_Q, W_K, W_V]': (16, 64, 4)\n W_O: (16, 4, 64)\n '[b_Q, b_K, b_V]': (16, 4)\n b_O: (64,)\n mask: (512, 512)\n IGNORE: ()\n mlp:\n W_in: (64, 256)\n b_in: (256,)\n W_out: (256, 64)\n b_out: (64,)\nln_final:\n '[w, b]': (64,)\nunembed:\n W_U: (64, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 64)"},"pos_embed":{"W_pos":"(512, 64)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(64,)"},"ln2":{"[w, b]":"(64,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 64, 4)","W_O":"(16, 4, 64)","[b_Q, b_K, b_V]":"(16, 4)","b_O":"(64,)","mask":"(512, 512)","IGNORE":"()"},"mlp":{"W_in":"(64, 256)","b_in":"(256,)","W_out":"(256, 64)","b_out":"(64,)"}}},"ln_final":{"[w, b]":"(64,)"},"unembed":{"W_U":"(64, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 64)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 4)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 64)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 256)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 64)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 64)\nunembed:\n hook_in: (batch, seq_len, 64)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 64)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 64)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 4)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 64)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 256)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 64)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 64)"},"unembed":{"hook_in":"(batch, seq_len, 64)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 64)"}} -{"name.default_alias":"tiny-stories-instruct-28M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-Instruct-28M","n_params.as_str":"25M","n_params.as_int":25165824,"n_params.from_name":"28M","cfg.n_params":25165824,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":512,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":32,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":2048,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-Instruct-28M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-Instruct-28M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":25165824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 32\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 2048\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-Instruct-28M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-Instruct-28M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 25165824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-Instruct-28M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 512)\npos_embed:\n W_pos: (512, 512)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (16, 512, 32)\n W_O: (16, 32, 512)\n '[b_Q, b_K, b_V]': (16, 32)\n b_O: (512,)\n mask: (512, 512)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 512)"},"pos_embed":{"W_pos":"(512, 512)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 512, 32)","W_O":"(16, 32, 512)","[b_Q, b_K, b_V]":"(16, 32)","b_O":"(512,)","mask":"(512, 512)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 32)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 32)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} -{"name.default_alias":"tiny-stories-instruct-2L-33M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-Instruct-2Layers-33M","n_params.as_str":"25M","n_params.as_int":25165824,"n_params.from_name":"33M","cfg.n_params":25165824,"cfg.n_layers":2,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":2,"n_ctx":512,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-Instruct-2Layers-33M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-Instruct-2Layers-33M","window_size":256,"attn_types":["global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":25165824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 2\nn_ctx: 512\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-Instruct-2Layers-33M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-Instruct-2Layers-33M\nwindow_size: 256\nattn_types:\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 25165824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-Instruct-2Layers-33M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (512, 1024)\nblocks:\n '[0-1]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (512, 512)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(512, 1024)"},"blocks":{"[0-1]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(512, 512)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-1]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-1]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} -{"name.default_alias":"tiny-stories-instruct-33M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-Instruct-33M","n_params.as_str":"28M","n_params.as_int":28311552,"n_params.from_name":"33M","cfg.n_params":28311552,"cfg.n_layers":4,"cfg.n_heads":16,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":48,"n_layers":4,"n_ctx":512,"n_heads":16,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-Instruct-33M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-Instruct-33M","window_size":256,"attn_types":["global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":28311552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 48\nn_layers: 4\nn_ctx: 512\nn_heads: 16\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-Instruct-33M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-Instruct-33M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 28311552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-Instruct-33M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (512, 768)\nblocks:\n '[0-3]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (16, 768, 48)\n W_O: (16, 48, 768)\n '[b_Q, b_K, b_V]': (16, 48)\n b_O: (768,)\n mask: (512, 512)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(512, 768)"},"blocks":{"[0-3]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 768, 48)","W_O":"(16, 48, 768)","[b_Q, b_K, b_V]":"(16, 48)","b_O":"(768,)","mask":"(512, 512)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-3]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 48)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-3]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 48)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} -{"name.default_alias":"tiny-stories-instruct-3M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-Instruct-3M","n_params.as_str":"1.6M","n_params.as_int":1572864,"n_params.from_name":"3M","cfg.n_params":1572864,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":128,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":128,"d_head":8,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":512,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-Instruct-3M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-Instruct-3M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0707106781,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1572864,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 128\nd_head: 8\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 512\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-Instruct-3M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-Instruct-3M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.07071067811865475\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1572864\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-Instruct-3M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 128)\npos_embed:\n W_pos: (512, 128)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (128,)\n ln2:\n '[w, b]': (128,)\n attn:\n '[W_Q, W_K, W_V]': (16, 128, 8)\n W_O: (16, 8, 128)\n '[b_Q, b_K, b_V]': (16, 8)\n b_O: (128,)\n mask: (512, 512)\n IGNORE: ()\n mlp:\n W_in: (128, 512)\n b_in: (512,)\n W_out: (512, 128)\n b_out: (128,)\nln_final:\n '[w, b]': (128,)\nunembed:\n W_U: (128, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 128)"},"pos_embed":{"W_pos":"(512, 128)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(128,)"},"ln2":{"[w, b]":"(128,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 128, 8)","W_O":"(16, 8, 128)","[b_Q, b_K, b_V]":"(16, 8)","b_O":"(128,)","mask":"(512, 512)","IGNORE":"()"},"mlp":{"W_in":"(128, 512)","b_in":"(512,)","W_out":"(512, 128)","b_out":"(128,)"}}},"ln_final":{"[w, b]":"(128,)"},"unembed":{"W_U":"(128, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 8)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 512)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 128)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\nunembed:\n hook_in: (batch, seq_len, 128)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 128)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 8)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 512)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 128)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"unembed":{"hook_in":"(batch, seq_len, 128)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 128)"}} -{"name.default_alias":"tiny-stories-instruct-8M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-Instruct-8M","n_params.as_str":"6.3M","n_params.as_int":6291456,"n_params.from_name":"8M","cfg.n_params":6291456,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":256,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":256,"d_head":16,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":1024,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-Instruct-8M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-Instruct-8M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.05,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":6291456,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 256\nd_head: 16\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 1024\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-Instruct-8M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-Instruct-8M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.05\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 6291456\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-Instruct-8M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 256)\npos_embed:\n W_pos: (512, 256)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (256,)\n ln2:\n '[w, b]': (256,)\n attn:\n '[W_Q, W_K, W_V]': (16, 256, 16)\n W_O: (16, 16, 256)\n '[b_Q, b_K, b_V]': (16, 16)\n b_O: (256,)\n mask: (512, 512)\n IGNORE: ()\n mlp:\n W_in: (256, 1024)\n b_in: (1024,)\n W_out: (1024, 256)\n b_out: (256,)\nln_final:\n '[w, b]': (256,)\nunembed:\n W_U: (256, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 256)"},"pos_embed":{"W_pos":"(512, 256)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(256,)"},"ln2":{"[w, b]":"(256,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 256, 16)","W_O":"(16, 16, 256)","[b_Q, b_K, b_V]":"(16, 16)","b_O":"(256,)","mask":"(512, 512)","IGNORE":"()"},"mlp":{"W_in":"(256, 1024)","b_in":"(1024,)","W_out":"(1024, 256)","b_out":"(256,)"}}},"ln_final":{"[w, b]":"(256,)"},"unembed":{"W_U":"(256, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 16)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 1024)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 256)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\nunembed:\n hook_in: (batch, seq_len, 256)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 256)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 16)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 1024)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 256)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"unembed":{"hook_in":"(batch, seq_len, 256)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 256)"}} -{"name.default_alias":"tiny-stories-instruct-1L-21M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-Instuct-1Layer-21M","n_params.as_str":"13M","n_params.as_int":12582912,"n_params.from_name":"21M","cfg.n_params":12582912,"cfg.n_layers":1,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":1,"n_ctx":512,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-Instuct-1Layer-21M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-Instuct-1Layer-21M","window_size":256,"attn_types":["global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 1\nn_ctx: 512\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-Instuct-1Layer-21M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-Instuct-1Layer-21M\nwindow_size: 256\nattn_types:\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-Instuct-1Layer-21M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (512, 1024)\nblocks:\n '0':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (512, 512)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(512, 1024)"},"blocks":{"0":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(512, 512)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '0':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"0":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} -{"name.default_alias":"stablelm-base-alpha-3b","name.huggingface":"stabilityai\/stablelm-base-alpha-3b","name.aliases":"stablelm-base-alpha-3b, stablelm-base-3b","model_type":"stablelm","name.from_cfg":"stablelm-base-alpha-3b","n_params.as_str":"3.2B","n_params.as_int":3221225472,"n_params.from_name":"3b","cfg.n_params":3221225472,"cfg.n_layers":16,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50688,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":16,"n_ctx":4096,"n_heads":32,"d_mlp":16384,"d_vocab":50688,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"stablelm-base-alpha-3b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stabilityai\/stablelm-base-alpha-3b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50688,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":3221225472,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 16\nn_ctx: 4096\nn_heads: 32\nd_mlp: 16384\nd_vocab: 50688\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: stablelm-base-alpha-3b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stabilityai\/stablelm-base-alpha-3b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50688\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 3221225472\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"stabilityai\/stablelm-base-alpha-3b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50688, 4096)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 32)\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50688)\n b_U: (50688,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50688, 4096)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 32)"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50688)","b_U":"(50688,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50688)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50688)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"stablelm-base-alpha-7b","name.huggingface":"stabilityai\/stablelm-base-alpha-7b","name.aliases":"stablelm-base-alpha-7b, stablelm-base-7b","model_type":"stablelm","name.from_cfg":"stablelm-base-alpha-7b","n_params.as_str":"7.2B","n_params.as_int":7247757312,"n_params.from_name":"7b","cfg.n_params":7247757312,"cfg.n_layers":16,"cfg.n_heads":48,"cfg.d_model":6144,"cfg.d_vocab":50432,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":6144,"d_head":128,"n_layers":16,"n_ctx":4096,"n_heads":48,"d_mlp":24576,"d_vocab":50432,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"stablelm-base-alpha-7b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stabilityai\/stablelm-base-alpha-7b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0102062073,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50432,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":7247757312,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 6144\nd_head: 128\nn_layers: 16\nn_ctx: 4096\nn_heads: 48\nd_mlp: 24576\nd_vocab: 50432\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: stablelm-base-alpha-7b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stabilityai\/stablelm-base-alpha-7b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.010206207261596576\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50432\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 7247757312\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"stabilityai\/stablelm-base-alpha-7b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50432, 6144)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (6144,)\n ln2:\n '[w, b]': (6144,)\n attn:\n '[W_Q, W_K, W_V]': (48, 6144, 128)\n W_O: (48, 128, 6144)\n '[b_Q, b_K, b_V]': (48, 128)\n b_O: (6144,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 32)\n mlp:\n W_in: (6144, 24576)\n b_in: (24576,)\n W_out: (24576, 6144)\n b_out: (6144,)\nln_final:\n '[w, b]': (6144,)\nunembed:\n W_U: (6144, 50432)\n b_U: (50432,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50432, 6144)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(6144,)"},"ln2":{"[w, b]":"(6144,)"},"attn":{"[W_Q, W_K, W_V]":"(48, 6144, 128)","W_O":"(48, 128, 6144)","[b_Q, b_K, b_V]":"(48, 128)","b_O":"(6144,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 32)"},"mlp":{"W_in":"(6144, 24576)","b_in":"(24576,)","W_out":"(24576, 6144)","b_out":"(6144,)"}}},"ln_final":{"[w, b]":"(6144,)"},"unembed":{"W_U":"(6144, 50432)","b_U":"(50432,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 48, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 48, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 24576)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 6144)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\nunembed:\n hook_in: (batch, seq_len, 6144)\n hook_out: (batch, seq_len, 50432)\nhook_embed: (batch, seq_len, 6144)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 48, 128)","[hook_attn_scores, hook_pattern]":"(batch, 48, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 24576)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 6144)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"unembed":{"hook_in":"(batch, seq_len, 6144)","hook_out":"(batch, seq_len, 50432)"},"hook_embed":"(batch, seq_len, 6144)"}} -{"name.default_alias":"stablelm-tuned-alpha-3b","name.huggingface":"stabilityai\/stablelm-tuned-alpha-3b","name.aliases":"stablelm-tuned-alpha-3b, stablelm-tuned-3b","model_type":"stablelm","name.from_cfg":"stablelm-tuned-alpha-3b","n_params.as_str":"3.2B","n_params.as_int":3221225472,"n_params.from_name":"3b","cfg.n_params":3221225472,"cfg.n_layers":16,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50688,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":16,"n_ctx":4096,"n_heads":32,"d_mlp":16384,"d_vocab":50688,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"stablelm-tuned-alpha-3b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stabilityai\/stablelm-tuned-alpha-3b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50688,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":3221225472,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 16\nn_ctx: 4096\nn_heads: 32\nd_mlp: 16384\nd_vocab: 50688\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: stablelm-tuned-alpha-3b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stabilityai\/stablelm-tuned-alpha-3b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50688\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 3221225472\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"stabilityai\/stablelm-tuned-alpha-3b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"RD3vcWSd_TiTpqo5dHyICzaXtGQ=","tensor_shapes.state_dict":"embed:\n W_E: (50688, 4096)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 32)\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50688)\n b_U: (50688,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50688, 4096)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 32)"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50688)","b_U":"(50688,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50688)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50688)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"stablelm-tuned-alpha-7b","name.huggingface":"stabilityai\/stablelm-tuned-alpha-7b","name.aliases":"stablelm-tuned-alpha-7b, stablelm-tuned-7b","model_type":"stablelm","name.from_cfg":"stablelm-tuned-alpha-7b","n_params.as_str":"7.2B","n_params.as_int":7247757312,"n_params.from_name":"7b","cfg.n_params":7247757312,"cfg.n_layers":16,"cfg.n_heads":48,"cfg.d_model":6144,"cfg.d_vocab":50432,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":6144,"d_head":128,"n_layers":16,"n_ctx":4096,"n_heads":48,"d_mlp":24576,"d_vocab":50432,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"stablelm-tuned-alpha-7b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stabilityai\/stablelm-tuned-alpha-7b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0102062073,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50432,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":7247757312,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 6144\nd_head: 128\nn_layers: 16\nn_ctx: 4096\nn_heads: 48\nd_mlp: 24576\nd_vocab: 50432\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: stablelm-tuned-alpha-7b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stabilityai\/stablelm-tuned-alpha-7b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.010206207261596576\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50432\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 7247757312\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"stabilityai\/stablelm-tuned-alpha-7b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"RD3vcWSd_TiTpqo5dHyICzaXtGQ=","tensor_shapes.state_dict":"embed:\n W_E: (50432, 6144)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (6144,)\n ln2:\n '[w, b]': (6144,)\n attn:\n '[W_Q, W_K, W_V]': (48, 6144, 128)\n W_O: (48, 128, 6144)\n '[b_Q, b_K, b_V]': (48, 128)\n b_O: (6144,)\n mask: (4096, 4096)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (4096, 32)\n mlp:\n W_in: (6144, 24576)\n b_in: (24576,)\n W_out: (24576, 6144)\n b_out: (6144,)\nln_final:\n '[w, b]': (6144,)\nunembed:\n W_U: (6144, 50432)\n b_U: (50432,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50432, 6144)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(6144,)"},"ln2":{"[w, b]":"(6144,)"},"attn":{"[W_Q, W_K, W_V]":"(48, 6144, 128)","W_O":"(48, 128, 6144)","[b_Q, b_K, b_V]":"(48, 128)","b_O":"(6144,)","mask":"(4096, 4096)","IGNORE":"()","[rotary_sin, rotary_cos]":"(4096, 32)"},"mlp":{"W_in":"(6144, 24576)","b_in":"(24576,)","W_out":"(24576, 6144)","b_out":"(6144,)"}}},"ln_final":{"[w, b]":"(6144,)"},"unembed":{"W_U":"(6144, 50432)","b_U":"(50432,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 48, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 48, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 24576)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 6144)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\nunembed:\n hook_in: (batch, seq_len, 6144)\n hook_out: (batch, seq_len, 50432)\nhook_embed: (batch, seq_len, 6144)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 48, 128)","[hook_attn_scores, hook_pattern]":"(batch, 48, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 24576)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 6144)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"unembed":{"hook_in":"(batch, seq_len, 6144)","hook_out":"(batch, seq_len, 50432)"},"hook_embed":"(batch, seq_len, 6144)"}} -{"name.default_alias":"stanford-gpt2-small-a","name.huggingface":"stanford-crfm\/alias-gpt2-small-x21","name.aliases":"stanford-gpt2-small-a, alias-gpt2-small-x21, gpt2-mistral-small-a, gpt2-stanford-small-a","model_type":"gpt2","name.from_cfg":"alias-gpt2-small-x21","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"alias-gpt2-small-x21","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/alias-gpt2-small-x21","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: alias-gpt2-small-x21\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/alias-gpt2-small-x21\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/alias-gpt2-small-x21","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} -{"name.default_alias":"stanford-gpt2-medium-a","name.huggingface":"stanford-crfm\/arwen-gpt2-medium-x21","name.aliases":"stanford-gpt2-medium-a, arwen-gpt2-medium-x21, gpt2-medium-small-a, gpt2-stanford-medium-a","model_type":"gpt2","name.from_cfg":"arwen-gpt2-medium-x21","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"arwen-gpt2-medium-x21","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/arwen-gpt2-medium-x21","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: arwen-gpt2-medium-x21\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/arwen-gpt2-medium-x21\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/arwen-gpt2-medium-x21","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} -{"name.default_alias":"stanford-gpt2-small-b","name.huggingface":"stanford-crfm\/battlestar-gpt2-small-x49","name.aliases":"stanford-gpt2-small-b, battlestar-gpt2-small-x49, gpt2-mistral-small-b, gpt2-mistral-small-b","model_type":"gpt2","name.from_cfg":"battlestar-gpt2-small-x49","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"battlestar-gpt2-small-x49","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/battlestar-gpt2-small-x49","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: battlestar-gpt2-small-x49\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/battlestar-gpt2-small-x49\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/battlestar-gpt2-small-x49","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} -{"name.default_alias":"stanford-gpt2-medium-b","name.huggingface":"stanford-crfm\/beren-gpt2-medium-x49","name.aliases":"stanford-gpt2-medium-b, beren-gpt2-medium-x49, gpt2-medium-small-b, gpt2-stanford-medium-b","model_type":"gpt2","name.from_cfg":"beren-gpt2-medium-x49","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"beren-gpt2-medium-x49","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/beren-gpt2-medium-x49","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: beren-gpt2-medium-x49\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/beren-gpt2-medium-x49\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/beren-gpt2-medium-x49","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} -{"name.default_alias":"stanford-gpt2-small-c","name.huggingface":"stanford-crfm\/caprica-gpt2-small-x81","name.aliases":"stanford-gpt2-small-c, caprica-gpt2-small-x81, gpt2-mistral-small-c, gpt2-stanford-small-c","model_type":"gpt2","name.from_cfg":"caprica-gpt2-small-x81","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"caprica-gpt2-small-x81","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/caprica-gpt2-small-x81","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: caprica-gpt2-small-x81\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/caprica-gpt2-small-x81\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/caprica-gpt2-small-x81","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} -{"name.default_alias":"stanford-gpt2-medium-c","name.huggingface":"stanford-crfm\/celebrimbor-gpt2-medium-x81","name.aliases":"stanford-gpt2-medium-c, celebrimbor-gpt2-medium-x81, gpt2-medium-small-c, gpt2-medium-small-c","model_type":"gpt2","name.from_cfg":"celebrimbor-gpt2-medium-x81","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"celebrimbor-gpt2-medium-x81","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/celebrimbor-gpt2-medium-x81","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: celebrimbor-gpt2-medium-x81\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/celebrimbor-gpt2-medium-x81\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/celebrimbor-gpt2-medium-x81","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} -{"name.default_alias":"stanford-gpt2-small-d","name.huggingface":"stanford-crfm\/darkmatter-gpt2-small-x343","name.aliases":"stanford-gpt2-small-d, darkmatter-gpt2-small-x343, gpt2-mistral-small-d, gpt2-mistral-small-d","model_type":"gpt2","name.from_cfg":"darkmatter-gpt2-small-x343","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"darkmatter-gpt2-small-x343","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/darkmatter-gpt2-small-x343","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: darkmatter-gpt2-small-x343\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/darkmatter-gpt2-small-x343\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/darkmatter-gpt2-small-x343","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} -{"name.default_alias":"stanford-gpt2-medium-d","name.huggingface":"stanford-crfm\/durin-gpt2-medium-x343","name.aliases":"stanford-gpt2-medium-d, durin-gpt2-medium-x343, gpt2-medium-small-d, gpt2-stanford-medium-d","model_type":"gpt2","name.from_cfg":"durin-gpt2-medium-x343","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"durin-gpt2-medium-x343","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/durin-gpt2-medium-x343","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: durin-gpt2-medium-x343\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/durin-gpt2-medium-x343\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/durin-gpt2-medium-x343","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} -{"name.default_alias":"stanford-gpt2-medium-e","name.huggingface":"stanford-crfm\/eowyn-gpt2-medium-x777","name.aliases":"stanford-gpt2-medium-e, eowyn-gpt2-medium-x777, gpt2-medium-small-e, gpt2-stanford-medium-e","model_type":"gpt2","name.from_cfg":"eowyn-gpt2-medium-x777","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"eowyn-gpt2-medium-x777","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/eowyn-gpt2-medium-x777","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: eowyn-gpt2-medium-x777\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/eowyn-gpt2-medium-x777\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/eowyn-gpt2-medium-x777","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} -{"name.default_alias":"stanford-gpt2-small-e","name.huggingface":"stanford-crfm\/expanse-gpt2-small-x777","name.aliases":"stanford-gpt2-small-e, expanse-gpt2-small-x777, gpt2-mistral-small-e, gpt2-mistral-small-e","model_type":"gpt2","name.from_cfg":"expanse-gpt2-small-x777","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"expanse-gpt2-small-x777","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/expanse-gpt2-small-x777","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: expanse-gpt2-small-x777\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/expanse-gpt2-small-x777\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/expanse-gpt2-small-x777","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (1024, 1024)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(1024, 1024)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} -{"name.default_alias":"apertus-8b","name.huggingface":"swiss-ai\/Apertus-8B-2509","name.aliases":"apertus-8b, apertus","model_type":null,"name.from_cfg":"Apertus-8B-2509","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"8b","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":131072,"cfg.act_fn":"xielu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"ApertusForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":65536,"n_heads":32,"d_mlp":21504,"d_vocab":131072,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"xielu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Apertus-8B-2509","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"ApertusForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"swiss-ai\/Apertus-8B-2509","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":131072,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":12000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 65536\nn_heads: 32\nd_mlp: 21504\nd_vocab: 131072\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: xielu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Apertus-8B-2509\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: ApertusForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: swiss-ai\/Apertus-8B-2509\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 131072\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 12000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"swiss-ai\/Apertus-8B-2509","tokenizer.vocab_size":131072.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"C46N1vCQdy2ADcCzoHO1wqva-9w=","tensor_shapes.state_dict":"embed:\n W_E: (131072, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (65536, 65536)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (65536, 128)\n mlp:\n act_fn:\n '[alpha_p, alpha_n, beta, eps]': ()\n W_in: (4096, 21504)\n b_in: (21504,)\n W_out: (21504, 4096)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 131072)\n b_U: (131072,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(131072, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(65536, 65536)","IGNORE":"()","[rotary_sin, rotary_cos]":"(65536, 128)"},"mlp":{"act_fn":{"[alpha_p, alpha_n, beta, eps]":"()"},"W_in":"(4096, 21504)","b_in":"(21504,)","W_out":"(21504, 4096)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 131072)","b_U":"(131072,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n q_norm:\n hook_scale: (1776098016, 1)\n hook_normalized: (1776098016, 128)\n k_norm:\n hook_scale: (444024504, 1)\n hook_normalized: (444024504, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 21504)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 131072)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"q_norm":{"hook_scale":"(1776098016, 1)","hook_normalized":"(1776098016, 128)"},"k_norm":{"hook_scale":"(444024504, 1)","hook_normalized":"(444024504, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 21504)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 131072)"},"hook_embed":"(batch, seq_len, 4096)"}} -{"name.default_alias":"apertus-8b-instruct","name.huggingface":"swiss-ai\/Apertus-8B-Instruct-2509","name.aliases":"apertus-8b-instruct, apertus-instruct","model_type":null,"name.from_cfg":"Apertus-8B-Instruct-2509","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"8b","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":131072,"cfg.act_fn":"xielu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"ApertusForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":65536,"n_heads":32,"d_mlp":21504,"d_vocab":131072,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"xielu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Apertus-8B-Instruct-2509","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"ApertusForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"swiss-ai\/Apertus-8B-Instruct-2509","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":131072,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":12000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 65536\nn_heads: 32\nd_mlp: 21504\nd_vocab: 131072\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: xielu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Apertus-8B-Instruct-2509\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy.core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: ApertusForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: swiss-ai\/Apertus-8B-Instruct-2509\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 131072\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 12000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nnorm_topk_prob: false\n","tokenizer.name":"swiss-ai\/Apertus-8B-Instruct-2509","tokenizer.vocab_size":131072.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"oobelwCFf_zbL7nyblhjYJWtf6c=","tensor_shapes.state_dict":"embed:\n W_E: (131072, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (65536, 65536)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (65536, 128)\n mlp:\n act_fn:\n '[alpha_p, alpha_n, beta, eps]': ()\n W_in: (4096, 21504)\n b_in: (21504,)\n W_out: (21504, 4096)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 131072)\n b_U: (131072,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(131072, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(65536, 65536)","IGNORE":"()","[rotary_sin, rotary_cos]":"(65536, 128)"},"mlp":{"act_fn":{"[alpha_p, alpha_n, beta, eps]":"()"},"W_in":"(4096, 21504)","b_in":"(21504,)","W_out":"(21504, 4096)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 131072)","b_U":"(131072,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n q_norm:\n hook_scale: (1776098016, 1)\n hook_normalized: (1776098016, 128)\n k_norm:\n hook_scale: (444024504, 1)\n hook_normalized: (444024504, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 21504)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 131072)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"q_norm":{"hook_scale":"(1776098016, 1)","hook_normalized":"(1776098016, 128)"},"k_norm":{"hook_scale":"(444024504, 1)","hook_normalized":"(444024504, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 21504)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 131072)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"yi-34b","name.huggingface":"01-ai\/Yi-34B","name.aliases":"yi-34b, Yi-34B","model_type":"yi","name.from_cfg":"Yi-34B","n_params.as_str":"39B","n_params.as_int":38755368960,"n_params.from_name":"34b","cfg.n_params":38755368960,"cfg.n_layers":60,"cfg.n_heads":56,"cfg.d_model":7168,"cfg.d_vocab":64000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":7168,"d_head":128,"n_layers":60,"n_ctx":4096,"n_heads":56,"d_mlp":20480,"d_vocab":64000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Yi-34B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"01-ai\/Yi-34B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0094491118,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":64000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":38755368960,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 7168\nd_head: 128\nn_layers: 60\nn_ctx: 4096\nn_heads: 56\nd_mlp: 20480\nd_vocab: 64000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Yi-34B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: 01-ai\/Yi-34B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.00944911182523068\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 64000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 38755368960\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"01-ai\/Yi-34B","tokenizer.vocab_size":64000.0,"tokenizer.max_len":4096.0,"tokenizer.class":"LlamaTokenizer","tokenizer.vocab_hash":"VBBPi7l7j0Xrv93YNq1tizlalWw=","tensor_shapes.state_dict":"embed:\n W_E: (64000, 7168)\nblocks:\n '[0-59]':\n ln1:\n w: (7168,)\n ln2:\n w: (7168,)\n attn:\n W_Q: (56, 7168, 128)\n W_O: (56, 128, 7168)\n b_Q: (56, 128)\n b_O: (7168,)\n '[_W_K, _W_V]': (8, 7168, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (7168, 20480)\n W_out: (20480, 7168)\n b_in: (20480,)\n b_out: (7168,)\nln_final:\n w: (7168,)\nunembed:\n W_U: (7168, 64000)\n b_U: (64000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(64000, 7168)"},"blocks":{"[0-59]":{"ln1":{"w":"(7168,)"},"ln2":{"w":"(7168,)"},"attn":{"W_Q":"(56, 7168, 128)","W_O":"(56, 128, 7168)","b_Q":"(56, 128)","b_O":"(7168,)","[_W_K, _W_V]":"(8, 7168, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(7168, 20480)","W_out":"(20480, 7168)","b_in":"(20480,)","b_out":"(7168,)"}}},"ln_final":{"w":"(7168,)"},"unembed":{"W_U":"(7168, 64000)","b_U":"(64000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-59]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 56, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 56, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 20480)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 7168)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\nunembed:\n hook_in: (batch, seq_len, 7168)\n hook_out: (batch, seq_len, 64000)\nhook_embed: (batch, seq_len, 7168)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-59]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 56, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 56, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 20480)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 7168)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"unembed":{"hook_in":"(batch, seq_len, 7168)","hook_out":"(batch, seq_len, 64000)"},"hook_embed":"(batch, seq_len, 7168)"}} +{"name.default_alias":"yi-34b-chat","name.huggingface":"01-ai\/Yi-34B-Chat","name.aliases":"yi-34b-chat, Yi-34B-Chat","model_type":"yi","name.from_cfg":"Yi-34B-Chat","n_params.as_str":"39B","n_params.as_int":38755368960,"n_params.from_name":"34b","cfg.n_params":38755368960,"cfg.n_layers":60,"cfg.n_heads":56,"cfg.d_model":7168,"cfg.d_vocab":64000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":7168,"d_head":128,"n_layers":60,"n_ctx":4096,"n_heads":56,"d_mlp":20480,"d_vocab":64000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Yi-34B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"01-ai\/Yi-34B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0094491118,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":64000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":38755368960,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 7168\nd_head: 128\nn_layers: 60\nn_ctx: 4096\nn_heads: 56\nd_mlp: 20480\nd_vocab: 64000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Yi-34B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: 01-ai\/Yi-34B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.00944911182523068\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 64000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 38755368960\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"01-ai\/Yi-34B-Chat","tokenizer.vocab_size":63992.0,"tokenizer.max_len":4096.0,"tokenizer.class":"LlamaTokenizer","tokenizer.vocab_hash":"VGXAFrTzytwGdUlX6AWH0NacncM=","tensor_shapes.state_dict":"embed:\n W_E: (64000, 7168)\nblocks:\n '[0-59]':\n ln1:\n w: (7168,)\n ln2:\n w: (7168,)\n attn:\n W_Q: (56, 7168, 128)\n W_O: (56, 128, 7168)\n b_Q: (56, 128)\n b_O: (7168,)\n '[_W_K, _W_V]': (8, 7168, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (7168, 20480)\n W_out: (20480, 7168)\n b_in: (20480,)\n b_out: (7168,)\nln_final:\n w: (7168,)\nunembed:\n W_U: (7168, 64000)\n b_U: (64000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(64000, 7168)"},"blocks":{"[0-59]":{"ln1":{"w":"(7168,)"},"ln2":{"w":"(7168,)"},"attn":{"W_Q":"(56, 7168, 128)","W_O":"(56, 128, 7168)","b_Q":"(56, 128)","b_O":"(7168,)","[_W_K, _W_V]":"(8, 7168, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(7168, 20480)","W_out":"(20480, 7168)","b_in":"(20480,)","b_out":"(7168,)"}}},"ln_final":{"w":"(7168,)"},"unembed":{"W_U":"(7168, 64000)","b_U":"(64000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-59]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 56, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 56, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 20480)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 7168)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\nunembed:\n hook_in: (batch, seq_len, 7168)\n hook_out: (batch, seq_len, 64000)\nhook_embed: (batch, seq_len, 7168)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-59]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 56, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 56, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 20480)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 7168)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"unembed":{"hook_in":"(batch, seq_len, 7168)","hook_out":"(batch, seq_len, 64000)"},"hook_embed":"(batch, seq_len, 7168)"}} +{"name.default_alias":"yi-6b","name.huggingface":"01-ai\/Yi-6B","name.aliases":"yi-6b, Yi-6B","model_type":"yi","name.from_cfg":"Yi-6B","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"6b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":64000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":64000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Yi-6B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"01-ai\/Yi-6B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":64000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 64000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Yi-6B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: 01-ai\/Yi-6B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 64000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"01-ai\/Yi-6B","tokenizer.vocab_size":63992.0,"tokenizer.max_len":4096.0,"tokenizer.class":"LlamaTokenizer","tokenizer.vocab_hash":"VGXAFrTzytwGdUlX6AWH0NacncM=","tensor_shapes.state_dict":"embed:\n W_E: (64000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (4, 4096, 128)\n '[_b_K, _b_V]': (4, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 64000)\n b_U: (64000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(64000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(4, 4096, 128)","[_b_K, _b_V]":"(4, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 64000)","b_U":"(64000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 64000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 64000)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"yi-6b-chat","name.huggingface":"01-ai\/Yi-6B-Chat","name.aliases":"yi-6b-chat, Yi-6B-Chat","model_type":"yi","name.from_cfg":"Yi-6B-Chat","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"6b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":64000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":64000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Yi-6B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"01-ai\/Yi-6B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":64000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 64000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Yi-6B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: 01-ai\/Yi-6B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 64000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"01-ai\/Yi-6B-Chat","tokenizer.vocab_size":63992.0,"tokenizer.max_len":4096.0,"tokenizer.class":"LlamaTokenizer","tokenizer.vocab_hash":"VGXAFrTzytwGdUlX6AWH0NacncM=","tensor_shapes.state_dict":"embed:\n W_E: (64000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (4, 4096, 128)\n '[_b_K, _b_V]': (4, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 64000)\n b_U: (64000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(64000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(4, 4096, 128)","[_b_K, _b_V]":"(4, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 64000)","b_U":"(64000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 64000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 64000)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"mGPT","name.huggingface":null,"name.aliases":"","model_type":null,"name.from_cfg":"mGPT","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":null,"cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":100000,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":100000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"mGPT","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"ai-forever\/mGPT","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100000,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 100000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: mGPT\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: ai-forever\/mGPT\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100000\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"ai-forever\/mGPT","tokenizer.vocab_size":100000.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"8j6CU_p3zgyeEBZ1Z3lu358tiy0=","tensor_shapes.state_dict":"embed:\n W_E: (100000, 2048)\npos_embed:\n W_pos: (2048, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 100000)\n b_U: (100000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100000, 2048)"},"pos_embed":{"W_pos":"(2048, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 100000)","b_U":"(100000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 100000)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 100000)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 2048)"}} +{"name.default_alias":"olmo-1b","name.huggingface":"allenai\/OLMo-1B-hf","name.aliases":"olmo-1b","model_type":null,"name.from_cfg":"OLMo-1B-hf","n_params.as_str":"1.1B","n_params.as_int":1073741824,"n_params.from_name":"1b","cfg.n_params":1073741824,"cfg.n_layers":16,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OlmoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":16,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"OLMo-1B-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OlmoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/OLMo-1B-hf","window_size":null,"attn_types":["global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1073741824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 16\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: OLMo-1B-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OlmoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/OLMo-1B-hf\nwindow_size: null\nattn_types:\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1073741824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/OLMo-1B-hf","tokenizer.vocab_size":50280.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"Zon1p_mdHoNTi0EKxz2EJK3tpZg=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (2048, 8192)\n W_out: (8192, 2048)\n b_in: (8192,)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(2048, 8192)","W_out":"(8192, 2048)","b_in":"(8192,)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"olmo-2-1b","name.huggingface":"allenai\/OLMo-2-0425-1B","name.aliases":"olmo-2-1b","model_type":null,"name.from_cfg":"OLMo-2-0425-1B","n_params.as_str":"1.1B","n_params.as_int":1073741824,"n_params.from_name":"1b","cfg.n_params":1073741824,"cfg.n_layers":16,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":100352,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":16,"n_ctx":4096,"n_heads":16,"d_mlp":8192,"d_vocab":100352,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"OLMo-2-0425-1B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/OLMo-2-0425-1B","window_size":null,"attn_types":["global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100352,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1073741824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 16\nn_ctx: 4096\nn_heads: 16\nd_mlp: 8192\nd_vocab: 100352\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: OLMo-2-0425-1B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/OLMo-2-0425-1B\nwindow_size: null\nattn_types:\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100352\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1073741824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/OLMo-2-0425-1B","tokenizer.vocab_size":100278.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"VuUSAx0rE_5hHVJSa7DTIutKX9s=","tensor_shapes.state_dict":"embed:\n W_E: (100352, 2048)\nblocks:\n '[0-15]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n q_norm:\n w: (2048,)\n k_norm:\n w: (2048,)\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (2048, 8192)\n W_out: (8192, 2048)\n b_in: (8192,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 100352)\n b_U: (100352,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100352, 2048)"},"blocks":{"[0-15]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"q_norm":{"w":"(2048,)"},"k_norm":{"w":"(2048,)"},"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(2048, 8192)","W_out":"(8192, 2048)","b_in":"(8192,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 100352)","b_U":"(100352,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8192)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 100352)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8192)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 100352)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"olmo-2-7b","name.huggingface":"allenai\/OLMo-2-1124-7B","name.aliases":"olmo-2-7b","model_type":null,"name.from_cfg":"OLMo-2-1124-7B","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":100352,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":100352,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"OLMo-2-1124-7B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/OLMo-2-1124-7B","window_size":null,"attn_types":["global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100352,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 100352\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: OLMo-2-1124-7B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/OLMo-2-1124-7B\nwindow_size: null\nattn_types:\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100352\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/OLMo-2-1124-7B","tokenizer.vocab_size":100278.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"VuUSAx0rE_5hHVJSa7DTIutKX9s=","tensor_shapes.state_dict":"embed:\n W_E: (100352, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n q_norm:\n w: (4096,)\n k_norm:\n w: (4096,)\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 100352)\n b_U: (100352,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100352, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"q_norm":{"w":"(4096,)"},"k_norm":{"w":"(4096,)"},"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 100352)","b_U":"(100352,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 100352)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 100352)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"olmo-3-1025-7b","name.huggingface":"allenai\/Olmo-3-1025-7B","name.aliases":"olmo-3-1025-7b","model_type":null,"name.from_cfg":"Olmo-3-1025-7B","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":100278,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":65536,"n_heads":32,"d_mlp":11008,"d_vocab":100278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":32,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Olmo-3-1025-7B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/Olmo-3-1025-7B","window_size":4096,"attn_types":["local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100278,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":true,"yarn_factor":8.0,"yarn_attention_factor":1.2079441542,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":8192,"yarn_truncate":true,"yarn_global_attn_only":true,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 65536\nn_heads: 32\nd_mlp: 11008\nd_vocab: 100278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 32\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Olmo-3-1025-7B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/Olmo-3-1025-7B\nwindow_size: 4096\nattn_types:\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100278\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: true\nyarn_factor: 8.0\nyarn_attention_factor: 1.2079441541679836\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 8192\nyarn_truncate: true\nyarn_global_attn_only: true\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/Olmo-3-1025-7B","tokenizer.vocab_size":100278.0,"tokenizer.max_len":65536.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"VuUSAx0rE_5hHVJSa7DTIutKX9s=","tensor_shapes.state_dict":"embed:\n W_E: (100278, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n q_norm:\n w: (4096,)\n k_norm:\n w: (4096,)\n '[W_Q, _W_K, _W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, _b_K, _b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 100278)\n b_U: (100278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100278, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"q_norm":{"w":"(4096,)"},"k_norm":{"w":"(4096,)"},"[W_Q, _W_K, _W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, _b_K, _b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 100278)","b_U":"(100278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 100278)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 100278)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"olmo-3-1125-32b","name.huggingface":"allenai\/Olmo-3-1125-32B","name.aliases":"olmo-3-1125-32b","model_type":null,"name.from_cfg":"Olmo-3-1125-32B","n_params.as_str":"34B","n_params.as_int":33889976320,"n_params.from_name":"32b","cfg.n_params":33889976320,"cfg.n_layers":64,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":100278,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":64,"n_ctx":65536,"n_heads":40,"d_mlp":27648,"d_vocab":100278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Olmo-3-1125-32B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/Olmo-3-1125-32B","window_size":4096,"attn_types":["local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100278,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":33889976320,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":true,"yarn_factor":8.0,"yarn_attention_factor":1.2079441542,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":8192,"yarn_truncate":true,"yarn_global_attn_only":true,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 64\nn_ctx: 65536\nn_heads: 40\nd_mlp: 27648\nd_vocab: 100278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Olmo-3-1125-32B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/Olmo-3-1125-32B\nwindow_size: 4096\nattn_types:\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100278\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 33889976320\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: true\nyarn_factor: 8.0\nyarn_attention_factor: 1.2079441541679836\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 8192\nyarn_truncate: true\nyarn_global_attn_only: true\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/Olmo-3-1125-32B","tokenizer.vocab_size":100278.0,"tokenizer.max_len":65536.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"VuUSAx0rE_5hHVJSa7DTIutKX9s=","tensor_shapes.state_dict":"embed:\n W_E: (100278, 5120)\nblocks:\n '[0-63]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n q_norm:\n w: (5120,)\n k_norm:\n w: (1024,)\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 27648)\n W_out: (27648, 5120)\n b_in: (27648,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 100278)\n b_U: (100278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100278, 5120)"},"blocks":{"[0-63]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"q_norm":{"w":"(5120,)"},"k_norm":{"w":"(1024,)"},"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 27648)","W_out":"(27648, 5120)","b_in":"(27648,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 100278)","b_U":"(100278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-63]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 27648)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 100278)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-63]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 27648)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 100278)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"olmo-3-32b-think","name.huggingface":"allenai\/Olmo-3-32B-Think","name.aliases":"olmo-3-32b-think","model_type":null,"name.from_cfg":"Olmo-3-32B-Think","n_params.as_str":"34B","n_params.as_int":33889976320,"n_params.from_name":"32b","cfg.n_params":33889976320,"cfg.n_layers":64,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":100278,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":64,"n_ctx":65536,"n_heads":40,"d_mlp":27648,"d_vocab":100278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Olmo-3-32B-Think","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/Olmo-3-32B-Think","window_size":4096,"attn_types":["local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100278,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":33889976320,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":true,"yarn_factor":8.0,"yarn_attention_factor":1.2079441542,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":8192,"yarn_truncate":true,"yarn_global_attn_only":true,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 64\nn_ctx: 65536\nn_heads: 40\nd_mlp: 27648\nd_vocab: 100278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Olmo-3-32B-Think\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/Olmo-3-32B-Think\nwindow_size: 4096\nattn_types:\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100278\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 33889976320\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: true\nyarn_factor: 8.0\nyarn_attention_factor: 1.2079441541679836\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 8192\nyarn_truncate: true\nyarn_global_attn_only: true\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/Olmo-3-32B-Think","tokenizer.vocab_size":100278.0,"tokenizer.max_len":65536.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"VuUSAx0rE_5hHVJSa7DTIutKX9s=","tensor_shapes.state_dict":"embed:\n W_E: (100278, 5120)\nblocks:\n '[0-63]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n q_norm:\n w: (5120,)\n k_norm:\n w: (1024,)\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 27648)\n W_out: (27648, 5120)\n b_in: (27648,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 100278)\n b_U: (100278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100278, 5120)"},"blocks":{"[0-63]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"q_norm":{"w":"(5120,)"},"k_norm":{"w":"(1024,)"},"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 27648)","W_out":"(27648, 5120)","b_in":"(27648,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 100278)","b_U":"(100278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-63]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 27648)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 100278)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-63]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 27648)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 100278)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"olmo-3-7b-instruct","name.huggingface":"allenai\/Olmo-3-7B-Instruct","name.aliases":"olmo-3-7b-instruct","model_type":null,"name.from_cfg":"Olmo-3-7B-Instruct","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":100278,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":65536,"n_heads":32,"d_mlp":11008,"d_vocab":100278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":32,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Olmo-3-7B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/Olmo-3-7B-Instruct","window_size":4096,"attn_types":["local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100278,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":true,"yarn_factor":8.0,"yarn_attention_factor":1.2079441542,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":8192,"yarn_truncate":true,"yarn_global_attn_only":true,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 65536\nn_heads: 32\nd_mlp: 11008\nd_vocab: 100278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 32\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Olmo-3-7B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/Olmo-3-7B-Instruct\nwindow_size: 4096\nattn_types:\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100278\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: true\nyarn_factor: 8.0\nyarn_attention_factor: 1.2079441541679836\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 8192\nyarn_truncate: true\nyarn_global_attn_only: true\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/Olmo-3-7B-Instruct","tokenizer.vocab_size":100278.0,"tokenizer.max_len":65536.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"RNowY96fjQ_x29qA8x5WywVkiVk=","tensor_shapes.state_dict":"embed:\n W_E: (100278, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n q_norm:\n w: (4096,)\n k_norm:\n w: (4096,)\n '[W_Q, _W_K, _W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, _b_K, _b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 100278)\n b_U: (100278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100278, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"q_norm":{"w":"(4096,)"},"k_norm":{"w":"(4096,)"},"[W_Q, _W_K, _W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, _b_K, _b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 100278)","b_U":"(100278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 100278)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 100278)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"olmo-3-7b-think","name.huggingface":"allenai\/Olmo-3-7B-Think","name.aliases":"olmo-3-7b-think","model_type":null,"name.from_cfg":"Olmo-3-7B-Think","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":100278,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":65536,"n_heads":32,"d_mlp":11008,"d_vocab":100278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":32,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Olmo-3-7B-Think","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/Olmo-3-7B-Think","window_size":4096,"attn_types":["local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100278,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":true,"yarn_factor":8.0,"yarn_attention_factor":1.2079441542,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":8192,"yarn_truncate":true,"yarn_global_attn_only":true,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 65536\nn_heads: 32\nd_mlp: 11008\nd_vocab: 100278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 32\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Olmo-3-7B-Think\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/Olmo-3-7B-Think\nwindow_size: 4096\nattn_types:\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100278\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: true\nyarn_factor: 8.0\nyarn_attention_factor: 1.2079441541679836\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 8192\nyarn_truncate: true\nyarn_global_attn_only: true\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/Olmo-3-7B-Think","tokenizer.vocab_size":100278.0,"tokenizer.max_len":65536.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"VuUSAx0rE_5hHVJSa7DTIutKX9s=","tensor_shapes.state_dict":"embed:\n W_E: (100278, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n q_norm:\n w: (4096,)\n k_norm:\n w: (4096,)\n '[W_Q, _W_K, _W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, _b_K, _b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 100278)\n b_U: (100278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100278, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"q_norm":{"w":"(4096,)"},"k_norm":{"w":"(4096,)"},"[W_Q, _W_K, _W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, _b_K, _b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 100278)","b_U":"(100278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 100278)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 100278)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"olmo-3.1-32b-instruct","name.huggingface":"allenai\/Olmo-3.1-32B-Instruct","name.aliases":"olmo-3.1-32b-instruct","model_type":null,"name.from_cfg":"Olmo-3.1-32B-Instruct","n_params.as_str":"34B","n_params.as_int":33889976320,"n_params.from_name":"32b","cfg.n_params":33889976320,"cfg.n_layers":64,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":100278,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":64,"n_ctx":65536,"n_heads":40,"d_mlp":27648,"d_vocab":100278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Olmo-3.1-32B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/Olmo-3.1-32B-Instruct","window_size":4096,"attn_types":["local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100278,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":33889976320,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":true,"yarn_factor":8.0,"yarn_attention_factor":1.2079441542,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":8192,"yarn_truncate":true,"yarn_global_attn_only":true,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 64\nn_ctx: 65536\nn_heads: 40\nd_mlp: 27648\nd_vocab: 100278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Olmo-3.1-32B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/Olmo-3.1-32B-Instruct\nwindow_size: 4096\nattn_types:\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100278\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 33889976320\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: true\nyarn_factor: 8.0\nyarn_attention_factor: 1.2079441541679836\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 8192\nyarn_truncate: true\nyarn_global_attn_only: true\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/Olmo-3.1-32B-Instruct","tokenizer.vocab_size":100278.0,"tokenizer.max_len":65536.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"RNowY96fjQ_x29qA8x5WywVkiVk=","tensor_shapes.state_dict":"embed:\n W_E: (100278, 5120)\nblocks:\n '[0-63]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n q_norm:\n w: (5120,)\n k_norm:\n w: (1024,)\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 27648)\n W_out: (27648, 5120)\n b_in: (27648,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 100278)\n b_U: (100278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100278, 5120)"},"blocks":{"[0-63]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"q_norm":{"w":"(5120,)"},"k_norm":{"w":"(1024,)"},"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 27648)","W_out":"(27648, 5120)","b_in":"(27648,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 100278)","b_U":"(100278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-63]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 27648)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 100278)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-63]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 27648)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 100278)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"olmo-3.1-32b-think","name.huggingface":"allenai\/Olmo-3.1-32B-Think","name.aliases":"olmo-3.1-32b-think","model_type":null,"name.from_cfg":"Olmo-3.1-32B-Think","n_params.as_str":"34B","n_params.as_int":33889976320,"n_params.from_name":"32b","cfg.n_params":33889976320,"cfg.n_layers":64,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":100278,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Olmo3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":64,"n_ctx":65536,"n_heads":40,"d_mlp":27648,"d_vocab":100278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Olmo-3.1-32B-Think","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Olmo3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/Olmo-3.1-32B-Think","window_size":4096,"attn_types":["local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100278,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":33889976320,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":true,"yarn_factor":8.0,"yarn_attention_factor":1.2079441542,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":8192,"yarn_truncate":true,"yarn_global_attn_only":true,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 64\nn_ctx: 65536\nn_heads: 40\nd_mlp: 27648\nd_vocab: 100278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Olmo-3.1-32B-Think\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Olmo3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/Olmo-3.1-32B-Think\nwindow_size: 4096\nattn_types:\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100278\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 33889976320\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: true\nyarn_factor: 8.0\nyarn_attention_factor: 1.2079441541679836\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 8192\nyarn_truncate: true\nyarn_global_attn_only: true\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/Olmo-3.1-32B-Think","tokenizer.vocab_size":100278.0,"tokenizer.max_len":65536.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"VuUSAx0rE_5hHVJSa7DTIutKX9s=","tensor_shapes.state_dict":"embed:\n W_E: (100278, 5120)\nblocks:\n '[0-63]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n q_norm:\n w: (5120,)\n k_norm:\n w: (1024,)\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 27648)\n W_out: (27648, 5120)\n b_in: (27648,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 100278)\n b_U: (100278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100278, 5120)"},"blocks":{"[0-63]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"q_norm":{"w":"(5120,)"},"k_norm":{"w":"(1024,)"},"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 27648)","W_out":"(27648, 5120)","b_in":"(27648,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 100278)","b_U":"(100278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-63]':\n attn:\n q_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n k_norm:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 27648)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 100278)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-63]":{"attn":{"q_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"k_norm":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 27648)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 100278)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"olmo-7b","name.huggingface":"allenai\/OLMo-7B-hf","name.aliases":"olmo-7b","model_type":null,"name.from_cfg":"OLMo-7B-hf","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50304,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OlmoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":11008,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"OLMo-7B-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OlmoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/OLMo-7B-hf","window_size":null,"attn_types":["global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 11008\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: OLMo-7B-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OlmoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/OLMo-7B-hf\nwindow_size: null\nattn_types:\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/OLMo-7B-hf","tokenizer.vocab_size":50280.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"Zon1p_mdHoNTi0EKxz2EJK3tpZg=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 4096)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 4096)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"olmoe","name.huggingface":"allenai\/OLMoE-1B-7B-0924","name.aliases":"olmoe","model_type":null,"name.from_cfg":"OLMoE-1B-7B-0924","n_params.as_str":"6.7B","n_params.as_int":6712983552,"n_params.from_name":null,"cfg.n_params":6712983552,"cfg.n_layers":16,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OlmoeForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":16,"n_ctx":4096,"n_heads":16,"d_mlp":1024,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":64,"experts_per_token":8,"final_rms":false,"dtype":"torch.float32","model_name":"OLMoE-1B-7B-0924","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OlmoeForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"allenai\/OLMoE-1B-7B-0924","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6712983552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 16\nn_ctx: 4096\nn_heads: 16\nd_mlp: 1024\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: 64\nexperts_per_token: 8\nfinal_rms: false\ndtype: torch.float32\nmodel_name: OLMoE-1B-7B-0924\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OlmoeForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: allenai\/OLMoE-1B-7B-0924\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6712983552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"allenai\/OLMoE-1B-7B-0924","tokenizer.vocab_size":50280.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"Zon1p_mdHoNTi0EKxz2EJK3tpZg=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-15]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n q_norm:\n w: (2048,)\n k_norm:\n w: (2048,)\n '[W_Q, _W_K, _W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, _b_K, _b_V]': (16, 128)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n experts:\n '[0-63]':\n W_in:\n weight: (1024, 2048)\n W_out:\n weight: (2048, 1024)\n W_gate:\n weight: (1024, 2048)\n W_gate:\n weight: (64, 2048)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-15]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"q_norm":{"w":"(2048,)"},"k_norm":{"w":"(2048,)"},"[W_Q, _W_K, _W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, _b_K, _b_V]":"(16, 128)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"experts":{"[0-63]":{"W_in":{"weight":"(1024, 2048)"},"W_out":{"weight":"(2048, 1024)"},"W_gate":{"weight":"(1024, 2048)"}}},"W_gate":{"weight":"(64, 2048)"}}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"redwood_attn_2l","name.huggingface":"ArthurConmy\/redwood_attn_2l","name.aliases":"redwood_attn_2l","model_type":null,"name.from_cfg":"redwood_attn_2l","n_params.as_str":"524K","n_params.as_int":524288,"n_params.from_name":null,"cfg.n_params":524288,"cfg.n_layers":2,"cfg.n_heads":8,"cfg.d_model":256,"cfg.d_vocab":50259,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"shortformer","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":256,"d_head":32,"n_layers":2,"n_ctx":2048,"n_heads":8,"d_mlp":-1,"d_vocab":50259,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"shortformer","n_key_value_heads":null,"attn_only":true,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"redwood_attn_2l","use_attn_scale":true,"attn_scale":5.6568542495,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"ArthurConmy\/redwood_tokenizer","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.05,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50259,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":524288,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 256\nd_head: 32\nn_layers: 2\nn_ctx: 2048\nn_heads: 8\nd_mlp: -1\nd_vocab: 50259\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: shortformer\nn_key_value_heads: null\nattn_only: true\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: redwood_attn_2l\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gFkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: ArthurConmy\/redwood_tokenizer\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.05\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50259\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 524288\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"othello-gpt","name.huggingface":"Baidicoot\/Othello-GPT-Transformer-Lens","name.aliases":"othello-gpt","model_type":null,"name.from_cfg":"Othello-GPT-Transformer-Lens","n_params.as_str":"25M","n_params.as_int":25165824,"n_params.from_name":null,"cfg.n_params":25165824,"cfg.n_layers":8,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":61,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"mingpt","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":8,"n_ctx":59,"n_heads":8,"d_mlp":2048,"d_vocab":61,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Othello-GPT-Transformer-Lens","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"mingpt","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":null,"window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":61,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":25165824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 8\nn_ctx: 59\nn_heads: 8\nd_mlp: 2048\nd_vocab: 61\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Othello-GPT-Transformer-Lens\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: mingpt\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: null\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 61\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 25165824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":"embed:\n W_E: (61, 512)\npos_embed:\n W_pos: (59, 512)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 61)\n b_U: (61,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(61, 512)"},"pos_embed":{"W_pos":"(59, 512)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 61)","b_U":"(61,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 61)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 61)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"bloom-1b1","name.huggingface":"bigscience\/bloom-1b1","name.aliases":"bloom-1b1","model_type":"bloom","name.from_cfg":"bloom-1b1","n_params.as_str":"679M","n_params.as_int":679477248,"n_params.from_name":null,"cfg.n_params":679477248,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1536,"cfg.d_vocab":250880,"cfg.act_fn":"gelu_fast","cfg.positional_embedding_type":"alibi","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BloomForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1536,"d_head":96,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":6144,"d_vocab":250880,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"alibi","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_fast","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bloom-1b1","use_attn_scale":true,"attn_scale":9.7979589711,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BloomForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"bigscience\/bloom-1b1","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0204124145,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":250880,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":679477248,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":true,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1536\nd_head: 96\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 6144\nd_vocab: 250880\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: alibi\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_fast\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bloom-1b1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n LiEJFI6YI0A=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BloomForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: bigscience\/bloom-1b1\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.020412414523193152\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 250880\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 679477248\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: true\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"bigscience\/bloom-1b1","tokenizer.vocab_size":250680.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"OO9NZoesMCpWsijo1O2DAbq9GqI=","tensor_shapes.state_dict":"embed:\n ln:\n '[w, b]': (1536,)\n W_E: (250880, 1536)\npos_embed:\n W_pos: (2048, 1536)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1536,)\n ln2:\n '[w, b]': (1536,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1536, 96)\n W_O: (16, 96, 1536)\n '[b_Q, b_K, b_V]': (16, 96)\n b_O: (1536,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1536, 6144)\n b_in: (6144,)\n W_out: (6144, 1536)\n b_out: (1536,)\nln_final:\n '[w, b]': (1536,)\nunembed:\n W_U: (1536, 250880)\n b_U: (250880,)\n","tensor_shapes.state_dict.raw__":{"embed":{"ln":{"[w, b]":"(1536,)"},"W_E":"(250880, 1536)"},"pos_embed":{"W_pos":"(2048, 1536)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1536,)"},"ln2":{"[w, b]":"(1536,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1536, 96)","W_O":"(16, 96, 1536)","[b_Q, b_K, b_V]":"(16, 96)","b_O":"(1536,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1536, 6144)","b_in":"(6144,)","W_out":"(6144, 1536)","b_out":"(1536,)"}}},"ln_final":{"[w, b]":"(1536,)"},"unembed":{"W_U":"(1536, 250880)","b_U":"(250880,)"}},"tensor_shapes.activation_cache":"embed:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\nblocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 96)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 6144)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1536)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\nunembed:\n hook_in: (batch, seq_len, 1536)\n hook_out: (batch, seq_len, 250880)\nhook_embed: (batch, seq_len, 1536)\n","tensor_shapes.activation_cache.raw__":{"embed":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"}},"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 96)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 6144)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1536)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"unembed":{"hook_in":"(batch, seq_len, 1536)","hook_out":"(batch, seq_len, 250880)"},"hook_embed":"(batch, seq_len, 1536)"}} +{"name.default_alias":"bloom-1b7","name.huggingface":"bigscience\/bloom-1b7","name.aliases":"bloom-1b7","model_type":"bloom","name.from_cfg":"bloom-1b7","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":null,"cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":250880,"cfg.act_fn":"gelu_fast","cfg.positional_embedding_type":"alibi","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BloomForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":250880,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"alibi","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_fast","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bloom-1b7","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BloomForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"bigscience\/bloom-1b7","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":250880,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":true,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 250880\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: alibi\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_fast\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bloom-1b7\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BloomForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: bigscience\/bloom-1b7\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 250880\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: true\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"bigscience\/bloom-1b7","tokenizer.vocab_size":250680.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"OO9NZoesMCpWsijo1O2DAbq9GqI=","tensor_shapes.state_dict":"embed:\n ln:\n '[w, b]': (2048,)\n W_E: (250880, 2048)\npos_embed:\n W_pos: (2048, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 250880)\n b_U: (250880,)\n","tensor_shapes.state_dict.raw__":{"embed":{"ln":{"[w, b]":"(2048,)"},"W_E":"(250880, 2048)"},"pos_embed":{"W_pos":"(2048, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 250880)","b_U":"(250880,)"}},"tensor_shapes.activation_cache":"embed:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nblocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 250880)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"embed":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"}},"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 250880)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"bloom-3b","name.huggingface":"bigscience\/bloom-3b","name.aliases":"bloom-3b","model_type":"bloom","name.from_cfg":"bloom-3b","n_params.as_str":"2.4B","n_params.as_int":2359296000,"n_params.from_name":"3b","cfg.n_params":2359296000,"cfg.n_layers":30,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":250880,"cfg.act_fn":"gelu_fast","cfg.positional_embedding_type":"alibi","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BloomForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":80,"n_layers":30,"n_ctx":2048,"n_heads":32,"d_mlp":10240,"d_vocab":250880,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"alibi","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_fast","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bloom-3b","use_attn_scale":true,"attn_scale":8.94427191,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BloomForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"bigscience\/bloom-3b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0158113883,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":250880,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":2359296000,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":true,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 80\nn_layers: 30\nn_ctx: 2048\nn_heads: 32\nd_mlp: 10240\nd_vocab: 250880\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: alibi\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_fast\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bloom-3b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n qPSXm3fjIUA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BloomForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: bigscience\/bloom-3b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.015811388300841896\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 250880\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 2359296000\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: true\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"bigscience\/bloom-3b","tokenizer.vocab_size":250680.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"OO9NZoesMCpWsijo1O2DAbq9GqI=","tensor_shapes.state_dict":"embed:\n ln:\n '[w, b]': (2560,)\n W_E: (250880, 2560)\npos_embed:\n W_pos: (2048, 2560)\nblocks:\n '[0-29]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2560, 80)\n W_O: (32, 80, 2560)\n '[b_Q, b_K, b_V]': (32, 80)\n b_O: (2560,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 250880)\n b_U: (250880,)\n","tensor_shapes.state_dict.raw__":{"embed":{"ln":{"[w, b]":"(2560,)"},"W_E":"(250880, 2560)"},"pos_embed":{"W_pos":"(2048, 2560)"},"blocks":{"[0-29]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2560, 80)","W_O":"(32, 80, 2560)","[b_Q, b_K, b_V]":"(32, 80)","b_O":"(2560,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 250880)","b_U":"(250880,)"}},"tensor_shapes.activation_cache":"embed:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nblocks:\n '[0-29]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 32, 80)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 250880)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"embed":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"}},"blocks":{"[0-29]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 32, 80)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 250880)"},"hook_embed":"(batch, seq_len, 2560)"}} +{"name.default_alias":"bloom-560m","name.huggingface":"bigscience\/bloom-560m","name.aliases":"bloom-560m","model_type":"bloom","name.from_cfg":"bloom-560m","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":"560m","cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":250880,"cfg.act_fn":"gelu_fast","cfg.positional_embedding_type":"alibi","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BloomForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":4096,"d_vocab":250880,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"alibi","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_fast","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bloom-560m","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BloomForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"bigscience\/bloom-560m","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":250880,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":true,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 4096\nd_vocab: 250880\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: alibi\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_fast\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bloom-560m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BloomForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: bigscience\/bloom-560m\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 250880\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: true\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"bigscience\/bloom-560m","tokenizer.vocab_size":250680.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"OO9NZoesMCpWsijo1O2DAbq9GqI=","tensor_shapes.state_dict":"embed:\n ln:\n '[w, b]': (1024,)\n W_E: (250880, 1024)\npos_embed:\n W_pos: (2048, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 250880)\n b_U: (250880,)\n","tensor_shapes.state_dict.raw__":{"embed":{"ln":{"[w, b]":"(1024,)"},"W_E":"(250880, 1024)"},"pos_embed":{"W_pos":"(2048, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 250880)","b_U":"(250880,)"}},"tensor_shapes.activation_cache":"embed:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nblocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 250880)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"embed":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"}},"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 250880)"},"hook_embed":"(batch, seq_len, 1024)"}} +{"name.default_alias":"bloom-7b1","name.huggingface":"bigscience\/bloom-7b1","name.aliases":"bloom-7b1","model_type":"bloom","name.from_cfg":"bloom-7b1","n_params.as_str":"6.0B","n_params.as_int":6039797760,"n_params.from_name":null,"cfg.n_params":6039797760,"cfg.n_layers":30,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":250880,"cfg.act_fn":"gelu_fast","cfg.positional_embedding_type":"alibi","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BloomForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":30,"n_ctx":2048,"n_heads":32,"d_mlp":16384,"d_vocab":250880,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"alibi","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_fast","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bloom-7b1","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BloomForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"bigscience\/bloom-7b1","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":250880,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":6039797760,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":true,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 30\nn_ctx: 2048\nn_heads: 32\nd_mlp: 16384\nd_vocab: 250880\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: alibi\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_fast\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bloom-7b1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BloomForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: bigscience\/bloom-7b1\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 250880\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 6039797760\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: true\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"bigscience\/bloom-7b1","tokenizer.vocab_size":250680.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"OO9NZoesMCpWsijo1O2DAbq9GqI=","tensor_shapes.state_dict":"embed:\n ln:\n '[w, b]': (4096,)\n W_E: (250880, 4096)\npos_embed:\n W_pos: (2048, 4096)\nblocks:\n '[0-29]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 250880)\n b_U: (250880,)\n","tensor_shapes.state_dict.raw__":{"embed":{"ln":{"[w, b]":"(4096,)"},"W_E":"(250880, 4096)"},"pos_embed":{"W_pos":"(2048, 4096)"},"blocks":{"[0-29]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 250880)","b_U":"(250880,)"}},"tensor_shapes.activation_cache":"embed:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nblocks:\n '[0-29]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 250880)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"embed":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"}},"blocks":{"[0-29]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 250880)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"CodeLlamallama-2-7b","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"CodeLlama-7b-hf","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32016,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":32016,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"CodeLlama-7b-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"codellama\/CodeLlama-7b-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32016,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 32016\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: CodeLlama-7b-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: codellama\/CodeLlama-7b-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32016\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"codellama\/CodeLlama-7b-hf","tokenizer.vocab_size":32016.0,"tokenizer.max_len":null,"tokenizer.class":"CodeLlamaTokenizer","tokenizer.vocab_hash":"TKLN0vmDArFwHMvDHh9FaoGA3_k=","tensor_shapes.state_dict":"embed:\n W_E: (32016, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32016)\n b_U: (32016,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32016, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32016)","b_U":"(32016,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32016)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32016)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"CodeLlama-7b-instruct","name.huggingface":null,"name.aliases":"","model_type":"CodeLlama","name.from_cfg":"CodeLlama-7b-Instruct-hf","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32016,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":32016,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"CodeLlama-7b-Instruct-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"codellama\/CodeLlama-7b-Instruct-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32016,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 32016\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: CodeLlama-7b-Instruct-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: codellama\/CodeLlama-7b-Instruct-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32016\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"codellama\/CodeLlama-7b-Instruct-hf","tokenizer.vocab_size":32016.0,"tokenizer.max_len":null,"tokenizer.class":"CodeLlamaTokenizer","tokenizer.vocab_hash":"TKLN0vmDArFwHMvDHh9FaoGA3_k=","tensor_shapes.state_dict":"embed:\n W_E: (32016, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32016)\n b_U: (32016,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32016, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32016)","b_U":"(32016,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32016)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32016)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"CodeLlama-7b-python","name.huggingface":null,"name.aliases":"","model_type":"CodeLlama","name.from_cfg":"CodeLlama-7b-Python-hf","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"CodeLlama-7b-Python-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"codellama\/CodeLlama-7b-Python-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: CodeLlama-7b-Python-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: codellama\/CodeLlama-7b-Python-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"codellama\/CodeLlama-7b-Python-hf","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"CodeLlamaTokenizer","tokenizer.vocab_hash":"demd-4_2c5nH6yyymVgGMcZU9lU=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"distillgpt2","name.huggingface":"distilgpt2","name.aliases":"distillgpt2, distill-gpt2, distil-gpt2, gpt2-xs","model_type":"gpt2","name.from_cfg":"distilgpt2","n_params.as_str":"42M","n_params.as_int":42467328,"n_params.from_name":null,"cfg.n_params":42467328,"cfg.n_layers":6,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":6,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"distilgpt2","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"distilgpt2","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":42467328,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 6\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: distilgpt2\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: distilgpt2\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 42467328\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"distilgpt2","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"gpt-j-6B","name.huggingface":null,"name.aliases":"","model_type":"gpt-j","name.from_cfg":"gpt-j-6B","n_params.as_str":"5.6B","n_params.as_int":5637144576,"n_params.from_name":"6B","cfg.n_params":5637144576,"cfg.n_layers":28,"cfg.n_heads":16,"cfg.d_model":4096,"cfg.d_vocab":50400,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTJForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":256,"n_layers":28,"n_ctx":2048,"n_heads":16,"d_mlp":16384,"d_vocab":50400,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt-j-6B","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTJForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-j-6B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50400,"parallel_attn_mlp":true,"rotary_dim":64,"n_params":5637144576,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":true,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 256\nn_layers: 28\nn_ctx: 2048\nn_heads: 16\nd_mlp: 16384\nd_vocab: 50400\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt-j-6B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTJForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-j-6B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50400\nparallel_attn_mlp: true\nrotary_dim: 64\nn_params: 5637144576\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: true\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-j-6B","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"aKfp-BCA9d3W27qknxFiS0DGC5s=","tensor_shapes.state_dict":"embed:\n W_E: (50400, 4096)\nblocks:\n '[0-27]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (16, 4096, 256)\n W_O: (16, 256, 4096)\n '[b_Q, b_K, b_V]': (16, 256)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50400)\n b_U: (50400,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50400, 4096)"},"blocks":{"[0-27]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 4096, 256)","W_O":"(16, 256, 4096)","[b_Q, b_K, b_V]":"(16, 256)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50400)","b_U":"(50400,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50400)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 256)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50400)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"gpt-neo-1.3B","name.huggingface":null,"name.aliases":"","model_type":"gpt-neo","name.from_cfg":"gpt-neo-1.3B","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":"1.3B","cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt-neo-1.3B","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neo-1.3B","window_size":256,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt-neo-1.3B\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neo-1.3B\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neo-1.3B","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 2048)\npos_embed:\n W_pos: (2048, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 2048)"},"pos_embed":{"W_pos":"(2048, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 2048)"}} +{"name.default_alias":"gpt-neo-125M","name.huggingface":null,"name.aliases":"","model_type":"gpt-neo","name.from_cfg":"gpt-neo-125M","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"125M","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt-neo-125M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neo-125M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt-neo-125M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neo-125M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neo-125M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (2048, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(2048, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"gpt-neo-2.7B","name.huggingface":null,"name.aliases":"","model_type":"gpt-neo","name.from_cfg":"gpt-neo-2.7B","n_params.as_str":"2.5B","n_params.as_int":2516582400,"n_params.from_name":"2.7B","cfg.n_params":2516582400,"cfg.n_layers":32,"cfg.n_heads":20,"cfg.d_model":2560,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":20,"d_mlp":10240,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt-neo-2.7B","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neo-2.7B","window_size":256,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0158113883,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":2516582400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 20\nd_mlp: 10240\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt-neo-2.7B\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neo-2.7B\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.015811388300841896\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 2516582400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neo-2.7B","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 2560)\npos_embed:\n W_pos: (2048, 2560)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (20, 2560, 128)\n W_O: (20, 128, 2560)\n '[b_Q, b_K, b_V]': (20, 128)\n b_O: (2560,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 2560)"},"pos_embed":{"W_pos":"(2048, 2560)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(20, 2560, 128)","W_O":"(20, 128, 2560)","[b_Q, b_K, b_V]":"(20, 128)","b_O":"(2560,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 20, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 20, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 20, 128)","[hook_attn_scores, hook_pattern]":"(batch, 20, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 2560)"}} +{"name.default_alias":"gpt-neox-20b","name.huggingface":"EleutherAI\/gpt-neox-20b","name.aliases":"gpt-neox-20b, gpt-neox, neox","model_type":"gpt-neo","name.from_cfg":"gpt-neox-20b","n_params.as_str":"20B","n_params.as_int":19931332608,"n_params.from_name":"20b","cfg.n_params":19931332608,"cfg.n_layers":44,"cfg.n_heads":64,"cfg.d_model":6144,"cfg.d_vocab":50432,"cfg.act_fn":"gelu_fast","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":6144,"d_head":96,"n_layers":44,"n_ctx":2048,"n_heads":64,"d_mlp":24576,"d_vocab":50432,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_fast","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt-neox-20b","use_attn_scale":true,"attn_scale":9.7979589711,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0102062073,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50432,"parallel_attn_mlp":true,"rotary_dim":24,"n_params":19931332608,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 6144\nd_head: 96\nn_layers: 44\nn_ctx: 2048\nn_heads: 64\nd_mlp: 24576\nd_vocab: 50432\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_fast\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt-neox-20b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n LiEJFI6YI0A=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.010206207261596576\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50432\nparallel_attn_mlp: true\nrotary_dim: 24\nn_params: 19931332608\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50432, 6144)\nblocks:\n '[0-43]':\n ln1:\n '[w, b]': (6144,)\n ln2:\n '[w, b]': (6144,)\n attn:\n '[W_Q, W_K, W_V]': (64, 6144, 96)\n W_O: (64, 96, 6144)\n '[b_Q, b_K, b_V]': (64, 96)\n b_O: (6144,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 24)\n mlp:\n W_in: (6144, 24576)\n b_in: (24576,)\n W_out: (24576, 6144)\n b_out: (6144,)\nln_final:\n '[w, b]': (6144,)\nunembed:\n W_U: (6144, 50432)\n b_U: (50432,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50432, 6144)"},"blocks":{"[0-43]":{"ln1":{"[w, b]":"(6144,)"},"ln2":{"[w, b]":"(6144,)"},"attn":{"[W_Q, W_K, W_V]":"(64, 6144, 96)","W_O":"(64, 96, 6144)","[b_Q, b_K, b_V]":"(64, 96)","b_O":"(6144,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 24)"},"mlp":{"W_in":"(6144, 24576)","b_in":"(24576,)","W_out":"(24576, 6144)","b_out":"(6144,)"}}},"ln_final":{"[w, b]":"(6144,)"},"unembed":{"W_U":"(6144, 50432)","b_U":"(50432,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-43]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 64, 96)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 24576)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 6144)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\nunembed:\n hook_in: (batch, seq_len, 6144)\n hook_out: (batch, seq_len, 50432)\nhook_embed: (batch, seq_len, 6144)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-43]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 64, 96)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 24576)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 6144)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"unembed":{"hook_in":"(batch, seq_len, 6144)","hook_out":"(batch, seq_len, 50432)"},"hook_embed":"(batch, seq_len, 6144)"}} +{"name.default_alias":"pythia-1.4b","name.huggingface":"EleutherAI\/pythia-1.4b","name.aliases":"pythia-1.4b, EleutherAI\/pythia-1.3b, pythia-1.3b","model_type":"pythia","name.from_cfg":"pythia-1.4b","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":"1.4b","cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1.4b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1.4b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1.4b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1.4b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1.4b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"pythia-1.4b-deduped","name.huggingface":"EleutherAI\/pythia-1.4b-deduped","name.aliases":"pythia-1.4b-deduped, EleutherAI\/pythia-1.3b-deduped, pythia-1.3b-deduped","model_type":"pythia","name.from_cfg":"pythia-1.4b-deduped","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":"1.4b","cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1.4b-deduped","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1.4b-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1.4b-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1.4b-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1.4b-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"pythia-1.4b-deduped-v0","name.huggingface":"EleutherAI\/pythia-1.4b-deduped-v0","name.aliases":"pythia-1.4b-deduped-v0, EleutherAI\/pythia-1.3b-deduped-v0, pythia-1.3b-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-1.4b-deduped-v0","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":"1.4b","cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1.4b-deduped-v0","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1.4b-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1.4b-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1.4b-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1.4b-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"pythia-1.4b-v0","name.huggingface":"EleutherAI\/pythia-1.4b-v0","name.aliases":"pythia-1.4b-v0, EleutherAI\/pythia-1.3b-v0, pythia-1.3b-v0","model_type":"pythia","name.from_cfg":"pythia-1.4b-v0","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":"1.4b","cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1.4b-v0","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1.4b-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1.4b-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1.4b-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1.4b-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, b_K, b_V]': (16, 128)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, b_K, b_V]":"(16, 128)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"pythia-12b","name.huggingface":"EleutherAI\/pythia-12b","name.aliases":"pythia-12b, EleutherAI\/pythia-13b, pythia-13b","model_type":"pythia","name.from_cfg":"pythia-12b","n_params.as_str":"11B","n_params.as_int":11324620800,"n_params.from_name":"12b","cfg.n_params":11324620800,"cfg.n_layers":36,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":50688,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":5120,"d_head":128,"n_layers":36,"n_ctx":2048,"n_heads":40,"d_mlp":20480,"d_vocab":50688,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-12b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-12b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50688,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":11324620800,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 36\nn_ctx: 2048\nn_heads: 40\nd_mlp: 20480\nd_vocab: 50688\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-12b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-12b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50688\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 11324620800\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-12b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50688, 5120)\nblocks:\n '[0-35]':\n ln1:\n '[w, b]': (5120,)\n ln2:\n '[w, b]': (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (5120, 20480)\n b_in: (20480,)\n W_out: (20480, 5120)\n b_out: (5120,)\nln_final:\n '[w, b]': (5120,)\nunembed:\n W_U: (5120, 50688)\n b_U: (50688,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50688, 5120)"},"blocks":{"[0-35]":{"ln1":{"[w, b]":"(5120,)"},"ln2":{"[w, b]":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(5120, 20480)","b_in":"(20480,)","W_out":"(20480, 5120)","b_out":"(5120,)"}}},"ln_final":{"[w, b]":"(5120,)"},"unembed":{"W_U":"(5120, 50688)","b_U":"(50688,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 20480)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 50688)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 20480)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 50688)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"pythia-12b-deduped","name.huggingface":"EleutherAI\/pythia-12b-deduped","name.aliases":"pythia-12b-deduped, EleutherAI\/pythia-13b-deduped, pythia-13b-deduped","model_type":"pythia","name.from_cfg":"pythia-12b-deduped","n_params.as_str":"11B","n_params.as_int":11324620800,"n_params.from_name":"12b","cfg.n_params":11324620800,"cfg.n_layers":36,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":50688,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":5120,"d_head":128,"n_layers":36,"n_ctx":2048,"n_heads":40,"d_mlp":20480,"d_vocab":50688,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-12b-deduped","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-12b-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50688,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":11324620800,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 36\nn_ctx: 2048\nn_heads: 40\nd_mlp: 20480\nd_vocab: 50688\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-12b-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-12b-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50688\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 11324620800\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-12b-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50688, 5120)\nblocks:\n '[0-35]':\n ln1:\n '[w, b]': (5120,)\n ln2:\n '[w, b]': (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (5120, 20480)\n b_in: (20480,)\n W_out: (20480, 5120)\n b_out: (5120,)\nln_final:\n '[w, b]': (5120,)\nunembed:\n W_U: (5120, 50688)\n b_U: (50688,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50688, 5120)"},"blocks":{"[0-35]":{"ln1":{"[w, b]":"(5120,)"},"ln2":{"[w, b]":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(5120, 20480)","b_in":"(20480,)","W_out":"(20480, 5120)","b_out":"(5120,)"}}},"ln_final":{"[w, b]":"(5120,)"},"unembed":{"W_U":"(5120, 50688)","b_U":"(50688,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 20480)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 50688)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 20480)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 50688)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"pythia-12b-deduped-v0","name.huggingface":"EleutherAI\/pythia-12b-deduped-v0","name.aliases":"pythia-12b-deduped-v0, EleutherAI\/pythia-13b-deduped-v0, pythia-13b-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-12b-deduped-v0","n_params.as_str":"11B","n_params.as_int":11324620800,"n_params.from_name":"12b","cfg.n_params":11324620800,"cfg.n_layers":36,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":50688,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":5120,"d_head":128,"n_layers":36,"n_ctx":2048,"n_heads":40,"d_mlp":20480,"d_vocab":50688,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-12b-deduped-v0","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-12b-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50688,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":11324620800,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 36\nn_ctx: 2048\nn_heads: 40\nd_mlp: 20480\nd_vocab: 50688\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-12b-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-12b-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50688\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 11324620800\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-12b-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50688, 5120)\nblocks:\n '[0-35]':\n ln1:\n '[w, b]': (5120,)\n ln2:\n '[w, b]': (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (5120, 20480)\n b_in: (20480,)\n W_out: (20480, 5120)\n b_out: (5120,)\nln_final:\n '[w, b]': (5120,)\nunembed:\n W_U: (5120, 50688)\n b_U: (50688,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50688, 5120)"},"blocks":{"[0-35]":{"ln1":{"[w, b]":"(5120,)"},"ln2":{"[w, b]":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(5120, 20480)","b_in":"(20480,)","W_out":"(20480, 5120)","b_out":"(5120,)"}}},"ln_final":{"[w, b]":"(5120,)"},"unembed":{"W_U":"(5120, 50688)","b_U":"(50688,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 20480)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 50688)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 20480)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 50688)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"pythia-12b-v0","name.huggingface":"EleutherAI\/pythia-12b-v0","name.aliases":"pythia-12b-v0, EleutherAI\/pythia-13b-v0, pythia-13b-v0","model_type":"pythia","name.from_cfg":"pythia-12b-v0","n_params.as_str":"11B","n_params.as_int":11324620800,"n_params.from_name":"12b","cfg.n_params":11324620800,"cfg.n_layers":36,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":50688,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":5120,"d_head":128,"n_layers":36,"n_ctx":2048,"n_heads":40,"d_mlp":20480,"d_vocab":50688,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-12b-v0","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-12b-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50688,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":11324620800,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 36\nn_ctx: 2048\nn_heads: 40\nd_mlp: 20480\nd_vocab: 50688\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-12b-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-12b-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50688\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 11324620800\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-12b-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50688, 5120)\nblocks:\n '[0-35]':\n ln1:\n '[w, b]': (5120,)\n ln2:\n '[w, b]': (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (5120, 20480)\n b_in: (20480,)\n W_out: (20480, 5120)\n b_out: (5120,)\nln_final:\n '[w, b]': (5120,)\nunembed:\n W_U: (5120, 50688)\n b_U: (50688,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50688, 5120)"},"blocks":{"[0-35]":{"ln1":{"[w, b]":"(5120,)"},"ln2":{"[w, b]":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(5120, 20480)","b_in":"(20480,)","W_out":"(20480, 5120)","b_out":"(5120,)"}}},"ln_final":{"[w, b]":"(5120,)"},"unembed":{"W_U":"(5120, 50688)","b_U":"(50688,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 20480)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 50688)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 20480)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 50688)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"pythia-14m","name.huggingface":"EleutherAI\/pythia-14m","name.aliases":"pythia-14m","model_type":"pythia","name.from_cfg":"pythia-14m","n_params.as_str":"1.2M","n_params.as_int":1179648,"n_params.from_name":"14m","cfg.n_params":1179648,"cfg.n_layers":6,"cfg.n_heads":4,"cfg.d_model":128,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":128,"d_head":32,"n_layers":6,"n_ctx":2048,"n_heads":4,"d_mlp":512,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-14m","use_attn_scale":true,"attn_scale":5.6568542495,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-14m","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0707106781,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":8,"n_params":1179648,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 128\nd_head: 32\nn_layers: 6\nn_ctx: 2048\nn_heads: 4\nd_mlp: 512\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-14m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gFkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-14m\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.07071067811865475\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 8\nn_params: 1179648\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-14m","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 128)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (128,)\n ln2:\n '[w, b]': (128,)\n attn:\n '[W_Q, W_K, W_V]': (4, 128, 32)\n W_O: (4, 32, 128)\n '[b_Q, b_K, b_V]': (4, 32)\n b_O: (128,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 8)\n mlp:\n W_in: (128, 512)\n b_in: (512,)\n W_out: (512, 128)\n b_out: (128,)\nln_final:\n '[w, b]': (128,)\nunembed:\n W_U: (128, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 128)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(128,)"},"ln2":{"[w, b]":"(128,)"},"attn":{"[W_Q, W_K, W_V]":"(4, 128, 32)","W_O":"(4, 32, 128)","[b_Q, b_K, b_V]":"(4, 32)","b_O":"(128,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 8)"},"mlp":{"W_in":"(128, 512)","b_in":"(512,)","W_out":"(512, 128)","b_out":"(128,)"}}},"ln_final":{"[w, b]":"(128,)"},"unembed":{"W_U":"(128, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 4, 32)\n '[hook_attn_scores, hook_pattern]': (batch, 4, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 512)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 128)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\nunembed:\n hook_in: (batch, seq_len, 128)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 128)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 4, 32)","[hook_attn_scores, hook_pattern]":"(batch, 4, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 512)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 128)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"unembed":{"hook_in":"(batch, seq_len, 128)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 128)"}} +{"name.default_alias":"pythia-160m","name.huggingface":"EleutherAI\/pythia-160m","name.aliases":"pythia-160m, EleutherAI\/pythia-125m, pythia-125m","model_type":"pythia","name.from_cfg":"pythia-160m","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"160m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-160m","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-160m","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-160m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-160m\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-160m","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 768)"}} +{"name.default_alias":"pythia-160m-deduped","name.huggingface":"EleutherAI\/pythia-160m-deduped","name.aliases":"pythia-160m-deduped, EleutherAI\/pythia-125m-deduped, pythia-125m-deduped","model_type":"pythia","name.from_cfg":"pythia-160m-deduped","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"160m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-160m-deduped","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-160m-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-160m-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-160m-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-160m-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 768)"}} +{"name.default_alias":"pythia-160m-deduped-v0","name.huggingface":"EleutherAI\/pythia-160m-deduped-v0","name.aliases":"pythia-160m-deduped-v0, EleutherAI\/pythia-125m-deduped-v0, pythia-125m-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-160m-deduped-v0","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"160m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-160m-deduped-v0","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-160m-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-160m-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-160m-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-160m-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 768)"}} +{"name.default_alias":"pythia-160m-seed1","name.huggingface":"EleutherAI\/pythia-160m-seed1","name.aliases":"pythia-160m-seed1, EleutherAI\/pythia-125m-seed1, pythia-125m-seed1","model_type":"pythia","name.from_cfg":"pythia-160m-seed1","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"160m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-160m-seed1","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-160m-seed1","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-160m-seed1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-160m-seed1\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-160m-seed1","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 768)"}} +{"name.default_alias":"pythia-160m-seed2","name.huggingface":"EleutherAI\/pythia-160m-seed2","name.aliases":"pythia-160m-seed2, EleutherAI\/pythia-125m-seed2, pythia-125m-seed2","model_type":"pythia","name.from_cfg":"pythia-160m-seed2","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"160m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-160m-seed2","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-160m-seed2","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-160m-seed2\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-160m-seed2\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-160m-seed2","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 768)"}} +{"name.default_alias":"pythia-160m-seed3","name.huggingface":"EleutherAI\/pythia-160m-seed3","name.aliases":"pythia-160m-seed3, EleutherAI\/pythia-125m-seed3, pythia-125m-seed3","model_type":"pythia","name.from_cfg":"pythia-160m-seed3","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"160m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-160m-seed3","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-160m-seed3","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-160m-seed3\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-160m-seed3\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-160m-seed3","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 768)"}} +{"name.default_alias":"pythia-160m-v0","name.huggingface":"EleutherAI\/pythia-160m-v0","name.aliases":"pythia-160m-v0, EleutherAI\/pythia-125m-v0, pythia-125m-v0","model_type":"pythia","name.from_cfg":"pythia-160m-v0","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"160m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-160m-v0","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-160m-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-160m-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-160m-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-160m-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 768)"}} +{"name.default_alias":"pythia-1b","name.huggingface":"EleutherAI\/pythia-1b","name.aliases":"pythia-1b, EleutherAI\/pythia-800m, pythia-800m","model_type":"pythia","name.from_cfg":"pythia-1b","n_params.as_str":"805M","n_params.as_int":805306368,"n_params.from_name":"1b","cfg.n_params":805306368,"cfg.n_layers":16,"cfg.n_heads":8,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":256,"n_layers":16,"n_ctx":2048,"n_heads":8,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1b","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":64,"n_params":805306368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 256\nn_layers: 16\nn_ctx: 2048\nn_heads: 8\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 64\nn_params: 805306368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (8, 2048, 256)\n W_O: (8, 256, 2048)\n '[b_Q, b_K, b_V]': (8, 256)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 2048, 256)","W_O":"(8, 256, 2048)","[b_Q, b_K, b_V]":"(8, 256)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"pythia-1b-deduped","name.huggingface":"EleutherAI\/pythia-1b-deduped","name.aliases":"pythia-1b-deduped, EleutherAI\/pythia-800m-deduped, pythia-800m-deduped","model_type":"pythia","name.from_cfg":"pythia-1b-deduped","n_params.as_str":"805M","n_params.as_int":805306368,"n_params.from_name":"1b","cfg.n_params":805306368,"cfg.n_layers":16,"cfg.n_heads":8,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":256,"n_layers":16,"n_ctx":2048,"n_heads":8,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1b-deduped","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1b-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":64,"n_params":805306368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 256\nn_layers: 16\nn_ctx: 2048\nn_heads: 8\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1b-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1b-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 64\nn_params: 805306368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1b-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (8, 2048, 256)\n W_O: (8, 256, 2048)\n '[b_Q, b_K, b_V]': (8, 256)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 2048, 256)","W_O":"(8, 256, 2048)","[b_Q, b_K, b_V]":"(8, 256)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"pythia-1b-deduped-v0","name.huggingface":"EleutherAI\/pythia-1b-deduped-v0","name.aliases":"pythia-1b-deduped-v0, EleutherAI\/pythia-800m-deduped-v0, pythia-800m-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-1b-deduped-v0","n_params.as_str":"805M","n_params.as_int":805306368,"n_params.from_name":"1b","cfg.n_params":805306368,"cfg.n_layers":16,"cfg.n_heads":8,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":256,"n_layers":16,"n_ctx":2048,"n_heads":8,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1b-deduped-v0","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1b-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":64,"n_params":805306368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 256\nn_layers: 16\nn_ctx: 2048\nn_heads: 8\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1b-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1b-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 64\nn_params: 805306368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1b-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (8, 2048, 256)\n W_O: (8, 256, 2048)\n '[b_Q, b_K, b_V]': (8, 256)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 2048, 256)","W_O":"(8, 256, 2048)","[b_Q, b_K, b_V]":"(8, 256)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"pythia-1b-v0","name.huggingface":"EleutherAI\/pythia-1b-v0","name.aliases":"pythia-1b-v0, EleutherAI\/pythia-800m-v0, pythia-800m-v0","model_type":"pythia","name.from_cfg":"pythia-1b-v0","n_params.as_str":"805M","n_params.as_int":805306368,"n_params.from_name":"1b","cfg.n_params":805306368,"cfg.n_layers":16,"cfg.n_heads":8,"cfg.d_model":2048,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":256,"n_layers":16,"n_ctx":2048,"n_heads":8,"d_mlp":8192,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-1b-v0","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-1b-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":64,"n_params":805306368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 256\nn_layers: 16\nn_ctx: 2048\nn_heads: 8\nd_mlp: 8192\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-1b-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-1b-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 64\nn_params: 805306368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-1b-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2048)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (8, 2048, 256)\n W_O: (8, 256, 2048)\n '[b_Q, b_K, b_V]': (8, 256)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2048)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 2048, 256)","W_O":"(8, 256, 2048)","[b_Q, b_K, b_V]":"(8, 256)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"pythia-2.8b","name.huggingface":"EleutherAI\/pythia-2.8b","name.aliases":"pythia-2.8b, EleutherAI\/pythia-2.7b, pythia-2.7b","model_type":"pythia","name.from_cfg":"pythia-2.8b","n_params.as_str":"2.5B","n_params.as_int":2516582400,"n_params.from_name":"2.8b","cfg.n_params":2516582400,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":80,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":10240,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-2.8b","use_attn_scale":true,"attn_scale":8.94427191,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-2.8b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0158113883,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":20,"n_params":2516582400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 80\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 10240\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-2.8b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n qPSXm3fjIUA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-2.8b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.015811388300841896\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 20\nn_params: 2516582400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-2.8b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2560)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2560, 80)\n W_O: (32, 80, 2560)\n '[b_Q, b_K, b_V]': (32, 80)\n b_O: (2560,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 20)\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2560)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2560, 80)","W_O":"(32, 80, 2560)","[b_Q, b_K, b_V]":"(32, 80)","b_O":"(2560,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 20)"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 80)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 80)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2560)"}} +{"name.default_alias":"pythia-2.8b-deduped","name.huggingface":"EleutherAI\/pythia-2.8b-deduped","name.aliases":"pythia-2.8b-deduped, EleutherAI\/pythia-2.7b-deduped, pythia-2.7b-deduped","model_type":"pythia","name.from_cfg":"pythia-2.8b-deduped","n_params.as_str":"2.5B","n_params.as_int":2516582400,"n_params.from_name":"2.8b","cfg.n_params":2516582400,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":80,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":10240,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-2.8b-deduped","use_attn_scale":true,"attn_scale":8.94427191,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-2.8b-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0158113883,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":20,"n_params":2516582400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 80\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 10240\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-2.8b-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n qPSXm3fjIUA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-2.8b-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.015811388300841896\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 20\nn_params: 2516582400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-2.8b-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2560)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2560, 80)\n W_O: (32, 80, 2560)\n '[b_Q, b_K, b_V]': (32, 80)\n b_O: (2560,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 20)\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2560)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2560, 80)","W_O":"(32, 80, 2560)","[b_Q, b_K, b_V]":"(32, 80)","b_O":"(2560,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 20)"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 80)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 80)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2560)"}} +{"name.default_alias":"pythia-2.8b-deduped-v0","name.huggingface":"EleutherAI\/pythia-2.8b-deduped-v0","name.aliases":"pythia-2.8b-deduped-v0, EleutherAI\/pythia-2.7b-deduped-v0, pythia-2.7b-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-2.8b-deduped-v0","n_params.as_str":"2.5B","n_params.as_int":2516582400,"n_params.from_name":"2.8b","cfg.n_params":2516582400,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":80,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":10240,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-2.8b-deduped-v0","use_attn_scale":true,"attn_scale":8.94427191,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-2.8b-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0158113883,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":20,"n_params":2516582400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 80\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 10240\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-2.8b-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n qPSXm3fjIUA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-2.8b-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.015811388300841896\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 20\nn_params: 2516582400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-2.8b-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2560)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2560, 80)\n W_O: (32, 80, 2560)\n '[b_Q, b_K, b_V]': (32, 80)\n b_O: (2560,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 20)\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2560)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2560, 80)","W_O":"(32, 80, 2560)","[b_Q, b_K, b_V]":"(32, 80)","b_O":"(2560,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 20)"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 80)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 80)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2560)"}} +{"name.default_alias":"pythia-2.8b-v0","name.huggingface":"EleutherAI\/pythia-2.8b-v0","name.aliases":"pythia-2.8b-v0, EleutherAI\/pythia-2.7b-v0, pythia-2.7b-v0","model_type":"pythia","name.from_cfg":"pythia-2.8b-v0","n_params.as_str":"2.5B","n_params.as_int":2516582400,"n_params.from_name":"2.8b","cfg.n_params":2516582400,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":80,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":10240,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-2.8b-v0","use_attn_scale":true,"attn_scale":8.94427191,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-2.8b-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0158113883,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":20,"n_params":2516582400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 80\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 10240\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-2.8b-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n qPSXm3fjIUA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-2.8b-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.015811388300841896\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 20\nn_params: 2516582400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-2.8b-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 2560)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2560, 80)\n W_O: (32, 80, 2560)\n '[b_Q, b_K, b_V]': (32, 80)\n b_O: (2560,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 20)\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 2560)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2560, 80)","W_O":"(32, 80, 2560)","[b_Q, b_K, b_V]":"(32, 80)","b_O":"(2560,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 20)"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 80)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 80)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 2560)"}} +{"name.default_alias":"pythia-31m","name.huggingface":"EleutherAI\/pythia-31m","name.aliases":"pythia-31m","model_type":"pythia","name.from_cfg":"pythia-31m","n_params.as_str":"4.7M","n_params.as_int":4718592,"n_params.from_name":"31m","cfg.n_params":4718592,"cfg.n_layers":6,"cfg.n_heads":8,"cfg.d_model":256,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":256,"d_head":32,"n_layers":6,"n_ctx":2048,"n_heads":8,"d_mlp":1024,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-31m","use_attn_scale":true,"attn_scale":5.6568542495,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-31m","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.05,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":8,"n_params":4718592,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 256\nd_head: 32\nn_layers: 6\nn_ctx: 2048\nn_heads: 8\nd_mlp: 1024\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-31m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gFkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-31m\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.05\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 8\nn_params: 4718592\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-31m","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 256)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (256,)\n ln2:\n '[w, b]': (256,)\n attn:\n '[W_Q, W_K, W_V]': (8, 256, 32)\n W_O: (8, 32, 256)\n '[b_Q, b_K, b_V]': (8, 32)\n b_O: (256,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 8)\n mlp:\n W_in: (256, 1024)\n b_in: (1024,)\n W_out: (1024, 256)\n b_out: (256,)\nln_final:\n '[w, b]': (256,)\nunembed:\n W_U: (256, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 256)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(256,)"},"ln2":{"[w, b]":"(256,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 256, 32)","W_O":"(8, 32, 256)","[b_Q, b_K, b_V]":"(8, 32)","b_O":"(256,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 8)"},"mlp":{"W_in":"(256, 1024)","b_in":"(1024,)","W_out":"(1024, 256)","b_out":"(256,)"}}},"ln_final":{"[w, b]":"(256,)"},"unembed":{"W_U":"(256, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 32)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 1024)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 256)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\nunembed:\n hook_in: (batch, seq_len, 256)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 256)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 32)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 1024)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 256)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"unembed":{"hook_in":"(batch, seq_len, 256)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 256)"}} +{"name.default_alias":"pythia-410m","name.huggingface":"EleutherAI\/pythia-410m","name.aliases":"pythia-410m, EleutherAI\/pythia-350m, pythia-350m","model_type":"pythia","name.from_cfg":"pythia-410m","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":"410m","cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":4096,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-410m","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-410m","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-410m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-410m\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-410m","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 1024)"}} +{"name.default_alias":"pythia-410m-deduped","name.huggingface":"EleutherAI\/pythia-410m-deduped","name.aliases":"pythia-410m-deduped, EleutherAI\/pythia-350m-deduped, pythia-350m-deduped","model_type":"pythia","name.from_cfg":"pythia-410m-deduped","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":"410m","cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":4096,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-410m-deduped","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-410m-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-410m-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-410m-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-410m-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 1024)"}} +{"name.default_alias":"pythia-410m-deduped-v0","name.huggingface":"EleutherAI\/pythia-410m-deduped-v0","name.aliases":"pythia-410m-deduped-v0, EleutherAI\/pythia-350m-deduped-v0, pythia-350m-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-410m-deduped-v0","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":"410m","cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":4096,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-410m-deduped-v0","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-410m-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-410m-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-410m-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-410m-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 1024)"}} +{"name.default_alias":"pythia-410m-v0","name.huggingface":"EleutherAI\/pythia-410m-v0","name.aliases":"pythia-410m-v0, EleutherAI\/pythia-350m-v0, pythia-350m-v0","model_type":"pythia","name.from_cfg":"pythia-410m-v0","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":"410m","cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":16,"d_mlp":4096,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-410m-v0","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-410m-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-410m-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-410m-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-410m-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 1024)"}} +{"name.default_alias":"pythia-6.9b","name.huggingface":"EleutherAI\/pythia-6.9b","name.aliases":"pythia-6.9b, EleutherAI\/pythia-6.7b, pythia-6.7b","model_type":"pythia","name.from_cfg":"pythia-6.9b","n_params.as_str":"6.4B","n_params.as_int":6442450944,"n_params.from_name":"6.9b","cfg.n_params":6442450944,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50432,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":16384,"d_vocab":50432,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-6.9b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-6.9b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50432,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":6442450944,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 16384\nd_vocab: 50432\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-6.9b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-6.9b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50432\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 6442450944\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-6.9b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50432, 4096)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50432)\n b_U: (50432,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50432, 4096)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50432)","b_U":"(50432,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50432)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50432)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"pythia-6.9b-deduped","name.huggingface":"EleutherAI\/pythia-6.9b-deduped","name.aliases":"pythia-6.9b-deduped, EleutherAI\/pythia-6.7b-deduped, pythia-6.7b-deduped","model_type":"pythia","name.from_cfg":"pythia-6.9b-deduped","n_params.as_str":"6.4B","n_params.as_int":6442450944,"n_params.from_name":"6.9b","cfg.n_params":6442450944,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50432,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":16384,"d_vocab":50432,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-6.9b-deduped","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-6.9b-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50432,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":6442450944,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 16384\nd_vocab: 50432\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-6.9b-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-6.9b-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50432\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 6442450944\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-6.9b-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50432, 4096)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50432)\n b_U: (50432,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50432, 4096)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50432)","b_U":"(50432,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50432)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50432)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"pythia-6.9b-deduped-v0","name.huggingface":"EleutherAI\/pythia-6.9b-deduped-v0","name.aliases":"pythia-6.9b-deduped-v0, EleutherAI\/pythia-6.7b-deduped-v0, pythia-6.7b-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-6.9b-deduped-v0","n_params.as_str":"6.4B","n_params.as_int":6442450944,"n_params.from_name":"6.9b","cfg.n_params":6442450944,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50432,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":16384,"d_vocab":50432,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-6.9b-deduped-v0","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-6.9b-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50432,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":6442450944,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 16384\nd_vocab: 50432\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-6.9b-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-6.9b-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50432\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 6442450944\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-6.9b-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50432, 4096)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50432)\n b_U: (50432,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50432, 4096)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50432)","b_U":"(50432,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50432)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50432)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"pythia-6.9b-v0","name.huggingface":"EleutherAI\/pythia-6.9b-v0","name.aliases":"pythia-6.9b-v0, EleutherAI\/pythia-6.7b-v0, pythia-6.7b-v0","model_type":"pythia","name.from_cfg":"pythia-6.9b-v0","n_params.as_str":"6.4B","n_params.as_int":6442450944,"n_params.from_name":"6.9b","cfg.n_params":6442450944,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50432,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":16384,"d_vocab":50432,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-6.9b-v0","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-6.9b-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50432,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":6442450944,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 16384\nd_vocab: 50432\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-6.9b-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-6.9b-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50432\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 6442450944\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-6.9b-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50432, 4096)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50432)\n b_U: (50432,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50432, 4096)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50432)","b_U":"(50432,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50432)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50432)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"pythia-70m","name.huggingface":"EleutherAI\/pythia-70m","name.aliases":"pythia-70m, pythia, EleutherAI\/pythia-19m, pythia-19m","model_type":"pythia","name.from_cfg":"pythia-70m","n_params.as_str":"19M","n_params.as_int":18874368,"n_params.from_name":"70m","cfg.n_params":18874368,"cfg.n_layers":6,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":6,"n_ctx":2048,"n_heads":8,"d_mlp":2048,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-70m","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-70m","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":18874368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 6\nn_ctx: 2048\nn_heads: 8\nd_mlp: 2048\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-70m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-70m\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 18874368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-70m","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 512)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 512)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 512)"}} +{"name.default_alias":"pythia-70m-deduped","name.huggingface":"EleutherAI\/pythia-70m-deduped","name.aliases":"pythia-70m-deduped, EleutherAI\/pythia-19m-deduped, pythia-19m-deduped","model_type":"pythia","name.from_cfg":"pythia-70m-deduped","n_params.as_str":"19M","n_params.as_int":18874368,"n_params.from_name":"70m","cfg.n_params":18874368,"cfg.n_layers":6,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":6,"n_ctx":2048,"n_heads":8,"d_mlp":2048,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-70m-deduped","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-70m-deduped","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":18874368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 6\nn_ctx: 2048\nn_heads: 8\nd_mlp: 2048\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-70m-deduped\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-70m-deduped\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 18874368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-70m-deduped","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 512)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 512)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 512)"}} +{"name.default_alias":"pythia-70m-deduped-v0","name.huggingface":"EleutherAI\/pythia-70m-deduped-v0","name.aliases":"pythia-70m-deduped-v0, EleutherAI\/pythia-19m-deduped-v0, pythia-19m-deduped-v0","model_type":"pythia","name.from_cfg":"pythia-70m-deduped-v0","n_params.as_str":"19M","n_params.as_int":18874368,"n_params.from_name":"70m","cfg.n_params":18874368,"cfg.n_layers":6,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":6,"n_ctx":2048,"n_heads":8,"d_mlp":2048,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-70m-deduped-v0","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-70m-deduped-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":18874368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 6\nn_ctx: 2048\nn_heads: 8\nd_mlp: 2048\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-70m-deduped-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-70m-deduped-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 18874368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-70m-deduped-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 512)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 512)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 512)"}} +{"name.default_alias":"pythia-70m-v0","name.huggingface":"EleutherAI\/pythia-70m-v0","name.aliases":"pythia-70m-v0, pythia-v0, EleutherAI\/pythia-19m-v0, pythia-19m-v0","model_type":"pythia","name.from_cfg":"pythia-70m-v0","n_params.as_str":"19M","n_params.as_int":18874368,"n_params.from_name":"70m","cfg.n_params":18874368,"cfg.n_layers":6,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":50304,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":6,"n_ctx":2048,"n_heads":8,"d_mlp":2048,"d_vocab":50304,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"pythia-70m-v0","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/pythia-70m-v0","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50304,"parallel_attn_mlp":true,"rotary_dim":16,"n_params":18874368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 6\nn_ctx: 2048\nn_heads: 8\nd_mlp: 2048\nd_vocab: 50304\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: pythia-70m-v0\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/pythia-70m-v0\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50304\nparallel_attn_mlp: true\nrotary_dim: 16\nn_params: 18874368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/pythia-70m-v0","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50304, 512)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 16)\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 50304)\n b_U: (50304,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50304, 512)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 16)"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 50304)","b_U":"(50304,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50304)\nhook_embed: (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50304)"},"hook_embed":"(batch, seq_len, 512)"}} +{"name.default_alias":"hubert-base-ls960","name.huggingface":"facebook\/hubert-base-ls960","name.aliases":"hubert-base-ls960","model_type":"bert","name.from_cfg":"hubert-base-ls960","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":-1,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"HubertModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":8192,"n_heads":12,"d_mlp":3072,"d_vocab":-1,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"hubert-base-ls960","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"HubertModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/hubert-base-ls960","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":-1,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 8192\nn_heads: 12\nd_mlp: 3072\nd_vocab: -1\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: hubert-base-ls960\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: HubertModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/hubert-base-ls960\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: -1\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"opt-1.3b","name.huggingface":"facebook\/opt-1.3b","name.aliases":"opt-1.3b, opt-medium","model_type":"opt","name.from_cfg":"opt-1.3b","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":"1.3b","cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":32,"cfg.d_model":2048,"cfg.d_vocab":50272,"cfg.act_fn":"relu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OPTForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":32,"d_mlp":8192,"d_vocab":50272,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"opt-1.3b","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OPTForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/opt-1.3b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50272,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 32\nd_mlp: 8192\nd_vocab: 50272\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: opt-1.3b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OPTForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/opt-1.3b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50272\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/opt-1.3b","tokenizer.vocab_size":50265.0,"tokenizer.max_len":null,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"f1FIzqnRiMYzke1CU0hp8TDxq7k=","tensor_shapes.state_dict":"embed:\n W_E: (50272, 2048)\npos_embed:\n W_pos: (2048, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2048, 64)\n W_O: (32, 64, 2048)\n '[b_Q, b_K, b_V]': (32, 64)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 50272)\n b_U: (50272,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50272, 2048)"},"pos_embed":{"W_pos":"(2048, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2048, 64)","W_O":"(32, 64, 2048)","[b_Q, b_K, b_V]":"(32, 64)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 50272)","b_U":"(50272,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 32, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 50272)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 32, 64)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 50272)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 2048)"}} +{"name.default_alias":"opt-125m","name.huggingface":"facebook\/opt-125m","name.aliases":"opt-125m, opt-small, opt","model_type":"opt","name.from_cfg":"opt-125m","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":"125m","cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50272,"cfg.act_fn":"relu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OPTForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":2048,"n_heads":12,"d_mlp":3072,"d_vocab":50272,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"opt-125m","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OPTForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/opt-125m","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50272,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 2048\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50272\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: opt-125m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OPTForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/opt-125m\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50272\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/opt-125m","tokenizer.vocab_size":50265.0,"tokenizer.max_len":null,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"f1FIzqnRiMYzke1CU0hp8TDxq7k=","tensor_shapes.state_dict":"embed:\n W_E: (50272, 768)\npos_embed:\n W_pos: (2048, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50272)\n b_U: (50272,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50272, 768)"},"pos_embed":{"W_pos":"(2048, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50272)","b_U":"(50272,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50272)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50272)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"opt-13b","name.huggingface":"facebook\/opt-13b","name.aliases":"opt-13b, opt-xxl","model_type":"opt","name.from_cfg":"opt-13b","n_params.as_str":"13B","n_params.as_int":12582912000,"n_params.from_name":"13b","cfg.n_params":12582912000,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":50272,"cfg.act_fn":"relu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OPTForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":40,"d_mlp":20480,"d_vocab":50272,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"opt-13b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OPTForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/opt-13b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50272,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912000,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 40\nd_mlp: 20480\nd_vocab: 50272\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: opt-13b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OPTForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/opt-13b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50272\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912000\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/opt-13b","tokenizer.vocab_size":50265.0,"tokenizer.max_len":null,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"f1FIzqnRiMYzke1CU0hp8TDxq7k=","tensor_shapes.state_dict":"embed:\n W_E: (50272, 5120)\npos_embed:\n W_pos: (2048, 5120)\nblocks:\n '[0-39]':\n ln1:\n '[w, b]': (5120,)\n ln2:\n '[w, b]': (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (5120, 20480)\n b_in: (20480,)\n W_out: (20480, 5120)\n b_out: (5120,)\nln_final:\n '[w, b]': (5120,)\nunembed:\n W_U: (5120, 50272)\n b_U: (50272,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50272, 5120)"},"pos_embed":{"W_pos":"(2048, 5120)"},"blocks":{"[0-39]":{"ln1":{"[w, b]":"(5120,)"},"ln2":{"[w, b]":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(5120, 20480)","b_in":"(20480,)","W_out":"(20480, 5120)","b_out":"(5120,)"}}},"ln_final":{"[w, b]":"(5120,)"},"unembed":{"W_U":"(5120, 50272)","b_U":"(50272,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 20480)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 50272)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 20480)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 50272)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 5120)"}} +{"name.default_alias":"opt-2.7b","name.huggingface":"facebook\/opt-2.7b","name.aliases":"opt-2.7b, opt-large","model_type":"opt","name.from_cfg":"opt-2.7b","n_params.as_str":"2.5B","n_params.as_int":2516582400,"n_params.from_name":"2.7b","cfg.n_params":2516582400,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":50272,"cfg.act_fn":"relu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OPTForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":80,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":10240,"d_vocab":50272,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"opt-2.7b","use_attn_scale":true,"attn_scale":8.94427191,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OPTForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/opt-2.7b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0158113883,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50272,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":2516582400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 80\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 10240\nd_vocab: 50272\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: opt-2.7b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n qPSXm3fjIUA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OPTForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/opt-2.7b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.015811388300841896\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50272\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 2516582400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/opt-2.7b","tokenizer.vocab_size":50265.0,"tokenizer.max_len":null,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"f1FIzqnRiMYzke1CU0hp8TDxq7k=","tensor_shapes.state_dict":"embed:\n W_E: (50272, 2560)\npos_embed:\n W_pos: (2048, 2560)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2560, 80)\n W_O: (32, 80, 2560)\n '[b_Q, b_K, b_V]': (32, 80)\n b_O: (2560,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 50272)\n b_U: (50272,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50272, 2560)"},"pos_embed":{"W_pos":"(2048, 2560)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2560, 80)","W_O":"(32, 80, 2560)","[b_Q, b_K, b_V]":"(32, 80)","b_O":"(2560,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 50272)","b_U":"(50272,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 32, 80)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 50272)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 32, 80)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 50272)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 2560)"}} +{"name.default_alias":"opt-30b","name.huggingface":"facebook\/opt-30b","name.aliases":"opt-30b, opt-xxxl","model_type":"opt","name.from_cfg":"opt-30b","n_params.as_str":"30B","n_params.as_int":29595009024,"n_params.from_name":"30b","cfg.n_params":29595009024,"cfg.n_layers":48,"cfg.n_heads":56,"cfg.d_model":7168,"cfg.d_vocab":50272,"cfg.act_fn":"relu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OPTForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":7168,"d_head":128,"n_layers":48,"n_ctx":2048,"n_heads":56,"d_mlp":28672,"d_vocab":50272,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"opt-30b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OPTForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/opt-30b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0094491118,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50272,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":29595009024,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 7168\nd_head: 128\nn_layers: 48\nn_ctx: 2048\nn_heads: 56\nd_mlp: 28672\nd_vocab: 50272\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: opt-30b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OPTForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/opt-30b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.00944911182523068\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50272\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 29595009024\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/opt-30b","tokenizer.vocab_size":50265.0,"tokenizer.max_len":null,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"f1FIzqnRiMYzke1CU0hp8TDxq7k=","tensor_shapes.state_dict":"embed:\n W_E: (50272, 7168)\npos_embed:\n W_pos: (2048, 7168)\nblocks:\n '[0-47]':\n ln1:\n '[w, b]': (7168,)\n ln2:\n '[w, b]': (7168,)\n attn:\n '[W_Q, W_K, W_V]': (56, 7168, 128)\n W_O: (56, 128, 7168)\n '[b_Q, b_K, b_V]': (56, 128)\n b_O: (7168,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (7168, 28672)\n b_in: (28672,)\n W_out: (28672, 7168)\n b_out: (7168,)\nln_final:\n '[w, b]': (7168,)\nunembed:\n W_U: (7168, 50272)\n b_U: (50272,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50272, 7168)"},"pos_embed":{"W_pos":"(2048, 7168)"},"blocks":{"[0-47]":{"ln1":{"[w, b]":"(7168,)"},"ln2":{"[w, b]":"(7168,)"},"attn":{"[W_Q, W_K, W_V]":"(56, 7168, 128)","W_O":"(56, 128, 7168)","[b_Q, b_K, b_V]":"(56, 128)","b_O":"(7168,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(7168, 28672)","b_in":"(28672,)","W_out":"(28672, 7168)","b_out":"(7168,)"}}},"ln_final":{"[w, b]":"(7168,)"},"unembed":{"W_U":"(7168, 50272)","b_U":"(50272,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-47]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 56, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 56, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 28672)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 7168)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 7168)\nunembed:\n hook_in: (batch, seq_len, 7168)\n hook_out: (batch, seq_len, 50272)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 7168)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-47]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 56, 128)","[hook_attn_scores, hook_pattern]":"(batch, 56, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 28672)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 7168)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 7168)"},"unembed":{"hook_in":"(batch, seq_len, 7168)","hook_out":"(batch, seq_len, 50272)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 7168)"}} +{"name.default_alias":"opt-6.7b","name.huggingface":"facebook\/opt-6.7b","name.aliases":"opt-6.7b, opt-xl","model_type":"opt","name.from_cfg":"opt-6.7b","n_params.as_str":"6.4B","n_params.as_int":6442450944,"n_params.from_name":"6.7b","cfg.n_params":6442450944,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50272,"cfg.act_fn":"relu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OPTForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":16384,"d_vocab":50272,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"opt-6.7b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OPTForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/opt-6.7b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50272,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":6442450944,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 16384\nd_vocab: 50272\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: opt-6.7b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OPTForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/opt-6.7b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50272\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 6442450944\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/opt-6.7b","tokenizer.vocab_size":50265.0,"tokenizer.max_len":null,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"f1FIzqnRiMYzke1CU0hp8TDxq7k=","tensor_shapes.state_dict":"embed:\n W_E: (50272, 4096)\npos_embed:\n W_pos: (2048, 4096)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50272)\n b_U: (50272,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50272, 4096)"},"pos_embed":{"W_pos":"(2048, 4096)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50272)","b_U":"(50272,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50272)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50272)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 4096)"}} +{"name.default_alias":"opt-66b","name.huggingface":"facebook\/opt-66b","name.aliases":"opt-66b, opt-xxxxl","model_type":"opt","name.from_cfg":"opt-66b","n_params.as_str":"65B","n_params.as_int":65229815808,"n_params.from_name":"66b","cfg.n_params":65229815808,"cfg.n_layers":64,"cfg.n_heads":72,"cfg.d_model":9216,"cfg.d_vocab":50272,"cfg.act_fn":"relu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"OPTForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":9216,"d_head":128,"n_layers":64,"n_ctx":2048,"n_heads":72,"d_mlp":36864,"d_vocab":50272,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"opt-66b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"OPTForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/opt-66b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0083333333,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50272,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":65229815808,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 9216\nd_head: 128\nn_layers: 64\nn_ctx: 2048\nn_heads: 72\nd_mlp: 36864\nd_vocab: 50272\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: opt-66b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: OPTForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/opt-66b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008333333333333333\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50272\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 65229815808\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/opt-66b","tokenizer.vocab_size":50265.0,"tokenizer.max_len":null,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"f1FIzqnRiMYzke1CU0hp8TDxq7k=","tensor_shapes.state_dict":"embed:\n W_E: (50272, 9216)\npos_embed:\n W_pos: (2048, 9216)\nblocks:\n '[0-63]':\n ln1:\n '[w, b]': (9216,)\n ln2:\n '[w, b]': (9216,)\n attn:\n '[W_Q, W_K, W_V]': (72, 9216, 128)\n W_O: (72, 128, 9216)\n '[b_Q, b_K, b_V]': (72, 128)\n b_O: (9216,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (9216, 36864)\n b_in: (36864,)\n W_out: (36864, 9216)\n b_out: (9216,)\nln_final:\n '[w, b]': (9216,)\nunembed:\n W_U: (9216, 50272)\n b_U: (50272,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50272, 9216)"},"pos_embed":{"W_pos":"(2048, 9216)"},"blocks":{"[0-63]":{"ln1":{"[w, b]":"(9216,)"},"ln2":{"[w, b]":"(9216,)"},"attn":{"[W_Q, W_K, W_V]":"(72, 9216, 128)","W_O":"(72, 128, 9216)","[b_Q, b_K, b_V]":"(72, 128)","b_O":"(9216,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(9216, 36864)","b_in":"(36864,)","W_out":"(36864, 9216)","b_out":"(9216,)"}}},"ln_final":{"[w, b]":"(9216,)"},"unembed":{"W_U":"(9216, 50272)","b_U":"(50272,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-63]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 9216)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 72, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 72, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 9216)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 36864)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 9216)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 9216)\nunembed:\n hook_in: (batch, seq_len, 9216)\n hook_out: (batch, seq_len, 50272)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 9216)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-63]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 9216)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 72, 128)","[hook_attn_scores, hook_pattern]":"(batch, 72, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 9216)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 36864)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 9216)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 9216)"},"unembed":{"hook_in":"(batch, seq_len, 9216)","hook_out":"(batch, seq_len, 50272)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 9216)"}} +{"name.default_alias":"wav2vec2-base","name.huggingface":"facebook\/wav2vec2-base","name.aliases":"wav2vec2-base, w2v2-base","model_type":null,"name.from_cfg":"wav2vec2-base","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":-1,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Wav2Vec2ForPreTraining","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":8192,"n_heads":12,"d_mlp":3072,"d_vocab":-1,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"wav2vec2-base","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Wav2Vec2ForPreTraining","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/wav2vec2-base","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":-1,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 8192\nn_heads: 12\nd_mlp: 3072\nd_vocab: -1\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: wav2vec2-base\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Wav2Vec2ForPreTraining\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/wav2vec2-base\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: -1\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"facebook\/wav2vec2-base","tokenizer.vocab_size":32.0,"tokenizer.max_len":null,"tokenizer.class":"Wav2Vec2CTCTokenizer","tokenizer.vocab_hash":"tJ24E7IRYtfON5KH_y7lgJl6WV4=","tensor_shapes.state_dict":"embed:\n W_E: (32, 768)\npos_embed:\n W_pos: (8192, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 32)\n b_U: (32,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32, 768)"},"pos_embed":{"W_pos":"(8192, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 32)","b_U":"(32,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 32)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 32)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"wav2vec2-large","name.huggingface":"facebook\/wav2vec2-large","name.aliases":"wav2vec2-large, w2v2-large","model_type":null,"name.from_cfg":"wav2vec2-large","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":-1,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Wav2Vec2ForPreTraining","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":8192,"n_heads":16,"d_mlp":4096,"d_vocab":-1,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"wav2vec2-large","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Wav2Vec2ForPreTraining","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"facebook\/wav2vec2-large","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":-1,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 8192\nn_heads: 16\nd_mlp: 4096\nd_vocab: -1\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: wav2vec2-large\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Wav2Vec2ForPreTraining\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: facebook\/wav2vec2-large\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: -1\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"bert-base-cased","name.huggingface":"google-bert\/bert-base-cased","name.aliases":"bert-base-cased","model_type":"bert","name.from_cfg":"bert-base-cased","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":28996,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BertForMaskedLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":512,"n_heads":12,"d_mlp":3072,"d_vocab":28996,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.0,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bert-base-cased","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BertForMaskedLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google-bert\/bert-base-cased","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":28996,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 512\nn_heads: 12\nd_mlp: 3072\nd_vocab: 28996\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-12\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bert-base-cased\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BertForMaskedLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google-bert\/bert-base-cased\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 28996\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google-bert\/bert-base-cased","tokenizer.vocab_size":28996.0,"tokenizer.max_len":512.0,"tokenizer.class":"BertTokenizer","tokenizer.vocab_hash":"SSKvHuFYtPbvgwMSLSIhfFE_kF8=","tensor_shapes.state_dict":"embed:\n W_E: (28996, 768)\npos_embed:\n W_pos: (512, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 28996)\n b_U: (28996,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(28996, 768)"},"pos_embed":{"W_pos":"(512, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 28996)","b_U":"(28996,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 28996)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 28996)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"bert-base-uncased","name.huggingface":"google-bert\/bert-base-uncased","name.aliases":"bert-base-uncased","model_type":"bert","name.from_cfg":"bert-base-uncased","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":30522,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BertForMaskedLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":512,"n_heads":12,"d_mlp":3072,"d_vocab":30522,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.0,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bert-base-uncased","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BertForMaskedLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google-bert\/bert-base-uncased","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":30522,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 512\nn_heads: 12\nd_mlp: 3072\nd_vocab: 30522\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-12\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bert-base-uncased\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BertForMaskedLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google-bert\/bert-base-uncased\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 30522\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google-bert\/bert-base-uncased","tokenizer.vocab_size":30522.0,"tokenizer.max_len":512.0,"tokenizer.class":"BertTokenizer","tokenizer.vocab_hash":"G9iEWgpI_JY73i8Lym9gBVhq4BI=","tensor_shapes.state_dict":"embed:\n W_E: (30522, 768)\npos_embed:\n W_pos: (512, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 30522)\n b_U: (30522,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(30522, 768)"},"pos_embed":{"W_pos":"(512, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 30522)","b_U":"(30522,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 30522)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 30522)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"bert-large-cased","name.huggingface":"google-bert\/bert-large-cased","name.aliases":"bert-large-cased","model_type":"bert","name.from_cfg":"bert-large-cased","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":28996,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BertForMaskedLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":512,"n_heads":16,"d_mlp":4096,"d_vocab":28996,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.0,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bert-large-cased","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BertForMaskedLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google-bert\/bert-large-cased","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":28996,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 512\nn_heads: 16\nd_mlp: 4096\nd_vocab: 28996\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-12\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bert-large-cased\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BertForMaskedLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google-bert\/bert-large-cased\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 28996\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google-bert\/bert-large-cased","tokenizer.vocab_size":28996.0,"tokenizer.max_len":512.0,"tokenizer.class":"BertTokenizer","tokenizer.vocab_hash":"SSKvHuFYtPbvgwMSLSIhfFE_kF8=","tensor_shapes.state_dict":"embed:\n W_E: (28996, 1024)\npos_embed:\n W_pos: (512, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 28996)\n b_U: (28996,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(28996, 1024)"},"pos_embed":{"W_pos":"(512, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 28996)","b_U":"(28996,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 28996)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 28996)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"bert-large-uncased","name.huggingface":"google-bert\/bert-large-uncased","name.aliases":"bert-large-uncased","model_type":"bert","name.from_cfg":"bert-large-uncased","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":30522,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"BertForMaskedLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":512,"n_heads":16,"d_mlp":4096,"d_vocab":30522,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.0,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"bert-large-uncased","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"BertForMaskedLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google-bert\/bert-large-uncased","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":30522,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 512\nn_heads: 16\nd_mlp: 4096\nd_vocab: 30522\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-12\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: bert-large-uncased\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: BertForMaskedLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google-bert\/bert-large-uncased\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 30522\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google-bert\/bert-large-uncased","tokenizer.vocab_size":30522.0,"tokenizer.max_len":512.0,"tokenizer.class":"BertTokenizer","tokenizer.vocab_hash":"G9iEWgpI_JY73i8Lym9gBVhq4BI=","tensor_shapes.state_dict":"embed:\n W_E: (30522, 1024)\npos_embed:\n W_pos: (512, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 30522)\n b_U: (30522,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(30522, 1024)"},"pos_embed":{"W_pos":"(512, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 30522)","b_U":"(30522,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 30522)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 30522)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"t5-base","name.huggingface":"google-t5\/t5-base","name.aliases":"t5-base","model_type":"t5","name.from_cfg":"t5-base","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":32128,"cfg.act_fn":"relu","cfg.positional_embedding_type":"relative_positional_bias","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"T5ForConditionalGeneration","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":512,"n_heads":12,"d_mlp":3072,"d_vocab":32128,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"relative_positional_bias","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"t5-base","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"T5ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google-t5\/t5-base","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32128,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":128,"relative_attention_num_buckets":32,"decoder_start_token_id":0,"tie_word_embeddings":true,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 512\nn_heads: 12\nd_mlp: 3072\nd_vocab: 32128\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: relative_positional_bias\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: t5-base\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: T5ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google-t5\/t5-base\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32128\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: 128\nrelative_attention_num_buckets: 32\ndecoder_start_token_id: 0\ntie_word_embeddings: true\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google-t5\/t5-base","tokenizer.vocab_size":32100.0,"tokenizer.max_len":null,"tokenizer.class":"T5Tokenizer","tokenizer.vocab_hash":"jQeywCyCMVL_vza2wKfpuwjNVys=","tensor_shapes.state_dict":"embed:\n W_E: (32128, 768)\npos_embed:\n W_pos: (512, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 32128)\n b_U: (32128,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32128, 768)"},"pos_embed":{"W_pos":"(512, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 32128)","b_U":"(32128,)"}},"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"t5-large","name.huggingface":"google-t5\/t5-large","name.aliases":"t5-large","model_type":"t5","name.from_cfg":"t5-large","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":32128,"cfg.act_fn":"relu","cfg.positional_embedding_type":"relative_positional_bias","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"T5ForConditionalGeneration","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":512,"n_heads":16,"d_mlp":4096,"d_vocab":32128,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"relative_positional_bias","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"t5-large","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"T5ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google-t5\/t5-large","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32128,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":128,"relative_attention_num_buckets":32,"decoder_start_token_id":0,"tie_word_embeddings":true,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 512\nn_heads: 16\nd_mlp: 4096\nd_vocab: 32128\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: relative_positional_bias\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: t5-large\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: T5ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google-t5\/t5-large\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32128\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: 128\nrelative_attention_num_buckets: 32\ndecoder_start_token_id: 0\ntie_word_embeddings: true\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google-t5\/t5-large","tokenizer.vocab_size":32100.0,"tokenizer.max_len":null,"tokenizer.class":"T5Tokenizer","tokenizer.vocab_hash":"jQeywCyCMVL_vza2wKfpuwjNVys=","tensor_shapes.state_dict":"embed:\n W_E: (32128, 1024)\npos_embed:\n W_pos: (512, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 32128)\n b_U: (32128,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32128, 1024)"},"pos_embed":{"W_pos":"(512, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 32128)","b_U":"(32128,)"}},"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"t5-small","name.huggingface":"google-t5\/t5-small","name.aliases":"t5-small","model_type":"t5","name.from_cfg":"t5-small","n_params.as_str":"19M","n_params.as_int":18874368,"n_params.from_name":null,"cfg.n_params":18874368,"cfg.n_layers":6,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":32128,"cfg.act_fn":"relu","cfg.positional_embedding_type":"relative_positional_bias","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"T5ForConditionalGeneration","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":6,"n_ctx":512,"n_heads":8,"d_mlp":2048,"d_vocab":32128,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"relative_positional_bias","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"relu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"t5-small","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"T5ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google-t5\/t5-small","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"bidirectional","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32128,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":18874368,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":128,"relative_attention_num_buckets":32,"decoder_start_token_id":0,"tie_word_embeddings":true,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 6\nn_ctx: 512\nn_heads: 8\nd_mlp: 2048\nd_vocab: 32128\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: relative_positional_bias\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: relu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: t5-small\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: T5ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google-t5\/t5-small\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: bidirectional\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32128\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 18874368\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: 128\nrelative_attention_num_buckets: 32\ndecoder_start_token_id: 0\ntie_word_embeddings: true\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google-t5\/t5-small","tokenizer.vocab_size":32100.0,"tokenizer.max_len":512.0,"tokenizer.class":"T5Tokenizer","tokenizer.vocab_hash":"jQeywCyCMVL_vza2wKfpuwjNVys=","tensor_shapes.state_dict":"embed:\n W_E: (32128, 512)\npos_embed:\n W_pos: (512, 512)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 32128)\n b_U: (32128,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32128, 512)"},"pos_embed":{"W_pos":"(512, 512)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 32128)","b_U":"(32128,)"}},"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"gemma-2-27b","name.huggingface":"google\/gemma-2-27b","name.aliases":"gemma-2-27b","model_type":"gemma","name.from_cfg":"gemma-2-27b","n_params.as_str":"27B","n_params.as_int":26914848768,"n_params.from_name":"27b","cfg.n_params":26914848768,"cfg.n_layers":46,"cfg.n_heads":32,"cfg.d_model":4608,"cfg.d_vocab":256000,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4608,"d_head":128,"n_layers":46,"n_ctx":8192,"n_heads":32,"d_mlp":36864,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2-27b","use_attn_scale":true,"attn_scale":12.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2-27b","window_size":4096,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":26914848768,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":50.0,"output_logits_soft_cap":30.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4608\nd_head: 128\nn_layers: 46\nn_ctx: 8192\nn_heads: 32\nd_mlp: 36864\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2-27b\nuse_attn_scale: true\nattn_scale: 12.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2-27b\nwindow_size: 4096\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 26914848768\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: 50.0\noutput_logits_soft_cap: 30.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2-27b","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 4608)\nblocks:\n '[0-45]':\n ln1:\n w: (4608,)\n ln1_post:\n w: (4608,)\n ln2:\n w: (4608,)\n ln2_post:\n w: (4608,)\n attn:\n W_Q: (32, 4608, 128)\n W_O: (32, 128, 4608)\n b_Q: (32, 128)\n b_O: (4608,)\n '[_W_K, _W_V]': (16, 4608, 128)\n '[_b_K, _b_V]': (16, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4608, 36864)\n W_out: (36864, 4608)\n b_in: (36864,)\n b_out: (4608,)\nln_final:\n w: (4608,)\nunembed:\n W_U: (4608, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 4608)"},"blocks":{"[0-45]":{"ln1":{"w":"(4608,)"},"ln1_post":{"w":"(4608,)"},"ln2":{"w":"(4608,)"},"ln2_post":{"w":"(4608,)"},"attn":{"W_Q":"(32, 4608, 128)","W_O":"(32, 128, 4608)","b_Q":"(32, 128)","b_O":"(4608,)","[_W_K, _W_V]":"(16, 4608, 128)","[_b_K, _b_V]":"(16, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4608, 36864)","W_out":"(36864, 4608)","b_in":"(36864,)","b_out":"(4608,)"}}},"ln_final":{"w":"(4608,)"},"unembed":{"W_U":"(4608, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-45]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 36864)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4608)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\nunembed:\n hook_in: (batch, seq_len, 4608)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 4608)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-45]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 36864)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4608)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"unembed":{"hook_in":"(batch, seq_len, 4608)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 4608)"}} +{"name.default_alias":"gemma-2-27b-it","name.huggingface":"google\/gemma-2-27b-it","name.aliases":"gemma-2-27b-it","model_type":"gemma","name.from_cfg":"gemma-2-27b-it","n_params.as_str":"27B","n_params.as_int":26914848768,"n_params.from_name":"27b","cfg.n_params":26914848768,"cfg.n_layers":46,"cfg.n_heads":32,"cfg.d_model":4608,"cfg.d_vocab":256000,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4608,"d_head":128,"n_layers":46,"n_ctx":8192,"n_heads":32,"d_mlp":36864,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2-27b-it","use_attn_scale":true,"attn_scale":12.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2-27b-it","window_size":4096,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":26914848768,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":50.0,"output_logits_soft_cap":30.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4608\nd_head: 128\nn_layers: 46\nn_ctx: 8192\nn_heads: 32\nd_mlp: 36864\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2-27b-it\nuse_attn_scale: true\nattn_scale: 12.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2-27b-it\nwindow_size: 4096\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 26914848768\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: 50.0\noutput_logits_soft_cap: 30.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2-27b-it","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 4608)\nblocks:\n '[0-45]':\n ln1:\n w: (4608,)\n ln1_post:\n w: (4608,)\n ln2:\n w: (4608,)\n ln2_post:\n w: (4608,)\n attn:\n W_Q: (32, 4608, 128)\n W_O: (32, 128, 4608)\n b_Q: (32, 128)\n b_O: (4608,)\n '[_W_K, _W_V]': (16, 4608, 128)\n '[_b_K, _b_V]': (16, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4608, 36864)\n W_out: (36864, 4608)\n b_in: (36864,)\n b_out: (4608,)\nln_final:\n w: (4608,)\nunembed:\n W_U: (4608, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 4608)"},"blocks":{"[0-45]":{"ln1":{"w":"(4608,)"},"ln1_post":{"w":"(4608,)"},"ln2":{"w":"(4608,)"},"ln2_post":{"w":"(4608,)"},"attn":{"W_Q":"(32, 4608, 128)","W_O":"(32, 128, 4608)","b_Q":"(32, 128)","b_O":"(4608,)","[_W_K, _W_V]":"(16, 4608, 128)","[_b_K, _b_V]":"(16, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4608, 36864)","W_out":"(36864, 4608)","b_in":"(36864,)","b_out":"(4608,)"}}},"ln_final":{"w":"(4608,)"},"unembed":{"W_U":"(4608, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-45]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 36864)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4608)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4608)\nunembed:\n hook_in: (batch, seq_len, 4608)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 4608)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-45]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 36864)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4608)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4608)"},"unembed":{"hook_in":"(batch, seq_len, 4608)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 4608)"}} +{"name.default_alias":"gemma-2-2b","name.huggingface":"google\/gemma-2-2b","name.aliases":"gemma-2-2b","model_type":"gemma","name.from_cfg":"gemma-2-2b","n_params.as_str":"2.1B","n_params.as_int":2146959360,"n_params.from_name":"2b","cfg.n_params":2146959360,"cfg.n_layers":26,"cfg.n_heads":8,"cfg.d_model":2304,"cfg.d_vocab":256000,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2304,"d_head":256,"n_layers":26,"n_ctx":8192,"n_heads":8,"d_mlp":9216,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2-2b","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2-2b","window_size":4096,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":2146959360,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":50.0,"output_logits_soft_cap":30.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2304\nd_head: 256\nn_layers: 26\nn_ctx: 8192\nn_heads: 8\nd_mlp: 9216\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2-2b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2-2b\nwindow_size: 4096\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 2146959360\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: 50.0\noutput_logits_soft_cap: 30.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2-2b","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 2304)\nblocks:\n '[0-25]':\n ln1:\n w: (2304,)\n ln1_post:\n w: (2304,)\n ln2:\n w: (2304,)\n ln2_post:\n w: (2304,)\n attn:\n W_Q: (8, 2304, 256)\n W_O: (8, 256, 2304)\n b_Q: (8, 256)\n b_O: (2304,)\n '[_W_K, _W_V]': (4, 2304, 256)\n '[_b_K, _b_V]': (4, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (2304, 9216)\n W_out: (9216, 2304)\n b_in: (9216,)\n b_out: (2304,)\nln_final:\n w: (2304,)\nunembed:\n W_U: (2304, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 2304)"},"blocks":{"[0-25]":{"ln1":{"w":"(2304,)"},"ln1_post":{"w":"(2304,)"},"ln2":{"w":"(2304,)"},"ln2_post":{"w":"(2304,)"},"attn":{"W_Q":"(8, 2304, 256)","W_O":"(8, 256, 2304)","b_Q":"(8, 256)","b_O":"(2304,)","[_W_K, _W_V]":"(4, 2304, 256)","[_b_K, _b_V]":"(4, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(2304, 9216)","W_out":"(9216, 2304)","b_in":"(9216,)","b_out":"(2304,)"}}},"ln_final":{"w":"(2304,)"},"unembed":{"W_U":"(2304, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-25]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 8, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 9216)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2304)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\nunembed:\n hook_in: (batch, seq_len, 2304)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 2304)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-25]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 8, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 9216)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2304)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"unembed":{"hook_in":"(batch, seq_len, 2304)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 2304)"}} +{"name.default_alias":"gemma-2-2b-it","name.huggingface":"google\/gemma-2-2b-it","name.aliases":"gemma-2-2b-it","model_type":"gemma","name.from_cfg":"gemma-2-2b-it","n_params.as_str":"2.1B","n_params.as_int":2146959360,"n_params.from_name":"2b","cfg.n_params":2146959360,"cfg.n_layers":26,"cfg.n_heads":8,"cfg.d_model":2304,"cfg.d_vocab":256000,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2304,"d_head":256,"n_layers":26,"n_ctx":8192,"n_heads":8,"d_mlp":9216,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2-2b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2-2b-it","window_size":4096,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":2146959360,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":50.0,"output_logits_soft_cap":30.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2304\nd_head: 256\nn_layers: 26\nn_ctx: 8192\nn_heads: 8\nd_mlp: 9216\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2-2b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2-2b-it\nwindow_size: 4096\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 2146959360\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: 50.0\noutput_logits_soft_cap: 30.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2-2b-it","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 2304)\nblocks:\n '[0-25]':\n ln1:\n w: (2304,)\n ln1_post:\n w: (2304,)\n ln2:\n w: (2304,)\n ln2_post:\n w: (2304,)\n attn:\n W_Q: (8, 2304, 256)\n W_O: (8, 256, 2304)\n b_Q: (8, 256)\n b_O: (2304,)\n '[_W_K, _W_V]': (4, 2304, 256)\n '[_b_K, _b_V]': (4, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (2304, 9216)\n W_out: (9216, 2304)\n b_in: (9216,)\n b_out: (2304,)\nln_final:\n w: (2304,)\nunembed:\n W_U: (2304, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 2304)"},"blocks":{"[0-25]":{"ln1":{"w":"(2304,)"},"ln1_post":{"w":"(2304,)"},"ln2":{"w":"(2304,)"},"ln2_post":{"w":"(2304,)"},"attn":{"W_Q":"(8, 2304, 256)","W_O":"(8, 256, 2304)","b_Q":"(8, 256)","b_O":"(2304,)","[_W_K, _W_V]":"(4, 2304, 256)","[_b_K, _b_V]":"(4, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(2304, 9216)","W_out":"(9216, 2304)","b_in":"(9216,)","b_out":"(2304,)"}}},"ln_final":{"w":"(2304,)"},"unembed":{"W_U":"(2304, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-25]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 8, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 9216)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2304)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2304)\nunembed:\n hook_in: (batch, seq_len, 2304)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 2304)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-25]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 8, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 9216)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2304)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2304)"},"unembed":{"hook_in":"(batch, seq_len, 2304)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 2304)"}} +{"name.default_alias":"gemma-2-9b","name.huggingface":"google\/gemma-2-9b","name.aliases":"gemma-2-9b","model_type":"gemma","name.from_cfg":"gemma-2-9b","n_params.as_str":"8.9B","n_params.as_int":8940158976,"n_params.from_name":"9b","cfg.n_params":8940158976,"cfg.n_layers":42,"cfg.n_heads":16,"cfg.d_model":3584,"cfg.d_vocab":256000,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3584,"d_head":256,"n_layers":42,"n_ctx":8192,"n_heads":16,"d_mlp":14336,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2-9b","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2-9b","window_size":4096,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":8940158976,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":50.0,"output_logits_soft_cap":30.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 3584\nd_head: 256\nn_layers: 42\nn_ctx: 8192\nn_heads: 16\nd_mlp: 14336\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2-9b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2-9b\nwindow_size: 4096\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 8940158976\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: 50.0\noutput_logits_soft_cap: 30.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2-9b","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 3584)\nblocks:\n '[0-41]':\n ln1:\n w: (3584,)\n ln1_post:\n w: (3584,)\n ln2:\n w: (3584,)\n ln2_post:\n w: (3584,)\n attn:\n W_Q: (16, 3584, 256)\n W_O: (16, 256, 3584)\n b_Q: (16, 256)\n b_O: (3584,)\n '[_W_K, _W_V]': (8, 3584, 256)\n '[_b_K, _b_V]': (8, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (3584, 14336)\n W_out: (14336, 3584)\n b_in: (14336,)\n b_out: (3584,)\nln_final:\n w: (3584,)\nunembed:\n W_U: (3584, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 3584)"},"blocks":{"[0-41]":{"ln1":{"w":"(3584,)"},"ln1_post":{"w":"(3584,)"},"ln2":{"w":"(3584,)"},"ln2_post":{"w":"(3584,)"},"attn":{"W_Q":"(16, 3584, 256)","W_O":"(16, 256, 3584)","b_Q":"(16, 256)","b_O":"(3584,)","[_W_K, _W_V]":"(8, 3584, 256)","[_b_K, _b_V]":"(8, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(3584, 14336)","W_out":"(14336, 3584)","b_in":"(14336,)","b_out":"(3584,)"}}},"ln_final":{"w":"(3584,)"},"unembed":{"W_U":"(3584, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-41]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3584)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\nunembed:\n hook_in: (batch, seq_len, 3584)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 3584)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-41]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3584)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"unembed":{"hook_in":"(batch, seq_len, 3584)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 3584)"}} +{"name.default_alias":"gemma-2-9b-it","name.huggingface":"google\/gemma-2-9b-it","name.aliases":"gemma-2-9b-it","model_type":"gemma","name.from_cfg":"gemma-2-9b-it","n_params.as_str":"8.9B","n_params.as_int":8940158976,"n_params.from_name":"9b","cfg.n_params":8940158976,"cfg.n_layers":42,"cfg.n_heads":16,"cfg.d_model":3584,"cfg.d_vocab":256000,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3584,"d_head":256,"n_layers":42,"n_ctx":8192,"n_heads":16,"d_mlp":14336,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2-9b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2-9b-it","window_size":4096,"attn_types":["global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":8940158976,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":50.0,"output_logits_soft_cap":30.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 3584\nd_head: 256\nn_layers: 42\nn_ctx: 8192\nn_heads: 16\nd_mlp: 14336\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2-9b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2-9b-it\nwindow_size: 4096\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 8940158976\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: 50.0\noutput_logits_soft_cap: 30.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2-9b-it","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 3584)\nblocks:\n '[0-41]':\n ln1:\n w: (3584,)\n ln1_post:\n w: (3584,)\n ln2:\n w: (3584,)\n ln2_post:\n w: (3584,)\n attn:\n W_Q: (16, 3584, 256)\n W_O: (16, 256, 3584)\n b_Q: (16, 256)\n b_O: (3584,)\n '[_W_K, _W_V]': (8, 3584, 256)\n '[_b_K, _b_V]': (8, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (3584, 14336)\n W_out: (14336, 3584)\n b_in: (14336,)\n b_out: (3584,)\nln_final:\n w: (3584,)\nunembed:\n W_U: (3584, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 3584)"},"blocks":{"[0-41]":{"ln1":{"w":"(3584,)"},"ln1_post":{"w":"(3584,)"},"ln2":{"w":"(3584,)"},"ln2_post":{"w":"(3584,)"},"attn":{"W_Q":"(16, 3584, 256)","W_O":"(16, 256, 3584)","b_Q":"(16, 256)","b_O":"(3584,)","[_W_K, _W_V]":"(8, 3584, 256)","[_b_K, _b_V]":"(8, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(3584, 14336)","W_out":"(14336, 3584)","b_in":"(14336,)","b_out":"(3584,)"}}},"ln_final":{"w":"(3584,)"},"unembed":{"W_U":"(3584, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-41]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3584)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\nunembed:\n hook_in: (batch, seq_len, 3584)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 3584)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-41]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3584)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"unembed":{"hook_in":"(batch, seq_len, 3584)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 3584)"}} +{"name.default_alias":"gemma-2b","name.huggingface":"google\/gemma-2b","name.aliases":"gemma-2b","model_type":"gemma","name.from_cfg":"gemma-2b","n_params.as_str":"2.1B","n_params.as_int":2113929216,"n_params.from_name":"2b","cfg.n_params":2113929216,"cfg.n_layers":18,"cfg.n_heads":8,"cfg.d_model":2048,"cfg.d_vocab":256000,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GemmaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":256,"n_layers":18,"n_ctx":8192,"n_heads":8,"d_mlp":16384,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":1,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2b","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GemmaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":2113929216,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 256\nn_layers: 18\nn_ctx: 8192\nn_heads: 8\nd_mlp: 16384\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 1\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GemmaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 2113929216\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2b","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 2048)\nblocks:\n '[0-17]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n W_Q: (8, 2048, 256)\n W_O: (8, 256, 2048)\n b_Q: (8, 256)\n b_O: (2048,)\n '[_W_K, _W_V]': (1, 2048, 256)\n '[_b_K, _b_V]': (1, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (2048, 16384)\n W_out: (16384, 2048)\n b_in: (16384,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 2048)"},"blocks":{"[0-17]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"W_Q":"(8, 2048, 256)","W_O":"(8, 256, 2048)","b_Q":"(8, 256)","b_O":"(2048,)","[_W_K, _W_V]":"(1, 2048, 256)","[_b_K, _b_V]":"(1, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(2048, 16384)","W_out":"(16384, 2048)","b_in":"(16384,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-17]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 8, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 1, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-17]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 8, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 1, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"gemma-2b-it","name.huggingface":"google\/gemma-2b-it","name.aliases":"gemma-2b-it","model_type":"gemma","name.from_cfg":"gemma-2b-it","n_params.as_str":"2.1B","n_params.as_int":2113929216,"n_params.from_name":"2b","cfg.n_params":2113929216,"cfg.n_layers":18,"cfg.n_heads":8,"cfg.d_model":2048,"cfg.d_vocab":256000,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GemmaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":256,"n_layers":18,"n_ctx":8192,"n_heads":8,"d_mlp":16384,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":1,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-2b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GemmaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-2b-it","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":2113929216,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 256\nn_layers: 18\nn_ctx: 8192\nn_heads: 8\nd_mlp: 16384\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 1\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-2b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GemmaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-2b-it\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 2113929216\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-2b-it","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 2048)\nblocks:\n '[0-17]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n W_Q: (8, 2048, 256)\n W_O: (8, 256, 2048)\n b_Q: (8, 256)\n b_O: (2048,)\n '[_W_K, _W_V]': (1, 2048, 256)\n '[_b_K, _b_V]': (1, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (2048, 16384)\n W_out: (16384, 2048)\n b_in: (16384,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 2048)"},"blocks":{"[0-17]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"W_Q":"(8, 2048, 256)","W_O":"(8, 256, 2048)","b_Q":"(8, 256)","b_O":"(2048,)","[_W_K, _W_V]":"(1, 2048, 256)","[_b_K, _b_V]":"(1, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(2048, 16384)","W_out":"(16384, 2048)","b_in":"(16384,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-17]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 8, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 1, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-17]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 8, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 1, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"gemma-3-12b-it","name.huggingface":"google\/gemma-3-12b-it","name.aliases":"gemma-3-12b-it","model_type":"gemma","name.from_cfg":"gemma-3-12b-it","n_params.as_str":"12B","n_params.as_int":11513364480,"n_params.from_name":"12b","cfg.n_params":11513364480,"cfg.n_layers":48,"cfg.n_heads":16,"cfg.d_model":3840,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":3840,"d_head":256,"n_layers":48,"n_ctx":8192,"n_heads":16,"d_mlp":15360,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-12b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-12b-it","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":11513364480,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 3840\nd_head: 256\nn_layers: 48\nn_ctx: 8192\nn_heads: 16\nd_mlp: 15360\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-12b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-12b-it\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 11513364480\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-12b-it","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262208, 3840)\nblocks:\n '[0-47]':\n ln1:\n w: (3840,)\n ln1_post:\n w: (3840,)\n ln2:\n w: (3840,)\n ln2_post:\n w: (3840,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (16, 3840, 256)\n W_O: (16, 256, 3840)\n b_Q: (16, 256)\n b_O: (3840,)\n '[_W_K, _W_V]': (8, 3840, 256)\n '[_b_K, _b_V]': (8, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (3840, 15360)\n W_out: (15360, 3840)\n b_in: (15360,)\n b_out: (3840,)\nln_final:\n w: (3840,)\nunembed:\n W_U: (3840, 262208)\n b_U: (262208,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262208, 3840)"},"blocks":{"[0-47]":{"ln1":{"w":"(3840,)"},"ln1_post":{"w":"(3840,)"},"ln2":{"w":"(3840,)"},"ln2_post":{"w":"(3840,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(16, 3840, 256)","W_O":"(16, 256, 3840)","b_Q":"(16, 256)","b_O":"(3840,)","[_W_K, _W_V]":"(8, 3840, 256)","[_b_K, _b_V]":"(8, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(3840, 15360)","W_out":"(15360, 3840)","b_in":"(15360,)","b_out":"(3840,)"}}},"ln_final":{"w":"(3840,)"},"unembed":{"W_U":"(3840, 262208)","b_U":"(262208,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-47]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n attn:\n q_norm:\n hook_scale: (110923120, 1)\n hook_normalized: (110923120, 256)\n k_norm:\n hook_scale: (55461560, 1)\n hook_normalized: (55461560, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 15360)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3840)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\nunembed:\n hook_in: (batch, seq_len, 3840)\n hook_out: (batch, seq_len, 262208)\nhook_embed: (batch, seq_len, 3840)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-47]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"attn":{"q_norm":{"hook_scale":"(110923120, 1)","hook_normalized":"(110923120, 256)"},"k_norm":{"hook_scale":"(55461560, 1)","hook_normalized":"(55461560, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 15360)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3840)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"unembed":{"hook_in":"(batch, seq_len, 3840)","hook_out":"(batch, seq_len, 262208)"},"hook_embed":"(batch, seq_len, 3840)"}} +{"name.default_alias":"gemma-3-12b-pt","name.huggingface":"google\/gemma-3-12b-pt","name.aliases":"gemma-3-12b-pt","model_type":"gemma","name.from_cfg":"gemma-3-12b-pt","n_params.as_str":"12B","n_params.as_int":11513364480,"n_params.from_name":"12b","cfg.n_params":11513364480,"cfg.n_layers":48,"cfg.n_heads":16,"cfg.d_model":3840,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":3840,"d_head":256,"n_layers":48,"n_ctx":8192,"n_heads":16,"d_mlp":15360,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-12b-pt","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-12b-pt","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":11513364480,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 3840\nd_head: 256\nn_layers: 48\nn_ctx: 8192\nn_heads: 16\nd_mlp: 15360\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-12b-pt\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-12b-pt\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 11513364480\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-12b-pt","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262208, 3840)\nblocks:\n '[0-47]':\n ln1:\n w: (3840,)\n ln1_post:\n w: (3840,)\n ln2:\n w: (3840,)\n ln2_post:\n w: (3840,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (16, 3840, 256)\n W_O: (16, 256, 3840)\n b_Q: (16, 256)\n b_O: (3840,)\n '[_W_K, _W_V]': (8, 3840, 256)\n '[_b_K, _b_V]': (8, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (3840, 15360)\n W_out: (15360, 3840)\n b_in: (15360,)\n b_out: (3840,)\nln_final:\n w: (3840,)\nunembed:\n W_U: (3840, 262208)\n b_U: (262208,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262208, 3840)"},"blocks":{"[0-47]":{"ln1":{"w":"(3840,)"},"ln1_post":{"w":"(3840,)"},"ln2":{"w":"(3840,)"},"ln2_post":{"w":"(3840,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(16, 3840, 256)","W_O":"(16, 256, 3840)","b_Q":"(16, 256)","b_O":"(3840,)","[_W_K, _W_V]":"(8, 3840, 256)","[_b_K, _b_V]":"(8, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(3840, 15360)","W_out":"(15360, 3840)","b_in":"(15360,)","b_out":"(3840,)"}}},"ln_final":{"w":"(3840,)"},"unembed":{"W_U":"(3840, 262208)","b_U":"(262208,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-47]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n attn:\n q_norm:\n hook_scale: (110923120, 1)\n hook_normalized: (110923120, 256)\n k_norm:\n hook_scale: (55461560, 1)\n hook_normalized: (55461560, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 15360)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3840)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3840)\nunembed:\n hook_in: (batch, seq_len, 3840)\n hook_out: (batch, seq_len, 262208)\nhook_embed: (batch, seq_len, 3840)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-47]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"attn":{"q_norm":{"hook_scale":"(110923120, 1)","hook_normalized":"(110923120, 256)"},"k_norm":{"hook_scale":"(55461560, 1)","hook_normalized":"(55461560, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 256)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 15360)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3840)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3840)"},"unembed":{"hook_in":"(batch, seq_len, 3840)","hook_out":"(batch, seq_len, 262208)"},"hook_embed":"(batch, seq_len, 3840)"}} +{"name.default_alias":"gemma-3-1b-it","name.huggingface":"google\/gemma-3-1b-it","name.aliases":"gemma-3-1b-it","model_type":"gemma","name.from_cfg":"gemma-3-1b-it","n_params.as_str":"744M","n_params.as_int":743768064,"n_params.from_name":"1b","cfg.n_params":743768064,"cfg.n_layers":26,"cfg.n_heads":4,"cfg.d_model":1152,"cfg.d_vocab":262144,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1152,"d_head":256,"n_layers":26,"n_ctx":8192,"n_heads":4,"d_mlp":6912,"d_vocab":262144,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":1,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-1b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-1b-it","window_size":512,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262144,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":743768064,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1152\nd_head: 256\nn_layers: 26\nn_ctx: 8192\nn_heads: 4\nd_mlp: 6912\nd_vocab: 262144\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 1\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-1b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-1b-it\nwindow_size: 512\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262144\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 743768064\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-1b-it","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262144, 1152)\nblocks:\n '[0-25]':\n ln1:\n w: (1152,)\n ln1_post:\n w: (1152,)\n ln2:\n w: (1152,)\n ln2_post:\n w: (1152,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (4, 1152, 256)\n W_O: (4, 256, 1152)\n b_Q: (4, 256)\n b_O: (1152,)\n '[_W_K, _W_V]': (1, 1152, 256)\n '[_b_K, _b_V]': (1, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (1152, 6912)\n W_out: (6912, 1152)\n b_in: (6912,)\n b_out: (1152,)\nln_final:\n w: (1152,)\nunembed:\n W_U: (1152, 262144)\n b_U: (262144,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262144, 1152)"},"blocks":{"[0-25]":{"ln1":{"w":"(1152,)"},"ln1_post":{"w":"(1152,)"},"ln2":{"w":"(1152,)"},"ln2_post":{"w":"(1152,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(4, 1152, 256)","W_O":"(4, 256, 1152)","b_Q":"(4, 256)","b_O":"(1152,)","[_W_K, _W_V]":"(1, 1152, 256)","[_b_K, _b_V]":"(1, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(1152, 6912)","W_out":"(6912, 1152)","b_in":"(6912,)","b_out":"(1152,)"}}},"ln_final":{"w":"(1152,)"},"unembed":{"W_U":"(1152, 262144)","b_U":"(262144,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-25]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n attn:\n q_norm:\n hook_scale: (27730780, 1)\n hook_normalized: (27730780, 256)\n k_norm:\n hook_scale: (6932695, 1)\n hook_normalized: (6932695, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 4, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 1, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 4, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 6912)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1152)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\nunembed:\n hook_in: (batch, seq_len, 1152)\n hook_out: (batch, seq_len, 262144)\nhook_embed: (batch, seq_len, 1152)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-25]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"attn":{"q_norm":{"hook_scale":"(27730780, 1)","hook_normalized":"(27730780, 256)"},"k_norm":{"hook_scale":"(6932695, 1)","hook_normalized":"(6932695, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 4, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 1, 256)","[hook_attn_scores, hook_pattern]":"(batch, 4, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 6912)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1152)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"unembed":{"hook_in":"(batch, seq_len, 1152)","hook_out":"(batch, seq_len, 262144)"},"hook_embed":"(batch, seq_len, 1152)"}} +{"name.default_alias":"gemma-3-1b-pt","name.huggingface":"google\/gemma-3-1b-pt","name.aliases":"gemma-3-1b-pt","model_type":"gemma","name.from_cfg":"gemma-3-1b-pt","n_params.as_str":"744M","n_params.as_int":743768064,"n_params.from_name":"1b","cfg.n_params":743768064,"cfg.n_layers":26,"cfg.n_heads":4,"cfg.d_model":1152,"cfg.d_vocab":262144,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1152,"d_head":256,"n_layers":26,"n_ctx":8192,"n_heads":4,"d_mlp":6912,"d_vocab":262144,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":1,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-1b-pt","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-1b-pt","window_size":512,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262144,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":743768064,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1152\nd_head: 256\nn_layers: 26\nn_ctx: 8192\nn_heads: 4\nd_mlp: 6912\nd_vocab: 262144\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 1\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-1b-pt\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-1b-pt\nwindow_size: 512\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262144\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 743768064\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-1b-pt","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262144, 1152)\nblocks:\n '[0-25]':\n ln1:\n w: (1152,)\n ln1_post:\n w: (1152,)\n ln2:\n w: (1152,)\n ln2_post:\n w: (1152,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (4, 1152, 256)\n W_O: (4, 256, 1152)\n b_Q: (4, 256)\n b_O: (1152,)\n '[_W_K, _W_V]': (1, 1152, 256)\n '[_b_K, _b_V]': (1, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (1152, 6912)\n W_out: (6912, 1152)\n b_in: (6912,)\n b_out: (1152,)\nln_final:\n w: (1152,)\nunembed:\n W_U: (1152, 262144)\n b_U: (262144,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262144, 1152)"},"blocks":{"[0-25]":{"ln1":{"w":"(1152,)"},"ln1_post":{"w":"(1152,)"},"ln2":{"w":"(1152,)"},"ln2_post":{"w":"(1152,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(4, 1152, 256)","W_O":"(4, 256, 1152)","b_Q":"(4, 256)","b_O":"(1152,)","[_W_K, _W_V]":"(1, 1152, 256)","[_b_K, _b_V]":"(1, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(1152, 6912)","W_out":"(6912, 1152)","b_in":"(6912,)","b_out":"(1152,)"}}},"ln_final":{"w":"(1152,)"},"unembed":{"W_U":"(1152, 262144)","b_U":"(262144,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-25]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n attn:\n q_norm:\n hook_scale: (27730780, 1)\n hook_normalized: (27730780, 256)\n k_norm:\n hook_scale: (6932695, 1)\n hook_normalized: (6932695, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 4, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 1, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 4, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 6912)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1152)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1152)\nunembed:\n hook_in: (batch, seq_len, 1152)\n hook_out: (batch, seq_len, 262144)\nhook_embed: (batch, seq_len, 1152)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-25]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"attn":{"q_norm":{"hook_scale":"(27730780, 1)","hook_normalized":"(27730780, 256)"},"k_norm":{"hook_scale":"(6932695, 1)","hook_normalized":"(6932695, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 4, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 1, 256)","[hook_attn_scores, hook_pattern]":"(batch, 4, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 6912)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1152)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1152)"},"unembed":{"hook_in":"(batch, seq_len, 1152)","hook_out":"(batch, seq_len, 262144)"},"hook_embed":"(batch, seq_len, 1152)"}} +{"name.default_alias":"gemma-3-270m","name.huggingface":"google\/gemma-3-270m","name.aliases":"gemma-3-270m","model_type":"gemma","name.from_cfg":"gemma-3-270m","n_params.as_str":"118M","n_params.as_int":117964800,"n_params.from_name":"270m","cfg.n_params":117964800,"cfg.n_layers":18,"cfg.n_heads":4,"cfg.d_model":640,"cfg.d_vocab":262144,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":640,"d_head":256,"n_layers":18,"n_ctx":8192,"n_heads":4,"d_mlp":2048,"d_vocab":262144,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":1,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-270m","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-270m","window_size":512,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262144,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":117964800,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 640\nd_head: 256\nn_layers: 18\nn_ctx: 8192\nn_heads: 4\nd_mlp: 2048\nd_vocab: 262144\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 1\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-270m\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-270m\nwindow_size: 512\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262144\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 117964800\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-270m","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262144, 640)\nblocks:\n '[0-17]':\n ln1:\n w: (640,)\n ln1_post:\n w: (640,)\n ln2:\n w: (640,)\n ln2_post:\n w: (640,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (4, 640, 256)\n W_O: (4, 256, 640)\n b_Q: (4, 256)\n b_O: (640,)\n '[_W_K, _W_V]': (1, 640, 256)\n '[_b_K, _b_V]': (1, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (640, 2048)\n W_out: (2048, 640)\n b_in: (2048,)\n b_out: (640,)\nln_final:\n w: (640,)\nunembed:\n W_U: (640, 262144)\n b_U: (262144,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262144, 640)"},"blocks":{"[0-17]":{"ln1":{"w":"(640,)"},"ln1_post":{"w":"(640,)"},"ln2":{"w":"(640,)"},"ln2_post":{"w":"(640,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(4, 640, 256)","W_O":"(4, 256, 640)","b_Q":"(4, 256)","b_O":"(640,)","[_W_K, _W_V]":"(1, 640, 256)","[_b_K, _b_V]":"(1, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(640, 2048)","W_out":"(2048, 640)","b_in":"(2048,)","b_out":"(640,)"}}},"ln_final":{"w":"(640,)"},"unembed":{"W_U":"(640, 262144)","b_U":"(262144,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-17]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n attn:\n q_norm:\n hook_scale: (27730780, 1)\n hook_normalized: (27730780, 256)\n k_norm:\n hook_scale: (6932695, 1)\n hook_normalized: (6932695, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 4, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 1, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 4, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 2048)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 640)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\nunembed:\n hook_in: (batch, seq_len, 640)\n hook_out: (batch, seq_len, 262144)\nhook_embed: (batch, seq_len, 640)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-17]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"attn":{"q_norm":{"hook_scale":"(27730780, 1)","hook_normalized":"(27730780, 256)"},"k_norm":{"hook_scale":"(6932695, 1)","hook_normalized":"(6932695, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 4, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 1, 256)","[hook_attn_scores, hook_pattern]":"(batch, 4, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 2048)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 640)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"unembed":{"hook_in":"(batch, seq_len, 640)","hook_out":"(batch, seq_len, 262144)"},"hook_embed":"(batch, seq_len, 640)"}} +{"name.default_alias":"gemma-3-270m-it","name.huggingface":"google\/gemma-3-270m-it","name.aliases":"gemma-3-270m-it","model_type":"gemma","name.from_cfg":"gemma-3-270m-it","n_params.as_str":"118M","n_params.as_int":117964800,"n_params.from_name":"270m","cfg.n_params":117964800,"cfg.n_layers":18,"cfg.n_heads":4,"cfg.d_model":640,"cfg.d_vocab":262144,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":640,"d_head":256,"n_layers":18,"n_ctx":8192,"n_heads":4,"d_mlp":2048,"d_vocab":262144,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":1,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-270m-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-270m-it","window_size":512,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262144,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":117964800,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 640\nd_head: 256\nn_layers: 18\nn_ctx: 8192\nn_heads: 4\nd_mlp: 2048\nd_vocab: 262144\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 1\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-270m-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-270m-it\nwindow_size: 512\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262144\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 117964800\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-270m-it","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262144, 640)\nblocks:\n '[0-17]':\n ln1:\n w: (640,)\n ln1_post:\n w: (640,)\n ln2:\n w: (640,)\n ln2_post:\n w: (640,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (4, 640, 256)\n W_O: (4, 256, 640)\n b_Q: (4, 256)\n b_O: (640,)\n '[_W_K, _W_V]': (1, 640, 256)\n '[_b_K, _b_V]': (1, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (640, 2048)\n W_out: (2048, 640)\n b_in: (2048,)\n b_out: (640,)\nln_final:\n w: (640,)\nunembed:\n W_U: (640, 262144)\n b_U: (262144,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262144, 640)"},"blocks":{"[0-17]":{"ln1":{"w":"(640,)"},"ln1_post":{"w":"(640,)"},"ln2":{"w":"(640,)"},"ln2_post":{"w":"(640,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(4, 640, 256)","W_O":"(4, 256, 640)","b_Q":"(4, 256)","b_O":"(640,)","[_W_K, _W_V]":"(1, 640, 256)","[_b_K, _b_V]":"(1, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(640, 2048)","W_out":"(2048, 640)","b_in":"(2048,)","b_out":"(640,)"}}},"ln_final":{"w":"(640,)"},"unembed":{"W_U":"(640, 262144)","b_U":"(262144,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-17]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n attn:\n q_norm:\n hook_scale: (27730780, 1)\n hook_normalized: (27730780, 256)\n k_norm:\n hook_scale: (6932695, 1)\n hook_normalized: (6932695, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 4, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 1, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 4, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 2048)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 640)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 640)\nunembed:\n hook_in: (batch, seq_len, 640)\n hook_out: (batch, seq_len, 262144)\nhook_embed: (batch, seq_len, 640)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-17]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"attn":{"q_norm":{"hook_scale":"(27730780, 1)","hook_normalized":"(27730780, 256)"},"k_norm":{"hook_scale":"(6932695, 1)","hook_normalized":"(6932695, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 4, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 1, 256)","[hook_attn_scores, hook_pattern]":"(batch, 4, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 2048)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 640)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 640)"},"unembed":{"hook_in":"(batch, seq_len, 640)","hook_out":"(batch, seq_len, 262144)"},"hook_embed":"(batch, seq_len, 640)"}} +{"name.default_alias":"gemma-3-27b-it","name.huggingface":"google\/gemma-3-27b-it","name.aliases":"gemma-3-27b-it","model_type":"gemma","name.from_cfg":"gemma-3-27b-it","n_params.as_str":"27B","n_params.as_int":26963607552,"n_params.from_name":"27b","cfg.n_params":26963607552,"cfg.n_layers":62,"cfg.n_heads":32,"cfg.d_model":5376,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":5376,"d_head":128,"n_layers":62,"n_ctx":8192,"n_heads":32,"d_mlp":21504,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-27b-it","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-27b-it","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":26963607552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5376\nd_head: 128\nn_layers: 62\nn_ctx: 8192\nn_heads: 32\nd_mlp: 21504\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-27b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-27b-it\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 26963607552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-27b-it","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262208, 5376)\nblocks:\n '[0-61]':\n ln1:\n w: (5376,)\n ln1_post:\n w: (5376,)\n ln2:\n w: (5376,)\n ln2_post:\n w: (5376,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (32, 5376, 128)\n W_O: (32, 128, 5376)\n b_Q: (32, 128)\n b_O: (5376,)\n '[_W_K, _W_V]': (16, 5376, 128)\n '[_b_K, _b_V]': (16, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5376, 21504)\n W_out: (21504, 5376)\n b_in: (21504,)\n b_out: (5376,)\nln_final:\n w: (5376,)\nunembed:\n W_U: (5376, 262208)\n b_U: (262208,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262208, 5376)"},"blocks":{"[0-61]":{"ln1":{"w":"(5376,)"},"ln1_post":{"w":"(5376,)"},"ln2":{"w":"(5376,)"},"ln2_post":{"w":"(5376,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(32, 5376, 128)","W_O":"(32, 128, 5376)","b_Q":"(32, 128)","b_O":"(5376,)","[_W_K, _W_V]":"(16, 5376, 128)","[_b_K, _b_V]":"(16, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5376, 21504)","W_out":"(21504, 5376)","b_in":"(21504,)","b_out":"(5376,)"}}},"ln_final":{"w":"(5376,)"},"unembed":{"W_U":"(5376, 262208)","b_U":"(262208,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-61]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n attn:\n q_norm:\n hook_scale: (221846240, 1)\n hook_normalized: (221846240, 128)\n k_norm:\n hook_scale: (110923120, 1)\n hook_normalized: (110923120, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 21504)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5376)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\nunembed:\n hook_in: (batch, seq_len, 5376)\n hook_out: (batch, seq_len, 262208)\nhook_embed: (batch, seq_len, 5376)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-61]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"attn":{"q_norm":{"hook_scale":"(221846240, 1)","hook_normalized":"(221846240, 128)"},"k_norm":{"hook_scale":"(110923120, 1)","hook_normalized":"(110923120, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 21504)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5376)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"unembed":{"hook_in":"(batch, seq_len, 5376)","hook_out":"(batch, seq_len, 262208)"},"hook_embed":"(batch, seq_len, 5376)"}} +{"name.default_alias":"gemma-3-27b-pt","name.huggingface":"google\/gemma-3-27b-pt","name.aliases":"gemma-3-27b-pt","model_type":"gemma","name.from_cfg":"gemma-3-27b-pt","n_params.as_str":"27B","n_params.as_int":26963607552,"n_params.from_name":"27b","cfg.n_params":26963607552,"cfg.n_layers":62,"cfg.n_heads":32,"cfg.d_model":5376,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":5376,"d_head":128,"n_layers":62,"n_ctx":8192,"n_heads":32,"d_mlp":21504,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-27b-pt","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-27b-pt","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":26963607552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5376\nd_head: 128\nn_layers: 62\nn_ctx: 8192\nn_heads: 32\nd_mlp: 21504\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-27b-pt\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-27b-pt\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 26963607552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-27b-pt","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262208, 5376)\nblocks:\n '[0-61]':\n ln1:\n w: (5376,)\n ln1_post:\n w: (5376,)\n ln2:\n w: (5376,)\n ln2_post:\n w: (5376,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (32, 5376, 128)\n W_O: (32, 128, 5376)\n b_Q: (32, 128)\n b_O: (5376,)\n '[_W_K, _W_V]': (16, 5376, 128)\n '[_b_K, _b_V]': (16, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5376, 21504)\n W_out: (21504, 5376)\n b_in: (21504,)\n b_out: (5376,)\nln_final:\n w: (5376,)\nunembed:\n W_U: (5376, 262208)\n b_U: (262208,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262208, 5376)"},"blocks":{"[0-61]":{"ln1":{"w":"(5376,)"},"ln1_post":{"w":"(5376,)"},"ln2":{"w":"(5376,)"},"ln2_post":{"w":"(5376,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(32, 5376, 128)","W_O":"(32, 128, 5376)","b_Q":"(32, 128)","b_O":"(5376,)","[_W_K, _W_V]":"(16, 5376, 128)","[_b_K, _b_V]":"(16, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5376, 21504)","W_out":"(21504, 5376)","b_in":"(21504,)","b_out":"(5376,)"}}},"ln_final":{"w":"(5376,)"},"unembed":{"W_U":"(5376, 262208)","b_U":"(262208,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-61]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n attn:\n q_norm:\n hook_scale: (221846240, 1)\n hook_normalized: (221846240, 128)\n k_norm:\n hook_scale: (110923120, 1)\n hook_normalized: (110923120, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 21504)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5376)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5376)\nunembed:\n hook_in: (batch, seq_len, 5376)\n hook_out: (batch, seq_len, 262208)\nhook_embed: (batch, seq_len, 5376)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-61]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"attn":{"q_norm":{"hook_scale":"(221846240, 1)","hook_normalized":"(221846240, 128)"},"k_norm":{"hook_scale":"(110923120, 1)","hook_normalized":"(110923120, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 21504)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5376)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5376)"},"unembed":{"hook_in":"(batch, seq_len, 5376)","hook_out":"(batch, seq_len, 262208)"},"hook_embed":"(batch, seq_len, 5376)"}} +{"name.default_alias":"gemma-3-4b-it","name.huggingface":"google\/gemma-3-4b-it","name.aliases":"gemma-3-4b-it","model_type":"gemma","name.from_cfg":"gemma-3-4b-it","n_params.as_str":"3.4B","n_params.as_int":3386900480,"n_params.from_name":"4b","cfg.n_params":3386900480,"cfg.n_layers":34,"cfg.n_heads":8,"cfg.d_model":2560,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":2560,"d_head":256,"n_layers":34,"n_ctx":8192,"n_heads":8,"d_mlp":10240,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-4b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-4b-it","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":3386900480,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 256\nn_layers: 34\nn_ctx: 8192\nn_heads: 8\nd_mlp: 10240\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-4b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-4b-it\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 3386900480\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-4b-it","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262208, 2560)\nblocks:\n '[0-33]':\n ln1:\n w: (2560,)\n ln1_post:\n w: (2560,)\n ln2:\n w: (2560,)\n ln2_post:\n w: (2560,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (8, 2560, 256)\n W_O: (8, 256, 2560)\n b_Q: (8, 256)\n b_O: (2560,)\n '[_W_K, _W_V]': (4, 2560, 256)\n '[_b_K, _b_V]': (4, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (2560, 10240)\n W_out: (10240, 2560)\n b_in: (10240,)\n b_out: (2560,)\nln_final:\n w: (2560,)\nunembed:\n W_U: (2560, 262208)\n b_U: (262208,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262208, 2560)"},"blocks":{"[0-33]":{"ln1":{"w":"(2560,)"},"ln1_post":{"w":"(2560,)"},"ln2":{"w":"(2560,)"},"ln2_post":{"w":"(2560,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(8, 2560, 256)","W_O":"(8, 256, 2560)","b_Q":"(8, 256)","b_O":"(2560,)","[_W_K, _W_V]":"(4, 2560, 256)","[_b_K, _b_V]":"(4, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(2560, 10240)","W_out":"(10240, 2560)","b_in":"(10240,)","b_out":"(2560,)"}}},"ln_final":{"w":"(2560,)"},"unembed":{"W_U":"(2560, 262208)","b_U":"(262208,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-33]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n q_norm:\n hook_scale: (55461560, 1)\n hook_normalized: (55461560, 256)\n k_norm:\n hook_scale: (27730780, 1)\n hook_normalized: (27730780, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 8, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 10240)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 262208)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-33]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"q_norm":{"hook_scale":"(55461560, 1)","hook_normalized":"(55461560, 256)"},"k_norm":{"hook_scale":"(27730780, 1)","hook_normalized":"(27730780, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 8, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 10240)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 262208)"},"hook_embed":"(batch, seq_len, 2560)"}} +{"name.default_alias":"gemma-3-4b-pt","name.huggingface":"google\/gemma-3-4b-pt","name.aliases":"gemma-3-4b-pt","model_type":"gemma","name.from_cfg":"gemma-3-4b-pt","n_params.as_str":"3.4B","n_params.as_int":3386900480,"n_params.from_name":"4b","cfg.n_params":3386900480,"cfg.n_layers":34,"cfg.n_heads":8,"cfg.d_model":2560,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":2560,"d_head":256,"n_layers":34,"n_ctx":8192,"n_heads":8,"d_mlp":10240,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-3-4b-pt","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-3-4b-pt","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":3386900480,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 256\nn_layers: 34\nn_ctx: 8192\nn_heads: 8\nd_mlp: 10240\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-3-4b-pt\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-3-4b-pt\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 3386900480\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-3-4b-pt","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262208, 2560)\nblocks:\n '[0-33]':\n ln1:\n w: (2560,)\n ln1_post:\n w: (2560,)\n ln2:\n w: (2560,)\n ln2_post:\n w: (2560,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (8, 2560, 256)\n W_O: (8, 256, 2560)\n b_Q: (8, 256)\n b_O: (2560,)\n '[_W_K, _W_V]': (4, 2560, 256)\n '[_b_K, _b_V]': (4, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (2560, 10240)\n W_out: (10240, 2560)\n b_in: (10240,)\n b_out: (2560,)\nln_final:\n w: (2560,)\nunembed:\n W_U: (2560, 262208)\n b_U: (262208,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262208, 2560)"},"blocks":{"[0-33]":{"ln1":{"w":"(2560,)"},"ln1_post":{"w":"(2560,)"},"ln2":{"w":"(2560,)"},"ln2_post":{"w":"(2560,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(8, 2560, 256)","W_O":"(8, 256, 2560)","b_Q":"(8, 256)","b_O":"(2560,)","[_W_K, _W_V]":"(4, 2560, 256)","[_b_K, _b_V]":"(4, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(2560, 10240)","W_out":"(10240, 2560)","b_in":"(10240,)","b_out":"(2560,)"}}},"ln_final":{"w":"(2560,)"},"unembed":{"W_U":"(2560, 262208)","b_U":"(262208,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-33]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n q_norm:\n hook_scale: (55461560, 1)\n hook_normalized: (55461560, 256)\n k_norm:\n hook_scale: (27730780, 1)\n hook_normalized: (27730780, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 8, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 10240)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 262208)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-33]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"q_norm":{"hook_scale":"(55461560, 1)","hook_normalized":"(55461560, 256)"},"k_norm":{"hook_scale":"(27730780, 1)","hook_normalized":"(27730780, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 8, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 10240)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 262208)"},"hook_embed":"(batch, seq_len, 2560)"}} +{"name.default_alias":"gemma-7b","name.huggingface":"google\/gemma-7b","name.aliases":"gemma-7b","model_type":"gemma","name.from_cfg":"gemma-7b","n_params.as_str":"7.8B","n_params.as_int":7751073792,"n_params.from_name":"7b","cfg.n_params":7751073792,"cfg.n_layers":28,"cfg.n_heads":16,"cfg.d_model":3072,"cfg.d_vocab":256000,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GemmaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3072,"d_head":256,"n_layers":28,"n_ctx":8192,"n_heads":16,"d_mlp":24576,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-7b","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GemmaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-7b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":7751073792,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 3072\nd_head: 256\nn_layers: 28\nn_ctx: 8192\nn_heads: 16\nd_mlp: 24576\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-7b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GemmaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-7b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 7751073792\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-7b","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 3072)\nblocks:\n '[0-27]':\n ln1:\n w: (3072,)\n ln2:\n w: (3072,)\n attn:\n '[W_Q, _W_K, _W_V]': (16, 3072, 256)\n W_O: (16, 256, 3072)\n '[b_Q, _b_K, _b_V]': (16, 256)\n b_O: (3072,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (3072, 24576)\n W_out: (24576, 3072)\n b_in: (24576,)\n b_out: (3072,)\nln_final:\n w: (3072,)\nunembed:\n W_U: (3072, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 3072)"},"blocks":{"[0-27]":{"ln1":{"w":"(3072,)"},"ln2":{"w":"(3072,)"},"attn":{"[W_Q, _W_K, _W_V]":"(16, 3072, 256)","W_O":"(16, 256, 3072)","[b_Q, _b_K, _b_V]":"(16, 256)","b_O":"(3072,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(3072, 24576)","W_out":"(24576, 3072)","b_in":"(24576,)","b_out":"(3072,)"}}},"ln_final":{"w":"(3072,)"},"unembed":{"W_U":"(3072, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 24576)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3072)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\nunembed:\n hook_in: (batch, seq_len, 3072)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 3072)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 256)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 24576)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3072)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"unembed":{"hook_in":"(batch, seq_len, 3072)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 3072)"}} +{"name.default_alias":"gemma-7b-it","name.huggingface":"google\/gemma-7b-it","name.aliases":"gemma-7b-it","model_type":"gemma","name.from_cfg":"gemma-7b-it","n_params.as_str":"7.8B","n_params.as_int":7751073792,"n_params.from_name":"7b","cfg.n_params":7751073792,"cfg.n_layers":28,"cfg.n_heads":16,"cfg.d_model":3072,"cfg.d_vocab":256000,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GemmaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3072,"d_head":256,"n_layers":28,"n_ctx":8192,"n_heads":16,"d_mlp":24576,"d_vocab":256000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"gemma-7b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GemmaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/gemma-7b-it","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":256000,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":7751073792,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 3072\nd_head: 256\nn_layers: 28\nn_ctx: 8192\nn_heads: 16\nd_mlp: 24576\nd_vocab: 256000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gemma-7b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GemmaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/gemma-7b-it\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 256000\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 7751073792\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/gemma-7b-it","tokenizer.vocab_size":256000.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"9GdVxqTUEa0Qt2kBBEHf3ebUisw=","tensor_shapes.state_dict":"embed:\n W_E: (256000, 3072)\nblocks:\n '[0-27]':\n ln1:\n w: (3072,)\n ln2:\n w: (3072,)\n attn:\n '[W_Q, _W_K, _W_V]': (16, 3072, 256)\n W_O: (16, 256, 3072)\n '[b_Q, _b_K, _b_V]': (16, 256)\n b_O: (3072,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (3072, 24576)\n W_out: (24576, 3072)\n b_in: (24576,)\n b_out: (3072,)\nln_final:\n w: (3072,)\nunembed:\n W_U: (3072, 256000)\n b_U: (256000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(256000, 3072)"},"blocks":{"[0-27]":{"ln1":{"w":"(3072,)"},"ln2":{"w":"(3072,)"},"attn":{"[W_Q, _W_K, _W_V]":"(16, 3072, 256)","W_O":"(16, 256, 3072)","[b_Q, _b_K, _b_V]":"(16, 256)","b_O":"(3072,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(3072, 24576)","W_out":"(24576, 3072)","b_in":"(24576,)","b_out":"(3072,)"}}},"ln_final":{"w":"(3072,)"},"unembed":{"W_U":"(3072, 256000)","b_U":"(256000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 24576)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3072)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\nunembed:\n hook_in: (batch, seq_len, 3072)\n hook_out: (batch, seq_len, 256000)\nhook_embed: (batch, seq_len, 3072)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 256)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 24576)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3072)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"unembed":{"hook_in":"(batch, seq_len, 3072)","hook_out":"(batch, seq_len, 256000)"},"hook_embed":"(batch, seq_len, 3072)"}} +{"name.default_alias":"medgemma-27b-it","name.huggingface":"google\/medgemma-27b-it","name.aliases":"medgemma-27b-it","model_type":"gemma","name.from_cfg":"medgemma-27b-it","n_params.as_str":"27B","n_params.as_int":26963607552,"n_params.from_name":"27b","cfg.n_params":26963607552,"cfg.n_layers":62,"cfg.n_heads":32,"cfg.d_model":5376,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":5376,"d_head":128,"n_layers":62,"n_ctx":8192,"n_heads":32,"d_mlp":21504,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"medgemma-27b-it","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/medgemma-27b-it","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":26963607552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5376\nd_head: 128\nn_layers: 62\nn_ctx: 8192\nn_heads: 32\nd_mlp: 21504\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: medgemma-27b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/medgemma-27b-it\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 26963607552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"medgemma-27b-text-it","name.huggingface":"google\/medgemma-27b-text-it","name.aliases":"medgemma-27b-text-it","model_type":"gemma","name.from_cfg":"medgemma-27b-text-it","n_params.as_str":"27B","n_params.as_int":26963607552,"n_params.from_name":"27b","cfg.n_params":26963607552,"cfg.n_layers":62,"cfg.n_heads":32,"cfg.d_model":5376,"cfg.d_vocab":262144,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5376,"d_head":128,"n_layers":62,"n_ctx":8192,"n_heads":32,"d_mlp":21504,"d_vocab":262144,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"medgemma-27b-text-it","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/medgemma-27b-text-it","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262144,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":26963607552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5376\nd_head: 128\nn_layers: 62\nn_ctx: 8192\nn_heads: 32\nd_mlp: 21504\nd_vocab: 262144\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: medgemma-27b-text-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/medgemma-27b-text-it\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262144\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 26963607552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"medgemma-4b-it","name.huggingface":"google\/medgemma-4b-it","name.aliases":"medgemma-4b-it","model_type":"gemma","name.from_cfg":"medgemma-4b-it","n_params.as_str":"3.4B","n_params.as_int":3386900480,"n_params.from_name":"4b","cfg.n_params":3386900480,"cfg.n_layers":34,"cfg.n_heads":8,"cfg.d_model":2560,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":2560,"d_head":256,"n_layers":34,"n_ctx":8192,"n_heads":8,"d_mlp":10240,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"medgemma-4b-it","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/medgemma-4b-it","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":3386900480,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 256\nn_layers: 34\nn_ctx: 8192\nn_heads: 8\nd_mlp: 10240\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: medgemma-4b-it\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/medgemma-4b-it\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 3386900480\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"medgemma-4b-pt","name.huggingface":"google\/medgemma-4b-pt","name.aliases":"medgemma-4b-pt","model_type":"gemma","name.from_cfg":"medgemma-4b-pt","n_params.as_str":"3.4B","n_params.as_int":3386900480,"n_params.from_name":"4b","cfg.n_params":3386900480,"cfg.n_layers":34,"cfg.n_heads":8,"cfg.d_model":2560,"cfg.d_vocab":262208,"cfg.act_fn":"gelu_pytorch_tanh","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Gemma3ForConditionalGeneration","cfg.normalization_type":"RMS","config.raw__":{"d_model":2560,"d_head":256,"n_layers":34,"n_ctx":8192,"n_heads":8,"d_mlp":10240,"d_vocab":262208,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"gelu_pytorch_tanh","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"medgemma-4b-pt","use_attn_scale":true,"attn_scale":16.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"Gemma3ForConditionalGeneration","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"google\/medgemma-4b-pt","window_size":1024,"attn_types":["local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local","local","global","local","local","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":262208,"parallel_attn_mlp":false,"rotary_dim":256,"n_params":3386900480,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":10000,"rotary_scaling_factor":8.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":true,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 256\nn_layers: 34\nn_ctx: 8192\nn_heads: 8\nd_mlp: 10240\nd_vocab: 262208\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: gelu_pytorch_tanh\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: medgemma-4b-pt\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAMEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: Gemma3ForConditionalGeneration\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: google\/medgemma-4b-pt\nwindow_size: 1024\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\n- local\n- global\n- local\n- local\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 262208\nparallel_attn_mlp: false\nrotary_dim: 256\nn_params: 3386900480\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: 10000\nrotary_scaling_factor: 8.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: true\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"google\/medgemma-4b-pt","tokenizer.vocab_size":262144.0,"tokenizer.max_len":null,"tokenizer.class":"GemmaTokenizer","tokenizer.vocab_hash":"cTyOT2pw07eF2vN0sn4RYJBX-mY=","tensor_shapes.state_dict":"embed:\n W_E: (262208, 2560)\nblocks:\n '[0-33]':\n ln1:\n w: (2560,)\n ln1_post:\n w: (2560,)\n ln2:\n w: (2560,)\n ln2_post:\n w: (2560,)\n attn:\n q_norm:\n w: (256,)\n k_norm:\n w: (256,)\n W_Q: (8, 2560, 256)\n W_O: (8, 256, 2560)\n b_Q: (8, 256)\n b_O: (2560,)\n '[_W_K, _W_V]': (4, 2560, 256)\n '[_b_K, _b_V]': (4, 256)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 256)\n mlp:\n '[W_in, W_gate]': (2560, 10240)\n W_out: (10240, 2560)\n b_in: (10240,)\n b_out: (2560,)\nln_final:\n w: (2560,)\nunembed:\n W_U: (2560, 262208)\n b_U: (262208,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(262208, 2560)"},"blocks":{"[0-33]":{"ln1":{"w":"(2560,)"},"ln1_post":{"w":"(2560,)"},"ln2":{"w":"(2560,)"},"ln2_post":{"w":"(2560,)"},"attn":{"q_norm":{"w":"(256,)"},"k_norm":{"w":"(256,)"},"W_Q":"(8, 2560, 256)","W_O":"(8, 256, 2560)","b_Q":"(8, 256)","b_O":"(2560,)","[_W_K, _W_V]":"(4, 2560, 256)","[_b_K, _b_V]":"(4, 256)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 256)"},"mlp":{"[W_in, W_gate]":"(2560, 10240)","W_out":"(10240, 2560)","b_in":"(10240,)","b_out":"(2560,)"}}},"ln_final":{"w":"(2560,)"},"unembed":{"W_U":"(2560, 262208)","b_U":"(262208,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-33]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n q_norm:\n hook_scale: (55461560, 1)\n hook_normalized: (55461560, 256)\n k_norm:\n hook_scale: (27730780, 1)\n hook_normalized: (27730780, 256)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 8, 256)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 256)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln1_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 10240)\n ln2_post:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 262208)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-33]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"q_norm":{"hook_scale":"(55461560, 1)","hook_normalized":"(55461560, 256)"},"k_norm":{"hook_scale":"(27730780, 1)","hook_normalized":"(27730780, 256)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 8, 256)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 256)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln1_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 10240)"},"ln2_post":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 262208)"},"hook_embed":"(batch, seq_len, 2560)"}} +{"name.default_alias":"gpt2-small","name.huggingface":"gpt2","name.aliases":"gpt2-small","model_type":"gpt2","name.from_cfg":"gpt2","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt2","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"gpt2","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt2\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: gpt2\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"gpt2","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"gpt2-large","name.huggingface":"gpt2-large","name.aliases":"","model_type":"gpt2","name.from_cfg":"gpt2-large","n_params.as_str":"708M","n_params.as_int":707788800,"n_params.from_name":null,"cfg.n_params":707788800,"cfg.n_layers":36,"cfg.n_heads":20,"cfg.d_model":1280,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1280,"d_head":64,"n_layers":36,"n_ctx":1024,"n_heads":20,"d_mlp":5120,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt2-large","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"gpt2-large","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0223606798,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":707788800,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1280\nd_head: 64\nn_layers: 36\nn_ctx: 1024\nn_heads: 20\nd_mlp: 5120\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt2-large\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: gpt2-large\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.022360679774997897\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 707788800\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"gpt2-large","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1280)\npos_embed:\n W_pos: (1024, 1280)\nblocks:\n '[0-35]':\n ln1:\n '[w, b]': (1280,)\n ln2:\n '[w, b]': (1280,)\n attn:\n '[W_Q, W_K, W_V]': (20, 1280, 64)\n W_O: (20, 64, 1280)\n '[b_Q, b_K, b_V]': (20, 64)\n b_O: (1280,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1280, 5120)\n b_in: (5120,)\n W_out: (5120, 1280)\n b_out: (1280,)\nln_final:\n '[w, b]': (1280,)\nunembed:\n W_U: (1280, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1280)"},"pos_embed":{"W_pos":"(1024, 1280)"},"blocks":{"[0-35]":{"ln1":{"[w, b]":"(1280,)"},"ln2":{"[w, b]":"(1280,)"},"attn":{"[W_Q, W_K, W_V]":"(20, 1280, 64)","W_O":"(20, 64, 1280)","[b_Q, b_K, b_V]":"(20, 64)","b_O":"(1280,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1280, 5120)","b_in":"(5120,)","W_out":"(5120, 1280)","b_out":"(1280,)"}}},"ln_final":{"[w, b]":"(1280,)"},"unembed":{"W_U":"(1280, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 20, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 20, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 5120)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1280)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\nunembed:\n hook_in: (batch, seq_len, 1280)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1280)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 20, 64)","[hook_attn_scores, hook_pattern]":"(batch, 20, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 5120)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1280)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"unembed":{"hook_in":"(batch, seq_len, 1280)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1280)"}} +{"name.default_alias":"gpt2-medium","name.huggingface":"gpt2-medium","name.aliases":"","model_type":"gpt2","name.from_cfg":"gpt2-medium","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt2-medium","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"gpt2-medium","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt2-medium\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: gpt2-medium\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"gpt2-medium","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"gpt2-xl","name.huggingface":"gpt2-xl","name.aliases":"","model_type":"gpt2","name.from_cfg":"gpt2-xl","n_params.as_str":"1.5B","n_params.as_int":1474560000,"n_params.from_name":null,"cfg.n_params":1474560000,"cfg.n_layers":48,"cfg.n_heads":25,"cfg.d_model":1600,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1600,"d_head":64,"n_layers":48,"n_ctx":1024,"n_heads":25,"d_mlp":6400,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"gpt2-xl","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"gpt2-xl","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1474560000,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1600\nd_head: 64\nn_layers: 48\nn_ctx: 1024\nn_heads: 25\nd_mlp: 6400\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: gpt2-xl\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: gpt2-xl\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1474560000\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"gpt2-xl","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1600)\npos_embed:\n W_pos: (1024, 1600)\nblocks:\n '[0-47]':\n ln1:\n '[w, b]': (1600,)\n ln2:\n '[w, b]': (1600,)\n attn:\n '[W_Q, W_K, W_V]': (25, 1600, 64)\n W_O: (25, 64, 1600)\n '[b_Q, b_K, b_V]': (25, 64)\n b_O: (1600,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1600, 6400)\n b_in: (6400,)\n W_out: (6400, 1600)\n b_out: (1600,)\nln_final:\n '[w, b]': (1600,)\nunembed:\n W_U: (1600, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1600)"},"pos_embed":{"W_pos":"(1024, 1600)"},"blocks":{"[0-47]":{"ln1":{"[w, b]":"(1600,)"},"ln2":{"[w, b]":"(1600,)"},"attn":{"[W_Q, W_K, W_V]":"(25, 1600, 64)","W_O":"(25, 64, 1600)","[b_Q, b_K, b_V]":"(25, 64)","b_O":"(1600,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1600, 6400)","b_in":"(6400,)","W_out":"(6400, 1600)","b_out":"(1600,)"}}},"ln_final":{"[w, b]":"(1600,)"},"unembed":{"W_U":"(1600, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-47]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1600)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 25, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 25, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1600)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 6400)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1600)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1600)\nunembed:\n hook_in: (batch, seq_len, 1600)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1600)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-47]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1600)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 25, 64)","[hook_attn_scores, hook_pattern]":"(batch, 25, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1600)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 6400)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1600)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1600)"},"unembed":{"hook_in":"(batch, seq_len, 1600)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1600)"}} +{"name.default_alias":"llama-13b","name.huggingface":"llama-13b-hf","name.aliases":"llama-13b","model_type":"llama","name.from_cfg":"llama-13b-hf","n_params.as_str":"13B","n_params.as_int":12687769600,"n_params.from_name":"13b","cfg.n_params":12687769600,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":40,"d_mlp":13824,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"llama-13b-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"llama-13b-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12687769600,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 40\nd_mlp: 13824\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: llama-13b-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: llama-13b-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12687769600\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"huggyllama\/llama-13b","tokenizer.vocab_size":32000.0,"tokenizer.max_len":2048.0,"tokenizer.class":"LlamaTokenizer","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 13824)\n W_out: (13824, 5120)\n b_in: (13824,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 13824)","W_out":"(13824, 5120)","b_in":"(13824,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 13824)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 13824)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"llama-30b","name.huggingface":"llama-30b-hf","name.aliases":"llama-30b","model_type":"llama","name.from_cfg":"llama-30b-hf","n_params.as_str":"32B","n_params.as_int":32102154240,"n_params.from_name":"30b","cfg.n_params":32102154240,"cfg.n_layers":60,"cfg.n_heads":52,"cfg.d_model":6656,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":6656,"d_head":128,"n_layers":60,"n_ctx":2048,"n_heads":52,"d_mlp":17920,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"llama-30b-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"llama-30b-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0098058068,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":32102154240,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 6656\nd_head: 128\nn_layers: 60\nn_ctx: 2048\nn_heads: 52\nd_mlp: 17920\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: llama-30b-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: llama-30b-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.009805806756909202\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 32102154240\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"huggyllama\/llama-30b","tokenizer.vocab_size":32000.0,"tokenizer.max_len":2048.0,"tokenizer.class":"LlamaTokenizer","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 6656)\nblocks:\n '[0-59]':\n ln1:\n w: (6656,)\n ln2:\n w: (6656,)\n attn:\n '[W_Q, W_K, W_V]': (52, 6656, 128)\n W_O: (52, 128, 6656)\n '[b_Q, b_K, b_V]': (52, 128)\n b_O: (6656,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (6656, 17920)\n W_out: (17920, 6656)\n b_in: (17920,)\n b_out: (6656,)\nln_final:\n w: (6656,)\nunembed:\n W_U: (6656, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 6656)"},"blocks":{"[0-59]":{"ln1":{"w":"(6656,)"},"ln2":{"w":"(6656,)"},"attn":{"[W_Q, W_K, W_V]":"(52, 6656, 128)","W_O":"(52, 128, 6656)","[b_Q, b_K, b_V]":"(52, 128)","b_O":"(6656,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(6656, 17920)","W_out":"(17920, 6656)","b_in":"(17920,)","b_out":"(6656,)"}}},"ln_final":{"w":"(6656,)"},"unembed":{"W_U":"(6656, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-59]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6656)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 52, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 52, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6656)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 17920)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 6656)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6656)\nunembed:\n hook_in: (batch, seq_len, 6656)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 6656)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-59]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6656)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 52, 128)","[hook_attn_scores, hook_pattern]":"(batch, 52, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6656)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 17920)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 6656)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6656)"},"unembed":{"hook_in":"(batch, seq_len, 6656)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 6656)"}} +{"name.default_alias":"llama-65b","name.huggingface":"llama-65b-hf","name.aliases":"llama-65b","model_type":"llama","name.from_cfg":"llama-65b-hf","n_params.as_str":"65B","n_params.as_int":64760053760,"n_params.from_name":"65b","cfg.n_params":64760053760,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":2048,"n_heads":64,"d_mlp":22016,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"llama-65b-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"llama-65b-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0088388348,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":64760053760,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 2048\nn_heads: 64\nd_mlp: 22016\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: llama-65b-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: llama-65b-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008838834764831844\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 64760053760\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"huggyllama\/llama-65b","tokenizer.vocab_size":32000.0,"tokenizer.max_len":2048.0,"tokenizer.class":"LlamaTokenizer","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n '[W_Q, W_K, W_V]': (64, 8192, 128)\n W_O: (64, 128, 8192)\n '[b_Q, b_K, b_V]': (64, 128)\n b_O: (8192,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (8192, 22016)\n W_out: (22016, 8192)\n b_in: (22016,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"[W_Q, W_K, W_V]":"(64, 8192, 128)","W_O":"(64, 128, 8192)","[b_Q, b_K, b_V]":"(64, 128)","b_O":"(8192,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 22016)","W_out":"(22016, 8192)","b_in":"(22016,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 64, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 22016)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 64, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 22016)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 8192)"}} +{"name.default_alias":"llama-7b","name.huggingface":"llama-7b-hf","name.aliases":"llama-7b","model_type":"llama","name.from_cfg":"llama-7b-hf","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":11008,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"llama-7b-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"llama-7b-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 11008\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: llama-7b-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: llama-7b-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"huggyllama\/llama-7b","tokenizer.vocab_size":32000.0,"tokenizer.max_len":2048.0,"tokenizer.class":"LlamaTokenizer","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"Llama-2-13b-chat","name.huggingface":null,"name.aliases":"","model_type":"Llama-2","name.from_cfg":"Llama-2-13b-chat-hf","n_params.as_str":"13B","n_params.as_int":12687769600,"n_params.from_name":"13b","cfg.n_params":12687769600,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":4096,"n_heads":40,"d_mlp":13824,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-2-13b-chat-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-2-13b-chat-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12687769600,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 4096\nn_heads: 40\nd_mlp: 13824\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-2-13b-chat-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-2-13b-chat-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12687769600\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-2-13b-chat-hf","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"LlamaTokenizer","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 13824)\n W_out: (13824, 5120)\n b_in: (13824,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 13824)","W_out":"(13824, 5120)","b_in":"(13824,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 13824)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 13824)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"Llama-2-13b","name.huggingface":null,"name.aliases":"","model_type":"Llama-2","name.from_cfg":"Llama-2-13b-hf","n_params.as_str":"13B","n_params.as_int":12687769600,"n_params.from_name":"13b","cfg.n_params":12687769600,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":4096,"n_heads":40,"d_mlp":13824,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-2-13b-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-2-13b-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12687769600,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 4096\nn_heads: 40\nd_mlp: 13824\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-2-13b-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-2-13b-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12687769600\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-2-13b-hf","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"LlamaTokenizer","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n '[W_Q, W_K, W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, b_K, b_V]': (40, 128)\n b_O: (5120,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 13824)\n W_out: (13824, 5120)\n b_in: (13824,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"[W_Q, W_K, W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, b_K, b_V]":"(40, 128)","b_O":"(5120,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 13824)","W_out":"(13824, 5120)","b_in":"(13824,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 13824)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 13824)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"Llama-2-70b-chat","name.huggingface":null,"name.aliases":"","model_type":"Llama-2","name.from_cfg":"Llama-2-70b-chat-hf","n_params.as_str":"78B","n_params.as_int":77846282240,"n_params.from_name":"70b","cfg.n_params":77846282240,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":4096,"n_heads":64,"d_mlp":28672,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-2-70b-chat-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-2-70b-chat-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0088388348,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":77846282240,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 4096\nn_heads: 64\nd_mlp: 28672\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-2-70b-chat-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-2-70b-chat-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008838834764831844\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 77846282240\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-2-70b-chat-hf","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"LlamaTokenizer","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n W_Q: (64, 8192, 128)\n W_O: (64, 128, 8192)\n b_Q: (64, 128)\n b_O: (8192,)\n '[_W_K, _W_V]': (8, 8192, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (8192, 28672)\n W_out: (28672, 8192)\n b_in: (28672,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"W_Q":"(64, 8192, 128)","W_O":"(64, 128, 8192)","b_Q":"(64, 128)","b_O":"(8192,)","[_W_K, _W_V]":"(8, 8192, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 28672)","W_out":"(28672, 8192)","b_in":"(28672,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 64, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 28672)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 64, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 28672)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 8192)"}} +{"name.default_alias":"Llama-2-7b-chat","name.huggingface":null,"name.aliases":"","model_type":"Llama-2","name.from_cfg":"Llama-2-7b-chat-hf","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-2-7b-chat-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-2-7b-chat-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-2-7b-chat-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-2-7b-chat-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-2-7b-chat-hf","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"LlamaTokenizer","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"Llama-2-7b","name.huggingface":null,"name.aliases":"","model_type":"Llama-2","name.from_cfg":"Llama-2-7b-hf","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":11008,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-2-7b-hf","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-2-7b-hf","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 11008\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-2-7b-hf\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-2-7b-hf\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-2-7b-hf","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"LlamaTokenizer","tokenizer.vocab_hash":"e3A7wYziNQPAWcJ15GMAQY8qZqw=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"meta-llama\/Llama-3.1-70B","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.1-70B","n_params.as_str":"78B","n_params.as_int":77846282240,"n_params.from_name":"70B","cfg.n_params":77846282240,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":2048,"n_heads":64,"d_mlp":28672,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.1-70B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.1-70B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0088388348,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":77846282240,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 2048\nn_heads: 64\nd_mlp: 28672\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.1-70B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.1-70B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008838834764831844\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 77846282240\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.1-70B","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n W_Q: (64, 8192, 128)\n W_O: (64, 128, 8192)\n b_Q: (64, 128)\n b_O: (8192,)\n '[_W_K, _W_V]': (8, 8192, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (8192, 28672)\n W_out: (28672, 8192)\n b_in: (28672,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"W_Q":"(64, 8192, 128)","W_O":"(64, 128, 8192)","b_Q":"(64, 128)","b_O":"(8192,)","[_W_K, _W_V]":"(8, 8192, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 28672)","W_out":"(28672, 8192)","b_in":"(28672,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 64, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 28672)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 64, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 28672)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 8192)"}} +{"name.default_alias":"meta-llama\/Llama-3.1-70B-Instruct","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.1-70B-Instruct","n_params.as_str":"78B","n_params.as_int":77846282240,"n_params.from_name":"70B","cfg.n_params":77846282240,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":2048,"n_heads":64,"d_mlp":28672,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.1-70B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.1-70B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0088388348,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":77846282240,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 2048\nn_heads: 64\nd_mlp: 28672\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.1-70B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.1-70B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008838834764831844\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 77846282240\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.1-70B-Instruct","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n W_Q: (64, 8192, 128)\n W_O: (64, 128, 8192)\n b_Q: (64, 128)\n b_O: (8192,)\n '[_W_K, _W_V]': (8, 8192, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (8192, 28672)\n W_out: (28672, 8192)\n b_in: (28672,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"W_Q":"(64, 8192, 128)","W_O":"(64, 128, 8192)","b_Q":"(64, 128)","b_O":"(8192,)","[_W_K, _W_V]":"(8, 8192, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 28672)","W_out":"(28672, 8192)","b_in":"(28672,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 64, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 28672)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 64, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 28672)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 8192)"}} +{"name.default_alias":"meta-llama\/Llama-3.1-8B","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.1-8B","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"8B","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":14336,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.1-8B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.1-8B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 14336\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.1-8B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.1-8B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.1-8B","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 14336)\n W_out: (14336, 4096)\n b_in: (14336,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 14336)","W_out":"(14336, 4096)","b_in":"(14336,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"meta-llama\/Llama-3.1-8B-Instruct","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.1-8B-Instruct","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"8B","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":14336,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.1-8B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.1-8B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 14336\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.1-8B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.1-8B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.1-8B-Instruct","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 14336)\n W_out: (14336, 4096)\n b_in: (14336,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 14336)","W_out":"(14336, 4096)","b_in":"(14336,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"meta-llama\/Llama-3.2-1B","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.2-1B","n_params.as_str":"1.1B","n_params.as_int":1073741824,"n_params.from_name":"1B","cfg.n_params":1073741824,"cfg.n_layers":16,"cfg.n_heads":32,"cfg.d_model":2048,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":64,"n_layers":16,"n_ctx":2048,"n_heads":32,"d_mlp":8192,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.2-1B","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.2-1B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":1073741824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":32.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 64\nn_layers: 16\nn_ctx: 2048\nn_heads: 32\nd_mlp: 8192\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.2-1B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.2-1B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 1073741824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 32.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.2-1B","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 2048)\nblocks:\n '[0-15]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n W_Q: (32, 2048, 64)\n W_O: (32, 64, 2048)\n b_Q: (32, 64)\n b_O: (2048,)\n '[_W_K, _W_V]': (8, 2048, 64)\n '[_b_K, _b_V]': (8, 64)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n '[W_in, W_gate]': (2048, 8192)\n W_out: (8192, 2048)\n b_in: (8192,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 2048)"},"blocks":{"[0-15]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"W_Q":"(32, 2048, 64)","W_O":"(32, 64, 2048)","b_Q":"(32, 64)","b_O":"(2048,)","[_W_K, _W_V]":"(8, 2048, 64)","[_b_K, _b_V]":"(8, 64)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"[W_in, W_gate]":"(2048, 8192)","W_out":"(8192, 2048)","b_in":"(8192,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 64)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 64)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"meta-llama\/Llama-3.2-1B-Instruct","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.2-1B-Instruct","n_params.as_str":"1.1B","n_params.as_int":1073741824,"n_params.from_name":"1B","cfg.n_params":1073741824,"cfg.n_layers":16,"cfg.n_heads":32,"cfg.d_model":2048,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":64,"n_layers":16,"n_ctx":2048,"n_heads":32,"d_mlp":8192,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.2-1B-Instruct","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.2-1B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0176776695,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":1073741824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":32.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 64\nn_layers: 16\nn_ctx: 2048\nn_heads: 32\nd_mlp: 8192\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.2-1B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.2-1B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.017677669529663688\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 1073741824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 32.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.2-1B-Instruct","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 2048)\nblocks:\n '[0-15]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n W_Q: (32, 2048, 64)\n W_O: (32, 64, 2048)\n b_Q: (32, 64)\n b_O: (2048,)\n '[_W_K, _W_V]': (8, 2048, 64)\n '[_b_K, _b_V]': (8, 64)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n '[W_in, W_gate]': (2048, 8192)\n W_out: (8192, 2048)\n b_in: (8192,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 2048)"},"blocks":{"[0-15]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"W_Q":"(32, 2048, 64)","W_O":"(32, 64, 2048)","b_Q":"(32, 64)","b_O":"(2048,)","[_W_K, _W_V]":"(8, 2048, 64)","[_b_K, _b_V]":"(8, 64)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"[W_in, W_gate]":"(2048, 8192)","W_out":"(8192, 2048)","b_in":"(8192,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 64)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 64)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"meta-llama\/Llama-3.2-3B","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.2-3B","n_params.as_str":"3.2B","n_params.as_int":3170893824,"n_params.from_name":"3B","cfg.n_params":3170893824,"cfg.n_layers":28,"cfg.n_heads":24,"cfg.d_model":3072,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3072,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":24,"d_mlp":8192,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.2-3B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.2-3B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0144337567,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":3170893824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":32.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 3072\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 24\nd_mlp: 8192\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.2-3B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.2-3B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.014433756729740645\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 3170893824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 32.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.2-3B","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 3072)\nblocks:\n '[0-27]':\n ln1:\n w: (3072,)\n ln2:\n w: (3072,)\n attn:\n W_Q: (24, 3072, 128)\n W_O: (24, 128, 3072)\n b_Q: (24, 128)\n b_O: (3072,)\n '[_W_K, _W_V]': (8, 3072, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (3072, 8192)\n W_out: (8192, 3072)\n b_in: (8192,)\n b_out: (3072,)\nln_final:\n w: (3072,)\nunembed:\n W_U: (3072, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 3072)"},"blocks":{"[0-27]":{"ln1":{"w":"(3072,)"},"ln2":{"w":"(3072,)"},"attn":{"W_Q":"(24, 3072, 128)","W_O":"(24, 128, 3072)","b_Q":"(24, 128)","b_O":"(3072,)","[_W_K, _W_V]":"(8, 3072, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(3072, 8192)","W_out":"(8192, 3072)","b_in":"(8192,)","b_out":"(3072,)"}}},"ln_final":{"w":"(3072,)"},"unembed":{"W_U":"(3072, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 24, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 24, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3072)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\nunembed:\n hook_in: (batch, seq_len, 3072)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 3072)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 24, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 24, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3072)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"unembed":{"hook_in":"(batch, seq_len, 3072)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 3072)"}} +{"name.default_alias":"meta-llama\/Llama-3.2-3B-Instruct","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.2-3B-Instruct","n_params.as_str":"3.2B","n_params.as_int":3170893824,"n_params.from_name":"3B","cfg.n_params":3170893824,"cfg.n_layers":28,"cfg.n_heads":24,"cfg.d_model":3072,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3072,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":24,"d_mlp":8192,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.2-3B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.2-3B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0144337567,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":3170893824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":32.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 3072\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 24\nd_mlp: 8192\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.2-3B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.2-3B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.014433756729740645\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 3170893824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 32.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.2-3B-Instruct","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 3072)\nblocks:\n '[0-27]':\n ln1:\n w: (3072,)\n ln2:\n w: (3072,)\n attn:\n W_Q: (24, 3072, 128)\n W_O: (24, 128, 3072)\n b_Q: (24, 128)\n b_O: (3072,)\n '[_W_K, _W_V]': (8, 3072, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (3072, 8192)\n W_out: (8192, 3072)\n b_in: (8192,)\n b_out: (3072,)\nln_final:\n w: (3072,)\nunembed:\n W_U: (3072, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 3072)"},"blocks":{"[0-27]":{"ln1":{"w":"(3072,)"},"ln2":{"w":"(3072,)"},"attn":{"W_Q":"(24, 3072, 128)","W_O":"(24, 128, 3072)","b_Q":"(24, 128)","b_O":"(3072,)","[_W_K, _W_V]":"(8, 3072, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(3072, 8192)","W_out":"(8192, 3072)","b_in":"(8192,)","b_out":"(3072,)"}}},"ln_final":{"w":"(3072,)"},"unembed":{"W_U":"(3072, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 24, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 24, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3072)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\nunembed:\n hook_in: (batch, seq_len, 3072)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 3072)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 24, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 24, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3072)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"unembed":{"hook_in":"(batch, seq_len, 3072)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 3072)"}} +{"name.default_alias":"meta-llama\/Llama-3.3-70B-Instruct","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Llama-3.3-70B-Instruct","n_params.as_str":"78B","n_params.as_int":77846282240,"n_params.from_name":"70B","cfg.n_params":77846282240,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":2048,"n_heads":64,"d_mlp":28672,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Llama-3.3-70B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Llama-3.3-70B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0088388348,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":77846282240,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 2048\nn_heads: 64\nd_mlp: 28672\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Llama-3.3-70B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Llama-3.3-70B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008838834764831844\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 77846282240\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Llama-3.3-70B-Instruct","tokenizer.vocab_size":128000.0,"tokenizer.max_len":131072.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"j9N50ddC7mjCgS4GseU9LmKZDKk=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n W_Q: (64, 8192, 128)\n W_O: (64, 128, 8192)\n b_Q: (64, 128)\n b_O: (8192,)\n '[_W_K, _W_V]': (8, 8192, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (8192, 28672)\n W_out: (28672, 8192)\n b_in: (28672,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"W_Q":"(64, 8192, 128)","W_O":"(64, 128, 8192)","b_Q":"(64, 128)","b_O":"(8192,)","[_W_K, _W_V]":"(8, 8192, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 28672)","W_out":"(28672, 8192)","b_in":"(28672,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 64, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 28672)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 64, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 28672)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 8192)"}} +{"name.default_alias":"meta-llama\/Meta-Llama-3-70B","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Meta-Llama-3-70B","n_params.as_str":"78B","n_params.as_int":77846282240,"n_params.from_name":"70B","cfg.n_params":77846282240,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":8192,"n_heads":64,"d_mlp":28672,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Meta-Llama-3-70B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Meta-Llama-3-70B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0088388348,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":77846282240,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 8192\nn_heads: 64\nd_mlp: 28672\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Meta-Llama-3-70B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Meta-Llama-3-70B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008838834764831844\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 77846282240\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Meta-Llama-3-70B","tokenizer.vocab_size":128000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"RnzNv9w_ITBp6b2dcibKR7_l85I=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n W_Q: (64, 8192, 128)\n W_O: (64, 128, 8192)\n b_Q: (64, 128)\n b_O: (8192,)\n '[_W_K, _W_V]': (8, 8192, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (8192, 28672)\n W_out: (28672, 8192)\n b_in: (28672,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"W_Q":"(64, 8192, 128)","W_O":"(64, 128, 8192)","b_Q":"(64, 128)","b_O":"(8192,)","[_W_K, _W_V]":"(8, 8192, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 28672)","W_out":"(28672, 8192)","b_in":"(28672,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 64, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 28672)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 64, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 28672)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 8192)"}} +{"name.default_alias":"meta-llama\/Meta-Llama-3-70B-Instruct","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Meta-Llama-3-70B-Instruct","n_params.as_str":"78B","n_params.as_int":77846282240,"n_params.from_name":"70B","cfg.n_params":77846282240,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":8192,"n_heads":64,"d_mlp":28672,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Meta-Llama-3-70B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Meta-Llama-3-70B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0088388348,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":77846282240,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 8192\nn_heads: 64\nd_mlp: 28672\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Meta-Llama-3-70B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Meta-Llama-3-70B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.008838834764831844\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 77846282240\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Meta-Llama-3-70B-Instruct","tokenizer.vocab_size":128000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"RnzNv9w_ITBp6b2dcibKR7_l85I=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n W_Q: (64, 8192, 128)\n W_O: (64, 128, 8192)\n b_Q: (64, 128)\n b_O: (8192,)\n '[_W_K, _W_V]': (8, 8192, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (8192, 28672)\n W_out: (28672, 8192)\n b_in: (28672,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"W_Q":"(64, 8192, 128)","W_O":"(64, 128, 8192)","b_Q":"(64, 128)","b_O":"(8192,)","[_W_K, _W_V]":"(8, 8192, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 28672)","W_out":"(28672, 8192)","b_in":"(28672,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 64, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 28672)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 64, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 28672)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 8192)"}} +{"name.default_alias":"meta-llama\/Meta-Llama-3-8B","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Meta-Llama-3-8B","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"8B","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":8192,"n_heads":32,"d_mlp":14336,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Meta-Llama-3-8B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Meta-Llama-3-8B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 8192\nn_heads: 32\nd_mlp: 14336\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Meta-Llama-3-8B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Meta-Llama-3-8B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Meta-Llama-3-8B","tokenizer.vocab_size":128000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"RnzNv9w_ITBp6b2dcibKR7_l85I=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 14336)\n W_out: (14336, 4096)\n b_in: (14336,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 14336)","W_out":"(14336, 4096)","b_in":"(14336,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"meta-llama\/Meta-Llama-3-8B-Instruct","name.huggingface":null,"name.aliases":"","model_type":"llama","name.from_cfg":"Meta-Llama-3-8B-Instruct","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"8B","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":128256,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"LlamaForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":8192,"n_heads":32,"d_mlp":14336,"d_vocab":128256,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Meta-Llama-3-8B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"LlamaForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"meta-llama\/Meta-Llama-3-8B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":128256,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":500000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 8192\nn_heads: 32\nd_mlp: 14336\nd_vocab: 128256\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Meta-Llama-3-8B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: LlamaForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: meta-llama\/Meta-Llama-3-8B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 128256\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 500000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"meta-llama\/Meta-Llama-3-8B-Instruct","tokenizer.vocab_size":128000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"RnzNv9w_ITBp6b2dcibKR7_l85I=","tensor_shapes.state_dict":"embed:\n W_E: (128256, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 14336)\n W_out: (14336, 4096)\n b_in: (14336,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 128256)\n b_U: (128256,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(128256, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 14336)","W_out":"(14336, 4096)","b_in":"(14336,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 128256)","b_U":"(128256,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 128256)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 128256)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"phi-1","name.huggingface":"microsoft\/phi-1","name.aliases":"phi-1","model_type":"phi","name.from_cfg":"phi-1","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":null,"cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":32,"cfg.d_model":2048,"cfg.d_vocab":51200,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"PhiForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":32,"d_mlp":8192,"d_vocab":51200,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"phi-1","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"PhiForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"microsoft\/phi-1","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":51200,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 32\nd_mlp: 8192\nd_vocab: 51200\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: phi-1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: PhiForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: microsoft\/phi-1\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 51200\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"microsoft\/phi-1","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"CodeGenTokenizer","tokenizer.vocab_hash":"TYk6J3OrqdU2F7JYiSfFXtd-vB4=","tensor_shapes.state_dict":"embed:\n W_E: (51200, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2048, 64)\n W_O: (32, 64, 2048)\n '[b_Q, b_K, b_V]': (32, 64)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 51200)\n b_U: (51200,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(51200, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2048, 64)","W_O":"(32, 64, 2048)","[b_Q, b_K, b_V]":"(32, 64)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 51200)","b_U":"(51200,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 51200)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 64)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 51200)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"phi-1_5","name.huggingface":"microsoft\/phi-1_5","name.aliases":"phi-1_5","model_type":"phi","name.from_cfg":"phi-1_5","n_params.as_str":"1.2B","n_params.as_int":1207959552,"n_params.from_name":null,"cfg.n_params":1207959552,"cfg.n_layers":24,"cfg.n_heads":32,"cfg.d_model":2048,"cfg.d_vocab":51200,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"PhiForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2048,"d_head":64,"n_layers":24,"n_ctx":2048,"n_heads":32,"d_mlp":8192,"d_vocab":51200,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"phi-1_5","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"PhiForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"microsoft\/phi-1_5","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":51200,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":1207959552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 64\nn_layers: 24\nn_ctx: 2048\nn_heads: 32\nd_mlp: 8192\nd_vocab: 51200\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: phi-1_5\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: PhiForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: microsoft\/phi-1_5\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 51200\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 1207959552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"microsoft\/phi-1_5","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"CodeGenTokenizer","tokenizer.vocab_hash":"TYk6J3OrqdU2F7JYiSfFXtd-vB4=","tensor_shapes.state_dict":"embed:\n W_E: (51200, 2048)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (2048,)\n ln2:\n '[w, b]': (2048,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2048, 64)\n W_O: (32, 64, 2048)\n '[b_Q, b_K, b_V]': (32, 64)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (2048, 8192)\n b_in: (8192,)\n W_out: (8192, 2048)\n b_out: (2048,)\nln_final:\n '[w, b]': (2048,)\nunembed:\n W_U: (2048, 51200)\n b_U: (51200,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(51200, 2048)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(2048,)"},"ln2":{"[w, b]":"(2048,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2048, 64)","W_O":"(32, 64, 2048)","[b_Q, b_K, b_V]":"(32, 64)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(2048, 8192)","b_in":"(8192,)","W_out":"(8192, 2048)","b_out":"(2048,)"}}},"ln_final":{"[w, b]":"(2048,)"},"unembed":{"W_U":"(2048, 51200)","b_U":"(51200,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 51200)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 64)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 51200)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"phi-2","name.huggingface":"microsoft\/phi-2","name.aliases":"phi-2","model_type":"phi","name.from_cfg":"phi-2","n_params.as_str":"2.5B","n_params.as_int":2516582400,"n_params.from_name":null,"cfg.n_params":2516582400,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":51200,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"PhiForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":2560,"d_head":80,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":10240,"d_vocab":51200,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"phi-2","use_attn_scale":true,"attn_scale":8.94427191,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"PhiForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"microsoft\/phi-2","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":51200,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":2516582400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 80\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 10240\nd_vocab: 51200\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: phi-2\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n qPSXm3fjIUA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: PhiForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: microsoft\/phi-2\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 51200\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 2516582400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"microsoft\/phi-2","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"CodeGenTokenizer","tokenizer.vocab_hash":"TYk6J3OrqdU2F7JYiSfFXtd-vB4=","tensor_shapes.state_dict":"embed:\n W_E: (51200, 2560)\nblocks:\n '[0-31]':\n ln1:\n '[w, b]': (2560,)\n ln2:\n '[w, b]': (2560,)\n attn:\n '[W_Q, W_K, W_V]': (32, 2560, 80)\n W_O: (32, 80, 2560)\n '[b_Q, b_K, b_V]': (32, 80)\n b_O: (2560,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (2560, 10240)\n b_in: (10240,)\n W_out: (10240, 2560)\n b_out: (2560,)\nln_final:\n '[w, b]': (2560,)\nunembed:\n W_U: (2560, 51200)\n b_U: (51200,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(51200, 2560)"},"blocks":{"[0-31]":{"ln1":{"[w, b]":"(2560,)"},"ln2":{"[w, b]":"(2560,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 2560, 80)","W_O":"(32, 80, 2560)","[b_Q, b_K, b_V]":"(32, 80)","b_O":"(2560,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(2560, 10240)","b_in":"(10240,)","W_out":"(10240, 2560)","b_out":"(2560,)"}}},"ln_final":{"[w, b]":"(2560,)"},"unembed":{"W_U":"(2560, 51200)","b_U":"(51200,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 80)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 10240)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 51200)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 80)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 10240)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 51200)"},"hook_embed":"(batch, seq_len, 2560)"}} +{"name.default_alias":"phi-3","name.huggingface":"microsoft\/Phi-3-mini-4k-instruct","name.aliases":"phi-3","model_type":"phi","name.from_cfg":"Phi-3-mini-4k-instruct","n_params.as_str":"3.6B","n_params.as_int":3623878656,"n_params.from_name":null,"cfg.n_params":3623878656,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":3072,"cfg.d_vocab":32064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Phi3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3072,"d_head":96,"n_layers":32,"n_ctx":4096,"n_heads":32,"d_mlp":8192,"d_vocab":32064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Phi-3-mini-4k-instruct","use_attn_scale":true,"attn_scale":9.7979589711,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Phi3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"microsoft\/Phi-3-mini-4k-instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32064,"parallel_attn_mlp":false,"rotary_dim":96,"n_params":3623878656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 3072\nd_head: 96\nn_layers: 32\nn_ctx: 4096\nn_heads: 32\nd_mlp: 8192\nd_vocab: 32064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Phi-3-mini-4k-instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n LiEJFI6YI0A=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Phi3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: microsoft\/Phi-3-mini-4k-instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32064\nparallel_attn_mlp: false\nrotary_dim: 96\nn_params: 3623878656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"microsoft\/Phi-3-mini-4k-instruct","tokenizer.vocab_size":32000.0,"tokenizer.max_len":4096.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"2BcGXsWoZjuOkMtb6uTbGL68fbc=","tensor_shapes.state_dict":"embed:\n W_E: (32064, 3072)\nblocks:\n '[0-31]':\n ln1:\n w: (3072,)\n ln2:\n w: (3072,)\n attn:\n '[W_Q, W_K, W_V]': (32, 3072, 96)\n W_O: (32, 96, 3072)\n '[b_Q, b_K, b_V]': (32, 96)\n b_O: (3072,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 96)\n mlp:\n '[W_in, W_gate]': (3072, 8192)\n W_out: (8192, 3072)\n b_in: (8192,)\n b_out: (3072,)\nln_final:\n w: (3072,)\nunembed:\n W_U: (3072, 32064)\n b_U: (32064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32064, 3072)"},"blocks":{"[0-31]":{"ln1":{"w":"(3072,)"},"ln2":{"w":"(3072,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 3072, 96)","W_O":"(32, 96, 3072)","[b_Q, b_K, b_V]":"(32, 96)","b_O":"(3072,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 96)"},"mlp":{"[W_in, W_gate]":"(3072, 8192)","W_out":"(8192, 3072)","b_in":"(8192,)","b_out":"(3072,)"}}},"ln_final":{"w":"(3072,)"},"unembed":{"W_U":"(3072, 32064)","b_U":"(32064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 96)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8192)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3072)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\nunembed:\n hook_in: (batch, seq_len, 3072)\n hook_out: (batch, seq_len, 32064)\nhook_embed: (batch, seq_len, 3072)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 96)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8192)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3072)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"unembed":{"hook_in":"(batch, seq_len, 3072)","hook_out":"(batch, seq_len, 32064)"},"hook_embed":"(batch, seq_len, 3072)"}} +{"name.default_alias":"phi-4","name.huggingface":"microsoft\/phi-4","name.aliases":"phi-4","model_type":"phi","name.from_cfg":"phi-4","n_params.as_str":"15B","n_params.as_int":15204352000,"n_params.from_name":null,"cfg.n_params":15204352000,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":100352,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Phi3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":16384,"n_heads":40,"d_mlp":17920,"d_vocab":100352,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":10,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"phi-4","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Phi3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"microsoft\/phi-4","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":100352,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":15204352000,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":250000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 16384\nn_heads: 40\nd_mlp: 17920\nd_vocab: 100352\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 10\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: phi-4\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Phi3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: microsoft\/phi-4\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 100352\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 15204352000\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 250000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"microsoft\/phi-4","tokenizer.vocab_size":100352.0,"tokenizer.max_len":16384.0,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"uJZqWk6gqn6tO_nlSJEZsP9MITQ=","tensor_shapes.state_dict":"embed:\n W_E: (100352, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (10, 5120, 128)\n '[_b_K, _b_V]': (10, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 17920)\n W_out: (17920, 5120)\n b_in: (17920,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 100352)\n b_U: (100352,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(100352, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(10, 5120, 128)","[_b_K, _b_V]":"(10, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 17920)","W_out":"(17920, 5120)","b_in":"(17920,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 100352)","b_U":"(100352,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 10, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 17920)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 100352)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 10, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 17920)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 100352)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"mistral-7b-instruct","name.huggingface":"mistralai\/Mistral-7B-Instruct-v0.1","name.aliases":"mistral-7b-instruct","model_type":"mistral","name.from_cfg":"Mistral-7B-Instruct-v0.1","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"7b","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"MistralForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":14336,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Mistral-7B-Instruct-v0.1","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"MistralForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"mistralai\/Mistral-7B-Instruct-v0.1","window_size":4096,"attn_types":["local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 14336\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Mistral-7B-Instruct-v0.1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: MistralForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: mistralai\/Mistral-7B-Instruct-v0.1\nwindow_size: 4096\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"mistralai\/Mistral-7B-Instruct-v0.1","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"kkCQxUk-PF9Ay_ZKDdKCh02YaGQ=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 14336)\n W_out: (14336, 4096)\n b_in: (14336,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 14336)","W_out":"(14336, 4096)","b_in":"(14336,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"mistral-7b","name.huggingface":"mistralai\/Mistral-7B-v0.1","name.aliases":"mistral-7b","model_type":"mistral","name.from_cfg":"Mistral-7B-v0.1","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"7b","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"MistralForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":2048,"n_heads":32,"d_mlp":14336,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Mistral-7B-v0.1","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"MistralForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"mistralai\/Mistral-7B-v0.1","window_size":4096,"attn_types":["local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 2048\nn_heads: 32\nd_mlp: 14336\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Mistral-7B-v0.1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: MistralForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: mistralai\/Mistral-7B-v0.1\nwindow_size: 4096\nattn_types:\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"mistralai\/Mistral-7B-v0.1","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"kkCQxUk-PF9Ay_ZKDdKCh02YaGQ=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 14336)\n W_out: (14336, 4096)\n b_in: (14336,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 14336)","W_out":"(14336, 4096)","b_in":"(14336,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 32000)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 32000)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"mistral-nemo-base-2407","name.huggingface":"mistralai\/Mistral-Nemo-Base-2407","name.aliases":"mistral-nemo-base-2407","model_type":"mistral","name.from_cfg":"Mistral-Nemo-Base-2407","n_params.as_str":"12B","n_params.as_int":12163481600,"n_params.from_name":null,"cfg.n_params":12163481600,"cfg.n_layers":40,"cfg.n_heads":32,"cfg.d_model":5120,"cfg.d_vocab":131072,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"MistralForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":32,"d_mlp":14336,"d_vocab":131072,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Mistral-Nemo-Base-2407","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"MistralForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"mistralai\/Mistral-Nemo-Base-2407","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":131072,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12163481600,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 32\nd_mlp: 14336\nd_vocab: 131072\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Mistral-Nemo-Base-2407\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: MistralForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: mistralai\/Mistral-Nemo-Base-2407\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 131072\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12163481600\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"mistralai\/Mistral-Nemo-Base-2407","tokenizer.vocab_size":131072.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"0xs_eSvVgsyZGbcLSIpGUn4Gdms=","tensor_shapes.state_dict":"embed:\n W_E: (131072, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n W_Q: (32, 5120, 128)\n W_O: (32, 128, 5120)\n b_Q: (32, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 14336)\n W_out: (14336, 5120)\n b_in: (14336,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 131072)\n b_U: (131072,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(131072, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"W_Q":"(32, 5120, 128)","W_O":"(32, 128, 5120)","b_Q":"(32, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 14336)","W_out":"(14336, 5120)","b_in":"(14336,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 131072)","b_U":"(131072,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 14336)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 131072)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 14336)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 131072)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"mistralai\/Mistral-Small-24B-Base-2501","name.huggingface":null,"name.aliases":"","model_type":"mistral","name.from_cfg":"Mistral-Small-24B-Base-2501","n_params.as_str":"23B","n_params.as_int":23488102400,"n_params.from_name":"24B","cfg.n_params":23488102400,"cfg.n_layers":40,"cfg.n_heads":32,"cfg.d_model":5120,"cfg.d_vocab":131072,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"MistralForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":32,"d_mlp":32768,"d_vocab":131072,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Mistral-Small-24B-Base-2501","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"MistralForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"mistralai\/Mistral-Small-24B-Base-2501","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0111803399,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":131072,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":23488102400,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":100000000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 32\nd_mlp: 32768\nd_vocab: 131072\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Mistral-Small-24B-Base-2501\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: MistralForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: mistralai\/Mistral-Small-24B-Base-2501\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.011180339887498949\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 131072\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 23488102400\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 100000000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"mistralai\/Mistral-Small-24B-Base-2501","tokenizer.vocab_size":131072.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"GEwgZayWxpmhQxtMq-WrVFJEfqM=","tensor_shapes.state_dict":"embed:\n W_E: (131072, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n W_Q: (32, 5120, 128)\n W_O: (32, 128, 5120)\n b_Q: (32, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 32768)\n W_out: (32768, 5120)\n b_in: (32768,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 131072)\n b_U: (131072,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(131072, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"W_Q":"(32, 5120, 128)","W_O":"(32, 128, 5120)","b_Q":"(32, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 32768)","W_out":"(32768, 5120)","b_in":"(32768,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 131072)","b_U":"(131072,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 32768)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 131072)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 32768)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 131072)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"mixtral-instruct","name.huggingface":"mistralai\/Mixtral-8x7B-Instruct-v0.1","name.aliases":"mixtral-instruct, mixtral-8x7b-instruct","model_type":"mixtral","name.from_cfg":"Mixtral-8x7B-Instruct-v0.1","n_params.as_str":"47B","n_params.as_int":47245688832,"n_params.from_name":null,"cfg.n_params":47245688832,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"MixtralForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":32768,"n_heads":32,"d_mlp":14336,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":8,"experts_per_token":2,"final_rms":false,"dtype":"torch.float32","model_name":"Mixtral-8x7B-Instruct-v0.1","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"MixtralForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"mistralai\/Mixtral-8x7B-Instruct-v0.1","window_size":null,"attn_types":["global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":47245688832,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 32768\nn_heads: 32\nd_mlp: 14336\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: 8\nexperts_per_token: 2\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Mixtral-8x7B-Instruct-v0.1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: MixtralForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: mistralai\/Mixtral-8x7B-Instruct-v0.1\nwindow_size: null\nattn_types:\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 47245688832\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"mistralai\/Mixtral-8x7B-Instruct-v0.1","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"kkCQxUk-PF9Ay_ZKDdKCh02YaGQ=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n experts:\n '[0-7]':\n W_in:\n weight: (14336, 4096)\n W_out:\n weight: (4096, 14336)\n W_gate:\n weight: (14336, 4096)\n W_gate:\n weight: (8, 4096)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"experts":{"[0-7]":{"W_in":{"weight":"(14336, 4096)"},"W_out":{"weight":"(4096, 14336)"},"W_gate":{"weight":"(14336, 4096)"}}},"W_gate":{"weight":"(8, 4096)"}}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"mixtral","name.huggingface":"mistralai\/Mixtral-8x7B-v0.1","name.aliases":"mixtral, mixtral-8x7b","model_type":"mixtral","name.from_cfg":"Mixtral-8x7B-v0.1","n_params.as_str":"47B","n_params.as_int":47245688832,"n_params.from_name":null,"cfg.n_params":47245688832,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":32000,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"MixtralForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":32768,"n_heads":32,"d_mlp":14336,"d_vocab":32000,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":8,"experts_per_token":2,"final_rms":false,"dtype":"torch.float32","model_name":"Mixtral-8x7B-v0.1","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"MixtralForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"mistralai\/Mixtral-8x7B-v0.1","window_size":null,"attn_types":["global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":32000,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":47245688832,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":1000000.0,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 32768\nn_heads: 32\nd_mlp: 14336\nd_vocab: 32000\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: 8\nexperts_per_token: 2\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Mixtral-8x7B-v0.1\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: MixtralForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: mistralai\/Mixtral-8x7B-v0.1\nwindow_size: null\nattn_types:\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 32000\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 47245688832\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 1000000.0\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"mistralai\/Mixtral-8x7B-v0.1","tokenizer.vocab_size":32000.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"kkCQxUk-PF9Ay_ZKDdKCh02YaGQ=","tensor_shapes.state_dict":"embed:\n W_E: (32000, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n experts:\n '[0-7]':\n W_in:\n weight: (14336, 4096)\n W_out:\n weight: (4096, 14336)\n W_gate:\n weight: (14336, 4096)\n W_gate:\n weight: (8, 4096)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 32000)\n b_U: (32000,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(32000, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"experts":{"[0-7]":{"W_in":{"weight":"(14336, 4096)"},"W_out":{"weight":"(4096, 14336)"},"W_gate":{"weight":"(14336, 4096)"}}},"W_gate":{"weight":"(8, 4096)"}}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 32000)","b_U":"(32000,)"}},"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"attn-only-2l-demo","name.huggingface":"NeelNanda\/Attn-Only-2L512W-Shortformer-6B-big-lr","name.aliases":"attn-only-2l-demo, attn-only-2l-shortformer-6b-big-lr, attn-only-2l-induction-demo, attn-only-demo","model_type":"attn-only","name.from_cfg":"Attn-Only-2L512W-Shortformer-6B-big-lr","n_params.as_str":"2.1M","n_params.as_int":2097152,"n_params.from_name":null,"cfg.n_params":2097152,"cfg.n_layers":2,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":50277,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"shortformer","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":null,"config.raw__":{"d_model":512,"d_head":64,"n_layers":2,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":50277,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"shortformer","n_key_value_heads":null,"attn_only":true,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":null,"num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Attn-Only-2L512W-Shortformer-6B-big-lr","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50277,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":2097152,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 2\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 50277\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: shortformer\nn_key_value_heads: null\nattn_only: true\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: null\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Attn-Only-2L512W-Shortformer-6B-big-lr\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50277\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 2097152\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50277, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-1]':\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\nunembed:\n W_U: (512, 50277)\n b_U: (50277,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50277, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-1]":{"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"}}},"unembed":{"W_U":"(512, 50277)","b_U":"(50277,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-1]':\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n '[hook_resid_pre, hook_attn_out, hook_resid_post]': (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50277)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-1]":{"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"[hook_resid_pre, hook_attn_out, hook_resid_post]":"(batch, seq_len, 512)"}},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50277)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"attn-only-1l","name.huggingface":"NeelNanda\/Attn_Only_1L512W_C4_Code","name.aliases":"attn-only-1l, attn-only-1l-new, attn-only-1l-c4-code","model_type":"attn-only","name.from_cfg":"Attn_Only_1L512W_C4_Code","n_params.as_str":"1.0M","n_params.as_int":1048576,"n_params.from_name":null,"cfg.n_params":1048576,"cfg.n_layers":1,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":1,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":true,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Attn_Only_1L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1048576,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 1\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: true\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Attn_Only_1L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1048576\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '0':\n ln1:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"0":{"ln1":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '0':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n '[hook_resid_pre, hook_attn_out, hook_resid_post]': (batch, seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"0":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"[hook_resid_pre, hook_attn_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"attn-only-2l","name.huggingface":"NeelNanda\/Attn_Only_2L512W_C4_Code","name.aliases":"attn-only-2l, attn-only-2l-new, attn-only-2l-c4-code","model_type":"attn-only","name.from_cfg":"Attn_Only_2L512W_C4_Code","n_params.as_str":"2.1M","n_params.as_int":2097152,"n_params.from_name":null,"cfg.n_params":2097152,"cfg.n_layers":2,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":2,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":true,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Attn_Only_2L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":2097152,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 2\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: true\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Attn_Only_2L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 2097152\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-1]':\n ln1:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-1]":{"ln1":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-1]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n '[hook_resid_pre, hook_attn_out, hook_resid_post]': (batch, seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-1]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"[hook_resid_pre, hook_attn_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"attn-only-3l","name.huggingface":"NeelNanda\/Attn_Only_3L512W_C4_Code","name.aliases":"attn-only-3l, attn-only-3l-new, attn-only-3l-c4-code","model_type":"attn-only","name.from_cfg":"Attn_Only_3L512W_C4_Code","n_params.as_str":"3.1M","n_params.as_int":3145728,"n_params.from_name":null,"cfg.n_params":3145728,"cfg.n_layers":3,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":3,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":true,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Attn_Only_3L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":3145728,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 3\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: true\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Attn_Only_3L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 3145728\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-2]':\n ln1:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-2]":{"ln1":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-2]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n '[hook_resid_pre, hook_attn_out, hook_resid_post]': (batch, seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-2]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"[hook_resid_pre, hook_attn_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"attn-only-4l","name.huggingface":"NeelNanda\/Attn_Only_4L512W_C4_Code","name.aliases":"attn-only-4l, attn-only-4l-new, attn-only-4l-c4-code","model_type":"attn-only","name.from_cfg":"Attn_Only_4L512W_C4_Code","n_params.as_str":"4.2M","n_params.as_int":4194304,"n_params.from_name":null,"cfg.n_params":4194304,"cfg.n_layers":4,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":4,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":true,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"Attn_Only_4L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":4194304,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 4\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: true\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: Attn_Only_4L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 4194304\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-3]':\n ln1:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-3]":{"ln1":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-3]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n '[hook_resid_pre, hook_attn_out, hook_resid_post]': (batch, seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-3]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"[hook_resid_pre, hook_attn_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"gelu-1l","name.huggingface":"NeelNanda\/GELU_1L512W_C4_Code","name.aliases":"gelu-1l, gelu-1l-new, gelu-1l-c4-code","model_type":"gelu","name.from_cfg":"GELU_1L512W_C4_Code","n_params.as_str":"3.1M","n_params.as_int":3145728,"n_params.from_name":null,"cfg.n_params":3145728,"cfg.n_layers":1,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":1,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"GELU_1L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":3145728,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 1\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: GELU_1L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 3145728\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '0':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"0":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '0':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"0":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"gelu-2l","name.huggingface":"NeelNanda\/GELU_2L512W_C4_Code","name.aliases":"gelu-2l, gelu-2l-new, gelu-2l-c4-code","model_type":"gelu","name.from_cfg":"GELU_2L512W_C4_Code","n_params.as_str":"6.3M","n_params.as_int":6291456,"n_params.from_name":null,"cfg.n_params":6291456,"cfg.n_layers":2,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":2,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"GELU_2L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":6291456,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 2\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: GELU_2L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 6291456\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-1]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-1]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-1]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-1]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"gelu-3l","name.huggingface":"NeelNanda\/GELU_3L512W_C4_Code","name.aliases":"gelu-3l, gelu-3l-new, gelu-3l-c4-code","model_type":"gelu","name.from_cfg":"GELU_3L512W_C4_Code","n_params.as_str":"9.4M","n_params.as_int":9437184,"n_params.from_name":null,"cfg.n_params":9437184,"cfg.n_layers":3,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":3,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"GELU_3L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":9437184,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 3\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: GELU_3L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 9437184\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-2]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-2]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-2]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-2]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"gelu-4l","name.huggingface":"NeelNanda\/GELU_4L512W_C4_Code","name.aliases":"gelu-4l, gelu-4l-new, gelu-4l-c4-code","model_type":"gelu","name.from_cfg":"GELU_4L512W_C4_Code","n_params.as_str":"13M","n_params.as_int":12582912,"n_params.from_name":null,"cfg.n_params":12582912,"cfg.n_layers":4,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":4,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"GELU_4L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 4\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: GELU_4L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-3]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-3]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-3]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-3]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"solu-10l","name.huggingface":"NeelNanda\/SoLU_10L1280W_C4_Code","name.aliases":"solu-10l, solu-10l-new, solu-10l-c4-code","model_type":"solu","name.from_cfg":"SoLU_10L1280W_C4_Code","n_params.as_str":"197M","n_params.as_int":196608000,"n_params.from_name":null,"cfg.n_params":196608000,"cfg.n_layers":10,"cfg.n_heads":20,"cfg.d_model":1280,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":1280,"d_head":64,"n_layers":10,"n_ctx":1024,"n_heads":20,"d_mlp":5120,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_10L1280W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0223606798,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":196608000,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1280\nd_head: 64\nn_layers: 10\nn_ctx: 1024\nn_heads: 20\nd_mlp: 5120\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_10L1280W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.022360679774997897\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 196608000\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 1280)\npos_embed:\n W_pos: (1024, 1280)\nblocks:\n '[0-9]':\n ln1:\n '[w, b]': (1280,)\n ln2:\n '[w, b]': (1280,)\n attn:\n '[W_Q, W_K, W_V]': (20, 1280, 64)\n W_O: (20, 64, 1280)\n '[b_Q, b_K, b_V]': (20, 64)\n b_O: (1280,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (5120,)\n W_in: (1280, 5120)\n b_in: (5120,)\n W_out: (5120, 1280)\n b_out: (1280,)\nln_final:\n '[w, b]': (1280,)\nunembed:\n W_U: (1280, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 1280)"},"pos_embed":{"W_pos":"(1024, 1280)"},"blocks":{"[0-9]":{"ln1":{"[w, b]":"(1280,)"},"ln2":{"[w, b]":"(1280,)"},"attn":{"[W_Q, W_K, W_V]":"(20, 1280, 64)","W_O":"(20, 64, 1280)","[b_Q, b_K, b_V]":"(20, 64)","b_O":"(1280,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(5120,)"},"W_in":"(1280, 5120)","b_in":"(5120,)","W_out":"(5120, 1280)","b_out":"(1280,)"}}},"ln_final":{"[w, b]":"(1280,)"},"unembed":{"W_U":"(1280, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-9]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 20, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 20, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 5120)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1280)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\nunembed:\n hook_in: (batch, seq_len, 1280)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1280)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-9]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 20, 64)","[hook_attn_scores, hook_pattern]":"(batch, 20, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 5120)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1280)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"unembed":{"hook_in":"(batch, seq_len, 1280)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1280)"}} +{"name.default_alias":"solu-10l-pile","name.huggingface":"NeelNanda\/SoLU_10L_v22_old","name.aliases":"solu-10l-pile, solu-10l-old","model_type":"solu","name.from_cfg":"SoLU_10L_v22_old","n_params.as_str":"197M","n_params.as_int":196608000,"n_params.from_name":null,"cfg.n_params":196608000,"cfg.n_layers":10,"cfg.n_heads":20,"cfg.d_model":1280,"cfg.d_vocab":50278,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel-solu-old","cfg.normalization_type":"LNPre","config.raw__":{"d_model":1280,"d_head":64,"n_layers":10,"n_ctx":1024,"n_heads":20,"d_mlp":5120,"d_vocab":50278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LNPre","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_10L_v22_old","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel-solu-old","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0223606798,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50278,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":196608000,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1280\nd_head: 64\nn_layers: 10\nn_ctx: 1024\nn_heads: 20\nd_mlp: 5120\nd_vocab: 50278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LNPre\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_10L_v22_old\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel-solu-old\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.022360679774997897\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50278\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 196608000\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50278, 1280)\npos_embed:\n W_pos: (1024, 1280)\nblocks:\n '[0-9]':\n attn:\n '[W_Q, W_K, W_V]': (20, 1280, 64)\n W_O: (20, 64, 1280)\n '[b_Q, b_K, b_V]': (20, 64)\n b_O: (1280,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1280, 5120)\n b_in: (5120,)\n W_out: (5120, 1280)\n b_out: (1280,)\nunembed:\n W_U: (1280, 50278)\n b_U: (50278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50278, 1280)"},"pos_embed":{"W_pos":"(1024, 1280)"},"blocks":{"[0-9]":{"attn":{"[W_Q, W_K, W_V]":"(20, 1280, 64)","W_O":"(20, 64, 1280)","[b_Q, b_K, b_V]":"(20, 64)","b_O":"(1280,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1280, 5120)","b_in":"(5120,)","W_out":"(5120, 1280)","b_out":"(1280,)"}}},"unembed":{"W_U":"(1280, 50278)","b_U":"(50278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-9]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 20, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 20, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 5120)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1280)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1280)\nunembed:\n hook_in: (batch, seq_len, 1280)\n hook_out: (batch, seq_len, 50278)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1280)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-9]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 20, 64)","[hook_attn_scores, hook_pattern]":"(batch, 20, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 5120)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1280)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1280)"},"unembed":{"hook_in":"(batch, seq_len, 1280)","hook_out":"(batch, seq_len, 50278)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1280)"}} +{"name.default_alias":"solu-12l","name.huggingface":"NeelNanda\/SoLU_12L1536W_C4_Code","name.aliases":"solu-12l, solu-12l-new, solu-12l-c4-code","model_type":"solu","name.from_cfg":"SoLU_12L1536W_C4_Code","n_params.as_str":"340M","n_params.as_int":339738624,"n_params.from_name":null,"cfg.n_params":339738624,"cfg.n_layers":12,"cfg.n_heads":24,"cfg.d_model":1536,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":1536,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":24,"d_mlp":6144,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_12L1536W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0204124145,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":339738624,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1536\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 24\nd_mlp: 6144\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_12L1536W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.020412414523193152\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 339738624\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 1536)\npos_embed:\n W_pos: (1024, 1536)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (1536,)\n ln2:\n '[w, b]': (1536,)\n attn:\n '[W_Q, W_K, W_V]': (24, 1536, 64)\n W_O: (24, 64, 1536)\n '[b_Q, b_K, b_V]': (24, 64)\n b_O: (1536,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (6144,)\n W_in: (1536, 6144)\n b_in: (6144,)\n W_out: (6144, 1536)\n b_out: (1536,)\nln_final:\n '[w, b]': (1536,)\nunembed:\n W_U: (1536, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 1536)"},"pos_embed":{"W_pos":"(1024, 1536)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(1536,)"},"ln2":{"[w, b]":"(1536,)"},"attn":{"[W_Q, W_K, W_V]":"(24, 1536, 64)","W_O":"(24, 64, 1536)","[b_Q, b_K, b_V]":"(24, 64)","b_O":"(1536,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(6144,)"},"W_in":"(1536, 6144)","b_in":"(6144,)","W_out":"(6144, 1536)","b_out":"(1536,)"}}},"ln_final":{"[w, b]":"(1536,)"},"unembed":{"W_U":"(1536, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 24, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 24, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 6144)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1536)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\nunembed:\n hook_in: (batch, seq_len, 1536)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1536)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 24, 64)","[hook_attn_scores, hook_pattern]":"(batch, 24, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 6144)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1536)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"unembed":{"hook_in":"(batch, seq_len, 1536)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1536)"}} +{"name.default_alias":"solu-12l-pile","name.huggingface":"NeelNanda\/SoLU_12L_v23_old","name.aliases":"solu-12l-pile, solu-12l-old","model_type":"solu","name.from_cfg":"SoLU_12L_v23_old","n_params.as_str":"340M","n_params.as_int":339738624,"n_params.from_name":null,"cfg.n_params":339738624,"cfg.n_layers":12,"cfg.n_heads":24,"cfg.d_model":1536,"cfg.d_vocab":50278,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel-solu-old","cfg.normalization_type":"LN","config.raw__":{"d_model":1536,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":24,"d_mlp":6144,"d_vocab":50278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_12L_v23_old","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel-solu-old","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0204124145,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50278,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":339738624,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1536\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 24\nd_mlp: 6144\nd_vocab: 50278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_12L_v23_old\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel-solu-old\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.020412414523193152\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50278\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 339738624\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50278, 1536)\npos_embed:\n W_pos: (1024, 1536)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (1536,)\n ln2:\n '[w, b]': (1536,)\n attn:\n '[W_Q, W_K, W_V]': (24, 1536, 64)\n W_O: (24, 64, 1536)\n '[b_Q, b_K, b_V]': (24, 64)\n b_O: (1536,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (6144,)\n W_in: (1536, 6144)\n b_in: (6144,)\n W_out: (6144, 1536)\n b_out: (1536,)\nln_final:\n '[w, b]': (1536,)\nunembed:\n W_U: (1536, 50278)\n b_U: (50278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50278, 1536)"},"pos_embed":{"W_pos":"(1024, 1536)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(1536,)"},"ln2":{"[w, b]":"(1536,)"},"attn":{"[W_Q, W_K, W_V]":"(24, 1536, 64)","W_O":"(24, 64, 1536)","[b_Q, b_K, b_V]":"(24, 64)","b_O":"(1536,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(6144,)"},"W_in":"(1536, 6144)","b_in":"(6144,)","W_out":"(6144, 1536)","b_out":"(1536,)"}}},"ln_final":{"[w, b]":"(1536,)"},"unembed":{"W_U":"(1536, 50278)","b_U":"(50278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 24, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 24, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 6144)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1536)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\nunembed:\n hook_in: (batch, seq_len, 1536)\n hook_out: (batch, seq_len, 50278)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1536)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 24, 64)","[hook_attn_scores, hook_pattern]":"(batch, 24, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 6144)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1536)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"unembed":{"hook_in":"(batch, seq_len, 1536)","hook_out":"(batch, seq_len, 50278)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1536)"}} +{"name.default_alias":"solu-1l","name.huggingface":"NeelNanda\/SoLU_1L512W_C4_Code","name.aliases":"solu-1l, solu-1l-new, solu-1l-c4-code","model_type":"solu","name.from_cfg":"SoLU_1L512W_C4_Code","n_params.as_str":"3.1M","n_params.as_int":3145728,"n_params.from_name":null,"cfg.n_params":3145728,"cfg.n_layers":1,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":1,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_1L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":3145728,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 1\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_1L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 3145728\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '0':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (2048,)\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"0":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(2048,)"},"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '0':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"0":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"solu-1l-wiki","name.huggingface":"NeelNanda\/SoLU_1L512W_Wiki_Finetune","name.aliases":"solu-1l-wiki, solu-1l-wiki-finetune, solu-1l-finetune","model_type":"solu","name.from_cfg":"SoLU_1L512W_Wiki_Finetune","n_params.as_str":"3.1M","n_params.as_int":3145728,"n_params.from_name":null,"cfg.n_params":3145728,"cfg.n_layers":1,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":1,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_1L512W_Wiki_Finetune","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":3145728,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 1\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_1L512W_Wiki_Finetune\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 3145728\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '0':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (2048,)\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"0":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(2048,)"},"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '0':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"0":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"solu-1l-pile","name.huggingface":"NeelNanda\/SoLU_1L_v9_old","name.aliases":"solu-1l-pile, solu-1l-old","model_type":"solu","name.from_cfg":"SoLU_1L_v9_old","n_params.as_str":"13M","n_params.as_int":12582912,"n_params.from_name":null,"cfg.n_params":12582912,"cfg.n_layers":1,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50278,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel-solu-old","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":1,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"SoLU_1L_v9_old","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel-solu-old","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50278,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 1\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: SoLU_1L_v9_old\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel-solu-old\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50278\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50278, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '0':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (4096,)\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n w: (1024,)\nunembed:\n W_U: (1024, 50278)\n b_U: (50278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50278, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"0":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(4096,)"},"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"w":"(1024,)"},"unembed":{"W_U":"(1024, 50278)","b_U":"(50278,)"}},"tensor_shapes.activation_cache":"blocks:\n '0':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50278)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"0":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50278)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"solu-2l","name.huggingface":"NeelNanda\/SoLU_2L512W_C4_Code","name.aliases":"solu-2l, solu-2l-new, solu-2l-c4-code","model_type":"solu","name.from_cfg":"SoLU_2L512W_C4_Code","n_params.as_str":"6.3M","n_params.as_int":6291456,"n_params.from_name":null,"cfg.n_params":6291456,"cfg.n_layers":2,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":2,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_2L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":6291456,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 2\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_2L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 6291456\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-1]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (2048,)\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-1]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(2048,)"},"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-1]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-1]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"solu-2l-pile","name.huggingface":"NeelNanda\/SoLU_2L_v10_old","name.aliases":"solu-2l-pile, solu-2l-old","model_type":"solu","name.from_cfg":"SoLU_2L_v10_old","n_params.as_str":"13M","n_params.as_int":12812288,"n_params.from_name":null,"cfg.n_params":12812288,"cfg.n_layers":2,"cfg.n_heads":11,"cfg.d_model":736,"cfg.d_vocab":50278,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel-solu-old","cfg.normalization_type":"LNPre","config.raw__":{"d_model":736,"d_head":64,"n_layers":2,"n_ctx":1024,"n_heads":11,"d_mlp":2944,"d_vocab":50278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LNPre","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"SoLU_2L_v10_old","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel-solu-old","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0294883912,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50278,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12812288,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 736\nd_head: 64\nn_layers: 2\nn_ctx: 1024\nn_heads: 11\nd_mlp: 2944\nd_vocab: 50278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LNPre\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: SoLU_2L_v10_old\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel-solu-old\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02948839123097943\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50278\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12812288\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50278, 736)\npos_embed:\n W_pos: (1024, 736)\nblocks:\n '[0-1]':\n attn:\n '[W_Q, W_K, W_V]': (11, 736, 64)\n W_O: (11, 64, 736)\n '[b_Q, b_K, b_V]': (11, 64)\n b_O: (736,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (736, 2944)\n b_in: (2944,)\n W_out: (2944, 736)\n b_out: (736,)\nunembed:\n W_U: (736, 50278)\n b_U: (50278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50278, 736)"},"pos_embed":{"W_pos":"(1024, 736)"},"blocks":{"[0-1]":{"attn":{"[W_Q, W_K, W_V]":"(11, 736, 64)","W_O":"(11, 64, 736)","[b_Q, b_K, b_V]":"(11, 64)","b_O":"(736,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(736, 2944)","b_in":"(2944,)","W_out":"(2944, 736)","b_out":"(736,)"}}},"unembed":{"W_U":"(736, 50278)","b_U":"(50278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-1]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 736)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 11, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 11, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 736)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2944)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2944)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 736)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 736)\nunembed:\n hook_in: (batch, seq_len, 736)\n hook_out: (batch, seq_len, 50278)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 736)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-1]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 736)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 11, 64)","[hook_attn_scores, hook_pattern]":"(batch, 11, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 736)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2944)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2944)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 736)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 736)"},"unembed":{"hook_in":"(batch, seq_len, 736)","hook_out":"(batch, seq_len, 50278)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 736)"}} +{"name.default_alias":"solu-3l","name.huggingface":"NeelNanda\/SoLU_3L512W_C4_Code","name.aliases":"solu-3l, solu-3l-new, solu-3l-c4-code","model_type":"solu","name.from_cfg":"SoLU_3L512W_C4_Code","n_params.as_str":"9.4M","n_params.as_int":9437184,"n_params.from_name":null,"cfg.n_params":9437184,"cfg.n_layers":3,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":3,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_3L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":9437184,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 3\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_3L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 9437184\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-2]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (2048,)\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-2]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(2048,)"},"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-2]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-2]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"solu-4l","name.huggingface":"NeelNanda\/SoLU_4L512W_C4_Code","name.aliases":"solu-4l, solu-4l-new, solu-4l-c4-code","model_type":"solu","name.from_cfg":"SoLU_4L512W_C4_Code","n_params.as_str":"13M","n_params.as_int":12582912,"n_params.from_name":null,"cfg.n_params":12582912,"cfg.n_layers":4,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":4,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_4L512W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 4\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_4L512W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-3]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (2048,)\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-3]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(2048,)"},"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-3]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-3]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"solu-4l-wiki","name.huggingface":"NeelNanda\/SoLU_4L512W_Wiki_Finetune","name.aliases":"solu-4l-wiki, solu-4l-wiki-finetune, solu-4l-finetune","model_type":"solu","name.from_cfg":"SoLU_4L512W_Wiki_Finetune","n_params.as_str":"13M","n_params.as_int":12582912,"n_params.from_name":null,"cfg.n_params":12582912,"cfg.n_layers":4,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":64,"n_layers":4,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_4L512W_Wiki_Finetune","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 4\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_4L512W_Wiki_Finetune\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-3]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (2048,)\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-3]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(2048,)"},"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-3]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-3]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"solu-4l-pile","name.huggingface":"NeelNanda\/SoLU_4L_v11_old","name.aliases":"solu-4l-pile, solu-4l-old","model_type":"solu","name.from_cfg":"SoLU_4L_v11_old","n_params.as_str":"13M","n_params.as_int":12582912,"n_params.from_name":null,"cfg.n_params":12582912,"cfg.n_layers":4,"cfg.n_heads":8,"cfg.d_model":512,"cfg.d_vocab":50278,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel-solu-old","cfg.normalization_type":"LNPre","config.raw__":{"d_model":512,"d_head":64,"n_layers":4,"n_ctx":1024,"n_heads":8,"d_mlp":2048,"d_vocab":50278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LNPre","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"SoLU_4L_v11_old","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel-solu-old","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50278,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 64\nn_layers: 4\nn_ctx: 1024\nn_heads: 8\nd_mlp: 2048\nd_vocab: 50278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LNPre\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: SoLU_4L_v11_old\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel-solu-old\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50278\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50278, 512)\npos_embed:\n W_pos: (1024, 512)\nblocks:\n '[0-3]':\n attn:\n '[W_Q, W_K, W_V]': (8, 512, 64)\n W_O: (8, 64, 512)\n '[b_Q, b_K, b_V]': (8, 64)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nunembed:\n W_U: (512, 50278)\n b_U: (50278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50278, 512)"},"pos_embed":{"W_pos":"(1024, 512)"},"blocks":{"[0-3]":{"attn":{"[W_Q, W_K, W_V]":"(8, 512, 64)","W_O":"(8, 64, 512)","[b_Q, b_K, b_V]":"(8, 64)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"unembed":{"W_U":"(512, 50278)","b_U":"(50278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-3]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 8, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 8, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50278)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-3]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 8, 64)","[hook_attn_scores, hook_pattern]":"(batch, 8, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50278)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"solu-6l","name.huggingface":"NeelNanda\/SoLU_6L768W_C4_Code","name.aliases":"solu-6l, solu-6l-new, solu-6l-c4-code","model_type":"solu","name.from_cfg":"SoLU_6L768W_C4_Code","n_params.as_str":"42M","n_params.as_int":42467328,"n_params.from_name":null,"cfg.n_params":42467328,"cfg.n_layers":6,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":6,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_6L768W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":42467328,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 6\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_6L768W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 42467328\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-5]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (3072,)\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-5]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(3072,)"},"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"solu-6l-pile","name.huggingface":"NeelNanda\/SoLU_6L_v13_old","name.aliases":"solu-6l-pile, solu-6l-old","model_type":"solu","name.from_cfg":"SoLU_6L_v13_old","n_params.as_str":"42M","n_params.as_int":42467328,"n_params.from_name":null,"cfg.n_params":42467328,"cfg.n_layers":6,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50278,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel-solu-old","cfg.normalization_type":"LNPre","config.raw__":{"d_model":768,"d_head":64,"n_layers":6,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LNPre","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"SoLU_6L_v13_old","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel-solu-old","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50278,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":42467328,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 6\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LNPre\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: SoLU_6L_v13_old\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel-solu-old\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50278\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 42467328\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50278, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-5]':\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nunembed:\n W_U: (768, 50278)\n b_U: (50278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50278, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-5]":{"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"unembed":{"W_U":"(768, 50278)","b_U":"(50278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-5]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3072)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50278)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-5]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3072)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50278)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"solu-8l","name.huggingface":"NeelNanda\/SoLU_8L1024W_C4_Code","name.aliases":"solu-8l, solu-8l-new, solu-8l-c4-code","model_type":"solu","name.from_cfg":"SoLU_8L1024W_C4_Code","n_params.as_str":"101M","n_params.as_int":100663296,"n_params.from_name":null,"cfg.n_params":100663296,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":48262,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":8,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":48262,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_8L1024W_C4_Code","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"NeelNanda\/gpt-neox-tokenizer-digits","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":48262,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":100663296,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 8\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 48262\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_8L1024W_C4_Code\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: NeelNanda\/gpt-neox-tokenizer-digits\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 48262\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 100663296\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"NeelNanda\/gpt-neox-tokenizer-digits","tokenizer.vocab_size":48262.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"AsGo9tS8Sq4-rlVHM2o3-GyDkJU=","tensor_shapes.state_dict":"embed:\n W_E: (48262, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n ln:\n '[w, b]': (4096,)\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 48262)\n b_U: (48262,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(48262, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"ln":{"[w, b]":"(4096,)"},"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 48262)","b_U":"(48262,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 48262)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 48262)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"solu-8l-pile","name.huggingface":"NeelNanda\/SoLU_8L_v21_old","name.aliases":"solu-8l-pile, solu-8l-old","model_type":"solu","name.from_cfg":"SoLU_8L_v21_old","n_params.as_str":"101M","n_params.as_int":100663296,"n_params.from_name":null,"cfg.n_params":100663296,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50278,"cfg.act_fn":"solu_ln","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"neel-solu-old","cfg.normalization_type":"LNPre","config.raw__":{"d_model":1024,"d_head":64,"n_layers":8,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50278,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"solu_ln","normalization_type":"LNPre","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"SoLU_8L_v21_old","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"neel-solu-old","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"EleutherAI\/gpt-neox-20b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50278,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":100663296,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 8\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50278\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: solu_ln\nnormalization_type: LNPre\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: SoLU_8L_v21_old\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: neel-solu-old\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: EleutherAI\/gpt-neox-20b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50278\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 100663296\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"EleutherAI\/gpt-neox-20b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50278, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-7]':\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nunembed:\n W_U: (1024, 50278)\n b_U: (50278,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50278, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-7]":{"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"unembed":{"W_U":"(1024, 50278)","b_U":"(50278,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n ln:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n '[hook_pre, hook_mid, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50278)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"ln":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"[hook_pre, hook_mid, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50278)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"gpt-oss-20b","name.huggingface":"openai\/gpt-oss-20b","name.aliases":"gpt-oss-20b, gpt-oss","model_type":null,"name.from_cfg":"gpt-oss-20b","n_params.as_str":"20B","n_params.as_int":20244971520,"n_params.from_name":"20b","cfg.n_params":20244971520,"cfg.n_layers":24,"cfg.n_heads":64,"cfg.d_model":2880,"cfg.d_vocab":201088,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GptOssForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2880,"d_head":64,"n_layers":24,"n_ctx":131072,"n_heads":64,"d_mlp":2880,"d_vocab":201088,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":32,"experts_per_token":4,"final_rms":true,"dtype":"torch.float32","model_name":"gpt-oss-20b","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GptOssForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"openai\/gpt-oss-20b","window_size":128,"attn_types":["local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global","local","global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0149071198,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":201088,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":20244971520,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":150000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":true,"yarn_factor":32.0,"yarn_attention_factor":1.3465735903,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":false,"yarn_global_attn_only":false,"use_attention_sinks":true,"norm_topk_prob":false},"config":"d_model: 2880\nd_head: 64\nn_layers: 24\nn_ctx: 131072\nn_heads: 64\nd_mlp: 2880\nd_vocab: 201088\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: 32\nexperts_per_token: 4\nfinal_rms: true\ndtype: torch.float32\nmodel_name: gpt-oss-20b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GptOssForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: openai\/gpt-oss-20b\nwindow_size: 128\nattn_types:\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.014907119849998597\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 201088\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 20244971520\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 150000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: true\nyarn_factor: 32.0\nyarn_attention_factor: 1.3465735902799727\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: false\nyarn_global_attn_only: false\nuse_attention_sinks: true\nnorm_topk_prob: false\n","tokenizer.name":"openai\/gpt-oss-20b","tokenizer.vocab_size":199998.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"c9AMJHU0SnHAXwla61AkYA78n-w=","tensor_shapes.state_dict":"embed:\n W_E: (201088, 2880)\nblocks:\n '[0-23]':\n ln1:\n w: (2880,)\n ln2:\n w: (2880,)\n attn:\n W_Q: (64, 2880, 64)\n W_O: (64, 64, 2880)\n b_Q: (64, 64)\n b_O: (2880,)\n sinks: (64,)\n '[_W_K, _W_V]': (8, 2880, 64)\n '[_b_K, _b_V]': (8, 64)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n experts:\n '[0-31]':\n W_gate:\n weight: (2880, 2880)\n bias: (2880,)\n W_in:\n weight: (2880, 2880)\n bias: (2880,)\n W_out:\n weight: (2880, 2880)\n bias: (2880,)\n W_gate:\n weight: (32, 2880)\n bias: (32,)\nln_final:\n w: (2880,)\nunembed:\n W_U: (2880, 201088)\n b_U: (201088,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(201088, 2880)"},"blocks":{"[0-23]":{"ln1":{"w":"(2880,)"},"ln2":{"w":"(2880,)"},"attn":{"W_Q":"(64, 2880, 64)","W_O":"(64, 64, 2880)","b_Q":"(64, 64)","b_O":"(2880,)","sinks":"(64,)","[_W_K, _W_V]":"(8, 2880, 64)","[_b_K, _b_V]":"(8, 64)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"experts":{"[0-31]":{"W_gate":{"weight":"(2880, 2880)","bias":"(2880,)"},"W_in":{"weight":"(2880, 2880)","bias":"(2880,)"},"W_out":{"weight":"(2880, 2880)","bias":"(2880,)"}}},"W_gate":{"weight":"(32, 2880)","bias":"(32,)"}}}},"ln_final":{"w":"(2880,)"},"unembed":{"W_U":"(2880, 201088)","b_U":"(201088,)"}},"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"qwen-14b","name.huggingface":"Qwen\/Qwen-14B","name.aliases":"qwen-14b","model_type":"qwen","name.from_cfg":"Qwen-14B","n_params.as_str":"13B","n_params.as_int":12609126400,"n_params.from_name":"14b","cfg.n_params":12609126400,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"QWenLMHeadModel","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":40,"d_mlp":13696,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen-14B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"QWenLMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen-14B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12609126400,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 40\nd_mlp: 13696\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen-14B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: QWenLMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen-14B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12609126400\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"qwen-14b-chat","name.huggingface":"Qwen\/Qwen-14B-Chat","name.aliases":"qwen-14b-chat","model_type":"qwen","name.from_cfg":"Qwen-14B-Chat","n_params.as_str":"13B","n_params.as_int":12609126400,"n_params.from_name":"14b","cfg.n_params":12609126400,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"QWenLMHeadModel","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":40,"d_mlp":13696,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen-14B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"QWenLMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen-14B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12609126400,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 40\nd_mlp: 13696\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen-14B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: QWenLMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen-14B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12609126400\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"qwen-1.8b","name.huggingface":"Qwen\/Qwen-1_8B","name.aliases":"qwen-1.8b","model_type":"qwen","name.from_cfg":"Qwen-1_8B","n_params.as_str":"1.2B","n_params.as_int":1214251008,"n_params.from_name":"1.8b","cfg.n_params":1214251008,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"QWenLMHeadModel","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":8192,"n_heads":16,"d_mlp":5504,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen-1_8B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"QWenLMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen-1_8B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1214251008,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 8192\nn_heads: 16\nd_mlp: 5504\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen-1_8B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: QWenLMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen-1_8B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1214251008\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"qwen-7b","name.huggingface":"Qwen\/Qwen-7B","name.aliases":"qwen-7b","model_type":"qwen","name.from_cfg":"Qwen-7B","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"QWenLMHeadModel","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":8192,"n_heads":32,"d_mlp":11008,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen-7B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"QWenLMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen-7B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 8192\nn_heads: 32\nd_mlp: 11008\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen-7B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: QWenLMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen-7B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"qwen-7b-chat","name.huggingface":"Qwen\/Qwen-7B-Chat","name.aliases":"qwen-7b-chat","model_type":"qwen","name.from_cfg":"Qwen-7B-Chat","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"QWenLMHeadModel","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":8192,"n_heads":32,"d_mlp":11008,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen-7B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"QWenLMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen-7B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 8192\nn_heads: 32\nd_mlp: 11008\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen-7B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: QWenLMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen-7B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":null,"tokenizer.vocab_size":null,"tokenizer.max_len":null,"tokenizer.class":null,"tokenizer.vocab_hash":null,"tensor_shapes.state_dict":null,"tensor_shapes.state_dict.raw__":null,"tensor_shapes.activation_cache":null,"tensor_shapes.activation_cache.raw__":null} +{"name.default_alias":"qwen1.5-0.5b","name.huggingface":"Qwen\/Qwen1.5-0.5B","name.aliases":"qwen1.5-0.5b","model_type":"qwen","name.from_cfg":"Qwen1.5-0.5B","n_params.as_str":"308M","n_params.as_int":308281344,"n_params.from_name":"0.5b","cfg.n_params":308281344,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":32768,"n_heads":16,"d_mlp":2816,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-0.5B","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-0.5B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":308281344,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 32768\nn_heads: 16\nd_mlp: 2816\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-0.5B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-0.5B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 308281344\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen1.5-0.5B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 1024)\nblocks:\n '[0-23]':\n ln1:\n w: (1024,)\n ln2:\n w: (1024,)\n attn:\n '[W_Q, _W_K, _W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, _b_K, _b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n '[W_in, W_gate]': (1024, 2816)\n W_out: (2816, 1024)\n b_in: (2816,)\n b_out: (1024,)\nln_final:\n w: (1024,)\nunembed:\n W_U: (1024, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 1024)"},"blocks":{"[0-23]":{"ln1":{"w":"(1024,)"},"ln2":{"w":"(1024,)"},"attn":{"[W_Q, _W_K, _W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, _b_K, _b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"[W_in, W_gate]":"(1024, 2816)","W_out":"(2816, 1024)","b_in":"(2816,)","b_out":"(1024,)"}}},"ln_final":{"w":"(1024,)"},"unembed":{"W_U":"(1024, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 2816)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 2816)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 1024)"}} +{"name.default_alias":"qwen1.5-0.5b-chat","name.huggingface":"Qwen\/Qwen1.5-0.5B-Chat","name.aliases":"qwen1.5-0.5b-chat","model_type":"qwen","name.from_cfg":"Qwen1.5-0.5B-Chat","n_params.as_str":"308M","n_params.as_int":308281344,"n_params.from_name":"0.5b","cfg.n_params":308281344,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":32768,"n_heads":16,"d_mlp":2816,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-0.5B-Chat","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-0.5B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":308281344,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 32768\nn_heads: 16\nd_mlp: 2816\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-0.5B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-0.5B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 308281344\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen1.5-0.5B-Chat","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 1024)\nblocks:\n '[0-23]':\n ln1:\n w: (1024,)\n ln2:\n w: (1024,)\n attn:\n '[W_Q, _W_K, _W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, _b_K, _b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n '[W_in, W_gate]': (1024, 2816)\n W_out: (2816, 1024)\n b_in: (2816,)\n b_out: (1024,)\nln_final:\n w: (1024,)\nunembed:\n W_U: (1024, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 1024)"},"blocks":{"[0-23]":{"ln1":{"w":"(1024,)"},"ln2":{"w":"(1024,)"},"attn":{"[W_Q, _W_K, _W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, _b_K, _b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"[W_in, W_gate]":"(1024, 2816)","W_out":"(2816, 1024)","b_in":"(2816,)","b_out":"(1024,)"}}},"ln_final":{"w":"(1024,)"},"unembed":{"W_U":"(1024, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 2816)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 2816)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 1024)"}} +{"name.default_alias":"qwen1.5-1.8b","name.huggingface":"Qwen\/Qwen1.5-1.8B","name.aliases":"qwen1.5-1.8b","model_type":"qwen","name.from_cfg":"Qwen1.5-1.8B","n_params.as_str":"1.2B","n_params.as_int":1214251008,"n_params.from_name":"1.8b","cfg.n_params":1214251008,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":32768,"n_heads":16,"d_mlp":5504,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-1.8B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-1.8B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1214251008,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 32768\nn_heads: 16\nd_mlp: 5504\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-1.8B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-1.8B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1214251008\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen1.5-1.8B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 2048)\nblocks:\n '[0-23]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n '[W_Q, _W_K, _W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, _b_K, _b_V]': (16, 128)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (2048, 5504)\n W_out: (5504, 2048)\n b_in: (5504,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 2048)"},"blocks":{"[0-23]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"[W_Q, _W_K, _W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, _b_K, _b_V]":"(16, 128)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(2048, 5504)","W_out":"(5504, 2048)","b_in":"(5504,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 5504)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 5504)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"qwen1.5-1.8b-chat","name.huggingface":"Qwen\/Qwen1.5-1.8B-Chat","name.aliases":"qwen1.5-1.8b-chat","model_type":"qwen","name.from_cfg":"Qwen1.5-1.8B-Chat","n_params.as_str":"1.2B","n_params.as_int":1214251008,"n_params.from_name":"1.8b","cfg.n_params":1214251008,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":24,"n_ctx":32768,"n_heads":16,"d_mlp":5504,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":16,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-1.8B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-1.8B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1214251008,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 24\nn_ctx: 32768\nn_heads: 16\nd_mlp: 5504\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 16\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-1.8B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-1.8B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1214251008\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen1.5-1.8B-Chat","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 2048)\nblocks:\n '[0-23]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n '[W_Q, _W_K, _W_V]': (16, 2048, 128)\n W_O: (16, 128, 2048)\n '[b_Q, _b_K, _b_V]': (16, 128)\n b_O: (2048,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (2048, 5504)\n W_out: (5504, 2048)\n b_in: (5504,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 2048)"},"blocks":{"[0-23]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"[W_Q, _W_K, _W_V]":"(16, 2048, 128)","W_O":"(16, 128, 2048)","[b_Q, _b_K, _b_V]":"(16, 128)","b_O":"(2048,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(2048, 5504)","W_out":"(5504, 2048)","b_in":"(5504,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 16, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 5504)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 16, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 5504)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"qwen1.5-14b","name.huggingface":"Qwen\/Qwen1.5-14B","name.aliases":"qwen1.5-14b","model_type":"qwen","name.from_cfg":"Qwen1.5-14B","n_params.as_str":"13B","n_params.as_int":12609126400,"n_params.from_name":"14b","cfg.n_params":12609126400,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":32768,"n_heads":40,"d_mlp":13696,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":40,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-14B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-14B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12609126400,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 32768\nn_heads: 40\nd_mlp: 13696\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 40\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-14B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-14B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12609126400\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen1.5-14B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (152064, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n '[W_Q, _W_K, _W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, _b_K, _b_V]': (40, 128)\n b_O: (5120,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 13696)\n W_out: (13696, 5120)\n b_in: (13696,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 152064)\n b_U: (152064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(152064, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"[W_Q, _W_K, _W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, _b_K, _b_V]":"(40, 128)","b_O":"(5120,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 13696)","W_out":"(13696, 5120)","b_in":"(13696,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 152064)","b_U":"(152064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 13696)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 152064)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 13696)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 152064)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"qwen1.5-14b-chat","name.huggingface":"Qwen\/Qwen1.5-14B-Chat","name.aliases":"qwen1.5-14b-chat","model_type":"qwen","name.from_cfg":"Qwen1.5-14B-Chat","n_params.as_str":"13B","n_params.as_int":12609126400,"n_params.from_name":"14b","cfg.n_params":12609126400,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":32768,"n_heads":40,"d_mlp":13696,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":40,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-14B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-14B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":12609126400,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 32768\nn_heads: 40\nd_mlp: 13696\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 40\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-14B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-14B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 12609126400\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen1.5-14B-Chat","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (152064, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n '[W_Q, _W_K, _W_V]': (40, 5120, 128)\n W_O: (40, 128, 5120)\n '[b_Q, _b_K, _b_V]': (40, 128)\n b_O: (5120,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 13696)\n W_out: (13696, 5120)\n b_in: (13696,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 152064)\n b_U: (152064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(152064, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"[W_Q, _W_K, _W_V]":"(40, 5120, 128)","W_O":"(40, 128, 5120)","[b_Q, _b_K, _b_V]":"(40, 128)","b_O":"(5120,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 13696)","W_out":"(13696, 5120)","b_in":"(13696,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 152064)","b_U":"(152064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 40, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 13696)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 152064)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 40, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 13696)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 152064)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"qwen1.5-4b","name.huggingface":"Qwen\/Qwen1.5-4B","name.aliases":"qwen1.5-4b","model_type":"qwen","name.from_cfg":"Qwen1.5-4B","n_params.as_str":"3.2B","n_params.as_int":3171942400,"n_params.from_name":"4b","cfg.n_params":3171942400,"cfg.n_layers":40,"cfg.n_heads":20,"cfg.d_model":2560,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2560,"d_head":128,"n_layers":40,"n_ctx":32768,"n_heads":20,"d_mlp":6912,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":20,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-4B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-4B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":3171942400,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":5000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 128\nn_layers: 40\nn_ctx: 32768\nn_heads: 20\nd_mlp: 6912\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 20\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-4B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-4B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 3171942400\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 5000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen1.5-4B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 2560)\nblocks:\n '[0-39]':\n ln1:\n w: (2560,)\n ln2:\n w: (2560,)\n attn:\n '[W_Q, _W_K, _W_V]': (20, 2560, 128)\n W_O: (20, 128, 2560)\n '[b_Q, _b_K, _b_V]': (20, 128)\n b_O: (2560,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (2560, 6912)\n W_out: (6912, 2560)\n b_in: (6912,)\n b_out: (2560,)\nln_final:\n w: (2560,)\nunembed:\n W_U: (2560, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 2560)"},"blocks":{"[0-39]":{"ln1":{"w":"(2560,)"},"ln2":{"w":"(2560,)"},"attn":{"[W_Q, _W_K, _W_V]":"(20, 2560, 128)","W_O":"(20, 128, 2560)","[b_Q, _b_K, _b_V]":"(20, 128)","b_O":"(2560,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(2560, 6912)","W_out":"(6912, 2560)","b_in":"(6912,)","b_out":"(2560,)"}}},"ln_final":{"w":"(2560,)"},"unembed":{"W_U":"(2560, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 20, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 20, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 6912)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 20, 128)","[hook_attn_scores, hook_pattern]":"(batch, 20, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 6912)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 2560)"}} +{"name.default_alias":"qwen1.5-4b-chat","name.huggingface":"Qwen\/Qwen1.5-4B-Chat","name.aliases":"qwen1.5-4b-chat","model_type":"qwen","name.from_cfg":"Qwen1.5-4B-Chat","n_params.as_str":"3.2B","n_params.as_int":3171942400,"n_params.from_name":"4b","cfg.n_params":3171942400,"cfg.n_layers":40,"cfg.n_heads":20,"cfg.d_model":2560,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2560,"d_head":128,"n_layers":40,"n_ctx":32768,"n_heads":20,"d_mlp":6912,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":20,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-4B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-4B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":3171942400,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":5000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 128\nn_layers: 40\nn_ctx: 32768\nn_heads: 20\nd_mlp: 6912\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 20\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-4B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-4B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 3171942400\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 5000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen1.5-4B-Chat","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 2560)\nblocks:\n '[0-39]':\n ln1:\n w: (2560,)\n ln2:\n w: (2560,)\n attn:\n '[W_Q, _W_K, _W_V]': (20, 2560, 128)\n W_O: (20, 128, 2560)\n '[b_Q, _b_K, _b_V]': (20, 128)\n b_O: (2560,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (2560, 6912)\n W_out: (6912, 2560)\n b_in: (6912,)\n b_out: (2560,)\nln_final:\n w: (2560,)\nunembed:\n W_U: (2560, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 2560)"},"blocks":{"[0-39]":{"ln1":{"w":"(2560,)"},"ln2":{"w":"(2560,)"},"attn":{"[W_Q, _W_K, _W_V]":"(20, 2560, 128)","W_O":"(20, 128, 2560)","[b_Q, _b_K, _b_V]":"(20, 128)","b_O":"(2560,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(2560, 6912)","W_out":"(6912, 2560)","b_in":"(6912,)","b_out":"(2560,)"}}},"ln_final":{"w":"(2560,)"},"unembed":{"W_U":"(2560, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 20, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 20, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 6912)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 20, 128)","[hook_attn_scores, hook_pattern]":"(batch, 20, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 6912)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 2560)"}} +{"name.default_alias":"qwen1.5-7b","name.huggingface":"Qwen\/Qwen1.5-7B","name.aliases":"qwen1.5-7b","model_type":"qwen","name.from_cfg":"Qwen1.5-7B","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":32768,"n_heads":32,"d_mlp":11008,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":32,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-7B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-7B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 32768\nn_heads: 32\nd_mlp: 11008\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 32\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-7B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-7B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen1.5-7B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n '[W_Q, _W_K, _W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, _b_K, _b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"[W_Q, _W_K, _W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, _b_K, _b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"qwen1.5-7b-chat","name.huggingface":"Qwen\/Qwen1.5-7B-Chat","name.aliases":"qwen1.5-7b-chat","model_type":"qwen","name.from_cfg":"Qwen1.5-7B-Chat","n_params.as_str":"6.5B","n_params.as_int":6476005376,"n_params.from_name":"7b","cfg.n_params":6476005376,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":32768,"n_heads":32,"d_mlp":11008,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":32,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen1.5-7B-Chat","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen1.5-7B-Chat","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":6476005376,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 32768\nn_heads: 32\nd_mlp: 11008\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 32\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen1.5-7B-Chat\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen1.5-7B-Chat\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 6476005376\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen1.5-7B-Chat","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n '[W_Q, _W_K, _W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, _b_K, _b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 11008)\n W_out: (11008, 4096)\n b_in: (11008,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"[W_Q, _W_K, _W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, _b_K, _b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 11008)","W_out":"(11008, 4096)","b_in":"(11008,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"qwen2-0.5b","name.huggingface":"Qwen\/Qwen2-0.5B","name.aliases":"qwen2-0.5b","model_type":"qwen","name.from_cfg":"Qwen2-0.5B","n_params.as_str":"391M","n_params.as_int":390856704,"n_params.from_name":"0.5b","cfg.n_params":390856704,"cfg.n_layers":24,"cfg.n_heads":14,"cfg.d_model":896,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":896,"d_head":64,"n_layers":24,"n_ctx":131072,"n_heads":14,"d_mlp":4864,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2-0.5B","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2-0.5B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":390856704,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 896\nd_head: 64\nn_layers: 24\nn_ctx: 131072\nn_heads: 14\nd_mlp: 4864\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2-0.5B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2-0.5B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 390856704\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2-0.5B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 896)\nblocks:\n '[0-23]':\n ln1:\n w: (896,)\n ln2:\n w: (896,)\n attn:\n W_Q: (14, 896, 64)\n W_O: (14, 64, 896)\n b_Q: (14, 64)\n b_O: (896,)\n '[_W_K, _W_V]': (2, 896, 64)\n '[_b_K, _b_V]': (2, 64)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n '[W_in, W_gate]': (896, 4864)\n W_out: (4864, 896)\n b_in: (4864,)\n b_out: (896,)\nln_final:\n w: (896,)\nunembed:\n W_U: (896, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 896)"},"blocks":{"[0-23]":{"ln1":{"w":"(896,)"},"ln2":{"w":"(896,)"},"attn":{"W_Q":"(14, 896, 64)","W_O":"(14, 64, 896)","b_Q":"(14, 64)","b_O":"(896,)","[_W_K, _W_V]":"(2, 896, 64)","[_b_K, _b_V]":"(2, 64)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"[W_in, W_gate]":"(896, 4864)","W_out":"(4864, 896)","b_in":"(4864,)","b_out":"(896,)"}}},"ln_final":{"w":"(896,)"},"unembed":{"W_U":"(896, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 896)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 14, 64)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 2, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 14, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 896)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 4864)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 896)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 896)\nunembed:\n hook_in: (batch, seq_len, 896)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 896)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 896)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 14, 64)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 2, 64)","[hook_attn_scores, hook_pattern]":"(batch, 14, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 896)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 4864)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 896)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 896)"},"unembed":{"hook_in":"(batch, seq_len, 896)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 896)"}} +{"name.default_alias":"qwen2-0.5b-instruct","name.huggingface":"Qwen\/Qwen2-0.5B-Instruct","name.aliases":"qwen2-0.5b-instruct","model_type":"qwen","name.from_cfg":"Qwen2-0.5B-Instruct","n_params.as_str":"391M","n_params.as_int":390856704,"n_params.from_name":"0.5b","cfg.n_params":390856704,"cfg.n_layers":24,"cfg.n_heads":14,"cfg.d_model":896,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":896,"d_head":64,"n_layers":24,"n_ctx":32768,"n_heads":14,"d_mlp":4864,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2-0.5B-Instruct","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2-0.5B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":390856704,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 896\nd_head: 64\nn_layers: 24\nn_ctx: 32768\nn_heads: 14\nd_mlp: 4864\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2-0.5B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2-0.5B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 390856704\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2-0.5B-Instruct","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 896)\nblocks:\n '[0-23]':\n ln1:\n w: (896,)\n ln2:\n w: (896,)\n attn:\n W_Q: (14, 896, 64)\n W_O: (14, 64, 896)\n b_Q: (14, 64)\n b_O: (896,)\n '[_W_K, _W_V]': (2, 896, 64)\n '[_b_K, _b_V]': (2, 64)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n '[W_in, W_gate]': (896, 4864)\n W_out: (4864, 896)\n b_in: (4864,)\n b_out: (896,)\nln_final:\n w: (896,)\nunembed:\n W_U: (896, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 896)"},"blocks":{"[0-23]":{"ln1":{"w":"(896,)"},"ln2":{"w":"(896,)"},"attn":{"W_Q":"(14, 896, 64)","W_O":"(14, 64, 896)","b_Q":"(14, 64)","b_O":"(896,)","[_W_K, _W_V]":"(2, 896, 64)","[_b_K, _b_V]":"(2, 64)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"[W_in, W_gate]":"(896, 4864)","W_out":"(4864, 896)","b_in":"(4864,)","b_out":"(896,)"}}},"ln_final":{"w":"(896,)"},"unembed":{"W_U":"(896, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 896)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 14, 64)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 2, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 14, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 896)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 4864)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 896)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 896)\nunembed:\n hook_in: (batch, seq_len, 896)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 896)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 896)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 14, 64)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 2, 64)","[hook_attn_scores, hook_pattern]":"(batch, 14, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 896)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 4864)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 896)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 896)"},"unembed":{"hook_in":"(batch, seq_len, 896)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 896)"}} +{"name.default_alias":"qwen2-1.5b","name.huggingface":"Qwen\/Qwen2-1.5B","name.aliases":"qwen2-1.5b","model_type":"qwen","name.from_cfg":"Qwen2-1.5B","n_params.as_str":"1.4B","n_params.as_int":1420296192,"n_params.from_name":"1.5b","cfg.n_params":1420296192,"cfg.n_layers":28,"cfg.n_heads":12,"cfg.d_model":1536,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1536,"d_head":128,"n_layers":28,"n_ctx":131072,"n_heads":12,"d_mlp":8960,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2-1.5B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2-1.5B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1420296192,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1536\nd_head: 128\nn_layers: 28\nn_ctx: 131072\nn_heads: 12\nd_mlp: 8960\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2-1.5B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2-1.5B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1420296192\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2-1.5B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 1536)\nblocks:\n '[0-27]':\n ln1:\n w: (1536,)\n ln2:\n w: (1536,)\n attn:\n W_Q: (12, 1536, 128)\n W_O: (12, 128, 1536)\n b_Q: (12, 128)\n b_O: (1536,)\n '[_W_K, _W_V]': (2, 1536, 128)\n '[_b_K, _b_V]': (2, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (1536, 8960)\n W_out: (8960, 1536)\n b_in: (8960,)\n b_out: (1536,)\nln_final:\n w: (1536,)\nunembed:\n W_U: (1536, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 1536)"},"blocks":{"[0-27]":{"ln1":{"w":"(1536,)"},"ln2":{"w":"(1536,)"},"attn":{"W_Q":"(12, 1536, 128)","W_O":"(12, 128, 1536)","b_Q":"(12, 128)","b_O":"(1536,)","[_W_K, _W_V]":"(2, 1536, 128)","[_b_K, _b_V]":"(2, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(1536, 8960)","W_out":"(8960, 1536)","b_in":"(8960,)","b_out":"(1536,)"}}},"ln_final":{"w":"(1536,)"},"unembed":{"W_U":"(1536, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 12, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 2, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8960)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1536)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\nunembed:\n hook_in: (batch, seq_len, 1536)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 1536)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 12, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 2, 128)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8960)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1536)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"unembed":{"hook_in":"(batch, seq_len, 1536)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 1536)"}} +{"name.default_alias":"qwen2-1.5b-instruct","name.huggingface":"Qwen\/Qwen2-1.5B-Instruct","name.aliases":"qwen2-1.5b-instruct","model_type":"qwen","name.from_cfg":"Qwen2-1.5B-Instruct","n_params.as_str":"1.4B","n_params.as_int":1420296192,"n_params.from_name":"1.5b","cfg.n_params":1420296192,"cfg.n_layers":28,"cfg.n_heads":12,"cfg.d_model":1536,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1536,"d_head":128,"n_layers":28,"n_ctx":32768,"n_heads":12,"d_mlp":8960,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2-1.5B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2-1.5B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1420296192,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1536\nd_head: 128\nn_layers: 28\nn_ctx: 32768\nn_heads: 12\nd_mlp: 8960\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2-1.5B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2-1.5B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1420296192\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2-1.5B-Instruct","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 1536)\nblocks:\n '[0-27]':\n ln1:\n w: (1536,)\n ln2:\n w: (1536,)\n attn:\n W_Q: (12, 1536, 128)\n W_O: (12, 128, 1536)\n b_Q: (12, 128)\n b_O: (1536,)\n '[_W_K, _W_V]': (2, 1536, 128)\n '[_b_K, _b_V]': (2, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (1536, 8960)\n W_out: (8960, 1536)\n b_in: (8960,)\n b_out: (1536,)\nln_final:\n w: (1536,)\nunembed:\n W_U: (1536, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 1536)"},"blocks":{"[0-27]":{"ln1":{"w":"(1536,)"},"ln2":{"w":"(1536,)"},"attn":{"W_Q":"(12, 1536, 128)","W_O":"(12, 128, 1536)","b_Q":"(12, 128)","b_O":"(1536,)","[_W_K, _W_V]":"(2, 1536, 128)","[_b_K, _b_V]":"(2, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(1536, 8960)","W_out":"(8960, 1536)","b_in":"(8960,)","b_out":"(1536,)"}}},"ln_final":{"w":"(1536,)"},"unembed":{"W_U":"(1536, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 12, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 2, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8960)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1536)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\nunembed:\n hook_in: (batch, seq_len, 1536)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 1536)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 12, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 2, 128)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8960)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1536)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"unembed":{"hook_in":"(batch, seq_len, 1536)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 1536)"}} +{"name.default_alias":"qwen2-7b","name.huggingface":"Qwen\/Qwen2-7B","name.aliases":"qwen2-7b","model_type":"qwen","name.from_cfg":"Qwen2-7B","n_params.as_str":"7.1B","n_params.as_int":7141851136,"n_params.from_name":"7b","cfg.n_params":7141851136,"cfg.n_layers":28,"cfg.n_heads":28,"cfg.d_model":3584,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3584,"d_head":128,"n_layers":28,"n_ctx":131072,"n_heads":28,"d_mlp":18944,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2-7B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2-7B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7141851136,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 3584\nd_head: 128\nn_layers: 28\nn_ctx: 131072\nn_heads: 28\nd_mlp: 18944\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2-7B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2-7B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7141851136\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2-7B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (152064, 3584)\nblocks:\n '[0-27]':\n ln1:\n w: (3584,)\n ln2:\n w: (3584,)\n attn:\n W_Q: (28, 3584, 128)\n W_O: (28, 128, 3584)\n b_Q: (28, 128)\n b_O: (3584,)\n '[_W_K, _W_V]': (4, 3584, 128)\n '[_b_K, _b_V]': (4, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (3584, 18944)\n W_out: (18944, 3584)\n b_in: (18944,)\n b_out: (3584,)\nln_final:\n w: (3584,)\nunembed:\n W_U: (3584, 152064)\n b_U: (152064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(152064, 3584)"},"blocks":{"[0-27]":{"ln1":{"w":"(3584,)"},"ln2":{"w":"(3584,)"},"attn":{"W_Q":"(28, 3584, 128)","W_O":"(28, 128, 3584)","b_Q":"(28, 128)","b_O":"(3584,)","[_W_K, _W_V]":"(4, 3584, 128)","[_b_K, _b_V]":"(4, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(3584, 18944)","W_out":"(18944, 3584)","b_in":"(18944,)","b_out":"(3584,)"}}},"ln_final":{"w":"(3584,)"},"unembed":{"W_U":"(3584, 152064)","b_U":"(152064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 28, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 28, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 18944)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3584)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\nunembed:\n hook_in: (batch, seq_len, 3584)\n hook_out: (batch, seq_len, 152064)\nhook_embed: (batch, seq_len, 3584)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 28, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 128)","[hook_attn_scores, hook_pattern]":"(batch, 28, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 18944)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3584)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"unembed":{"hook_in":"(batch, seq_len, 3584)","hook_out":"(batch, seq_len, 152064)"},"hook_embed":"(batch, seq_len, 3584)"}} +{"name.default_alias":"qwen2-7b-instruct","name.huggingface":"Qwen\/Qwen2-7B-Instruct","name.aliases":"qwen2-7b-instruct","model_type":"qwen","name.from_cfg":"Qwen2-7B-Instruct","n_params.as_str":"7.1B","n_params.as_int":7141851136,"n_params.from_name":"7b","cfg.n_params":7141851136,"cfg.n_layers":28,"cfg.n_heads":28,"cfg.d_model":3584,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3584,"d_head":128,"n_layers":28,"n_ctx":32768,"n_heads":28,"d_mlp":18944,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2-7B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2-7B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7141851136,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 3584\nd_head: 128\nn_layers: 28\nn_ctx: 32768\nn_heads: 28\nd_mlp: 18944\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2-7B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2-7B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7141851136\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2-7B-Instruct","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"vakQOjPaHpZ23kxcqX0tTXi2EzQ=","tensor_shapes.state_dict":"embed:\n W_E: (152064, 3584)\nblocks:\n '[0-27]':\n ln1:\n w: (3584,)\n ln2:\n w: (3584,)\n attn:\n W_Q: (28, 3584, 128)\n W_O: (28, 128, 3584)\n b_Q: (28, 128)\n b_O: (3584,)\n '[_W_K, _W_V]': (4, 3584, 128)\n '[_b_K, _b_V]': (4, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (3584, 18944)\n W_out: (18944, 3584)\n b_in: (18944,)\n b_out: (3584,)\nln_final:\n w: (3584,)\nunembed:\n W_U: (3584, 152064)\n b_U: (152064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(152064, 3584)"},"blocks":{"[0-27]":{"ln1":{"w":"(3584,)"},"ln2":{"w":"(3584,)"},"attn":{"W_Q":"(28, 3584, 128)","W_O":"(28, 128, 3584)","b_Q":"(28, 128)","b_O":"(3584,)","[_W_K, _W_V]":"(4, 3584, 128)","[_b_K, _b_V]":"(4, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(3584, 18944)","W_out":"(18944, 3584)","b_in":"(18944,)","b_out":"(3584,)"}}},"ln_final":{"w":"(3584,)"},"unembed":{"W_U":"(3584, 152064)","b_U":"(152064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 28, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 28, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 18944)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3584)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\nunembed:\n hook_in: (batch, seq_len, 3584)\n hook_out: (batch, seq_len, 152064)\nhook_embed: (batch, seq_len, 3584)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 28, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 128)","[hook_attn_scores, hook_pattern]":"(batch, 28, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 18944)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3584)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"unembed":{"hook_in":"(batch, seq_len, 3584)","hook_out":"(batch, seq_len, 152064)"},"hook_embed":"(batch, seq_len, 3584)"}} +{"name.default_alias":"qwen2.5-0.5b","name.huggingface":"Qwen\/Qwen2.5-0.5B","name.aliases":"qwen2.5-0.5b","model_type":"qwen","name.from_cfg":"Qwen2.5-0.5B","n_params.as_str":"391M","n_params.as_int":390856704,"n_params.from_name":"0.5b","cfg.n_params":390856704,"cfg.n_layers":24,"cfg.n_heads":14,"cfg.d_model":896,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":896,"d_head":64,"n_layers":24,"n_ctx":32768,"n_heads":14,"d_mlp":4864,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-0.5B","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-0.5B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":390856704,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 896\nd_head: 64\nn_layers: 24\nn_ctx: 32768\nn_heads: 14\nd_mlp: 4864\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-0.5B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-0.5B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 390856704\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2.5-0.5B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 896)\nblocks:\n '[0-23]':\n ln1:\n w: (896,)\n ln2:\n w: (896,)\n attn:\n W_Q: (14, 896, 64)\n W_O: (14, 64, 896)\n b_Q: (14, 64)\n b_O: (896,)\n '[_W_K, _W_V]': (2, 896, 64)\n '[_b_K, _b_V]': (2, 64)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n '[W_in, W_gate]': (896, 4864)\n W_out: (4864, 896)\n b_in: (4864,)\n b_out: (896,)\nln_final:\n w: (896,)\nunembed:\n W_U: (896, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 896)"},"blocks":{"[0-23]":{"ln1":{"w":"(896,)"},"ln2":{"w":"(896,)"},"attn":{"W_Q":"(14, 896, 64)","W_O":"(14, 64, 896)","b_Q":"(14, 64)","b_O":"(896,)","[_W_K, _W_V]":"(2, 896, 64)","[_b_K, _b_V]":"(2, 64)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"[W_in, W_gate]":"(896, 4864)","W_out":"(4864, 896)","b_in":"(4864,)","b_out":"(896,)"}}},"ln_final":{"w":"(896,)"},"unembed":{"W_U":"(896, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 896)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 14, 64)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 2, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 14, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 896)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 4864)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 896)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 896)\nunembed:\n hook_in: (batch, seq_len, 896)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 896)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 896)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 14, 64)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 2, 64)","[hook_attn_scores, hook_pattern]":"(batch, 14, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 896)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 4864)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 896)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 896)"},"unembed":{"hook_in":"(batch, seq_len, 896)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 896)"}} +{"name.default_alias":"qwen2.5-0.5b-instruct","name.huggingface":"Qwen\/Qwen2.5-0.5B-Instruct","name.aliases":"qwen2.5-0.5b-instruct","model_type":"qwen","name.from_cfg":"Qwen2.5-0.5B-Instruct","n_params.as_str":"391M","n_params.as_int":390856704,"n_params.from_name":"0.5b","cfg.n_params":390856704,"cfg.n_layers":24,"cfg.n_heads":14,"cfg.d_model":896,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":896,"d_head":64,"n_layers":24,"n_ctx":32768,"n_heads":14,"d_mlp":4864,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-0.5B-Instruct","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-0.5B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":64,"n_params":390856704,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 896\nd_head: 64\nn_layers: 24\nn_ctx: 32768\nn_heads: 14\nd_mlp: 4864\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-0.5B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-0.5B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 64\nn_params: 390856704\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2.5-0.5B-Instruct","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 896)\nblocks:\n '[0-23]':\n ln1:\n w: (896,)\n ln2:\n w: (896,)\n attn:\n W_Q: (14, 896, 64)\n W_O: (14, 64, 896)\n b_Q: (14, 64)\n b_O: (896,)\n '[_W_K, _W_V]': (2, 896, 64)\n '[_b_K, _b_V]': (2, 64)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 64)\n mlp:\n '[W_in, W_gate]': (896, 4864)\n W_out: (4864, 896)\n b_in: (4864,)\n b_out: (896,)\nln_final:\n w: (896,)\nunembed:\n W_U: (896, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 896)"},"blocks":{"[0-23]":{"ln1":{"w":"(896,)"},"ln2":{"w":"(896,)"},"attn":{"W_Q":"(14, 896, 64)","W_O":"(14, 64, 896)","b_Q":"(14, 64)","b_O":"(896,)","[_W_K, _W_V]":"(2, 896, 64)","[_b_K, _b_V]":"(2, 64)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 64)"},"mlp":{"[W_in, W_gate]":"(896, 4864)","W_out":"(4864, 896)","b_in":"(4864,)","b_out":"(896,)"}}},"ln_final":{"w":"(896,)"},"unembed":{"W_U":"(896, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 896)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 14, 64)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 2, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 14, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 896)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 4864)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 896)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 896)\nunembed:\n hook_in: (batch, seq_len, 896)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 896)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 896)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 14, 64)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 2, 64)","[hook_attn_scores, hook_pattern]":"(batch, 14, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 896)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 4864)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 896)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 896)"},"unembed":{"hook_in":"(batch, seq_len, 896)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 896)"}} +{"name.default_alias":"qwen2.5-1.5b","name.huggingface":"Qwen\/Qwen2.5-1.5B","name.aliases":"qwen2.5-1.5b","model_type":"qwen","name.from_cfg":"Qwen2.5-1.5B","n_params.as_str":"1.4B","n_params.as_int":1420296192,"n_params.from_name":"1.5b","cfg.n_params":1420296192,"cfg.n_layers":28,"cfg.n_heads":12,"cfg.d_model":1536,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1536,"d_head":128,"n_layers":28,"n_ctx":131072,"n_heads":12,"d_mlp":8960,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-1.5B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-1.5B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1420296192,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1536\nd_head: 128\nn_layers: 28\nn_ctx: 131072\nn_heads: 12\nd_mlp: 8960\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-1.5B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-1.5B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1420296192\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2.5-1.5B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 1536)\nblocks:\n '[0-27]':\n ln1:\n w: (1536,)\n ln2:\n w: (1536,)\n attn:\n W_Q: (12, 1536, 128)\n W_O: (12, 128, 1536)\n b_Q: (12, 128)\n b_O: (1536,)\n '[_W_K, _W_V]': (2, 1536, 128)\n '[_b_K, _b_V]': (2, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (1536, 8960)\n W_out: (8960, 1536)\n b_in: (8960,)\n b_out: (1536,)\nln_final:\n w: (1536,)\nunembed:\n W_U: (1536, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 1536)"},"blocks":{"[0-27]":{"ln1":{"w":"(1536,)"},"ln2":{"w":"(1536,)"},"attn":{"W_Q":"(12, 1536, 128)","W_O":"(12, 128, 1536)","b_Q":"(12, 128)","b_O":"(1536,)","[_W_K, _W_V]":"(2, 1536, 128)","[_b_K, _b_V]":"(2, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(1536, 8960)","W_out":"(8960, 1536)","b_in":"(8960,)","b_out":"(1536,)"}}},"ln_final":{"w":"(1536,)"},"unembed":{"W_U":"(1536, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 12, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 2, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8960)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1536)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\nunembed:\n hook_in: (batch, seq_len, 1536)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 1536)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 12, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 2, 128)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8960)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1536)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"unembed":{"hook_in":"(batch, seq_len, 1536)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 1536)"}} +{"name.default_alias":"qwen2.5-1.5b-instruct","name.huggingface":"Qwen\/Qwen2.5-1.5B-Instruct","name.aliases":"qwen2.5-1.5b-instruct","model_type":"qwen","name.from_cfg":"Qwen2.5-1.5B-Instruct","n_params.as_str":"1.4B","n_params.as_int":1420296192,"n_params.from_name":"1.5b","cfg.n_params":1420296192,"cfg.n_layers":28,"cfg.n_heads":12,"cfg.d_model":1536,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1536,"d_head":128,"n_layers":28,"n_ctx":32768,"n_heads":12,"d_mlp":8960,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-1.5B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-1.5B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1420296192,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1536\nd_head: 128\nn_layers: 28\nn_ctx: 32768\nn_heads: 12\nd_mlp: 8960\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-1.5B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-1.5B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1420296192\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2.5-1.5B-Instruct","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 1536)\nblocks:\n '[0-27]':\n ln1:\n w: (1536,)\n ln2:\n w: (1536,)\n attn:\n W_Q: (12, 1536, 128)\n W_O: (12, 128, 1536)\n b_Q: (12, 128)\n b_O: (1536,)\n '[_W_K, _W_V]': (2, 1536, 128)\n '[_b_K, _b_V]': (2, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (1536, 8960)\n W_out: (8960, 1536)\n b_in: (8960,)\n b_out: (1536,)\nln_final:\n w: (1536,)\nunembed:\n W_U: (1536, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 1536)"},"blocks":{"[0-27]":{"ln1":{"w":"(1536,)"},"ln2":{"w":"(1536,)"},"attn":{"W_Q":"(12, 1536, 128)","W_O":"(12, 128, 1536)","b_Q":"(12, 128)","b_O":"(1536,)","[_W_K, _W_V]":"(2, 1536, 128)","[_b_K, _b_V]":"(2, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(1536, 8960)","W_out":"(8960, 1536)","b_in":"(8960,)","b_out":"(1536,)"}}},"ln_final":{"w":"(1536,)"},"unembed":{"W_U":"(1536, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 12, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 2, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 8960)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1536)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1536)\nunembed:\n hook_in: (batch, seq_len, 1536)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 1536)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 12, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 2, 128)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 8960)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1536)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1536)"},"unembed":{"hook_in":"(batch, seq_len, 1536)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 1536)"}} +{"name.default_alias":"qwen2.5-14b","name.huggingface":"Qwen\/Qwen2.5-14B","name.aliases":"qwen2.5-14b","model_type":"qwen","name.from_cfg":"Qwen2.5-14B","n_params.as_str":"15B","n_params.as_int":15225323520,"n_params.from_name":"14b","cfg.n_params":15225323520,"cfg.n_layers":48,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":48,"n_ctx":131072,"n_heads":40,"d_mlp":13824,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-14B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-14B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":15225323520,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 48\nn_ctx: 131072\nn_heads: 40\nd_mlp: 13824\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-14B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-14B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 15225323520\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2.5-14B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (152064, 5120)\nblocks:\n '[0-47]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 13824)\n W_out: (13824, 5120)\n b_in: (13824,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 152064)\n b_U: (152064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(152064, 5120)"},"blocks":{"[0-47]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 13824)","W_out":"(13824, 5120)","b_in":"(13824,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 152064)","b_U":"(152064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-47]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 13824)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 152064)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-47]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 13824)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 152064)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"qwen2.5-14b-instruct","name.huggingface":"Qwen\/Qwen2.5-14B-Instruct","name.aliases":"qwen2.5-14b-instruct","model_type":"qwen","name.from_cfg":"Qwen2.5-14B-Instruct","n_params.as_str":"15B","n_params.as_int":15225323520,"n_params.from_name":"14b","cfg.n_params":15225323520,"cfg.n_layers":48,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":48,"n_ctx":32768,"n_heads":40,"d_mlp":13824,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-14B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-14B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":15225323520,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 48\nn_ctx: 32768\nn_heads: 40\nd_mlp: 13824\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-14B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-14B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 15225323520\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2.5-14B-Instruct","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (152064, 5120)\nblocks:\n '[0-47]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 13824)\n W_out: (13824, 5120)\n b_in: (13824,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 152064)\n b_U: (152064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(152064, 5120)"},"blocks":{"[0-47]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 13824)","W_out":"(13824, 5120)","b_in":"(13824,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 152064)","b_U":"(152064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-47]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 13824)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 152064)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-47]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 13824)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 152064)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"qwen2.5-32b","name.huggingface":"Qwen\/Qwen2.5-32B","name.aliases":"qwen2.5-32b","model_type":"qwen","name.from_cfg":"Qwen2.5-32B","n_params.as_str":"34B","n_params.as_int":33889976320,"n_params.from_name":"32b","cfg.n_params":33889976320,"cfg.n_layers":64,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":64,"n_ctx":131072,"n_heads":40,"d_mlp":27648,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-32B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-32B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":33889976320,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 64\nn_ctx: 131072\nn_heads: 40\nd_mlp: 27648\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-32B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-32B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 33889976320\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2.5-32B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (152064, 5120)\nblocks:\n '[0-63]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 27648)\n W_out: (27648, 5120)\n b_in: (27648,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 152064)\n b_U: (152064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(152064, 5120)"},"blocks":{"[0-63]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 27648)","W_out":"(27648, 5120)","b_in":"(27648,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 152064)","b_U":"(152064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-63]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 27648)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 152064)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-63]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 27648)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 152064)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"qwen2.5-32b-instruct","name.huggingface":"Qwen\/Qwen2.5-32B-Instruct","name.aliases":"qwen2.5-32b-instruct","model_type":"qwen","name.from_cfg":"Qwen2.5-32B-Instruct","n_params.as_str":"34B","n_params.as_int":33889976320,"n_params.from_name":"32b","cfg.n_params":33889976320,"cfg.n_layers":64,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":64,"n_ctx":32768,"n_heads":40,"d_mlp":27648,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-32B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-32B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":33889976320,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 64\nn_ctx: 32768\nn_heads: 40\nd_mlp: 27648\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-32B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-32B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 33889976320\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2.5-32B-Instruct","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (152064, 5120)\nblocks:\n '[0-63]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 27648)\n W_out: (27648, 5120)\n b_in: (27648,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 152064)\n b_U: (152064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(152064, 5120)"},"blocks":{"[0-63]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 27648)","W_out":"(27648, 5120)","b_in":"(27648,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 152064)","b_U":"(152064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-63]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 27648)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 152064)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-63]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 27648)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 152064)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"qwen2.5-3b","name.huggingface":"Qwen\/Qwen2.5-3B","name.aliases":"qwen2.5-3b","model_type":"qwen","name.from_cfg":"Qwen2.5-3B","n_params.as_str":"3.0B","n_params.as_int":3038773248,"n_params.from_name":"3b","cfg.n_params":3038773248,"cfg.n_layers":36,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":36,"n_ctx":32768,"n_heads":16,"d_mlp":11008,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-3B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-3B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":3038773248,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 36\nn_ctx: 32768\nn_heads: 16\nd_mlp: 11008\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-3B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-3B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 3038773248\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2.5-3B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 2048)\nblocks:\n '[0-35]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n W_Q: (16, 2048, 128)\n W_O: (16, 128, 2048)\n b_Q: (16, 128)\n b_O: (2048,)\n '[_W_K, _W_V]': (2, 2048, 128)\n '[_b_K, _b_V]': (2, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (2048, 11008)\n W_out: (11008, 2048)\n b_in: (11008,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 2048)"},"blocks":{"[0-35]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"W_Q":"(16, 2048, 128)","W_O":"(16, 128, 2048)","b_Q":"(16, 128)","b_O":"(2048,)","[_W_K, _W_V]":"(2, 2048, 128)","[_b_K, _b_V]":"(2, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(2048, 11008)","W_out":"(11008, 2048)","b_in":"(11008,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 2, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 2, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"qwen2.5-3b-instruct","name.huggingface":"Qwen\/Qwen2.5-3B-Instruct","name.aliases":"qwen2.5-3b-instruct","model_type":"qwen","name.from_cfg":"Qwen2.5-3B-Instruct","n_params.as_str":"3.0B","n_params.as_int":3038773248,"n_params.from_name":"3b","cfg.n_params":3038773248,"cfg.n_layers":36,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":36,"n_ctx":32768,"n_heads":16,"d_mlp":11008,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":2,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-3B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-3B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":3038773248,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 36\nn_ctx: 32768\nn_heads: 16\nd_mlp: 11008\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 2\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-3B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-3B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 3038773248\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2.5-3B-Instruct","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 2048)\nblocks:\n '[0-35]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n W_Q: (16, 2048, 128)\n W_O: (16, 128, 2048)\n b_Q: (16, 128)\n b_O: (2048,)\n '[_W_K, _W_V]': (2, 2048, 128)\n '[_b_K, _b_V]': (2, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (2048, 11008)\n W_out: (11008, 2048)\n b_in: (11008,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 2048)"},"blocks":{"[0-35]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"W_Q":"(16, 2048, 128)","W_O":"(16, 128, 2048)","b_Q":"(16, 128)","b_O":"(2048,)","[_W_K, _W_V]":"(2, 2048, 128)","[_b_K, _b_V]":"(2, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(2048, 11008)","W_out":"(11008, 2048)","b_in":"(11008,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 2, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 11008)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 2, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 11008)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"qwen2.5-72b","name.huggingface":"Qwen\/Qwen2.5-72B","name.aliases":"qwen2.5-72b","model_type":"qwen","name.from_cfg":"Qwen2.5-72B","n_params.as_str":"80B","n_params.as_int":79607889920,"n_params.from_name":"72b","cfg.n_params":79607889920,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":131072,"n_heads":64,"d_mlp":29568,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-72B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-72B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":79607889920,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 131072\nn_heads: 64\nd_mlp: 29568\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-72B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-72B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 79607889920\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2.5-72B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (152064, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n W_Q: (64, 8192, 128)\n W_O: (64, 128, 8192)\n b_Q: (64, 128)\n b_O: (8192,)\n '[_W_K, _W_V]': (8, 8192, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (8192, 29568)\n W_out: (29568, 8192)\n b_in: (29568,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 152064)\n b_U: (152064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(152064, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"W_Q":"(64, 8192, 128)","W_O":"(64, 128, 8192)","b_Q":"(64, 128)","b_O":"(8192,)","[_W_K, _W_V]":"(8, 8192, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 29568)","W_out":"(29568, 8192)","b_in":"(29568,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 152064)","b_U":"(152064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 64, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 29568)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 152064)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 64, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 29568)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 152064)"},"hook_embed":"(batch, seq_len, 8192)"}} +{"name.default_alias":"qwen2.5-72b-instruct","name.huggingface":"Qwen\/Qwen2.5-72B-Instruct","name.aliases":"qwen2.5-72b-instruct","model_type":"qwen","name.from_cfg":"Qwen2.5-72B-Instruct","n_params.as_str":"80B","n_params.as_int":79607889920,"n_params.from_name":"72b","cfg.n_params":79607889920,"cfg.n_layers":80,"cfg.n_heads":64,"cfg.d_model":8192,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":8192,"d_head":128,"n_layers":80,"n_ctx":32768,"n_heads":64,"d_mlp":29568,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-72B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-72B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":79607889920,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 8192\nd_head: 128\nn_layers: 80\nn_ctx: 32768\nn_heads: 64\nd_mlp: 29568\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-72B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-72B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 79607889920\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2.5-72B-Instruct","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (152064, 8192)\nblocks:\n '[0-79]':\n ln1:\n w: (8192,)\n ln2:\n w: (8192,)\n attn:\n W_Q: (64, 8192, 128)\n W_O: (64, 128, 8192)\n b_Q: (64, 128)\n b_O: (8192,)\n '[_W_K, _W_V]': (8, 8192, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (8192, 29568)\n W_out: (29568, 8192)\n b_in: (29568,)\n b_out: (8192,)\nln_final:\n w: (8192,)\nunembed:\n W_U: (8192, 152064)\n b_U: (152064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(152064, 8192)"},"blocks":{"[0-79]":{"ln1":{"w":"(8192,)"},"ln2":{"w":"(8192,)"},"attn":{"W_Q":"(64, 8192, 128)","W_O":"(64, 128, 8192)","b_Q":"(64, 128)","b_O":"(8192,)","[_W_K, _W_V]":"(8, 8192, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(8192, 29568)","W_out":"(29568, 8192)","b_in":"(29568,)","b_out":"(8192,)"}}},"ln_final":{"w":"(8192,)"},"unembed":{"W_U":"(8192, 152064)","b_U":"(152064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-79]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 64, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 64, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 29568)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 8192)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 8192)\nunembed:\n hook_in: (batch, seq_len, 8192)\n hook_out: (batch, seq_len, 152064)\nhook_embed: (batch, seq_len, 8192)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-79]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 64, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 64, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 29568)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 8192)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 8192)"},"unembed":{"hook_in":"(batch, seq_len, 8192)","hook_out":"(batch, seq_len, 152064)"},"hook_embed":"(batch, seq_len, 8192)"}} +{"name.default_alias":"qwen2.5-7b","name.huggingface":"Qwen\/Qwen2.5-7B","name.aliases":"qwen2.5-7b","model_type":"qwen","name.from_cfg":"Qwen2.5-7B","n_params.as_str":"7.1B","n_params.as_int":7141851136,"n_params.from_name":"7b","cfg.n_params":7141851136,"cfg.n_layers":28,"cfg.n_heads":28,"cfg.d_model":3584,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3584,"d_head":128,"n_layers":28,"n_ctx":131072,"n_heads":28,"d_mlp":18944,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-7B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-7B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7141851136,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 3584\nd_head: 128\nn_layers: 28\nn_ctx: 131072\nn_heads: 28\nd_mlp: 18944\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-7B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-7B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7141851136\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2.5-7B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (152064, 3584)\nblocks:\n '[0-27]':\n ln1:\n w: (3584,)\n ln2:\n w: (3584,)\n attn:\n W_Q: (28, 3584, 128)\n W_O: (28, 128, 3584)\n b_Q: (28, 128)\n b_O: (3584,)\n '[_W_K, _W_V]': (4, 3584, 128)\n '[_b_K, _b_V]': (4, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (3584, 18944)\n W_out: (18944, 3584)\n b_in: (18944,)\n b_out: (3584,)\nln_final:\n w: (3584,)\nunembed:\n W_U: (3584, 152064)\n b_U: (152064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(152064, 3584)"},"blocks":{"[0-27]":{"ln1":{"w":"(3584,)"},"ln2":{"w":"(3584,)"},"attn":{"W_Q":"(28, 3584, 128)","W_O":"(28, 128, 3584)","b_Q":"(28, 128)","b_O":"(3584,)","[_W_K, _W_V]":"(4, 3584, 128)","[_b_K, _b_V]":"(4, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(3584, 18944)","W_out":"(18944, 3584)","b_in":"(18944,)","b_out":"(3584,)"}}},"ln_final":{"w":"(3584,)"},"unembed":{"W_U":"(3584, 152064)","b_U":"(152064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 28, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 28, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 18944)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3584)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\nunembed:\n hook_in: (batch, seq_len, 3584)\n hook_out: (batch, seq_len, 152064)\nhook_embed: (batch, seq_len, 3584)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 28, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 128)","[hook_attn_scores, hook_pattern]":"(batch, 28, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 18944)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3584)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"unembed":{"hook_in":"(batch, seq_len, 3584)","hook_out":"(batch, seq_len, 152064)"},"hook_embed":"(batch, seq_len, 3584)"}} +{"name.default_alias":"qwen2.5-7b-instruct","name.huggingface":"Qwen\/Qwen2.5-7B-Instruct","name.aliases":"qwen2.5-7b-instruct","model_type":"qwen","name.from_cfg":"Qwen2.5-7B-Instruct","n_params.as_str":"7.1B","n_params.as_int":7141851136,"n_params.from_name":"7b","cfg.n_params":7141851136,"cfg.n_layers":28,"cfg.n_heads":28,"cfg.d_model":3584,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":3584,"d_head":128,"n_layers":28,"n_ctx":32768,"n_heads":28,"d_mlp":18944,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":4,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen2.5-7B-Instruct","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen2.5-7B-Instruct","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7141851136,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 3584\nd_head: 128\nn_layers: 28\nn_ctx: 32768\nn_heads: 28\nd_mlp: 18944\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 4\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen2.5-7B-Instruct\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen2.5-7B-Instruct\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7141851136\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen2.5-7B-Instruct","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (152064, 3584)\nblocks:\n '[0-27]':\n ln1:\n w: (3584,)\n ln2:\n w: (3584,)\n attn:\n W_Q: (28, 3584, 128)\n W_O: (28, 128, 3584)\n b_Q: (28, 128)\n b_O: (3584,)\n '[_W_K, _W_V]': (4, 3584, 128)\n '[_b_K, _b_V]': (4, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (3584, 18944)\n W_out: (18944, 3584)\n b_in: (18944,)\n b_out: (3584,)\nln_final:\n w: (3584,)\nunembed:\n W_U: (3584, 152064)\n b_U: (152064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(152064, 3584)"},"blocks":{"[0-27]":{"ln1":{"w":"(3584,)"},"ln2":{"w":"(3584,)"},"attn":{"W_Q":"(28, 3584, 128)","W_O":"(28, 128, 3584)","b_Q":"(28, 128)","b_O":"(3584,)","[_W_K, _W_V]":"(4, 3584, 128)","[_b_K, _b_V]":"(4, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(3584, 18944)","W_out":"(18944, 3584)","b_in":"(18944,)","b_out":"(3584,)"}}},"ln_final":{"w":"(3584,)"},"unembed":{"W_U":"(3584, 152064)","b_U":"(152064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 28, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 4, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 28, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 18944)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 3584)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 3584)\nunembed:\n hook_in: (batch, seq_len, 3584)\n hook_out: (batch, seq_len, 152064)\nhook_embed: (batch, seq_len, 3584)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 28, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 4, 128)","[hook_attn_scores, hook_pattern]":"(batch, 28, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 18944)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 3584)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 3584)"},"unembed":{"hook_in":"(batch, seq_len, 3584)","hook_out":"(batch, seq_len, 152064)"},"hook_embed":"(batch, seq_len, 3584)"}} +{"name.default_alias":"qwen3-0.6b","name.huggingface":"Qwen\/Qwen3-0.6B","name.aliases":"qwen3-0.6b","model_type":"qwen","name.from_cfg":"Qwen3-0.6B","n_params.as_str":"499M","n_params.as_int":499122176,"n_params.from_name":"0.6b","cfg.n_params":499122176,"cfg.n_layers":28,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1024,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":16,"d_mlp":3072,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen3-0.6B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen3-0.6B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":499122176,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 16\nd_mlp: 3072\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen3-0.6B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen3-0.6B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 499122176\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen3-0.6B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"OfOK7SjmrN4KFUqlDylX-Up77SM=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 1024)\nblocks:\n '[0-27]':\n ln1:\n w: (1024,)\n ln2:\n w: (1024,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (16, 1024, 128)\n W_O: (16, 128, 1024)\n b_Q: (16, 128)\n b_O: (1024,)\n '[_W_K, _W_V]': (8, 1024, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (1024, 3072)\n W_out: (3072, 1024)\n b_in: (3072,)\n b_out: (1024,)\nln_final:\n w: (1024,)\nunembed:\n W_U: (1024, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 1024)"},"blocks":{"[0-27]":{"ln1":{"w":"(1024,)"},"ln2":{"w":"(1024,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(16, 1024, 128)","W_O":"(16, 128, 1024)","b_Q":"(16, 128)","b_O":"(1024,)","[_W_K, _W_V]":"(8, 1024, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(1024, 3072)","W_out":"(3072, 1024)","b_in":"(3072,)","b_out":"(1024,)"}}},"ln_final":{"w":"(1024,)"},"unembed":{"W_U":"(1024, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n q_norm:\n hook_scale: (27659632, 1)\n hook_normalized: (27659632, 128)\n k_norm:\n hook_scale: (13829816, 1)\n hook_normalized: (13829816, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"q_norm":{"hook_scale":"(27659632, 1)","hook_normalized":"(27659632, 128)"},"k_norm":{"hook_scale":"(13829816, 1)","hook_normalized":"(13829816, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 1024)"}} +{"name.default_alias":"qwen3-0.6b-base","name.huggingface":"Qwen\/Qwen3-0.6B-Base","name.aliases":"qwen3-0.6b-base","model_type":"qwen","name.from_cfg":"Qwen3-0.6B-Base","n_params.as_str":"499M","n_params.as_int":499122176,"n_params.from_name":"0.6b","cfg.n_params":499122176,"cfg.n_layers":28,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":1024,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":16,"d_mlp":3072,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen3-0.6B-Base","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen3-0.6B-Base","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":499122176,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 16\nd_mlp: 3072\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen3-0.6B-Base\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen3-0.6B-Base\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 499122176\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen3-0.6B-Base","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"OfOK7SjmrN4KFUqlDylX-Up77SM=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 1024)\nblocks:\n '[0-27]':\n ln1:\n w: (1024,)\n ln2:\n w: (1024,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (16, 1024, 128)\n W_O: (16, 128, 1024)\n b_Q: (16, 128)\n b_O: (1024,)\n '[_W_K, _W_V]': (8, 1024, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (1024, 3072)\n W_out: (3072, 1024)\n b_in: (3072,)\n b_out: (1024,)\nln_final:\n w: (1024,)\nunembed:\n W_U: (1024, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 1024)"},"blocks":{"[0-27]":{"ln1":{"w":"(1024,)"},"ln2":{"w":"(1024,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(16, 1024, 128)","W_O":"(16, 128, 1024)","b_Q":"(16, 128)","b_O":"(1024,)","[_W_K, _W_V]":"(8, 1024, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(1024, 3072)","W_out":"(3072, 1024)","b_in":"(3072,)","b_out":"(1024,)"}}},"ln_final":{"w":"(1024,)"},"unembed":{"W_U":"(1024, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n q_norm:\n hook_scale: (27659632, 1)\n hook_normalized: (27659632, 128)\n k_norm:\n hook_scale: (13829816, 1)\n hook_normalized: (13829816, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"q_norm":{"hook_scale":"(27659632, 1)","hook_normalized":"(27659632, 128)"},"k_norm":{"hook_scale":"(13829816, 1)","hook_normalized":"(13829816, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 1024)"}} +{"name.default_alias":"qwen3-1.7b","name.huggingface":"Qwen\/Qwen3-1.7B","name.aliases":"qwen3-1.7b","model_type":"qwen","name.from_cfg":"Qwen3-1.7B","n_params.as_str":"1.5B","n_params.as_int":1526726656,"n_params.from_name":"1.7b","cfg.n_params":1526726656,"cfg.n_layers":28,"cfg.n_heads":16,"cfg.d_model":2048,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2048,"d_head":128,"n_layers":28,"n_ctx":2048,"n_heads":16,"d_mlp":6144,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen3-1.7B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen3-1.7B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":1526726656,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2048\nd_head: 128\nn_layers: 28\nn_ctx: 2048\nn_heads: 16\nd_mlp: 6144\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen3-1.7B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen3-1.7B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 1526726656\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen3-1.7B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"OfOK7SjmrN4KFUqlDylX-Up77SM=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 2048)\nblocks:\n '[0-27]':\n ln1:\n w: (2048,)\n ln2:\n w: (2048,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (16, 2048, 128)\n W_O: (16, 128, 2048)\n b_Q: (16, 128)\n b_O: (2048,)\n '[_W_K, _W_V]': (8, 2048, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (2048, 6144)\n W_out: (6144, 2048)\n b_in: (6144,)\n b_out: (2048,)\nln_final:\n w: (2048,)\nunembed:\n W_U: (2048, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 2048)"},"blocks":{"[0-27]":{"ln1":{"w":"(2048,)"},"ln2":{"w":"(2048,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(16, 2048, 128)","W_O":"(16, 128, 2048)","b_Q":"(16, 128)","b_O":"(2048,)","[_W_K, _W_V]":"(8, 2048, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(2048, 6144)","W_out":"(6144, 2048)","b_in":"(6144,)","b_out":"(2048,)"}}},"ln_final":{"w":"(2048,)"},"unembed":{"W_U":"(2048, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-27]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n attn:\n q_norm:\n hook_scale: (27659632, 1)\n hook_normalized: (27659632, 128)\n k_norm:\n hook_scale: (13829816, 1)\n hook_normalized: (13829816, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 16, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 6144)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2048)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2048)\nunembed:\n hook_in: (batch, seq_len, 2048)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 2048)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-27]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"attn":{"q_norm":{"hook_scale":"(27659632, 1)","hook_normalized":"(27659632, 128)"},"k_norm":{"hook_scale":"(13829816, 1)","hook_normalized":"(13829816, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 16, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 6144)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2048)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2048)"},"unembed":{"hook_in":"(batch, seq_len, 2048)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 2048)"}} +{"name.default_alias":"qwen3-14b","name.huggingface":"Qwen\/Qwen3-14B","name.aliases":"qwen3-14b","model_type":"qwen","name.from_cfg":"Qwen3-14B","n_params.as_str":"15B","n_params.as_int":14889779200,"n_params.from_name":"14b","cfg.n_params":14889779200,"cfg.n_layers":40,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":40,"n_ctx":2048,"n_heads":40,"d_mlp":17408,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen3-14B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen3-14B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":14889779200,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 40\nn_ctx: 2048\nn_heads: 40\nd_mlp: 17408\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen3-14B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen3-14B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 14889779200\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen3-14B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"OfOK7SjmrN4KFUqlDylX-Up77SM=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 5120)\nblocks:\n '[0-39]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 17408)\n W_out: (17408, 5120)\n b_in: (17408,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 5120)"},"blocks":{"[0-39]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 17408)","W_out":"(17408, 5120)","b_in":"(17408,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-39]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n q_norm:\n hook_scale: (69149080, 1)\n hook_normalized: (69149080, 128)\n k_norm:\n hook_scale: (13829816, 1)\n hook_normalized: (13829816, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 17408)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-39]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"q_norm":{"hook_scale":"(69149080, 1)","hook_normalized":"(69149080, 128)"},"k_norm":{"hook_scale":"(13829816, 1)","hook_normalized":"(13829816, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 17408)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"qwen3-4b","name.huggingface":"Qwen\/Qwen3-4B","name.aliases":"qwen3-4b","model_type":"qwen","name.from_cfg":"Qwen3-4B","n_params.as_str":"4.2B","n_params.as_int":4199546880,"n_params.from_name":"4b","cfg.n_params":4199546880,"cfg.n_layers":36,"cfg.n_heads":32,"cfg.d_model":2560,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":2560,"d_head":128,"n_layers":36,"n_ctx":2048,"n_heads":32,"d_mlp":9728,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen3-4B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen3-4B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":4199546880,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 2560\nd_head: 128\nn_layers: 36\nn_ctx: 2048\nn_heads: 32\nd_mlp: 9728\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen3-4B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen3-4B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 4199546880\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen3-4B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"OfOK7SjmrN4KFUqlDylX-Up77SM=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 2560)\nblocks:\n '[0-35]':\n ln1:\n w: (2560,)\n ln2:\n w: (2560,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (32, 2560, 128)\n W_O: (32, 128, 2560)\n b_Q: (32, 128)\n b_O: (2560,)\n '[_W_K, _W_V]': (8, 2560, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (2560, 9728)\n W_out: (9728, 2560)\n b_in: (9728,)\n b_out: (2560,)\nln_final:\n w: (2560,)\nunembed:\n W_U: (2560, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 2560)"},"blocks":{"[0-35]":{"ln1":{"w":"(2560,)"},"ln2":{"w":"(2560,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(32, 2560, 128)","W_O":"(32, 128, 2560)","b_Q":"(32, 128)","b_O":"(2560,)","[_W_K, _W_V]":"(8, 2560, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(2560, 9728)","W_out":"(9728, 2560)","b_in":"(9728,)","b_out":"(2560,)"}}},"ln_final":{"w":"(2560,)"},"unembed":{"W_U":"(2560, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n attn:\n q_norm:\n hook_scale: (55319264, 1)\n hook_normalized: (55319264, 128)\n k_norm:\n hook_scale: (13829816, 1)\n hook_normalized: (13829816, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 9728)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 2560)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 2560)\nunembed:\n hook_in: (batch, seq_len, 2560)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 2560)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"attn":{"q_norm":{"hook_scale":"(55319264, 1)","hook_normalized":"(55319264, 128)"},"k_norm":{"hook_scale":"(13829816, 1)","hook_normalized":"(13829816, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 9728)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 2560)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 2560)"},"unembed":{"hook_in":"(batch, seq_len, 2560)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 2560)"}} +{"name.default_alias":"qwen3-8b","name.huggingface":"Qwen\/Qwen3-8B","name.aliases":"qwen3-8b","model_type":"qwen","name.from_cfg":"Qwen3-8B","n_params.as_str":"7.9B","n_params.as_int":7851737088,"n_params.from_name":"8b","cfg.n_params":7851737088,"cfg.n_layers":36,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":151936,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen3ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":36,"n_ctx":2048,"n_heads":32,"d_mlp":12288,"d_vocab":151936,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.000001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Qwen3-8B","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen3ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/Qwen3-8B","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":151936,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7851737088,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 36\nn_ctx: 2048\nn_heads: 32\nd_mlp: 12288\nd_vocab: 151936\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-06\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Qwen3-8B\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen3ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/Qwen3-8B\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 151936\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7851737088\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/Qwen3-8B","tokenizer.vocab_size":151643.0,"tokenizer.max_len":131072.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"OfOK7SjmrN4KFUqlDylX-Up77SM=","tensor_shapes.state_dict":"embed:\n W_E: (151936, 4096)\nblocks:\n '[0-35]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (4096, 12288)\n W_out: (12288, 4096)\n b_in: (12288,)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 151936)\n b_U: (151936,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(151936, 4096)"},"blocks":{"[0-35]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(4096, 12288)","W_out":"(12288, 4096)","b_in":"(12288,)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 151936)","b_U":"(151936,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-35]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n q_norm:\n hook_scale: (55319264, 1)\n hook_normalized: (55319264, 128)\n k_norm:\n hook_scale: (13829816, 1)\n hook_normalized: (13829816, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 12288)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 151936)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-35]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"q_norm":{"hook_scale":"(55319264, 1)","hook_normalized":"(55319264, 128)"},"k_norm":{"hook_scale":"(13829816, 1)","hook_normalized":"(13829816, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 12288)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 151936)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"qwen-32b-preview","name.huggingface":"Qwen\/QwQ-32B-Preview","name.aliases":"qwen-32b-preview","model_type":"qwen","name.from_cfg":"QwQ-32B-Preview","n_params.as_str":"34B","n_params.as_int":33889976320,"n_params.from_name":"32b","cfg.n_params":33889976320,"cfg.n_layers":64,"cfg.n_heads":40,"cfg.d_model":5120,"cfg.d_vocab":152064,"cfg.act_fn":"silu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"Qwen2ForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":5120,"d_head":128,"n_layers":64,"n_ctx":32768,"n_heads":40,"d_mlp":27648,"d_vocab":152064,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":true,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"silu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"QwQ-32B-Preview","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"Qwen2ForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"Qwen\/QwQ-32B-Preview","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.02,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":152064,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":33889976320,"use_hook_tokens":false,"tokenizer_prepends_bos":true,"post_embedding_ln":false,"rotary_base":1000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 5120\nd_head: 128\nn_layers: 64\nn_ctx: 32768\nn_heads: 40\nd_mlp: 27648\nd_vocab: 152064\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: true\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: silu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: QwQ-32B-Preview\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: Qwen2ForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: Qwen\/QwQ-32B-Preview\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 152064\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 33889976320\nuse_hook_tokens: false\ntokenizer_prepends_bos: true\npost_embedding_ln: false\nrotary_base: 1000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"Qwen\/QwQ-32B-Preview","tokenizer.vocab_size":151643.0,"tokenizer.max_len":32768.0,"tokenizer.class":"Qwen2Tokenizer","tokenizer.vocab_hash":"NI384GYDfJidzgXg_-9habj8lOk=","tensor_shapes.state_dict":"embed:\n W_E: (152064, 5120)\nblocks:\n '[0-63]':\n ln1:\n w: (5120,)\n ln2:\n w: (5120,)\n attn:\n W_Q: (40, 5120, 128)\n W_O: (40, 128, 5120)\n b_Q: (40, 128)\n b_O: (5120,)\n '[_W_K, _W_V]': (8, 5120, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n '[W_in, W_gate]': (5120, 27648)\n W_out: (27648, 5120)\n b_in: (27648,)\n b_out: (5120,)\nln_final:\n w: (5120,)\nunembed:\n W_U: (5120, 152064)\n b_U: (152064,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(152064, 5120)"},"blocks":{"[0-63]":{"ln1":{"w":"(5120,)"},"ln2":{"w":"(5120,)"},"attn":{"W_Q":"(40, 5120, 128)","W_O":"(40, 128, 5120)","b_Q":"(40, 128)","b_O":"(5120,)","[_W_K, _W_V]":"(8, 5120, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"[W_in, W_gate]":"(5120, 27648)","W_out":"(27648, 5120)","b_in":"(27648,)","b_out":"(5120,)"}}},"ln_final":{"w":"(5120,)"},"unembed":{"W_U":"(5120, 152064)","b_U":"(152064,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-63]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n attn:\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 40, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 40, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\n mlp:\n '[hook_pre, hook_pre_linear, hook_post]': (batch, seq_len, 27648)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 5120)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 5120)\nunembed:\n hook_in: (batch, seq_len, 5120)\n hook_out: (batch, seq_len, 152064)\nhook_embed: (batch, seq_len, 5120)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-63]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"attn":{"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 40, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 40, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"mlp":{"[hook_pre, hook_pre_linear, hook_post]":"(batch, seq_len, 27648)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 5120)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 5120)"},"unembed":{"hook_in":"(batch, seq_len, 5120)","hook_out":"(batch, seq_len, 152064)"},"hook_embed":"(batch, seq_len, 5120)"}} +{"name.default_alias":"tiny-stories-1L-21M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-1Layer-21M","n_params.as_str":"13M","n_params.as_int":12582912,"n_params.from_name":"21M","cfg.n_params":12582912,"cfg.n_layers":1,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":1,"n_ctx":512,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-1Layer-21M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-1Layer-21M","window_size":256,"attn_types":["global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 1\nn_ctx: 512\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-1Layer-21M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-1Layer-21M\nwindow_size: 256\nattn_types:\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-1Layer-21M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (512, 1024)\nblocks:\n '0':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(512, 1024)"},"blocks":{"0":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '0':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"0":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"tiny-stories-1M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-1M","n_params.as_str":"393K","n_params.as_int":393216,"n_params.from_name":"1M","cfg.n_params":393216,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":64,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":64,"d_head":4,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":256,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-1M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-1M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.1,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":393216,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 64\nd_head: 4\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 256\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-1M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-1M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.1\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 393216\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-1M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 64)\npos_embed:\n W_pos: (512, 64)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (64,)\n ln2:\n '[w, b]': (64,)\n attn:\n '[W_Q, W_K, W_V]': (16, 64, 4)\n W_O: (16, 4, 64)\n '[b_Q, b_K, b_V]': (16, 4)\n b_O: (64,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (64, 256)\n b_in: (256,)\n W_out: (256, 64)\n b_out: (64,)\nln_final:\n '[w, b]': (64,)\nunembed:\n W_U: (64, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 64)"},"pos_embed":{"W_pos":"(512, 64)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(64,)"},"ln2":{"[w, b]":"(64,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 64, 4)","W_O":"(16, 4, 64)","[b_Q, b_K, b_V]":"(16, 4)","b_O":"(64,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(64, 256)","b_in":"(256,)","W_out":"(256, 64)","b_out":"(64,)"}}},"ln_final":{"[w, b]":"(64,)"},"unembed":{"W_U":"(64, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 64)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 4)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 64)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 256)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 64)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 64)\nunembed:\n hook_in: (batch, seq_len, 64)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 64)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 64)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 4)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 64)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 256)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 64)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 64)"},"unembed":{"hook_in":"(batch, seq_len, 64)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 64)"}} +{"name.default_alias":"tiny-stories-28M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-28M","n_params.as_str":"25M","n_params.as_int":25165824,"n_params.from_name":"28M","cfg.n_params":25165824,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":512,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":32,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":2048,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-28M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-28M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":25165824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 32\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 2048\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-28M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-28M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 25165824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-28M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 512)\npos_embed:\n W_pos: (512, 512)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (16, 512, 32)\n W_O: (16, 32, 512)\n '[b_Q, b_K, b_V]': (16, 32)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 512)"},"pos_embed":{"W_pos":"(512, 512)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 512, 32)","W_O":"(16, 32, 512)","[b_Q, b_K, b_V]":"(16, 32)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 32)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 32)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"tiny-stories-2L-33M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-2Layers-33M","n_params.as_str":"25M","n_params.as_int":25165824,"n_params.from_name":"33M","cfg.n_params":25165824,"cfg.n_layers":2,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":2,"n_ctx":512,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-2Layers-33M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-2Layers-33M","window_size":256,"attn_types":["global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":25165824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 2\nn_ctx: 512\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-2Layers-33M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-2Layers-33M\nwindow_size: 256\nattn_types:\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 25165824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-2Layers-33M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (512, 1024)\nblocks:\n '[0-1]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(512, 1024)"},"blocks":{"[0-1]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-1]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-1]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"tiny-stories-33M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-33M","n_params.as_str":"28M","n_params.as_int":28311552,"n_params.from_name":"33M","cfg.n_params":28311552,"cfg.n_layers":4,"cfg.n_heads":16,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":48,"n_layers":4,"n_ctx":512,"n_heads":16,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-33M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-33M","window_size":256,"attn_types":["global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":28311552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 48\nn_layers: 4\nn_ctx: 512\nn_heads: 16\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-33M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-33M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 28311552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-33M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (512, 768)\nblocks:\n '[0-3]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (16, 768, 48)\n W_O: (16, 48, 768)\n '[b_Q, b_K, b_V]': (16, 48)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(512, 768)"},"blocks":{"[0-3]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 768, 48)","W_O":"(16, 48, 768)","[b_Q, b_K, b_V]":"(16, 48)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-3]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 48)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-3]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 48)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"tiny-stories-3M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-3M","n_params.as_str":"1.6M","n_params.as_int":1572864,"n_params.from_name":"3M","cfg.n_params":1572864,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":128,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":128,"d_head":8,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":512,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-3M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-3M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0707106781,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1572864,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 128\nd_head: 8\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 512\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-3M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-3M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.07071067811865475\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1572864\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-3M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 128)\npos_embed:\n W_pos: (512, 128)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (128,)\n ln2:\n '[w, b]': (128,)\n attn:\n '[W_Q, W_K, W_V]': (16, 128, 8)\n W_O: (16, 8, 128)\n '[b_Q, b_K, b_V]': (16, 8)\n b_O: (128,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (128, 512)\n b_in: (512,)\n W_out: (512, 128)\n b_out: (128,)\nln_final:\n '[w, b]': (128,)\nunembed:\n W_U: (128, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 128)"},"pos_embed":{"W_pos":"(512, 128)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(128,)"},"ln2":{"[w, b]":"(128,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 128, 8)","W_O":"(16, 8, 128)","[b_Q, b_K, b_V]":"(16, 8)","b_O":"(128,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(128, 512)","b_in":"(512,)","W_out":"(512, 128)","b_out":"(128,)"}}},"ln_final":{"[w, b]":"(128,)"},"unembed":{"W_U":"(128, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 8)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 512)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 128)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\nunembed:\n hook_in: (batch, seq_len, 128)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 128)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 8)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 512)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 128)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"unembed":{"hook_in":"(batch, seq_len, 128)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 128)"}} +{"name.default_alias":"tiny-stories-8M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-8M","n_params.as_str":"6.3M","n_params.as_int":6291456,"n_params.from_name":"8M","cfg.n_params":6291456,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":256,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":256,"d_head":16,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":1024,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-8M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-8M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.05,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":6291456,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 256\nd_head: 16\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 1024\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-8M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-8M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.05\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 6291456\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-8M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 256)\npos_embed:\n W_pos: (512, 256)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (256,)\n ln2:\n '[w, b]': (256,)\n attn:\n '[W_Q, W_K, W_V]': (16, 256, 16)\n W_O: (16, 16, 256)\n '[b_Q, b_K, b_V]': (16, 16)\n b_O: (256,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (256, 1024)\n b_in: (1024,)\n W_out: (1024, 256)\n b_out: (256,)\nln_final:\n '[w, b]': (256,)\nunembed:\n W_U: (256, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 256)"},"pos_embed":{"W_pos":"(512, 256)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(256,)"},"ln2":{"[w, b]":"(256,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 256, 16)","W_O":"(16, 16, 256)","[b_Q, b_K, b_V]":"(16, 16)","b_O":"(256,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(256, 1024)","b_in":"(1024,)","W_out":"(1024, 256)","b_out":"(256,)"}}},"ln_final":{"[w, b]":"(256,)"},"unembed":{"W_U":"(256, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 16)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 1024)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 256)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\nunembed:\n hook_in: (batch, seq_len, 256)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 256)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 16)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 1024)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 256)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"unembed":{"hook_in":"(batch, seq_len, 256)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 256)"}} +{"name.default_alias":"tiny-stories-instruct-1M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-Instruct-1M","n_params.as_str":"393K","n_params.as_int":393216,"n_params.from_name":"1M","cfg.n_params":393216,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":64,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":64,"d_head":4,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":256,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-Instruct-1M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-Instruct-1M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.1,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":393216,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 64\nd_head: 4\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 256\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-Instruct-1M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-Instruct-1M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.1\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 393216\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-Instruct-1M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 64)\npos_embed:\n W_pos: (512, 64)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (64,)\n ln2:\n '[w, b]': (64,)\n attn:\n '[W_Q, W_K, W_V]': (16, 64, 4)\n W_O: (16, 4, 64)\n '[b_Q, b_K, b_V]': (16, 4)\n b_O: (64,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (64, 256)\n b_in: (256,)\n W_out: (256, 64)\n b_out: (64,)\nln_final:\n '[w, b]': (64,)\nunembed:\n W_U: (64, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 64)"},"pos_embed":{"W_pos":"(512, 64)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(64,)"},"ln2":{"[w, b]":"(64,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 64, 4)","W_O":"(16, 4, 64)","[b_Q, b_K, b_V]":"(16, 4)","b_O":"(64,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(64, 256)","b_in":"(256,)","W_out":"(256, 64)","b_out":"(64,)"}}},"ln_final":{"[w, b]":"(64,)"},"unembed":{"W_U":"(64, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 64)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 4)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 64)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 256)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 64)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 64)\nunembed:\n hook_in: (batch, seq_len, 64)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 64)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 64)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 4)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 64)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 256)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 64)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 64)"},"unembed":{"hook_in":"(batch, seq_len, 64)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 64)"}} +{"name.default_alias":"tiny-stories-instruct-28M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-Instruct-28M","n_params.as_str":"25M","n_params.as_int":25165824,"n_params.from_name":"28M","cfg.n_params":25165824,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":512,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":512,"d_head":32,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":2048,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-Instruct-28M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-Instruct-28M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0353553391,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":25165824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 512\nd_head: 32\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 2048\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-Instruct-28M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-Instruct-28M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.035355339059327376\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 25165824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-Instruct-28M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 512)\npos_embed:\n W_pos: (512, 512)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (512,)\n ln2:\n '[w, b]': (512,)\n attn:\n '[W_Q, W_K, W_V]': (16, 512, 32)\n W_O: (16, 32, 512)\n '[b_Q, b_K, b_V]': (16, 32)\n b_O: (512,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (512, 2048)\n b_in: (2048,)\n W_out: (2048, 512)\n b_out: (512,)\nln_final:\n '[w, b]': (512,)\nunembed:\n W_U: (512, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 512)"},"pos_embed":{"W_pos":"(512, 512)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(512,)"},"ln2":{"[w, b]":"(512,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 512, 32)","W_O":"(16, 32, 512)","[b_Q, b_K, b_V]":"(16, 32)","b_O":"(512,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(512, 2048)","b_in":"(2048,)","W_out":"(2048, 512)","b_out":"(512,)"}}},"ln_final":{"[w, b]":"(512,)"},"unembed":{"W_U":"(512, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 32)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 2048)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 512)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 512)\nunembed:\n hook_in: (batch, seq_len, 512)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 512)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 32)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 2048)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 512)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 512)"},"unembed":{"hook_in":"(batch, seq_len, 512)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 512)"}} +{"name.default_alias":"tiny-stories-instruct-2L-33M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-Instruct-2Layers-33M","n_params.as_str":"25M","n_params.as_int":25165824,"n_params.from_name":"33M","cfg.n_params":25165824,"cfg.n_layers":2,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":2,"n_ctx":512,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-Instruct-2Layers-33M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-Instruct-2Layers-33M","window_size":256,"attn_types":["global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":25165824,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 2\nn_ctx: 512\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-Instruct-2Layers-33M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-Instruct-2Layers-33M\nwindow_size: 256\nattn_types:\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 25165824\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-Instruct-2Layers-33M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (512, 1024)\nblocks:\n '[0-1]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(512, 1024)"},"blocks":{"[0-1]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-1]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-1]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"tiny-stories-instruct-33M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-Instruct-33M","n_params.as_str":"28M","n_params.as_int":28311552,"n_params.from_name":"33M","cfg.n_params":28311552,"cfg.n_layers":4,"cfg.n_heads":16,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":48,"n_layers":4,"n_ctx":512,"n_heads":16,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-Instruct-33M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-Instruct-33M","window_size":256,"attn_types":["global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":28311552,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 48\nn_layers: 4\nn_ctx: 512\nn_heads: 16\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-Instruct-33M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-Instruct-33M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 28311552\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-Instruct-33M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (512, 768)\nblocks:\n '[0-3]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (16, 768, 48)\n W_O: (16, 48, 768)\n '[b_Q, b_K, b_V]': (16, 48)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(512, 768)"},"blocks":{"[0-3]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 768, 48)","W_O":"(16, 48, 768)","[b_Q, b_K, b_V]":"(16, 48)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-3]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 48)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-3]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 48)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"tiny-stories-instruct-3M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-Instruct-3M","n_params.as_str":"1.6M","n_params.as_int":1572864,"n_params.from_name":"3M","cfg.n_params":1572864,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":128,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":128,"d_head":8,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":512,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-Instruct-3M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-Instruct-3M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0707106781,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":1572864,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 128\nd_head: 8\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 512\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-Instruct-3M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-Instruct-3M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.07071067811865475\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 1572864\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-Instruct-3M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 128)\npos_embed:\n W_pos: (512, 128)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (128,)\n ln2:\n '[w, b]': (128,)\n attn:\n '[W_Q, W_K, W_V]': (16, 128, 8)\n W_O: (16, 8, 128)\n '[b_Q, b_K, b_V]': (16, 8)\n b_O: (128,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (128, 512)\n b_in: (512,)\n W_out: (512, 128)\n b_out: (128,)\nln_final:\n '[w, b]': (128,)\nunembed:\n W_U: (128, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 128)"},"pos_embed":{"W_pos":"(512, 128)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(128,)"},"ln2":{"[w, b]":"(128,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 128, 8)","W_O":"(16, 8, 128)","[b_Q, b_K, b_V]":"(16, 8)","b_O":"(128,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(128, 512)","b_in":"(512,)","W_out":"(512, 128)","b_out":"(128,)"}}},"ln_final":{"[w, b]":"(128,)"},"unembed":{"W_U":"(128, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 8)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 512)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 128)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 128)\nunembed:\n hook_in: (batch, seq_len, 128)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 128)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 8)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 512)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 128)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 128)"},"unembed":{"hook_in":"(batch, seq_len, 128)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 128)"}} +{"name.default_alias":"tiny-stories-instruct-8M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-Instruct-8M","n_params.as_str":"6.3M","n_params.as_int":6291456,"n_params.from_name":"8M","cfg.n_params":6291456,"cfg.n_layers":8,"cfg.n_heads":16,"cfg.d_model":256,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":256,"d_head":16,"n_layers":8,"n_ctx":512,"n_heads":16,"d_mlp":1024,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-Instruct-8M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-Instruct-8M","window_size":256,"attn_types":["global","local","global","local","global","local","global","local"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.05,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":6291456,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 256\nd_head: 16\nn_layers: 8\nn_ctx: 512\nn_heads: 16\nd_mlp: 1024\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-Instruct-8M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-Instruct-8M\nwindow_size: 256\nattn_types:\n- global\n- local\n- global\n- local\n- global\n- local\n- global\n- local\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.05\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 6291456\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-Instruct-8M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 256)\npos_embed:\n W_pos: (512, 256)\nblocks:\n '[0-7]':\n ln1:\n '[w, b]': (256,)\n ln2:\n '[w, b]': (256,)\n attn:\n '[W_Q, W_K, W_V]': (16, 256, 16)\n W_O: (16, 16, 256)\n '[b_Q, b_K, b_V]': (16, 16)\n b_O: (256,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (256, 1024)\n b_in: (1024,)\n W_out: (1024, 256)\n b_out: (256,)\nln_final:\n '[w, b]': (256,)\nunembed:\n W_U: (256, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 256)"},"pos_embed":{"W_pos":"(512, 256)"},"blocks":{"[0-7]":{"ln1":{"[w, b]":"(256,)"},"ln2":{"[w, b]":"(256,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 256, 16)","W_O":"(16, 16, 256)","[b_Q, b_K, b_V]":"(16, 16)","b_O":"(256,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(256, 1024)","b_in":"(1024,)","W_out":"(1024, 256)","b_out":"(256,)"}}},"ln_final":{"[w, b]":"(256,)"},"unembed":{"W_U":"(256, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-7]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 16)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 1024)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 256)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 256)\nunembed:\n hook_in: (batch, seq_len, 256)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 256)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-7]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 16)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 1024)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 256)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 256)"},"unembed":{"hook_in":"(batch, seq_len, 256)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 256)"}} +{"name.default_alias":"tiny-stories-instruct-1L-21M","name.huggingface":null,"name.aliases":"","model_type":"tiny-stories","name.from_cfg":"TinyStories-Instuct-1Layer-21M","n_params.as_str":"13M","n_params.as_int":12582912,"n_params.from_name":"21M","cfg.n_params":12582912,"cfg.n_layers":1,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPTNeoForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":1,"n_ctx":512,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"TinyStories-Instuct-1Layer-21M","use_attn_scale":false,"attn_scale":-1.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":true,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"roneneldan\/TinyStories-Instuct-1Layer-21M","window_size":256,"attn_types":["global"],"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":12582912,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 1\nn_ctx: 512\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: TinyStories-Instuct-1Layer-21M\nuse_attn_scale: false\nattn_scale: -1.0\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: true\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: roneneldan\/TinyStories-Instuct-1Layer-21M\nwindow_size: 256\nattn_types:\n- global\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 12582912\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"roneneldan\/TinyStories-Instuct-1Layer-21M","tokenizer.vocab_size":50257.0,"tokenizer.max_len":2048.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (512, 1024)\nblocks:\n '0':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(512, 1024)"},"blocks":{"0":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '0':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"0":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"stablelm-base-alpha-3b","name.huggingface":"stabilityai\/stablelm-base-alpha-3b","name.aliases":"stablelm-base-alpha-3b, stablelm-base-3b","model_type":"stablelm","name.from_cfg":"stablelm-base-alpha-3b","n_params.as_str":"3.2B","n_params.as_int":3221225472,"n_params.from_name":"3b","cfg.n_params":3221225472,"cfg.n_layers":16,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50688,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":16,"n_ctx":4096,"n_heads":32,"d_mlp":16384,"d_vocab":50688,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"stablelm-base-alpha-3b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stabilityai\/stablelm-base-alpha-3b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50688,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":3221225472,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 16\nn_ctx: 4096\nn_heads: 32\nd_mlp: 16384\nd_vocab: 50688\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: stablelm-base-alpha-3b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stabilityai\/stablelm-base-alpha-3b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50688\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 3221225472\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"stabilityai\/stablelm-base-alpha-3b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50688, 4096)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50688)\n b_U: (50688,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50688, 4096)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50688)","b_U":"(50688,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50688)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50688)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"stablelm-base-alpha-7b","name.huggingface":"stabilityai\/stablelm-base-alpha-7b","name.aliases":"stablelm-base-alpha-7b, stablelm-base-7b","model_type":"stablelm","name.from_cfg":"stablelm-base-alpha-7b","n_params.as_str":"7.2B","n_params.as_int":7247757312,"n_params.from_name":"7b","cfg.n_params":7247757312,"cfg.n_layers":16,"cfg.n_heads":48,"cfg.d_model":6144,"cfg.d_vocab":50432,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":6144,"d_head":128,"n_layers":16,"n_ctx":4096,"n_heads":48,"d_mlp":24576,"d_vocab":50432,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"stablelm-base-alpha-7b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stabilityai\/stablelm-base-alpha-7b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0102062073,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50432,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":7247757312,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 6144\nd_head: 128\nn_layers: 16\nn_ctx: 4096\nn_heads: 48\nd_mlp: 24576\nd_vocab: 50432\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: stablelm-base-alpha-7b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stabilityai\/stablelm-base-alpha-7b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.010206207261596576\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50432\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 7247757312\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"stabilityai\/stablelm-base-alpha-7b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"96EawM8Lij99W7OBTk0KW2ELUrQ=","tensor_shapes.state_dict":"embed:\n W_E: (50432, 6144)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (6144,)\n ln2:\n '[w, b]': (6144,)\n attn:\n '[W_Q, W_K, W_V]': (48, 6144, 128)\n W_O: (48, 128, 6144)\n '[b_Q, b_K, b_V]': (48, 128)\n b_O: (6144,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (6144, 24576)\n b_in: (24576,)\n W_out: (24576, 6144)\n b_out: (6144,)\nln_final:\n '[w, b]': (6144,)\nunembed:\n W_U: (6144, 50432)\n b_U: (50432,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50432, 6144)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(6144,)"},"ln2":{"[w, b]":"(6144,)"},"attn":{"[W_Q, W_K, W_V]":"(48, 6144, 128)","W_O":"(48, 128, 6144)","[b_Q, b_K, b_V]":"(48, 128)","b_O":"(6144,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(6144, 24576)","b_in":"(24576,)","W_out":"(24576, 6144)","b_out":"(6144,)"}}},"ln_final":{"[w, b]":"(6144,)"},"unembed":{"W_U":"(6144, 50432)","b_U":"(50432,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 48, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 48, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 24576)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 6144)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\nunembed:\n hook_in: (batch, seq_len, 6144)\n hook_out: (batch, seq_len, 50432)\nhook_embed: (batch, seq_len, 6144)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 48, 128)","[hook_attn_scores, hook_pattern]":"(batch, 48, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 24576)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 6144)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"unembed":{"hook_in":"(batch, seq_len, 6144)","hook_out":"(batch, seq_len, 50432)"},"hook_embed":"(batch, seq_len, 6144)"}} +{"name.default_alias":"stablelm-tuned-alpha-3b","name.huggingface":"stabilityai\/stablelm-tuned-alpha-3b","name.aliases":"stablelm-tuned-alpha-3b, stablelm-tuned-3b","model_type":"stablelm","name.from_cfg":"stablelm-tuned-alpha-3b","n_params.as_str":"3.2B","n_params.as_int":3221225472,"n_params.from_name":"3b","cfg.n_params":3221225472,"cfg.n_layers":16,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":50688,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":4096,"d_head":128,"n_layers":16,"n_ctx":4096,"n_heads":32,"d_mlp":16384,"d_vocab":50688,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"stablelm-tuned-alpha-3b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stabilityai\/stablelm-tuned-alpha-3b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50688,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":3221225472,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 16\nn_ctx: 4096\nn_heads: 32\nd_mlp: 16384\nd_vocab: 50688\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: stablelm-tuned-alpha-3b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stabilityai\/stablelm-tuned-alpha-3b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50688\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 3221225472\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"stabilityai\/stablelm-tuned-alpha-3b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"RD3vcWSd_TiTpqo5dHyICzaXtGQ=","tensor_shapes.state_dict":"embed:\n W_E: (50688, 4096)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (4096,)\n ln2:\n '[w, b]': (4096,)\n attn:\n '[W_Q, W_K, W_V]': (32, 4096, 128)\n W_O: (32, 128, 4096)\n '[b_Q, b_K, b_V]': (32, 128)\n b_O: (4096,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (4096, 16384)\n b_in: (16384,)\n W_out: (16384, 4096)\n b_out: (4096,)\nln_final:\n '[w, b]': (4096,)\nunembed:\n W_U: (4096, 50688)\n b_U: (50688,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50688, 4096)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(4096,)"},"ln2":{"[w, b]":"(4096,)"},"attn":{"[W_Q, W_K, W_V]":"(32, 4096, 128)","W_O":"(32, 128, 4096)","[b_Q, b_K, b_V]":"(32, 128)","b_O":"(4096,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(4096, 16384)","b_in":"(16384,)","W_out":"(16384, 4096)","b_out":"(4096,)"}}},"ln_final":{"[w, b]":"(4096,)"},"unembed":{"W_U":"(4096, 50688)","b_U":"(50688,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 32, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 16384)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 50688)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 32, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 16384)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 50688)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"stablelm-tuned-alpha-7b","name.huggingface":"stabilityai\/stablelm-tuned-alpha-7b","name.aliases":"stablelm-tuned-alpha-7b, stablelm-tuned-7b","model_type":"stablelm","name.from_cfg":"stablelm-tuned-alpha-7b","n_params.as_str":"7.2B","n_params.as_int":7247757312,"n_params.from_name":"7b","cfg.n_params":7247757312,"cfg.n_layers":16,"cfg.n_heads":48,"cfg.d_model":6144,"cfg.d_vocab":50432,"cfg.act_fn":"gelu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":true,"cfg.original_architecture":"GPTNeoXForCausalLM","cfg.normalization_type":"LN","config.raw__":{"d_model":6144,"d_head":128,"n_layers":16,"n_ctx":4096,"n_heads":48,"d_mlp":24576,"d_vocab":50432,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":false,"positional_embedding_type":"rotary","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"stablelm-tuned-alpha-7b","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPTNeoXForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stabilityai\/stablelm-tuned-alpha-7b","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0102062073,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":50432,"parallel_attn_mlp":true,"rotary_dim":32,"n_params":7247757312,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 6144\nd_head: 128\nn_layers: 16\nn_ctx: 4096\nn_heads: 48\nd_mlp: 24576\nd_vocab: 50432\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: false\npositional_embedding_type: rotary\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: stablelm-tuned-alpha-7b\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPTNeoXForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stabilityai\/stablelm-tuned-alpha-7b\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.010206207261596576\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 50432\nparallel_attn_mlp: true\nrotary_dim: 32\nn_params: 7247757312\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"stabilityai\/stablelm-tuned-alpha-7b","tokenizer.vocab_size":50254.0,"tokenizer.max_len":null,"tokenizer.class":"GPTNeoXTokenizer","tokenizer.vocab_hash":"RD3vcWSd_TiTpqo5dHyICzaXtGQ=","tensor_shapes.state_dict":"embed:\n W_E: (50432, 6144)\nblocks:\n '[0-15]':\n ln1:\n '[w, b]': (6144,)\n ln2:\n '[w, b]': (6144,)\n attn:\n '[W_Q, W_K, W_V]': (48, 6144, 128)\n W_O: (48, 128, 6144)\n '[b_Q, b_K, b_V]': (48, 128)\n b_O: (6144,)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 32)\n mlp:\n W_in: (6144, 24576)\n b_in: (24576,)\n W_out: (24576, 6144)\n b_out: (6144,)\nln_final:\n '[w, b]': (6144,)\nunembed:\n W_U: (6144, 50432)\n b_U: (50432,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50432, 6144)"},"blocks":{"[0-15]":{"ln1":{"[w, b]":"(6144,)"},"ln2":{"[w, b]":"(6144,)"},"attn":{"[W_Q, W_K, W_V]":"(48, 6144, 128)","W_O":"(48, 128, 6144)","[b_Q, b_K, b_V]":"(48, 128)","b_O":"(6144,)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 32)"},"mlp":{"W_in":"(6144, 24576)","b_in":"(24576,)","W_out":"(24576, 6144)","b_out":"(6144,)"}}},"ln_final":{"[w, b]":"(6144,)"},"unembed":{"W_U":"(6144, 50432)","b_U":"(50432,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-15]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n attn:\n '[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]': (batch, seq_len,\n 48, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 48, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 24576)\n '[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]': (batch, seq_len,\n 6144)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 6144)\nunembed:\n hook_in: (batch, seq_len, 6144)\n hook_out: (batch, seq_len, 50432)\nhook_embed: (batch, seq_len, 6144)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-15]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"attn":{"[hook_q, hook_k, hook_v, hook_rot_q, hook_rot_k, hook_z]":"(batch, seq_len, 48, 128)","[hook_attn_scores, hook_pattern]":"(batch, 48, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 24576)"},"[hook_resid_pre, hook_attn_out, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 6144)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 6144)"},"unembed":{"hook_in":"(batch, seq_len, 6144)","hook_out":"(batch, seq_len, 50432)"},"hook_embed":"(batch, seq_len, 6144)"}} +{"name.default_alias":"stanford-gpt2-small-a","name.huggingface":"stanford-crfm\/alias-gpt2-small-x21","name.aliases":"stanford-gpt2-small-a, alias-gpt2-small-x21, gpt2-mistral-small-a, gpt2-stanford-small-a","model_type":"gpt2","name.from_cfg":"alias-gpt2-small-x21","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"alias-gpt2-small-x21","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/alias-gpt2-small-x21","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: alias-gpt2-small-x21\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/alias-gpt2-small-x21\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/alias-gpt2-small-x21","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"stanford-gpt2-medium-a","name.huggingface":"stanford-crfm\/arwen-gpt2-medium-x21","name.aliases":"stanford-gpt2-medium-a, arwen-gpt2-medium-x21, gpt2-medium-small-a, gpt2-stanford-medium-a","model_type":"gpt2","name.from_cfg":"arwen-gpt2-medium-x21","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"arwen-gpt2-medium-x21","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/arwen-gpt2-medium-x21","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: arwen-gpt2-medium-x21\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/arwen-gpt2-medium-x21\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/arwen-gpt2-medium-x21","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"stanford-gpt2-small-b","name.huggingface":"stanford-crfm\/battlestar-gpt2-small-x49","name.aliases":"stanford-gpt2-small-b, battlestar-gpt2-small-x49, gpt2-mistral-small-b","model_type":"gpt2","name.from_cfg":"battlestar-gpt2-small-x49","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"battlestar-gpt2-small-x49","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/battlestar-gpt2-small-x49","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: battlestar-gpt2-small-x49\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/battlestar-gpt2-small-x49\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/battlestar-gpt2-small-x49","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"stanford-gpt2-medium-b","name.huggingface":"stanford-crfm\/beren-gpt2-medium-x49","name.aliases":"stanford-gpt2-medium-b, beren-gpt2-medium-x49, gpt2-medium-small-b, gpt2-stanford-medium-b","model_type":"gpt2","name.from_cfg":"beren-gpt2-medium-x49","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"beren-gpt2-medium-x49","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/beren-gpt2-medium-x49","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: beren-gpt2-medium-x49\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/beren-gpt2-medium-x49\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/beren-gpt2-medium-x49","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"stanford-gpt2-small-c","name.huggingface":"stanford-crfm\/caprica-gpt2-small-x81","name.aliases":"stanford-gpt2-small-c, caprica-gpt2-small-x81, gpt2-mistral-small-c, gpt2-stanford-small-c","model_type":"gpt2","name.from_cfg":"caprica-gpt2-small-x81","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"caprica-gpt2-small-x81","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/caprica-gpt2-small-x81","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: caprica-gpt2-small-x81\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/caprica-gpt2-small-x81\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/caprica-gpt2-small-x81","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"stanford-gpt2-medium-c","name.huggingface":"stanford-crfm\/celebrimbor-gpt2-medium-x81","name.aliases":"stanford-gpt2-medium-c, celebrimbor-gpt2-medium-x81, gpt2-medium-small-c","model_type":"gpt2","name.from_cfg":"celebrimbor-gpt2-medium-x81","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"celebrimbor-gpt2-medium-x81","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/celebrimbor-gpt2-medium-x81","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: celebrimbor-gpt2-medium-x81\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/celebrimbor-gpt2-medium-x81\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/celebrimbor-gpt2-medium-x81","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"stanford-gpt2-small-d","name.huggingface":"stanford-crfm\/darkmatter-gpt2-small-x343","name.aliases":"stanford-gpt2-small-d, darkmatter-gpt2-small-x343, gpt2-mistral-small-d","model_type":"gpt2","name.from_cfg":"darkmatter-gpt2-small-x343","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"darkmatter-gpt2-small-x343","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/darkmatter-gpt2-small-x343","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: darkmatter-gpt2-small-x343\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/darkmatter-gpt2-small-x343\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/darkmatter-gpt2-small-x343","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"stanford-gpt2-medium-d","name.huggingface":"stanford-crfm\/durin-gpt2-medium-x343","name.aliases":"stanford-gpt2-medium-d, durin-gpt2-medium-x343, gpt2-medium-small-d, gpt2-stanford-medium-d","model_type":"gpt2","name.from_cfg":"durin-gpt2-medium-x343","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"durin-gpt2-medium-x343","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/durin-gpt2-medium-x343","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: durin-gpt2-medium-x343\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/durin-gpt2-medium-x343\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/durin-gpt2-medium-x343","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"stanford-gpt2-medium-e","name.huggingface":"stanford-crfm\/eowyn-gpt2-medium-x777","name.aliases":"stanford-gpt2-medium-e, eowyn-gpt2-medium-x777, gpt2-medium-small-e, gpt2-stanford-medium-e","model_type":"gpt2","name.from_cfg":"eowyn-gpt2-medium-x777","n_params.as_str":"302M","n_params.as_int":301989888,"n_params.from_name":null,"cfg.n_params":301989888,"cfg.n_layers":24,"cfg.n_heads":16,"cfg.d_model":1024,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":1024,"d_head":64,"n_layers":24,"n_ctx":1024,"n_heads":16,"d_mlp":4096,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"eowyn-gpt2-medium-x777","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/eowyn-gpt2-medium-x777","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.025,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":301989888,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 1024\nd_head: 64\nn_layers: 24\nn_ctx: 1024\nn_heads: 16\nd_mlp: 4096\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: eowyn-gpt2-medium-x777\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/eowyn-gpt2-medium-x777\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.025\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 301989888\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/eowyn-gpt2-medium-x777","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 1024)\npos_embed:\n W_pos: (1024, 1024)\nblocks:\n '[0-23]':\n ln1:\n '[w, b]': (1024,)\n ln2:\n '[w, b]': (1024,)\n attn:\n '[W_Q, W_K, W_V]': (16, 1024, 64)\n W_O: (16, 64, 1024)\n '[b_Q, b_K, b_V]': (16, 64)\n b_O: (1024,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (1024, 4096)\n b_in: (4096,)\n W_out: (4096, 1024)\n b_out: (1024,)\nln_final:\n '[w, b]': (1024,)\nunembed:\n W_U: (1024, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 1024)"},"pos_embed":{"W_pos":"(1024, 1024)"},"blocks":{"[0-23]":{"ln1":{"[w, b]":"(1024,)"},"ln2":{"[w, b]":"(1024,)"},"attn":{"[W_Q, W_K, W_V]":"(16, 1024, 64)","W_O":"(16, 64, 1024)","[b_Q, b_K, b_V]":"(16, 64)","b_O":"(1024,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(1024, 4096)","b_in":"(4096,)","W_out":"(4096, 1024)","b_out":"(1024,)"}}},"ln_final":{"[w, b]":"(1024,)"},"unembed":{"W_U":"(1024, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-23]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 16, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 16, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 4096)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 1024)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 1024)\nunembed:\n hook_in: (batch, seq_len, 1024)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 1024)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-23]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 16, 64)","[hook_attn_scores, hook_pattern]":"(batch, 16, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 4096)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 1024)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 1024)"},"unembed":{"hook_in":"(batch, seq_len, 1024)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 1024)"}} +{"name.default_alias":"stanford-gpt2-small-e","name.huggingface":"stanford-crfm\/expanse-gpt2-small-x777","name.aliases":"stanford-gpt2-small-e, expanse-gpt2-small-x777, gpt2-mistral-small-e","model_type":"gpt2","name.from_cfg":"expanse-gpt2-small-x777","n_params.as_str":"85M","n_params.as_int":84934656,"n_params.from_name":null,"cfg.n_params":84934656,"cfg.n_layers":12,"cfg.n_heads":12,"cfg.d_model":768,"cfg.d_vocab":50257,"cfg.act_fn":"gelu_new","cfg.positional_embedding_type":"standard","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"GPT2LMHeadModel","cfg.normalization_type":"LN","config.raw__":{"d_model":768,"d_head":64,"n_layers":12,"n_ctx":1024,"n_heads":12,"d_mlp":3072,"d_vocab":50257,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"standard","n_key_value_heads":null,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"gelu_new","normalization_type":"LN","num_experts":null,"experts_per_token":null,"final_rms":false,"dtype":"torch.float32","model_name":"expanse-gpt2-small-x777","use_attn_scale":true,"attn_scale":8.0,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":false,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"GPT2LMHeadModel","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"stanford-crfm\/expanse-gpt2-small-x777","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0288675135,"init_weights":false,"scale_attn_by_inverse_layer_idx":true,"d_vocab_out":50257,"parallel_attn_mlp":false,"rotary_dim":null,"n_params":84934656,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":10000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":false,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":false,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 768\nd_head: 64\nn_layers: 12\nn_ctx: 1024\nn_heads: 12\nd_mlp: 3072\nd_vocab: 50257\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: standard\nn_key_value_heads: null\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: gelu_new\nnormalization_type: LN\nnum_experts: null\nexperts_per_token: null\nfinal_rms: false\ndtype: torch.float32\nmodel_name: expanse-gpt2-small-x777\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n AAAAAAAAIEA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: false\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: GPT2LMHeadModel\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: stanford-crfm\/expanse-gpt2-small-x777\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.02886751345948129\ninit_weights: false\nscale_attn_by_inverse_layer_idx: true\nd_vocab_out: 50257\nparallel_attn_mlp: false\nrotary_dim: null\nn_params: 84934656\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 10000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: false\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: false\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"stanford-crfm\/expanse-gpt2-small-x777","tokenizer.vocab_size":50257.0,"tokenizer.max_len":1024.0,"tokenizer.class":"GPT2Tokenizer","tokenizer.vocab_hash":"v8xfIj5kwZX5RwgLU66lZNZUlE4=","tensor_shapes.state_dict":"embed:\n W_E: (50257, 768)\npos_embed:\n W_pos: (1024, 768)\nblocks:\n '[0-11]':\n ln1:\n '[w, b]': (768,)\n ln2:\n '[w, b]': (768,)\n attn:\n '[W_Q, W_K, W_V]': (12, 768, 64)\n W_O: (12, 64, 768)\n '[b_Q, b_K, b_V]': (12, 64)\n b_O: (768,)\n mask: (0, 0)\n IGNORE: ()\n mlp:\n W_in: (768, 3072)\n b_in: (3072,)\n W_out: (3072, 768)\n b_out: (768,)\nln_final:\n '[w, b]': (768,)\nunembed:\n W_U: (768, 50257)\n b_U: (50257,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(50257, 768)"},"pos_embed":{"W_pos":"(1024, 768)"},"blocks":{"[0-11]":{"ln1":{"[w, b]":"(768,)"},"ln2":{"[w, b]":"(768,)"},"attn":{"[W_Q, W_K, W_V]":"(12, 768, 64)","W_O":"(12, 64, 768)","[b_Q, b_K, b_V]":"(12, 64)","b_O":"(768,)","mask":"(0, 0)","IGNORE":"()"},"mlp":{"W_in":"(768, 3072)","b_in":"(3072,)","W_out":"(3072, 768)","b_out":"(768,)"}}},"ln_final":{"[w, b]":"(768,)"},"unembed":{"W_U":"(768, 50257)","b_U":"(50257,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-11]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n attn:\n '[hook_q, hook_k, hook_v, hook_z]': (batch, seq_len, 12, 64)\n '[hook_attn_scores, hook_pattern]': (batch, 12, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 3072)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 768)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 768)\nunembed:\n hook_in: (batch, seq_len, 768)\n hook_out: (batch, seq_len, 50257)\n'[hook_embed, hook_pos_embed]': (batch, seq_len, 768)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-11]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"attn":{"[hook_q, hook_k, hook_v, hook_z]":"(batch, seq_len, 12, 64)","[hook_attn_scores, hook_pattern]":"(batch, 12, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 3072)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 768)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 768)"},"unembed":{"hook_in":"(batch, seq_len, 768)","hook_out":"(batch, seq_len, 50257)"},"[hook_embed, hook_pos_embed]":"(batch, seq_len, 768)"}} +{"name.default_alias":"apertus-8b","name.huggingface":"swiss-ai\/Apertus-8B-2509","name.aliases":"apertus-8b, apertus","model_type":null,"name.from_cfg":"Apertus-8B-2509","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"8b","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":131072,"cfg.act_fn":"xielu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"ApertusForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":65536,"n_heads":32,"d_mlp":21504,"d_vocab":131072,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"xielu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Apertus-8B-2509","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"ApertusForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"swiss-ai\/Apertus-8B-2509","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":131072,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":12000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 65536\nn_heads: 32\nd_mlp: 21504\nd_vocab: 131072\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: xielu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Apertus-8B-2509\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: ApertusForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: swiss-ai\/Apertus-8B-2509\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 131072\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 12000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"swiss-ai\/Apertus-8B-2509","tokenizer.vocab_size":131072.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"C46N1vCQdy2ADcCzoHO1wqva-9w=","tensor_shapes.state_dict":"embed:\n W_E: (131072, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n act_fn:\n '[alpha_p, alpha_n, beta, eps]': ()\n W_in: (4096, 21504)\n b_in: (21504,)\n W_out: (21504, 4096)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 131072)\n b_U: (131072,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(131072, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"act_fn":{"[alpha_p, alpha_n, beta, eps]":"()"},"W_in":"(4096, 21504)","b_in":"(21504,)","W_out":"(21504, 4096)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 131072)","b_U":"(131072,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n q_norm:\n hook_scale: (1776098016, 1)\n hook_normalized: (1776098016, 128)\n k_norm:\n hook_scale: (444024504, 1)\n hook_normalized: (444024504, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 21504)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 131072)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"q_norm":{"hook_scale":"(1776098016, 1)","hook_normalized":"(1776098016, 128)"},"k_norm":{"hook_scale":"(444024504, 1)","hook_normalized":"(444024504, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 21504)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 131072)"},"hook_embed":"(batch, seq_len, 4096)"}} +{"name.default_alias":"apertus-8b-instruct","name.huggingface":"swiss-ai\/Apertus-8B-Instruct-2509","name.aliases":"apertus-8b-instruct, apertus-instruct","model_type":null,"name.from_cfg":"Apertus-8B-Instruct-2509","n_params.as_str":"7.8B","n_params.as_int":7784628224,"n_params.from_name":"8b","cfg.n_params":7784628224,"cfg.n_layers":32,"cfg.n_heads":32,"cfg.d_model":4096,"cfg.d_vocab":131072,"cfg.act_fn":"xielu","cfg.positional_embedding_type":"rotary","cfg.parallel_attn_mlp":false,"cfg.original_architecture":"ApertusForCausalLM","cfg.normalization_type":"RMS","config.raw__":{"d_model":4096,"d_head":128,"n_layers":32,"n_ctx":65536,"n_heads":32,"d_mlp":21504,"d_vocab":131072,"device":"cpu","use_attn_result":false,"use_split_qkv_input":false,"default_prepend_bos":true,"positional_embedding_type":"rotary","n_key_value_heads":8,"attn_only":false,"gated_mlp":false,"uses_rms_norm":false,"eps":0.00001,"layer_norm_folding":false,"act_fn":"xielu","normalization_type":"RMS","num_experts":null,"experts_per_token":null,"final_rms":true,"dtype":"torch.float32","model_name":"Apertus-8B-Instruct-2509","use_attn_scale":true,"attn_scale":11.313708499,"use_hook_mlp_in":false,"use_attn_in":false,"use_qk_norm":true,"use_local_attn":false,"ungroup_grouped_query_attention":false,"original_architecture":"ApertusForCausalLM","from_checkpoint":false,"checkpoint_index":null,"checkpoint_label_type":null,"checkpoint_value":null,"tokenizer_name":"swiss-ai\/Apertus-8B-Instruct-2509","window_size":null,"attn_types":null,"init_mode":"gpt2","n_devices":1,"attention_dir":"causal","seed":null,"initializer_range":0.0125,"init_weights":false,"scale_attn_by_inverse_layer_idx":false,"d_vocab_out":131072,"parallel_attn_mlp":false,"rotary_dim":128,"n_params":7784628224,"use_hook_tokens":false,"tokenizer_prepends_bos":null,"post_embedding_ln":false,"rotary_base":12000000,"rotary_base_local":null,"rotary_scaling_factor":1.0,"trust_remote_code":true,"rotary_adjacent_pairs":false,"load_in_4bit":false,"relative_attention_max_distance":null,"relative_attention_num_buckets":null,"decoder_start_token_id":null,"tie_word_embeddings":false,"use_normalization_before_and_after":false,"attn_scores_soft_cap":-1.0,"output_logits_soft_cap":-1.0,"use_NTK_by_parts_rope":true,"NTK_by_parts_low_freq_factor":1.0,"NTK_by_parts_high_freq_factor":4.0,"NTK_by_parts_factor":8.0,"NTK_original_ctx_len":8192,"use_yarn_rope":false,"yarn_factor":1.0,"yarn_attention_factor":1.0,"yarn_beta_fast":32.0,"yarn_beta_slow":1.0,"yarn_original_max_position_embeddings":4096,"yarn_truncate":true,"yarn_global_attn_only":false,"use_attention_sinks":false,"norm_topk_prob":false},"config":"d_model: 4096\nd_head: 128\nn_layers: 32\nn_ctx: 65536\nn_heads: 32\nd_mlp: 21504\nd_vocab: 131072\ndevice: cpu\nuse_attn_result: false\nuse_split_qkv_input: false\ndefault_prepend_bos: true\npositional_embedding_type: rotary\nn_key_value_heads: 8\nattn_only: false\ngated_mlp: false\nuses_rms_norm: false\neps: 1.0e-05\nlayer_norm_folding: false\nact_fn: xielu\nnormalization_type: RMS\nnum_experts: null\nexperts_per_token: null\nfinal_rms: true\ndtype: torch.float32\nmodel_name: Apertus-8B-Instruct-2509\nuse_attn_scale: true\nattn_scale: !!python\/object\/apply:numpy._core.multiarray.scalar\n- !!python\/object\/apply:numpy.dtype\n args:\n - f8\n - false\n - true\n state: !!python\/tuple\n - 3\n - <\n - null\n - null\n - null\n - -1\n - -1\n - 0\n- !!binary |\n zTt\/Zp6gJkA=\nuse_hook_mlp_in: false\nuse_attn_in: false\nuse_qk_norm: true\nuse_local_attn: false\nungroup_grouped_query_attention: false\noriginal_architecture: ApertusForCausalLM\nfrom_checkpoint: false\ncheckpoint_index: null\ncheckpoint_label_type: null\ncheckpoint_value: null\ntokenizer_name: swiss-ai\/Apertus-8B-Instruct-2509\nwindow_size: null\nattn_types: null\ninit_mode: gpt2\nn_devices: 1\nattention_dir: causal\nseed: null\ninitializer_range: 0.0125\ninit_weights: false\nscale_attn_by_inverse_layer_idx: false\nd_vocab_out: 131072\nparallel_attn_mlp: false\nrotary_dim: 128\nn_params: 7784628224\nuse_hook_tokens: false\ntokenizer_prepends_bos: null\npost_embedding_ln: false\nrotary_base: 12000000\nrotary_base_local: null\nrotary_scaling_factor: 1.0\ntrust_remote_code: true\nrotary_adjacent_pairs: false\nload_in_4bit: false\nrelative_attention_max_distance: null\nrelative_attention_num_buckets: null\ndecoder_start_token_id: null\ntie_word_embeddings: false\nuse_normalization_before_and_after: false\nattn_scores_soft_cap: -1.0\noutput_logits_soft_cap: -1.0\nuse_NTK_by_parts_rope: true\nNTK_by_parts_low_freq_factor: 1.0\nNTK_by_parts_high_freq_factor: 4.0\nNTK_by_parts_factor: 8.0\nNTK_original_ctx_len: 8192\nuse_yarn_rope: false\nyarn_factor: 1.0\nyarn_attention_factor: 1.0\nyarn_beta_fast: 32.0\nyarn_beta_slow: 1.0\nyarn_original_max_position_embeddings: 4096\nyarn_truncate: true\nyarn_global_attn_only: false\nuse_attention_sinks: false\nnorm_topk_prob: false\n","tokenizer.name":"swiss-ai\/Apertus-8B-Instruct-2509","tokenizer.vocab_size":131072.0,"tokenizer.max_len":null,"tokenizer.class":"TokenizersBackend","tokenizer.vocab_hash":"C46N1vCQdy2ADcCzoHO1wqva-9w=","tensor_shapes.state_dict":"embed:\n W_E: (131072, 4096)\nblocks:\n '[0-31]':\n ln1:\n w: (4096,)\n ln2:\n w: (4096,)\n attn:\n q_norm:\n w: (128,)\n k_norm:\n w: (128,)\n W_Q: (32, 4096, 128)\n W_O: (32, 128, 4096)\n b_Q: (32, 128)\n b_O: (4096,)\n '[_W_K, _W_V]': (8, 4096, 128)\n '[_b_K, _b_V]': (8, 128)\n mask: (0, 0)\n IGNORE: ()\n '[rotary_sin, rotary_cos]': (2048, 128)\n mlp:\n act_fn:\n '[alpha_p, alpha_n, beta, eps]': ()\n W_in: (4096, 21504)\n b_in: (21504,)\n W_out: (21504, 4096)\n b_out: (4096,)\nln_final:\n w: (4096,)\nunembed:\n W_U: (4096, 131072)\n b_U: (131072,)\n","tensor_shapes.state_dict.raw__":{"embed":{"W_E":"(131072, 4096)"},"blocks":{"[0-31]":{"ln1":{"w":"(4096,)"},"ln2":{"w":"(4096,)"},"attn":{"q_norm":{"w":"(128,)"},"k_norm":{"w":"(128,)"},"W_Q":"(32, 4096, 128)","W_O":"(32, 128, 4096)","b_Q":"(32, 128)","b_O":"(4096,)","[_W_K, _W_V]":"(8, 4096, 128)","[_b_K, _b_V]":"(8, 128)","mask":"(0, 0)","IGNORE":"()","[rotary_sin, rotary_cos]":"(2048, 128)"},"mlp":{"act_fn":{"[alpha_p, alpha_n, beta, eps]":"()"},"W_in":"(4096, 21504)","b_in":"(21504,)","W_out":"(21504, 4096)","b_out":"(4096,)"}}},"ln_final":{"w":"(4096,)"},"unembed":{"W_U":"(4096, 131072)","b_U":"(131072,)"}},"tensor_shapes.activation_cache":"blocks:\n '[0-31]':\n ln1:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n attn:\n q_norm:\n hook_scale: (1776098016, 1)\n hook_normalized: (1776098016, 128)\n k_norm:\n hook_scale: (444024504, 1)\n hook_normalized: (444024504, 128)\n '[hook_q, hook_rot_q, hook_z]': (batch, seq_len, 32, 128)\n '[hook_k, hook_v, hook_rot_k]': (batch, seq_len, 8, 128)\n '[hook_attn_scores, hook_pattern]': (batch, 32, seq_len, seq_len)\n ln2:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\n mlp:\n '[hook_pre, hook_post]': (batch, seq_len, 21504)\n '[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]': (batch,\n seq_len, 4096)\nln_final:\n hook_scale: (batch, seq_len, 1)\n hook_normalized: (batch, seq_len, 4096)\nunembed:\n hook_in: (batch, seq_len, 4096)\n hook_out: (batch, seq_len, 131072)\nhook_embed: (batch, seq_len, 4096)\n","tensor_shapes.activation_cache.raw__":{"blocks":{"[0-31]":{"ln1":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"attn":{"q_norm":{"hook_scale":"(1776098016, 1)","hook_normalized":"(1776098016, 128)"},"k_norm":{"hook_scale":"(444024504, 1)","hook_normalized":"(444024504, 128)"},"[hook_q, hook_rot_q, hook_z]":"(batch, seq_len, 32, 128)","[hook_k, hook_v, hook_rot_k]":"(batch, seq_len, 8, 128)","[hook_attn_scores, hook_pattern]":"(batch, 32, seq_len, seq_len)"},"ln2":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"mlp":{"[hook_pre, hook_post]":"(batch, seq_len, 21504)"},"[hook_resid_pre, hook_attn_out, hook_resid_mid, hook_mlp_out, hook_resid_post]":"(batch, seq_len, 4096)"}},"ln_final":{"hook_scale":"(batch, seq_len, 1)","hook_normalized":"(batch, seq_len, 4096)"},"unembed":{"hook_in":"(batch, seq_len, 4096)","hook_out":"(batch, seq_len, 131072)"},"hook_embed":"(batch, seq_len, 4096)"}} diff --git a/docs/source/content/adapter_development/adapter-builder-tool.md b/docs/source/content/adapter_development/adapter-builder-tool.md new file mode 100644 index 0000000000..6550899d9c --- /dev/null +++ b/docs/source/content/adapter_development/adapter-builder-tool.md @@ -0,0 +1,72 @@ +# Automated Adapter Builder + +The repo ships a contributor tool that automates the entire adapter-creation +workflow described in the [Adapter Creation Guide](adapter-creation-guide.md): +give it a HuggingFace architecture class name, and a pair of Claude Code +agents analyzes the HF source, plans the adapter, implements it +phase-by-phase with independent code review at every checkpoint, and verifies +it against real models. + +It lives at `devtools/adapter_builder/` — it is developer tooling, not part +of the shipped `transformer_lens` package. This page is an orientation; the +authoritative reference is the tool's own +[README](https://github.com/TransformerLensOrg/TransformerLens/blob/main/devtools/adapter_builder/README.md) +and the docs beside it. + +## When to use it + +- **Manual path**: for a single adapter you want to write yourself, follow + the [Adapter Creation Guide](adapter-creation-guide.md) (Claude Code users: + the `/add-model-support` slash command walks the same checklist). +- **Automated path**: for batch adapter work, or when you want an + autonomously built and reviewed first draft, use the builder. The output is + left **uncommitted in an isolated git worktree** for human review — the + agents cannot commit, push, or open PRs. + +## How it works, briefly + +The builder launches a **Programmer** and a **Reviewer** agent that work in a +dedicated git worktree of the repo (your main working tree is never touched). +Every lifecycle step is gated: the architecture brief is fact-checked against +the HF source before planning, each implementation phase is code-reviewed +before the next begins, and verification requires full HuggingFace output +parity — model targets are chosen for config diversity (always including the +architecture's smallest model, verified first) so numerical edge cases fail +in seconds. + +Two coordination modes exist: **agent-teams** (an orchestrator session; +requires Claude Code Max) and **solo** (two independent sessions coordinated +by a signal-routing daemon; works on any tier). + +Critical rules — no git writes, no edits to deprecated `HookedTransformer` +paths, no oversized or structural-only verification runs, mandatory +mypy/format gates — are enforced by Claude Code hooks at the framework +level, not just by prompt instructions. + +## Launching + +From the repo root: + +```bash +# Agent-teams mode (Max tier) +devtools/adapter_builder/agents/launch-agent-pair.sh --architecture CohereForCausalLM + +# Solo mode (any tier) +devtools/adapter_builder/agents/launch-agent-pair.sh --mode solo --architecture CohereForCausalLM + +# Check everything without creating anything +devtools/adapter_builder/agents/launch-agent-pair.sh --architecture CohereForCausalLM --dry-run +``` + +When a run completes, the worktree (default: `../worktrees/<branch>/`) +contains the adapter, its unit tests, registry entries, and a full audit +trail (`.adapter-workspace/`: brief, plan, every review round, verification +results, and a structured timeline of every tool call). Review it, then +commit and open the PR yourself — registration and testing expectations are +the same as the manual path. + +## Requirements + +Claude Code on `PATH` (Max tier for agent-teams mode; any tier for solo), +plus `git`, `jq`, `tmux`, and `uv`. Verification loads real model weights: +budget memory accordingly (`--max-memory`, default 96GB). diff --git a/docs/source/content/contributing.md b/docs/source/content/contributing.md index 59fececccf..b9f24323ad 100644 --- a/docs/source/content/contributing.md +++ b/docs/source/content/contributing.md @@ -277,6 +277,7 @@ Two guides walk through the process: - [Architecture Unit Test Suite Guide](adapter_development/adapter-unit-test-guide.md) — what to test (and what not to) in the per-adapter unit suite, so every test guards a real, adapter-specific regression. - [HuggingFace Model Analysis Guide](adapter_development/hf-model-analysis-guide.md) — a reference for reading an HF model's `config.json` and source files to extract the attributes you'll set on `self.cfg`. - [HuggingFace Model Scraper](adapter_development/hf-scraper.md) — how to run the scraper that discovers HF models for the registry, including the per-architecture targeted-scrape mode used after merging a new adapter. +- [Automated Adapter Builder](adapter_development/adapter-builder-tool.md) — the `devtools/adapter_builder/` agent tool that runs this whole workflow autonomously (analysis, phased implementation, review at every checkpoint, parity verification), leaving the result uncommitted in a worktree for human review. Adapters live in `transformer_lens/model_bridge/supported_architectures/<model_name>.py` and need to be registered in **four** places. Each registration site has a different consequence if you skip it, which is why the next section's invariant test is worth running before you open the PR. @@ -358,4 +359,5 @@ adapter_development/adapter-unit-test-guide adapter_development/hf-model-analysis-guide adapter_development/hf-scraper adapter_development/external-adapter-registration +adapter_development/adapter-builder-tool ``` From b67e6cd975bfc3a166beaf58b780e6a0dfa4cc58 Mon Sep 17 00:00:00 2001 From: emerardd <113128214+emerardd@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:35:22 +0800 Subject: [PATCH 50/87] Stabilize Bridge numerical CI tests (#1654) --- .../test_left_padding_positions.py | 19 ++++++++++--------- .../test_optimizer_compatibility.py | 3 ++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/integration/model_bridge/test_left_padding_positions.py b/tests/integration/model_bridge/test_left_padding_positions.py index 272561c9c3..46d9ab8ff5 100644 --- a/tests/integration/model_bridge/test_left_padding_positions.py +++ b/tests/integration/model_bridge/test_left_padding_positions.py @@ -180,22 +180,23 @@ def test_unshifted_rows_keep_default_positions(distilgpt2_bridge, tokens) -> Non def test_one_left_padded_row_does_not_perturb_its_neighbours(distilgpt2_bridge, tokens) -> None: - """A whole-batch predicate would hand derived positions to every row; the - rows that needed no correction must come out bit-identical to running alone.""" + """Derived positions for one row must not change its unshifted neighbours.""" n_pad = 3 batch, mask, (right, m_right), (plain, m_plain) = _mixed_batch(tokens, n_pad) + control_batch = torch.cat([right, right, plain], dim=0) + control_mask = torch.cat([m_right, m_right, m_plain], dim=0) with torch.no_grad(): mixed = distilgpt2_bridge(batch, attention_mask=mask, return_type="logits") - alone_right = distilgpt2_bridge(right, attention_mask=m_right, return_type="logits") - alone_plain = distilgpt2_bridge(plain, attention_mask=m_plain, return_type="logits") + control = distilgpt2_bridge( + control_batch, attention_mask=control_mask, return_type="logits" + ) unpadded = distilgpt2_bridge(tokens, return_type="logits") - # Not exact equality: batching alone perturbs float accumulation order. The - # regression this guards was 8e-01, so 1e-6 separates them decisively while - # staying above anything a different BLAS could introduce. - torch.testing.assert_close(mixed[0:1], alone_right, rtol=0, atol=1e-6) - torch.testing.assert_close(mixed[2:3], alone_plain, rtol=0, atol=1e-6) + # Matching batch shapes isolate derived-position handling from BLAS kernel + # changes caused by comparing batched and single-row matrix multiplications. + torch.testing.assert_close(mixed[0:1], control[0:1], rtol=0, atol=1e-6) + torch.testing.assert_close(mixed[2:3], control[2:3], rtol=0, atol=1e-6) # ...while the row that did need correcting still gets it. torch.testing.assert_close(mixed[1:2, n_pad:], unpadded, rtol=1e-3, atol=1e-3) diff --git a/tests/integration/model_bridge/test_optimizer_compatibility.py b/tests/integration/model_bridge/test_optimizer_compatibility.py index 34196853dc..f306f56e96 100644 --- a/tests/integration/model_bridge/test_optimizer_compatibility.py +++ b/tests/integration/model_bridge/test_optimizer_compatibility.py @@ -134,7 +134,8 @@ def test_bridge_hooked_parity_multi_step_optimization(): StepThresholds( step=1, initial_fwd=StageThresholds(logits_max=1e-3, logits_mean=1e-4, loss_relative=1e-6), - post_update_fwd=StageThresholds(logits_max=2.0, logits_mean=1e-3, loss_relative=1e-5), + # GitHub CPU runners repeatedly produce a 1.032e-3 mean difference here. + post_update_fwd=StageThresholds(logits_max=2.0, logits_mean=2e-3, loss_relative=1e-5), param_update=StageThresholds(params_max=1e-2, params_mean=1e-6), ), StepThresholds( From b8aca0f6d0bfaa41cada71ca8f3db2916b7cbc96 Mon Sep 17 00:00:00 2001 From: emerardd <113128214+emerardd@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:19:10 +0800 Subject: [PATCH 51/87] Fix gated Qwen W_Q analysis weights (#1653) --- .../test_qwen3_gated_query_weights.py | 180 ++++++++++++++++++ .../test_qwen3_5_adapter.py | 113 +---------- .../test_qwen3_5_multimodal_adapter.py | 32 ++-- .../test_qwen3_adapter.py | 51 ----- .../test_qwen3_next_adapter.py | 109 ----------- .../generalized_components/attention.py | 29 ++- .../supported_architectures/qwen3.py | 18 -- .../supported_architectures/qwen3_5.py | 13 +- .../qwen3_5_multimodal.py | 6 - .../supported_architectures/qwen3_next.py | 6 - 10 files changed, 227 insertions(+), 330 deletions(-) create mode 100644 tests/integration/model_bridge/test_qwen3_gated_query_weights.py diff --git a/tests/integration/model_bridge/test_qwen3_gated_query_weights.py b/tests/integration/model_bridge/test_qwen3_gated_query_weights.py new file mode 100644 index 0000000000..c490eab77f --- /dev/null +++ b/tests/integration/model_bridge/test_qwen3_gated_query_weights.py @@ -0,0 +1,180 @@ +"""Download-free integration tests for Qwen3 gated query projections.""" + +import copy +from typing import NamedTuple + +import pytest +import torch +from transformers import ( + Qwen3_5ForCausalLM, + Qwen3_5TextConfig, + Qwen3Config, + Qwen3ForCausalLM, + Qwen3NextConfig, + Qwen3NextForCausalLM, +) + +from transformer_lens.model_bridge.bridge import TransformerBridge +from transformer_lens.model_bridge.sources import build_bridge_from_module + +N_HEADS = 2 +D_HEAD = 8 +D_MODEL = 16 +N_LAYERS = 2 +VOCAB_SIZE = 32 + + +class GatedQueryCase(NamedTuple): + bridge: TransformerBridge + reference_logits: torch.Tensor + tokens: torch.Tensor + raw_q_weights: torch.Tensor + + +def _tiny_hybrid_config(architecture: str): + common = dict( + hidden_size=D_MODEL, + num_hidden_layers=N_LAYERS, + num_attention_heads=N_HEADS, + num_key_value_heads=1, + head_dim=D_HEAD, + intermediate_size=32, + vocab_size=VOCAB_SIZE, + rms_norm_eps=1e-6, + hidden_act="silu", + full_attention_interval=1, + linear_conv_kernel_dim=4, + linear_key_head_dim=8, + linear_value_head_dim=8, + linear_num_key_heads=N_HEADS, + linear_num_value_heads=N_HEADS, + rope_parameters={ + "rope_theta": 10000.0, + "partial_rotary_factor": 0.25, + "rope_type": "default", + }, + ) + if architecture == "Qwen3_5ForCausalLM": + return Qwen3_5TextConfig(**common) + return Qwen3NextConfig( + **common, + num_experts=2, + num_experts_per_tok=1, + moe_intermediate_size=16, + shared_expert_intermediate_size=16, + decoder_sparse_step=1, + mlp_only_layers=[], + ) + + +@pytest.fixture( + scope="module", + params=[ + ("Qwen3_5ForCausalLM", Qwen3_5ForCausalLM), + ("Qwen3NextForCausalLM", Qwen3NextForCausalLM), + ], + ids=["qwen3_5", "qwen3_next"], +) +def gated_query_case(request: pytest.FixtureRequest) -> GatedQueryCase: + architecture, model_cls = request.param + torch.manual_seed(0) + cfg = _tiny_hybrid_config(architecture) + hf_model = model_cls(cfg).eval() + with torch.no_grad(): + for layer_index, layer in enumerate(hf_model.model.layers): + q_weight = layer.self_attn.q_proj.weight + values = torch.arange(q_weight.numel(), dtype=q_weight.dtype).reshape_as(q_weight) + q_weight.copy_((values + layer_index * q_weight.numel()) / q_weight.numel()) + raw_q_weights = torch.stack( + [layer.self_attn.q_proj.weight.detach().clone() for layer in hf_model.model.layers] + ) + + tokens = torch.arange(4).unsqueeze(0) + with torch.no_grad(): + reference_logits = hf_model(tokens).logits + + bridge = build_bridge_from_module( + hf_model, + architecture, + hf_config=copy.deepcopy(cfg), + tokenizer=None, + device="cpu", + ).eval() + bridge.process_weights( + fold_ln=False, + center_writing_weights=False, + center_unembed=False, + fold_value_biases=False, + ) + return GatedQueryCase( + bridge=bridge, + reference_logits=reference_logits, + tokens=tokens, + raw_q_weights=raw_q_weights, + ) + + +def test_gated_w_q_exposes_query_rows_only(gated_query_case: GatedQueryCase) -> None: + expected = gated_query_case.raw_q_weights.view(N_LAYERS, N_HEADS, D_HEAD * 2, D_MODEL)[ + :, :, :D_HEAD, : + ].transpose(-1, -2) + + assert gated_query_case.bridge.W_Q.shape == (N_LAYERS, N_HEADS, D_MODEL, D_HEAD) + torch.testing.assert_close(gated_query_case.bridge.W_Q, expected) + + +def test_gated_w_q_access_preserves_forward_and_gate_hook( + gated_query_case: GatedQueryCase, +) -> None: + captured_gate: list[torch.Tensor] = [] + _ = gated_query_case.bridge.W_Q + + with torch.no_grad(): + bridge_logits = gated_query_case.bridge.run_with_hooks( + gated_query_case.tokens, + fwd_hooks=[ + ( + "blocks.0.attn.hook_q_gate", + lambda gate, hook: captured_gate.append(gate.detach().clone()), + ) + ], + ) + + torch.testing.assert_close( + bridge_logits, gated_query_case.reference_logits, atol=1e-5, rtol=1e-5 + ) + assert len(captured_gate) == 1 + assert captured_gate[0].shape == (1, gated_query_case.tokens.shape[1], N_HEADS * D_HEAD) + live_q_weights = torch.stack( + [ + gated_query_case.bridge.state_dict()[f"blocks.{layer}.attn.q.weight"] + for layer in range(N_LAYERS) + ] + ) + torch.testing.assert_close(live_q_weights, gated_query_case.raw_q_weights) + + +def test_standard_qwen3_w_q_is_unchanged() -> None: + cfg = Qwen3Config( + hidden_size=D_MODEL, + num_hidden_layers=1, + num_attention_heads=N_HEADS, + num_key_value_heads=1, + head_dim=D_HEAD, + intermediate_size=32, + vocab_size=VOCAB_SIZE, + max_position_embeddings=32, + ) + hf_model = Qwen3ForCausalLM(cfg).eval() + bridge = build_bridge_from_module( + hf_model, + "Qwen3ForCausalLM", + hf_config=copy.deepcopy(cfg), + tokenizer=None, + device="cpu", + ).eval() + raw_q_weight = hf_model.model.layers[0].self_attn.q_proj.weight.detach() + expected = raw_q_weight.view(N_HEADS, D_HEAD, D_MODEL).transpose(-1, -2).unsqueeze(0) + + assert bridge.W_Q.shape == (1, N_HEADS, D_MODEL, D_HEAD) + torch.testing.assert_close(bridge.W_Q, expected) diff --git a/tests/unit/model_bridge/supported_architectures/test_qwen3_5_adapter.py b/tests/unit/model_bridge/supported_architectures/test_qwen3_5_adapter.py index 91236361a2..30fb9aaf69 100644 --- a/tests/unit/model_bridge/supported_architectures/test_qwen3_5_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_qwen3_5_adapter.py @@ -335,117 +335,6 @@ def test_n_key_value_heads_not_set_when_absent(self, qwen3_5_dependency_availabl ) -class TestQwen3_5PreprocessWeights: - """q_proj rows are interleaved per-head (query, gate, query, gate, ...) — naive first-half slice is wrong.""" - - N_HEADS = 4 - D_HEAD = 8 - HIDDEN_SIZE = 32 - - @pytest.fixture - def adapter(self, qwen3_5_dependency_available): - from transformer_lens.model_bridge.supported_architectures.qwen3_5 import ( - Qwen3_5ArchitectureAdapter, - ) - - cfg = _make_bridge_cfg( - n_heads=self.N_HEADS, - d_head=self.D_HEAD, - d_model=self.HIDDEN_SIZE, - n_key_value_heads=self.N_HEADS, - ) - return Qwen3_5ArchitectureAdapter(cfg) - - def _make_q_proj_weight(self): - import torch - - total_rows = self.N_HEADS * self.D_HEAD * 2 - w = torch.zeros(total_rows, self.HIDDEN_SIZE) - for row_idx in range(total_rows): - w[row_idx] = float(row_idx) - return w - - def test_q_proj_output_shape(self, adapter): - import torch - - w = self._make_q_proj_weight() - state_dict = {"model.layers.3.self_attn.q_proj.weight": w} - result = adapter.preprocess_weights(state_dict) - out = result["model.layers.3.self_attn.q_proj.weight"] - assert out.shape == (self.N_HEADS * self.D_HEAD, self.HIDDEN_SIZE) - - def test_q_proj_selects_query_rows_not_naive_first_half(self, adapter): - import torch - - w = self._make_q_proj_weight() - state_dict = {"model.layers.0.self_attn.q_proj.weight": w} - result = adapter.preprocess_weights(state_dict) - out = result["model.layers.0.self_attn.q_proj.weight"] - - for head_idx in range(self.N_HEADS): - out_rows = out[head_idx * self.D_HEAD : (head_idx + 1) * self.D_HEAD] - expected_start = head_idx * self.D_HEAD * 2 - expected_rows = w[expected_start : expected_start + self.D_HEAD] - assert torch.equal(out_rows, expected_rows), ( - f"Head {head_idx}: output rows do not match expected query rows. " - f"Got row values starting at {out_rows[0, 0].item()}, " - f"expected starting at {expected_rows[0, 0].item()}" - ) - - def test_naive_slice_would_be_wrong(self, adapter): - import torch - - w = self._make_q_proj_weight() - state_dict = {"model.layers.0.self_attn.q_proj.weight": w} - result = adapter.preprocess_weights(state_dict) - correct_out = result["model.layers.0.self_attn.q_proj.weight"] - naive_out = w[: self.N_HEADS * self.D_HEAD] - - if self.N_HEADS > 1: - assert not torch.equal(correct_out, naive_out), ( - "Naive first-half slice gave the same result as per-head slice — " - "test setup may be wrong" - ) - - def test_non_q_proj_weights_unchanged(self, adapter): - import torch - - k_proj = torch.randn(self.N_HEADS * self.D_HEAD, self.HIDDEN_SIZE) - down_proj = torch.randn(self.HIDDEN_SIZE, self.N_HEADS * self.D_HEAD) - state_dict = { - "model.layers.0.self_attn.k_proj.weight": k_proj.clone(), - "model.layers.0.mlp.down_proj.weight": down_proj.clone(), - } - result = adapter.preprocess_weights(state_dict) - assert torch.equal(result["model.layers.0.self_attn.k_proj.weight"], k_proj) - assert torch.equal(result["model.layers.0.mlp.down_proj.weight"], down_proj) - - def test_multiple_layers_all_processed(self, adapter): - import torch - - w0 = self._make_q_proj_weight() - w3 = self._make_q_proj_weight() * 2 - state_dict = { - "model.layers.0.self_attn.q_proj.weight": w0, - "model.layers.3.self_attn.q_proj.weight": w3, - } - result = adapter.preprocess_weights(state_dict) - expected_shape = (self.N_HEADS * self.D_HEAD, self.HIDDEN_SIZE) - assert result["model.layers.0.self_attn.q_proj.weight"].shape == expected_shape - assert result["model.layers.3.self_attn.q_proj.weight"].shape == expected_shape - - def test_empty_state_dict_returns_empty(self, adapter): - assert adapter.preprocess_weights({}) == {} - - def test_state_dict_without_q_proj_unchanged(self, adapter): - import torch - - state_dict = {"model.embed_tokens.weight": torch.randn(100, self.HIDDEN_SIZE)} - original_keys = set(state_dict.keys()) - result = adapter.preprocess_weights(state_dict) - assert set(result.keys()) == original_keys - - @pytest.mark.skipif( not _QWEN3_5_AVAILABLE, reason="Qwen3_5TextConfig / Qwen3_5ForCausalLM not available in installed transformers", @@ -508,7 +397,7 @@ def adapter(self): return Qwen3_5ArchitectureAdapter(_make_bridge_cfg()) def test_gated_q_proj_flag_set(self, adapter): - """Flag drives preprocess_weights to slice the gated half of q_proj.""" + """Flag drives the query-only W_Q analysis view and gate hook path.""" assert getattr(adapter.cfg, "gated_q_proj", False) is True diff --git a/tests/unit/model_bridge/supported_architectures/test_qwen3_5_multimodal_adapter.py b/tests/unit/model_bridge/supported_architectures/test_qwen3_5_multimodal_adapter.py index 544c7f6bda..f76cfbf7a5 100644 --- a/tests/unit/model_bridge/supported_architectures/test_qwen3_5_multimodal_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_qwen3_5_multimodal_adapter.py @@ -9,7 +9,6 @@ import torch -from transformer_lens.config import TransformerBridgeConfig from transformer_lens.config.transformer_bridge_config import TransformerBridgeConfig from transformer_lens.model_bridge.generalized_components import ( LinearBridge, @@ -110,17 +109,26 @@ def test_vision_tower_decomposed(self): assert isinstance(block.submodules[comp].submodules[sub], LinearBridge) -def test_gated_q_proj_query_half_is_sliced_under_nested_path(): - """preprocess_weights slices the query half from the 2x-wide gated q_proj, matching the - nested model.language_model.* key.""" +def test_gated_q_proj_exposes_query_only_w_q_without_mutating_live_weight(): + """The multimodal adapter shares the non-mutating gated W_Q analysis view.""" adapter = Qwen3_5MultimodalArchitectureAdapter(_make_cfg()) + attention = adapter.component_mapping["blocks"].submodules["attn"] n_heads, d_head, hidden = adapter.cfg.n_heads, adapter.cfg.d_head, adapter.cfg.d_model - key = "model.language_model.layers.1.self_attn.q_proj.weight" - # Per head: rows [query(d_head), gate(d_head)] -> 2*d_head wide. - full = torch.randn(n_heads * d_head * 2, hidden) - out = adapter.preprocess_weights({key: full.clone()}) - assert out[key].shape == (n_heads * d_head, hidden) - expected = full.view(n_heads, d_head * 2, hidden)[:, :d_head, :].reshape( - n_heads * d_head, hidden + q_proj = torch.nn.Linear(hidden, n_heads * d_head * 2, bias=False) + with torch.no_grad(): + values = torch.arange(q_proj.weight.numel(), dtype=q_proj.weight.dtype) + q_proj.weight.copy_(values.reshape_as(q_proj.weight)) + original_weight = q_proj.weight.detach().clone() + query = attention.submodules["q"] + query.set_original_component(q_proj) + attention.add_module("q", query) + + expected = ( + original_weight.view(n_heads, d_head * 2, hidden)[:, :d_head, :] + .transpose(-1, -2) + .contiguous() ) - assert torch.equal(out[key], expected) + + assert attention.W_Q.shape == (n_heads, hidden, d_head) + torch.testing.assert_close(attention.W_Q, expected) + torch.testing.assert_close(q_proj.weight, original_weight) diff --git a/tests/unit/model_bridge/supported_architectures/test_qwen3_adapter.py b/tests/unit/model_bridge/supported_architectures/test_qwen3_adapter.py index f5508623f0..9c937b91de 100644 --- a/tests/unit/model_bridge/supported_architectures/test_qwen3_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_qwen3_adapter.py @@ -4,14 +4,12 @@ - Config attributes - Component mapping structure and HF module names (incl. q_norm/k_norm) - Weight conversion keys/types (GQA: k/v use n_key_value_heads) -- _preprocess_gated_q_proj static helper (gated q_proj slicing) - Factory registration """ from types import SimpleNamespace from typing import Any import pytest -import torch from tests.unit.model_bridge.supported_architectures.helpers import make_bridge_cfg from transformer_lens.config import TransformerBridgeConfig @@ -140,55 +138,6 @@ def test_no_linear_attn_when_dense(self, adapter: Qwen3ArchitectureAdapter) -> N assert "linear_attn" not in blocks.submodules -class TestPreprocessGatedQProj: - """Numerical correctness of the _preprocess_gated_q_proj static helper - on synthetic interleaved [query, gate] rows: asserts query-half slicing, - that unrelated state-dict keys are untouched, and that the rewrite - applies across all matching layers.""" - - def test_slices_query_half(self) -> None: - """Interleaved [query, gate] rows per head must be reduced to query-only.""" - n_heads, d_head, d_model = 4, 8, 16 - # Build q_proj.weight as (n_heads, d_head*2, d_model): query=1.0, gate=9.0 - w = torch.empty(n_heads, d_head * 2, d_model) - w[:, :d_head, :] = 1.0 - w[:, d_head:, :] = 9.0 - w_flat = w.reshape(n_heads * d_head * 2, d_model) - - state_dict = {"model.layers.0.self_attn.q_proj.weight": w_flat.clone()} - out = Qwen3ArchitectureAdapter._preprocess_gated_q_proj(state_dict, n_heads, d_head) - - result = out["model.layers.0.self_attn.q_proj.weight"] - assert result.shape == (n_heads * d_head, d_model) - assert torch.all(result == 1.0), "gate rows must be dropped" - - def test_only_q_proj_keys_modified(self) -> None: - n_heads, d_head, d_model = 2, 4, 8 - q_w = torch.ones(n_heads * d_head * 2, d_model) - other = torch.full((d_model, d_model), 7.0) - state_dict = { - "model.layers.0.self_attn.q_proj.weight": q_w, - "model.layers.0.self_attn.k_proj.weight": other.clone(), - "model.layers.0.mlp.gate_proj.weight": other.clone(), - } - out = Qwen3ArchitectureAdapter._preprocess_gated_q_proj(state_dict, n_heads, d_head) - assert torch.equal(out["model.layers.0.self_attn.k_proj.weight"], other) - assert torch.equal(out["model.layers.0.mlp.gate_proj.weight"], other) - - def test_multiple_layers(self) -> None: - n_heads, d_head, d_model = 2, 4, 8 - state_dict = { - f"model.layers.{i}.self_attn.q_proj.weight": torch.ones(n_heads * d_head * 2, d_model) - for i in range(3) - } - out = Qwen3ArchitectureAdapter._preprocess_gated_q_proj(state_dict, n_heads, d_head) - for i in range(3): - assert out[f"model.layers.{i}.self_attn.q_proj.weight"].shape == ( - n_heads * d_head, - d_model, - ) - - class TestQwen3HybridConstructor: """The hybrid=True constructor branch on the base class. The Qwen3_5 / Qwen3Next subclasses exercise this path transitively; pinning it here diff --git a/tests/unit/model_bridge/supported_architectures/test_qwen3_next_adapter.py b/tests/unit/model_bridge/supported_architectures/test_qwen3_next_adapter.py index 2b2ed53326..640f53e9fb 100644 --- a/tests/unit/model_bridge/supported_architectures/test_qwen3_next_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_qwen3_next_adapter.py @@ -123,115 +123,6 @@ def test_rotary_emb_bridge_type(self, adapter): assert isinstance(adapter.component_mapping["rotary_emb"], RotaryEmbeddingBridge) -class TestQwen3NextWeightConversions: - """q_proj rows are interleaved per-head (query, gate, query, gate, ...) — naive first-half slice is wrong.""" - - N_HEADS = 4 - D_HEAD = 8 - HIDDEN_SIZE = 32 - - @pytest.fixture - def adapter(self): - from transformer_lens.model_bridge.supported_architectures.qwen3_next import ( - Qwen3NextArchitectureAdapter, - ) - - cfg = _make_bridge_cfg( - n_heads=self.N_HEADS, - d_head=self.D_HEAD, - d_model=self.HIDDEN_SIZE, - n_key_value_heads=self.N_HEADS, - ) - return Qwen3NextArchitectureAdapter(cfg) - - def _make_q_proj_weight(self): - import torch - - total_rows = self.N_HEADS * self.D_HEAD * 2 - w = torch.zeros(total_rows, self.HIDDEN_SIZE) - for row_idx in range(total_rows): - w[row_idx] = float(row_idx) - return w - - def test_q_proj_output_shape(self, adapter): - import torch - - w = self._make_q_proj_weight() - state_dict = {"model.layers.3.self_attn.q_proj.weight": w} - - result = adapter.preprocess_weights(state_dict) - out = result["model.layers.3.self_attn.q_proj.weight"] - - assert out.shape == (self.N_HEADS * self.D_HEAD, self.HIDDEN_SIZE) - - def test_q_proj_selects_query_rows_not_naive_first_half(self, adapter): - import torch - - w = self._make_q_proj_weight() - state_dict = {"model.layers.0.self_attn.q_proj.weight": w} - - result = adapter.preprocess_weights(state_dict) - out = result["model.layers.0.self_attn.q_proj.weight"] - - for head_idx in range(self.N_HEADS): - out_rows = out[head_idx * self.D_HEAD : (head_idx + 1) * self.D_HEAD] - expected_start = head_idx * self.D_HEAD * 2 - expected_rows = w[expected_start : expected_start + self.D_HEAD] - assert torch.equal(out_rows, expected_rows), ( - f"Head {head_idx}: output rows do not match expected query rows. " - f"Got row values starting at {out_rows[0, 0].item()}, " - f"expected starting at {expected_rows[0, 0].item()}" - ) - - def test_non_q_proj_weights_unchanged(self, adapter): - import torch - - k_proj = torch.randn(self.N_HEADS * self.D_HEAD, self.HIDDEN_SIZE) - down_proj = torch.randn(self.HIDDEN_SIZE, self.N_HEADS * self.D_HEAD) - state_dict = { - "model.layers.0.self_attn.k_proj.weight": k_proj.clone(), - "model.layers.0.mlp.down_proj.weight": down_proj.clone(), - } - - result = adapter.preprocess_weights(state_dict) - - assert torch.equal(result["model.layers.0.self_attn.k_proj.weight"], k_proj) - assert torch.equal(result["model.layers.0.mlp.down_proj.weight"], down_proj) - - def test_multiple_layers_all_processed(self, adapter): - import torch - - w0 = self._make_q_proj_weight() - w3 = self._make_q_proj_weight() * 2 - - state_dict = { - "model.layers.0.self_attn.q_proj.weight": w0, - "model.layers.3.self_attn.q_proj.weight": w3, - } - - result = adapter.preprocess_weights(state_dict) - - expected_shape = (self.N_HEADS * self.D_HEAD, self.HIDDEN_SIZE) - assert result["model.layers.0.self_attn.q_proj.weight"].shape == expected_shape - assert result["model.layers.3.self_attn.q_proj.weight"].shape == expected_shape - - def test_empty_state_dict_returns_empty(self, adapter): - result = adapter.preprocess_weights({}) - assert result == {} - - def test_state_dict_without_q_proj_unchanged(self, adapter): - import torch - - state_dict = { - "model.embed_tokens.weight": torch.randn(100, self.HIDDEN_SIZE), - } - original_keys = set(state_dict.keys()) - - result = adapter.preprocess_weights(state_dict) - - assert set(result.keys()) == original_keys - - class TestQwen3NextConfigAttributes: """cfg attributes set by the adapter.""" diff --git a/transformer_lens/model_bridge/generalized_components/attention.py b/transformer_lens/model_bridge/generalized_components/attention.py index 9a79c3e2a3..bc9aa4a861 100644 --- a/transformer_lens/model_bridge/generalized_components/attention.py +++ b/transformer_lens/model_bridge/generalized_components/attention.py @@ -766,12 +766,33 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: @property def W_Q(self) -> torch.Tensor: - """Get W_Q in 3D format [n_heads, d_model, d_head].""" + """Get W_Q in 3D format [n_heads, d_model, d_head]. + + Gated query projections retain their live query-and-gate parameter; + this analysis view selects the query rows interleaved within each head. + """ weight = self.q.weight if weight.ndim == 2 and self.config is not None: - return self._reshape_weight_to_3d( - weight, self._get_n_heads(), in_out_layout=self._weight_layout_in_out(self.q) - ) + n_heads = self._get_n_heads() + in_out_layout = self._weight_layout_in_out(self.q) + if getattr(self.config, "gated_q_proj", False): + d_head = int(self.config.d_head) + gated_width = n_heads * d_head * 2 + if in_out_layout is True: + output_first_weight = weight.T + elif in_out_layout is False or weight.shape[0] == gated_width: + output_first_weight = weight + elif weight.shape[1] == gated_width: + output_first_weight = weight.T + else: + output_first_weight = None + + if output_first_weight is not None and output_first_weight.shape[0] == gated_width: + # Preserve the live query-gate projection; W_Q is an analysis-only query view. + per_head_weight = output_first_weight.reshape(n_heads, d_head * 2, -1) + return per_head_weight[:, :d_head, :].transpose(-1, -2) + + return self._reshape_weight_to_3d(weight, n_heads, in_out_layout=in_out_layout) return weight @property diff --git a/transformer_lens/model_bridge/supported_architectures/qwen3.py b/transformer_lens/model_bridge/supported_architectures/qwen3.py index 9551c4ebc0..a4481dfcc0 100644 --- a/transformer_lens/model_bridge/supported_architectures/qwen3.py +++ b/transformer_lens/model_bridge/supported_architectures/qwen3.py @@ -7,8 +7,6 @@ from typing import Any -import torch - from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter from transformer_lens.model_bridge.generalized_components import ( AttentionBridge, @@ -93,19 +91,3 @@ def _build_component_mapping(self, *, hybrid: bool = False, lm_prefix: str = "mo "ln_final": RMSNormalizationBridge(name=f"{lm_prefix}.norm", config=self.cfg), "unembed": UnembeddingBridge(name="lm_head"), } - - @staticmethod - def _preprocess_gated_q_proj( - state_dict: dict[str, torch.Tensor], n_heads: int, d_head: int - ) -> dict[str, torch.Tensor]: - """Slice query half from gated q_proj.weight (interleaved per-head layout). - - q_proj.weight has shape (n_heads * d_head * 2, hidden_size) with - interleaved [query, gate] rows per head. Extracts query-only half. - """ - keys_to_update = [k for k in state_dict if k.endswith(".self_attn.q_proj.weight")] - for key in keys_to_update: - w = state_dict[key] - w = w.view(n_heads, d_head * 2, -1) - state_dict[key] = w[:, :d_head, :].reshape(n_heads * d_head, -1) - return state_dict diff --git a/transformer_lens/model_bridge/supported_architectures/qwen3_5.py b/transformer_lens/model_bridge/supported_architectures/qwen3_5.py index 9b3a98bc32..a155055579 100644 --- a/transformer_lens/model_bridge/supported_architectures/qwen3_5.py +++ b/transformer_lens/model_bridge/supported_architectures/qwen3_5.py @@ -7,8 +7,6 @@ from typing import Any -import torch - from transformer_lens.model_bridge.supported_architectures.qwen3 import ( Qwen3ArchitectureAdapter, ) @@ -19,7 +17,7 @@ class Qwen3_5ArchitectureAdapter(Qwen3ArchitectureAdapter): Inherits Qwen3 config/attention/MLP structure. Differences: - Attention + linear_attn are optional (per-layer type) - - Gated q_proj (2x wide) sliced by preprocess_weights for weight analysis + - Gated q_proj (2x wide); AttentionBridge exposes a query-only W_Q view """ # Multimodal wrapper architecture this text-only adapter rejects; the MoE @@ -58,12 +56,3 @@ def prepare_model(self, hf_model: Any) -> None: f"TransformerBridge.boot_transformers(...) so {multimodal_arch} " f"checkpoints route to the multimodal adapter automatically." ) - - def preprocess_weights(self, state_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """Slice query half from gated q_proj.weight for weight-space analysis. - - In processed mode, W_Q is the pure query projection (for composition - scores, logit lens). Gate signal available in unprocessed mode on - full-attention layers via blocks.N.attn.hook_q_gate. - """ - return self._preprocess_gated_q_proj(state_dict, self.cfg.n_heads, self.cfg.d_head) diff --git a/transformer_lens/model_bridge/supported_architectures/qwen3_5_multimodal.py b/transformer_lens/model_bridge/supported_architectures/qwen3_5_multimodal.py index 6e2aa3b277..e5cbd8975b 100644 --- a/transformer_lens/model_bridge/supported_architectures/qwen3_5_multimodal.py +++ b/transformer_lens/model_bridge/supported_architectures/qwen3_5_multimodal.py @@ -7,8 +7,6 @@ from typing import Any -import torch - from transformer_lens.model_bridge.generalized_components import VisionProjectionBridge from transformer_lens.model_bridge.generalized_components.qwen3_5_vision_encoder import ( Qwen3_5VisionEncoderBridge, @@ -39,7 +37,3 @@ def __init__(self, cfg: Any) -> None: name="model.visual", config=self.cfg ) self.components["vision_projector"] = VisionProjectionBridge(name="model.visual.merger") - - def preprocess_weights(self, state_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """Slice query half from gated q_proj.weight (matcher is path-prefix-agnostic).""" - return self._preprocess_gated_q_proj(state_dict, self.cfg.n_heads, self.cfg.d_head) diff --git a/transformer_lens/model_bridge/supported_architectures/qwen3_next.py b/transformer_lens/model_bridge/supported_architectures/qwen3_next.py index 31e1be3cdc..d0649ace9d 100644 --- a/transformer_lens/model_bridge/supported_architectures/qwen3_next.py +++ b/transformer_lens/model_bridge/supported_architectures/qwen3_next.py @@ -7,8 +7,6 @@ from typing import Any -import torch - from transformer_lens.model_bridge.generalized_components import MoEBridge from transformer_lens.model_bridge.supported_architectures.qwen3 import ( Qwen3ArchitectureAdapter, @@ -28,7 +26,3 @@ def __init__(self, cfg: Any) -> None: def _build_mlp_bridge(self): """Sparse MoE MLP (router + batched experts + shared expert).""" return MoEBridge(name="mlp", config=self.cfg) - - def preprocess_weights(self, state_dict: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """Slice query half from gated q_proj.weight for weight-space analysis.""" - return self._preprocess_gated_q_proj(state_dict, self.cfg.n_heads, self.cfg.d_head) From 1fdc9550b8fc14cad346b92f5de50e8728af9f6b Mon Sep 17 00:00:00 2001 From: jlarson4 <jonahalarson@comcast.net> Date: Thu, 13 Aug 2026 11:54:09 -0500 Subject: [PATCH 52/87] cleaning up warning text to ensure it is warning the correct versions --- demos/BERT.ipynb | 8 +++++- docs/source/content/migrating_to_v3.md | 34 +++++++++++++++++++++--- transformer_lens/HookedAudioEncoder.py | 4 +-- transformer_lens/HookedEncoderDecoder.py | 4 +-- transformer_lens/HookedTransformer.py | 4 +-- 5 files changed, 44 insertions(+), 10 deletions(-) diff --git a/demos/BERT.ipynb b/demos/BERT.ipynb index 63404a85af..5c58c557c5 100644 --- a/demos/BERT.ipynb +++ b/demos/BERT.ipynb @@ -499,7 +499,13 @@ "\n", "inputs = tokenizer(sentence_a, sentence_b, return_tensors=\"pt\")\n", "device = next(nsp.parameters()).device\n", - "predictions = nsp(inputs[\"input_ids\"].to(device), return_type=\"predictions\")\n", + "# token_type_ids mark where sentence A ends and B begins — without them the NSP\n", + "# head sees one undifferentiated span and can return the wrong verdict.\n", + "predictions = nsp(\n", + " inputs[\"input_ids\"].to(device),\n", + " token_type_ids=inputs[\"token_type_ids\"].to(device),\n", + " return_type=\"predictions\",\n", + ")\n", "\n", "print(f\"Sentence A: {sentence_a}\")\n", "print(f\"Sentence B: {sentence_b}\")\n", diff --git a/docs/source/content/migrating_to_v3.md b/docs/source/content/migrating_to_v3.md index b61b60326f..f2e0d13262 100644 --- a/docs/source/content/migrating_to_v3.md +++ b/docs/source/content/migrating_to_v3.md @@ -4,7 +4,7 @@ TransformerLens 3 introduces **TransformerBridge**, a new way of loading and ins This page explains the differences and gives side-by-side migration recipes for the most common patterns. -> **Deprecation status.** `HookedTransformer.from_pretrained` — along with the `HookedEncoderDecoder` and `HookedAudioEncoder` load paths — now emits a `DeprecationWarning`. `HookedTransformer` and the other `Hooked*` classes are slated for removal in a future major release; every feature is being migrated to `TransformerBridge` and the driver system (features that aren't a fit for a driver, such as train-from-scratch, are moving to bridge-based homes rather than staying on `HookedTransformer`). New code should use `TransformerBridge.boot_transformers(...)`. Follow the migration progress in the deprecation plan. +> **Deprecation status.** `HookedTransformer.from_pretrained` — along with the `HookedEncoderDecoder` and `HookedAudioEncoder` load paths — now emits a `DeprecationWarning`. `HookedTransformer` and the other `Hooked*` classes are slated for removal in 4.0; every feature is being migrated to `TransformerBridge` and the driver system (features that aren't a fit for a driver, such as train-from-scratch, are moving to bridge-based homes rather than staying on `HookedTransformer`). New code should use `TransformerBridge.boot_transformers(...)`. Follow the migration progress in the deprecation plan. ## Why the change? @@ -150,7 +150,35 @@ If your code only touches these APIs, the migration is genuinely just the loadin ### BERT Next Sentence Prediction -The high-level NSP API (`BertNextSentencePrediction`) is not yet ported: it requires the legacy `HookedEncoder` surface (`encoder_output`, `pooler`, `nsp_head`), which `TransformerBridge` does not expose. The bridge's BERT adapter does load NSP HuggingFace checkpoints (it rewires the unembed to `cls.seq_relationship`), so the underlying weights are available — but sentence-pair tokenization, `[CLS]` pooling, and "sequential"/"not sequential" decoding are not. Until it is ported, NSP workflows need the legacy classes, which are slated for removal; if you rely on this API, please say so on the tracking issue. +NSP runs on the bridge today — load the NSP head via `model_class` and pass the +sentence-pair tokenization through: + +```python +from transformers import AutoTokenizer, BertForNextSentencePrediction +from transformer_lens.model_bridge import TransformerBridge + +tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased") +nsp = TransformerBridge.boot_transformers( + "google-bert/bert-base-cased", + model_class=BertForNextSentencePrediction, +) +nsp.enable_compatibility_mode() + +inputs = tokenizer("A man walked into a grocery store.", "He bought an apple.", return_tensors="pt") +nsp(inputs["input_ids"], token_type_ids=inputs["token_type_ids"], return_type="predictions") +# 'The sentences are sequential' +``` + +**Pass `token_type_ids`.** They are what tells BERT where the first sentence ends +and the second begins; without them the NSP head scores a single undifferentiated +span and can return the wrong verdict (on the pair above, dropping them collapses +the logits from ±4.37 to ±0.58, and a genuinely non-sequential pair flips to +"sequential"). With them, the bridge reproduces the raw HuggingFace NSP logits +exactly. + +The legacy `BertNextSentencePrediction` wrapper is deprecated and cannot wrap a +`TransformerBridge` — it reaches for `HookedEncoder`-only internals +(`encoder_output`, `pooler`, `nsp_head`). Use the recipe above instead. ### New in 3.x: streaming generation @@ -216,7 +244,7 @@ The cache, hook, and config APIs are the same. The only lines that had to change ## Migrating specific `HookedTransformer` APIs -`HookedTransformer` is deprecated and will be removed in a future major release. The compatibility layer keeps existing code running in the meantime, but new work should target `TransformerBridge`, and migrating existing projects is the long-term supported path. +`HookedTransformer` is deprecated and will be removed in 4.0. The compatibility layer keeps existing code running in the meantime, but new work should target `TransformerBridge`, and migrating existing projects is the long-term supported path. Most `HookedTransformer` methods and properties exist on `TransformerBridge` under the same name — see [APIs that are unchanged](#apis-that-are-unchanged). The table below covers the cases where the name or access path differs. diff --git a/transformer_lens/HookedAudioEncoder.py b/transformer_lens/HookedAudioEncoder.py index 5b5d6747a2..2f805c1bd1 100644 --- a/transformer_lens/HookedAudioEncoder.py +++ b/transformer_lens/HookedAudioEncoder.py @@ -364,8 +364,8 @@ def from_pretrained( import warnings warnings.warn( - "HookedAudioEncoder.from_pretrained is deprecated and will be removed in a " - "future major release. Use TransformerBridge.boot_transformers(...) instead — " + "HookedAudioEncoder.from_pretrained is deprecated and will be removed in " + "4.0. Use TransformerBridge.boot_transformers(...) instead — " "HuBERT/Wav2Vec2 are supported via the bridge's audio adapter. See " "docs/source/content/migrating_to_v3.md.", DeprecationWarning, diff --git a/transformer_lens/HookedEncoderDecoder.py b/transformer_lens/HookedEncoderDecoder.py index 2961190183..7bea005548 100644 --- a/transformer_lens/HookedEncoderDecoder.py +++ b/transformer_lens/HookedEncoderDecoder.py @@ -551,8 +551,8 @@ def from_pretrained( import warnings warnings.warn( - "HookedEncoderDecoder.from_pretrained is deprecated and will be removed in a " - "future major release. Use TransformerBridge.boot_transformers(...) instead — " + "HookedEncoderDecoder.from_pretrained is deprecated and will be removed in " + "4.0. Use TransformerBridge.boot_transformers(...) instead — " "the bridge supports T5-style encoder-decoder models. See " "docs/source/content/migrating_to_v3.md.", DeprecationWarning, diff --git a/transformer_lens/HookedTransformer.py b/transformer_lens/HookedTransformer.py index 6d2cc2071b..719f4dadb4 100644 --- a/transformer_lens/HookedTransformer.py +++ b/transformer_lens/HookedTransformer.py @@ -1324,8 +1324,8 @@ def from_pretrained( import warnings warnings.warn( - "HookedTransformer.from_pretrained is deprecated and will be removed in a " - "future major release. Use TransformerBridge.boot_transformers(...) instead, " + "HookedTransformer.from_pretrained is deprecated and will be removed in " + "4.0. Use TransformerBridge.boot_transformers(...) instead, " "then call enable_compatibility_mode() for HookedTransformer-equivalent " "numerics. See docs/source/content/migrating_to_v3.md.", DeprecationWarning, From 33501424002dadccf2250497258dbbca4f0e6b5b Mon Sep 17 00:00:00 2001 From: Liang Hu <35699841+LarryHu0217@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:01:29 -0400 Subject: [PATCH 53/87] Expose BERT token type embedding hook (#1664) --- .../test_bert_adapter.py | 33 +++++++++++++++++++ .../supported_architectures/bert.py | 1 + 2 files changed, 34 insertions(+) diff --git a/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py b/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py index a5d8e5ae31..06ce437333 100644 --- a/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py @@ -74,6 +74,7 @@ class TestBertComponentMapping: def test_top_level_keys(self, adapter: BertArchitectureAdapter) -> None: assert set(adapter.component_mapping.keys()) == { "embed", + "token_type_embed", "pos_embed", "blocks", "ln_final", @@ -83,6 +84,7 @@ def test_top_level_keys(self, adapter: BertArchitectureAdapter) -> None: def test_bridge_types(self, adapter: BertArchitectureAdapter) -> None: mapping = adapter.component_mapping assert isinstance(mapping["embed"], EmbeddingBridge) + assert isinstance(mapping["token_type_embed"], EmbeddingBridge) assert isinstance(mapping["pos_embed"], PosEmbedBridge) assert isinstance(mapping["blocks"], BlockBridge) assert isinstance(mapping["ln_final"], NormalizationBridge) @@ -91,11 +93,42 @@ def test_bridge_types(self, adapter: BertArchitectureAdapter) -> None: def test_top_level_hf_paths(self, adapter: BertArchitectureAdapter) -> None: mapping = adapter.component_mapping assert mapping["embed"].name == "bert.embeddings.word_embeddings" + assert mapping["token_type_embed"].name == "bert.embeddings.token_type_embeddings" assert mapping["pos_embed"].name == "bert.embeddings.position_embeddings" assert mapping["blocks"].name == "bert.encoder.layer" assert mapping["ln_final"].name == "cls.predictions.transform.LayerNorm" assert mapping["unembed"].name == "cls.predictions.decoder" + def test_token_type_embedding_is_cached_with_hf_output(self) -> None: + import torch + from transformers import BertForMaskedLM + + from transformer_lens.model_bridge.sources import build_bridge_from_module + + hf_model = BertForMaskedLM.from_pretrained("bert-base-cased").eval() + input_ids = torch.tensor([[101, 7592, 102, 2088, 102]]) + token_type_ids = torch.tensor([[0, 0, 0, 1, 1]]) + with torch.no_grad(): + expected = hf_model.bert.embeddings.token_type_embeddings(token_type_ids).clone() + + bridge = build_bridge_from_module( + hf_model, + "BertForMaskedLM", + hf_config=hf_model.config, + dtype=torch.float32, + device="cpu", + model_name="bert-base-cased", + ) + with torch.no_grad(): + _, cache = bridge.run_with_cache( + input_ids, + token_type_ids=token_type_ids, + names_filter=["token_type_embed.hook_out"], + ) + + assert "token_type_embed.hook_out" in cache + torch.testing.assert_close(cache["token_type_embed.hook_out"], expected) + def test_block_submodule_keys(self, adapter: BertArchitectureAdapter) -> None: assert set(adapter.component_mapping["blocks"].submodules.keys()) == { "ln1", diff --git a/transformer_lens/model_bridge/supported_architectures/bert.py b/transformer_lens/model_bridge/supported_architectures/bert.py index f58c841711..358fd38ec3 100644 --- a/transformer_lens/model_bridge/supported_architectures/bert.py +++ b/transformer_lens/model_bridge/supported_architectures/bert.py @@ -87,6 +87,7 @@ def __init__(self, cfg: Any) -> None: # MLM defaults; prepare_model() adjusts for other task heads (e.g., NSP). self.component_mapping = { "embed": EmbeddingBridge(name="bert.embeddings.word_embeddings"), + "token_type_embed": EmbeddingBridge(name="bert.embeddings.token_type_embeddings"), "pos_embed": PosEmbedBridge(name="bert.embeddings.position_embeddings"), "blocks": BlockBridge( name="bert.encoder.layer", From 3d59a5192729c80afa01517dfcae13ff663606a3 Mon Sep 17 00:00:00 2001 From: "Md.Sadiq" <mohammadsadiq4950@gmail.com> Date: Fri, 14 Aug 2026 00:52:38 +0530 Subject: [PATCH 54/87] migrate the othello off hooked transformer (#1667) * migrate the othello off hooked transformer * pipeline fix * pipeline fix --- demos/Othello_GPT.ipynb | 54 +++++++------------ transformer_lens/conversion_utils/__init__.py | 3 +- 2 files changed, 21 insertions(+), 36 deletions(-) diff --git a/demos/Othello_GPT.ipynb b/demos/Othello_GPT.ipynb index 42d381c9aa..004304b016 100644 --- a/demos/Othello_GPT.ipynb +++ b/demos/Othello_GPT.ipynb @@ -185,9 +185,10 @@ "import transformer_lens.utilities as utils\n", "from transformer_lens.hook_points import HookPoint\n", "from transformer_lens.HookedRootModule import HookedRootModule # Hooking utilities\n", + "from transformer_lens.config import TransformerBridgeConfig\n", + "from transformer_lens.model_bridge import TransformerBridge\n", + "from transformer_lens.utilities.tl_checkpoint_conversion import convert_tl_checkpoint\n", "from transformer_lens import (\n", - " HookedTransformer,\n", - " HookedTransformerConfig,\n", " FactoredMatrix,\n", " ActivationCache,\n", ")" @@ -279,22 +280,12 @@ "cell_type": "code", "execution_count": 56, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DeprecationWarning:\n", - "\n", - "HookedTransformer is deprecated and will be removed in 4.0. Use TransformerBridge.boot_transformers(...) instead, then call enable_compatibility_mode() for HookedTransformer-equivalent numerics.\n", - "\n" - ] - } - ], + "outputs": [], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", "import transformer_lens.utilities as utils\n", "\n", - "cfg = HookedTransformerConfig(\n", + "cfg = TransformerBridgeConfig(\n", " n_layers=8,\n", " d_model=512,\n", " d_head=64,\n", @@ -305,7 +296,7 @@ " act_fn=\"gelu\",\n", " normalization_type=\"LNPre\",\n", ")\n", - "model = HookedTransformer(cfg)" + "model = TransformerBridge.boot_native(cfg)" ] }, { @@ -324,22 +315,13 @@ "cell_type": "code", "execution_count": 58, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "<All keys matched successfully>" - ] - }, - "execution_count": 58, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", "\n", "# champion_ship_sd = utils.download_file_from_hf(\"NeelNanda/Othello-GPT-Transformer-Lens\", \"championship_model.pth\")\n", - "model.load_state_dict(sd)" + "native_sd = convert_tl_checkpoint(sd, cfg)\n", + "model.load_state_dict(native_sd)" ] }, { @@ -357,6 +339,7 @@ "outputs": [], "source": [ "def convert_to_transformer_lens_format(in_sd, n_layers=8, n_heads=8):\n", + " \"\"\"Convert author's checkpoint format to TL format.\"\"\"\n", " out_sd = {}\n", " out_sd[\"pos_embed.W_pos\"] = in_sd[\"pos_emb\"].squeeze(0)\n", " out_sd[\"embed.W_E\"] = in_sd[\"tok_emb.weight\"]\n", @@ -422,7 +405,7 @@ " if name.startswith(\"blocks.0\") or not name.startswith(\"blocks\"):\n", " print(name, param.shape)\n", "\n", - " cfg = HookedTransformerConfig(\n", + " cfg = TransformerBridgeConfig(\n", " n_layers=8,\n", " d_model=512,\n", " d_head=64,\n", @@ -433,11 +416,12 @@ " act_fn=\"gelu\",\n", " normalization_type=\"LNPre\",\n", " )\n", - " model = HookedTransformer(cfg)\n", + " model = TransformerBridge.boot_native(cfg)\n", "\n", - " model.load_and_process_state_dict(\n", - " convert_to_transformer_lens_format(synthetic_checkpoint)\n", - " )" + " # Convert author format -> TL format -> native format\n", + " tl_sd = convert_to_transformer_lens_format(synthetic_checkpoint)\n", + " native_sd = convert_tl_checkpoint(tl_sd, cfg)\n", + " model.load_state_dict(native_sd)" ] }, { @@ -626,4 +610,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/transformer_lens/conversion_utils/__init__.py b/transformer_lens/conversion_utils/__init__.py index ebfde4165b..e1e006fc15 100644 --- a/transformer_lens/conversion_utils/__init__.py +++ b/transformer_lens/conversion_utils/__init__.py @@ -6,5 +6,6 @@ from transformer_lens.conversion_utils.conversion_steps import ( TensorConversionSet, ) +from transformer_lens.utilities.tl_checkpoint_conversion import convert_tl_checkpoint -__all__ = ["TensorConversionSet"] +__all__ = ["TensorConversionSet", "convert_tl_checkpoint"] From 80d9f36b2084ed82bc1a5d04692bbc183e538bc5 Mon Sep 17 00:00:00 2001 From: emerardd <113128214+emerardd@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:43:23 +0800 Subject: [PATCH 55/87] Fix recursive TransformerBridge state dict composition (#1661) * Fix recursive TransformerBridge state dict composition * Clarify filtered checkpoint loading --- .../test_state_dict_composition.py | 171 ++++++++++++++++++ .../model_bridge/component_setup.py | 2 + .../generalized_components/block.py | 24 ++- .../joint_gate_up_mlp.py | 26 +++ .../joint_qkv_attention.py | 26 +++ .../model_bridge/transformer_bridge.py | 22 ++- 6 files changed, 261 insertions(+), 10 deletions(-) create mode 100644 tests/unit/model_bridge/test_state_dict_composition.py diff --git a/tests/unit/model_bridge/test_state_dict_composition.py b/tests/unit/model_bridge/test_state_dict_composition.py new file mode 100644 index 0000000000..a8f5a7401b --- /dev/null +++ b/tests/unit/model_bridge/test_state_dict_composition.py @@ -0,0 +1,171 @@ +"""Regression tests for recursive TransformerBridge checkpoint composition (#1655).""" + +from __future__ import annotations + +from collections import OrderedDict +from types import SimpleNamespace + +import pytest +import torch +from transformers import GPT2Config, GPT2LMHeadModel + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge import TransformerBridge +from transformer_lens.model_bridge.generalized_components import ( + JointGateUpMLPBridge, + JointQKVAttentionBridge, + LinearBridge, +) +from transformer_lens.model_bridge.sources import build_bridge_from_module + + +def _native_bridge() -> TransformerBridge: + cfg = TransformerBridgeConfig( + d_model=32, + d_head=16, + n_heads=2, + n_layers=2, + n_ctx=8, + d_vocab=16, + d_mlp=64, + act_fn="gelu", + normalization_type="LN", + seed=0, + ) + return TransformerBridge.boot_native(cfg) + + +def _parent_with_bridge(bridge: TransformerBridge) -> torch.nn.Module: + parent = torch.nn.Module() + parent.add_module("bridge", bridge) + return parent + + +def test_state_dict_with_destination_and_prefix_uses_recursive_semantics() -> None: + bridge = _native_bridge() + sentinel = torch.tensor(1) + destination: OrderedDict[str, torch.Tensor] = OrderedDict({"sentinel": sentinel}) + + returned = bridge.state_dict(destination=destination, prefix="nested.bridge.") + + assert returned is destination + assert destination["sentinel"] is sentinel + recursive_keys = set(destination) - {"sentinel"} + assert recursive_keys + assert all(key.startswith("nested.bridge.") for key in recursive_keys) + + +def test_parent_state_dict_strict_round_trip() -> None: + parent = _parent_with_bridge(_native_bridge()) + checkpoint = {key: value.clone() for key, value in parent.state_dict().items()} + + with torch.no_grad(): + for parameter in parent.parameters(): + parameter.zero_() + + result = parent.load_state_dict(checkpoint, strict=True) + + assert result.missing_keys == [] + assert result.unexpected_keys == [] + reloaded = parent.state_dict() + for key, value in checkpoint.items(): + assert torch.equal(reloaded[key], value), f"{key} did not round-trip" + + +def test_parent_registration_is_stable_across_first_forward() -> None: + bridge = _native_bridge() + parent = _parent_with_bridge(bridge) + + for block in bridge.blocks: + assert block.attn._ln1_module is block.ln1.original_component + + keys_before = tuple(parent.state_dict()) + assert not any("._ln1_module." in key for key in keys_before) + with torch.no_grad(): + bridge(torch.randint(0, bridge.cfg.d_vocab, (1, 4))) + keys_after = tuple(parent.state_dict()) + + assert keys_after == keys_before + assert not any("._ln1_module." in key for key in keys_after) + + +def test_nested_joint_qkv_bridge_strict_round_trip() -> None: + cfg = GPT2Config( + vocab_size=32, + n_positions=16, + n_embd=16, + n_layer=1, + n_head=2, + n_inner=32, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + ) + bridge = build_bridge_from_module( + GPT2LMHeadModel(cfg), + architecture="GPT2LMHeadModel", + hf_config=cfg, + ) + parent = _parent_with_bridge(bridge) + + checkpoint = {key: value.clone() for key, value in parent.state_dict().items()} + assert not any(".qkv." in key for key in checkpoint) + with torch.no_grad(): + for parameter in parent.parameters(): + parameter.zero_() + + result = parent.load_state_dict(checkpoint, strict=True) + + assert result.missing_keys == [] + assert result.unexpected_keys == [] + reloaded = parent.state_dict() + for key, value in checkpoint.items(): + assert torch.equal(reloaded[key], value), f"{key} did not round-trip" + + +def _filtered_joint_component(kind: str) -> torch.nn.Module: + filtered_child = LinearBridge(name=kind) + filtered_child.set_original_component(torch.nn.Linear(4, 8)) + cfg = SimpleNamespace(n_heads=2, d_head=4) + + if kind == "qkv": + qkv_component = JointQKVAttentionBridge( + name="attn", + config=cfg, + submodules={"qkv": filtered_child}, + ) + for child_name in ("q", "k", "v"): + getattr(qkv_component, child_name).set_original_component(torch.nn.Linear(4, 4)) + return qkv_component + gate_up_component = JointGateUpMLPBridge( + name="mlp", + config=cfg, + submodules={"gate_up": filtered_child}, + ) + gate_up_component.add_module("gate_up", filtered_child) + gate_up_component.gate.set_original_component(torch.nn.Linear(4, 4)) + getattr(gate_up_component, "in").set_original_component(torch.nn.Linear(4, 4)) + return gate_up_component + + +@pytest.mark.parametrize("filtered_child_name", ["qkv", "gate_up"]) +def test_filtered_joint_component_strict_round_trip(filtered_child_name: str) -> None: + component = _filtered_joint_component(filtered_child_name) + filtered_child = component.get_submodule(filtered_child_name) + checkpoint = {key: value.clone() for key, value in component.state_dict().items()} + + assert checkpoint + assert not any(key.startswith(f"{filtered_child_name}.") for key in checkpoint) + with torch.no_grad(): + for parameter in component.parameters(): + parameter.zero_() + + result = component.load_state_dict(checkpoint, strict=True) + + assert result.missing_keys == [] + assert result.unexpected_keys == [] + reloaded = component.state_dict() + for key, value in checkpoint.items(): + assert torch.equal(reloaded[key], value), f"{key} did not round-trip" + for parameter in filtered_child.parameters(): + assert torch.count_nonzero(parameter) == 0 diff --git a/transformer_lens/model_bridge/component_setup.py b/transformer_lens/model_bridge/component_setup.py index fdd41d0c8f..b58a8dcec6 100644 --- a/transformer_lens/model_bridge/component_setup.py +++ b/transformer_lens/model_bridge/component_setup.py @@ -303,6 +303,8 @@ def setup_blocks_bridge( block_bridge.name = f"{blocks_template.name}.{i}" block_bridge.set_original_component(original_block) setup_submodules(block_bridge, architecture_adapter, original_block) + if hasattr(block_bridge, "_wire_ln1_module"): + block_bridge._wire_ln1_module() bridged_blocks.append(block_bridge) replace_remote_component(bridged_blocks, blocks_template.name, original_model) return bridged_blocks diff --git a/transformer_lens/model_bridge/generalized_components/block.py b/transformer_lens/model_bridge/generalized_components/block.py index 1f89c7dcaf..d934b0c0c9 100644 --- a/transformer_lens/model_bridge/generalized_components/block.py +++ b/transformer_lens/model_bridge/generalized_components/block.py @@ -110,6 +110,28 @@ def __init__( # Fires pre-ln2 when use_hook_mlp_in is set. See #1317. self.hook_mlp_in = HookPoint() + def _wire_ln1_module(self) -> None: + """Keep the raw ln1 execution reference outside the ownership tree.""" + from transformer_lens.model_bridge.generalized_components.attention import ( + AttentionBridge, + ) + + ln1 = self.submodules.get("ln1") if self.submodules else None + attn = self.submodules.get("attn") if self.submodules else None + if not isinstance(attn, AttentionBridge): + return + + ln1_module = None + if ( + ln1 is not None + and getattr(attn, "supports_split_qkv_fork", False) + and getattr(ln1, "original_component", None) is not None + ): + ln1_module = ln1.original_component + + attn._modules.pop("_ln1_module", None) + object.__setattr__(attn, "_ln1_module", ln1_module) + def _maybe_wire_pre_ln_capture(self) -> None: """Install ln1/ln2 forward_pre_hooks that feed the bridge's pre-LN hooks (#1317). @@ -118,6 +140,7 @@ def _maybe_wire_pre_ln_capture(self) -> None: forward never calls the raw module, so a hook there would silently miss on most adapters. Idempotent. """ + self._wire_ln1_module() if self._pre_ln_capture_wired: return from transformer_lens.model_bridge.generalized_components.attention import ( @@ -140,7 +163,6 @@ def _capture_pre_ln1(_module: torch.nn.Module, args: tuple) -> None: handle = ln1.register_forward_pre_hook(_capture_pre_ln1) self._pre_ln_capture_handles.append(handle) - attn._ln1_module = ln1.original_component ln2 = self.submodules.get("ln2") if self.submodules else None if ln2 is not None and getattr(ln2, "original_component", None) is not None: diff --git a/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py b/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py index 391b68fa7e..5f2ac77b1c 100644 --- a/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py +++ b/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py @@ -60,6 +60,9 @@ def __init__( self._activation_fn: Any = None self._register_state_dict_hook(JointGateUpMLPBridge._filter_gate_up_state_dict) + self.register_load_state_dict_pre_hook( + JointGateUpMLPBridge._restore_filtered_gate_up_state_dict + ) @staticmethod def _filter_gate_up_state_dict( @@ -74,6 +77,29 @@ def _filter_gate_up_state_dict( for k in keys_to_remove: del state_dict[k] + @staticmethod + def _restore_filtered_gate_up_state_dict( + module: torch.nn.Module, + state_dict: Dict[str, Any], + prefix: str, + local_metadata: Dict[str, Any], + strict: bool, + missing_keys: list[str], + unexpected_keys: list[str], + error_msgs: list[str], + ) -> None: + """Insert current combined weights only to satisfy strict key matching. + + Production checkpoints restore authoritative values through the unfiltered + Hugging Face ``_original_component`` path. + """ + del local_metadata, strict, missing_keys, unexpected_keys, error_msgs + gate_up = module._modules.get("gate_up") + if gate_up is None: + return + for key, value in gate_up.state_dict(prefix=f"{prefix}gate_up.").items(): + state_dict.setdefault(key, value) + @staticmethod def _default_split_gate_up( original_mlp_component: Any, diff --git a/transformer_lens/model_bridge/generalized_components/joint_qkv_attention.py b/transformer_lens/model_bridge/generalized_components/joint_qkv_attention.py index 1aefc20d06..96798cff9a 100644 --- a/transformer_lens/model_bridge/generalized_components/joint_qkv_attention.py +++ b/transformer_lens/model_bridge/generalized_components/joint_qkv_attention.py @@ -100,6 +100,9 @@ def __init__( # Exclude stale qkv combined weights from state_dict after splitting. self._register_state_dict_hook(JointQKVAttentionBridge._filter_qkv_state_dict) + self.register_load_state_dict_pre_hook( + JointQKVAttentionBridge._restore_filtered_qkv_state_dict + ) def __deepcopy__(self, memo): """Share split_qkv_matrix and config across clones instead of copying. @@ -143,6 +146,29 @@ def _filter_qkv_state_dict( for k in keys_to_remove: del state_dict[k] + @staticmethod + def _restore_filtered_qkv_state_dict( + module: torch.nn.Module, + state_dict: Dict[str, Any], + prefix: str, + local_metadata: Dict[str, Any], + strict: bool, + missing_keys: list[str], + unexpected_keys: list[str], + error_msgs: list[str], + ) -> None: + """Insert current combined weights only to satisfy strict key matching. + + Production checkpoints restore authoritative values through the unfiltered + Hugging Face ``_original_component`` path. + """ + del local_metadata, strict, missing_keys, unexpected_keys, error_msgs + qkv = module._modules.get("qkv") + if qkv is None: + return + for key, value in qkv.state_dict(prefix=f"{prefix}qkv.").items(): + state_dict.setdefault(key, value) + def _create_qkv_conversion_rule(self) -> BaseTensorConversion: """Create the appropriate conversion rule for the individual q, k, and v matrices. diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index 331bde2e22..ebec6c20fa 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -3824,9 +3824,10 @@ def state_dict(self, destination=None, prefix="", keep_vars=False): Converts HuggingFace format keys to TransformerLens format and filters out _original_component references and nested HuggingFace components. - This returns a clean state dict with only bridge component paths converted to TL format, - excluding nested HF components (like c_fc, c_proj, c_attn) that exist inside - original_component modules. + A direct no-argument call returns a clean state dict with bridge component + paths converted to TL format. Calls that supply ``destination`` or + ``prefix`` use standard ``nn.Module`` recursive semantics so a Bridge can + compose inside a parent module. Args: destination: Optional dict to store state dict in @@ -3834,14 +3835,17 @@ def state_dict(self, destination=None, prefix="", keep_vars=False): keep_vars: Whether to keep variables as Variables instead of tensors Returns: - Dict containing the state dict with TransformerLens format keys + Direct calls return TransformerLens-format keys; recursive calls + return the supplied destination with standard module-tree keys. """ - if destination is not None: - raw_state_dict = self.original_model.state_dict( - destination=destination, prefix=prefix, keep_vars=keep_vars + if destination is not None or prefix: + return super().state_dict( + destination=destination, + prefix=prefix, + keep_vars=keep_vars, ) - else: - raw_state_dict = self.original_model.state_dict(prefix=prefix, keep_vars=keep_vars) + + raw_state_dict = self.original_model.state_dict(keep_vars=keep_vars) # Clean _original_component references and convert to TL format # Also filter out nested HuggingFace components that are wrapped by bridge components From bf89924de234a33a890b257c3aeb392e7a43aece Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Sat, 15 Aug 2026 08:45:16 +0800 Subject: [PATCH 56/87] test(model-bridge): retain native state dict round trips (#1591) Co-authored-by: jlarson4 <jonahalarson@comcast.net> --- tests/unit/model_bridge/test_boot_native.py | 36 +++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/unit/model_bridge/test_boot_native.py b/tests/unit/model_bridge/test_boot_native.py index d933e0e464..7ad0be924f 100644 --- a/tests/unit/model_bridge/test_boot_native.py +++ b/tests/unit/model_bridge/test_boot_native.py @@ -142,6 +142,42 @@ def test_native_state_dict_strict_rejects_unexpected_keys(): ) +def test_state_dict_round_trip_restores_native_bridge(): + bridge = TransformerBridge.boot_native(_cfg()) + expected = {key: value.clone() for key, value in bridge.state_dict().items()} + + with torch.no_grad(): + for parameter in bridge.parameters(): + parameter.zero_() + + incompatible_keys = bridge.load_state_dict(expected, strict=True) + + assert incompatible_keys.missing_keys == [] + assert incompatible_keys.unexpected_keys == [] + for key, expected_value in expected.items(): + assert torch.equal(bridge.state_dict()[key], expected_value), key + + +def test_state_dict_strict_load_rejects_missing_tl_key(): + bridge = TransformerBridge.boot_native(_cfg()) + state_dict = bridge.state_dict() + state_dict.pop("embed.weight") + + with pytest.raises(RuntimeError, match=r"Missing key\(s\)"): + bridge.load_state_dict(state_dict, strict=True) + + +def test_load_state_dict_preserves_raw_native_keys(): + bridge = TransformerBridge.boot_native(_cfg()) + raw_key = "layers.0.attn.k.weight" + actual_key = "layers.0.attn.k._original_component.weight" + replacement = torch.full_like(bridge.original_model.state_dict()[actual_key], 0.25) + + bridge.load_state_dict({raw_key: replacement}, strict=False) + + assert torch.equal(bridge.original_model.state_dict()[actual_key], replacement) + + def test_boot_native_accepts_dict_config(): cfg_dict = dict( d_model=32, From c5967eae9045ec5c29934803831c972f94832b56 Mon Sep 17 00:00:00 2001 From: Sohan Venkatesh <126096232+sohv@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:56:11 +0100 Subject: [PATCH 57/87] fix(bridge): gate batched-list position_ids on the target model (#1627) * fix(bridge): gate batched-list position_ids on the target model Batched list input builds an attention_mask and position_ids itself so pad tokens don't contaminate the forward. The mask is safe for any model, but the position_ids were handed over unchecked: a forward taking neither position_ids nor **kwargs raises TypeError where it would have returned logits. This is the gap jlarson4 raised while reviewing #1610. That PR added _accepts_derived_position_ids() and gated the main forward() derivation, but these two sites were left for a follow-up because no model could be shown to fail there. The LLaDA test harness builds a fixed-signature forward in process, which reproduces it: TypeError: TinyLLaDAModelLM.forward() got an unexpected keyword argument 'position_ids' Gate both sites on the same helper. The attention_mask stays unconditional -- it is safe everywhere, and withholding it would reintroduce the padding contamination this branch exists to prevent. A single unbatched string was never affected, and the test asserts that alongside the batched case. The regression test wraps its forward spy in functools.wraps: the gate reads that forward's signature, so a bare (*args, **kwargs) wrapper would look like it accepts position_ids and silently defeat the check under test. Fixes #1626 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> * fix(bridge): gate the cached-step position_ids on the target model too Review follow-up on #1627. Gating only the batched-list prompt derivation moved the injection rather than stopping it: every branch of the cached-step block supplies position_ids, so a refused model fell through to the torch.full(total_len - 1) fallback, which counts pad slots and is therefore wrong per row for a left-padded batch. Measured on hf-internal-testing/tiny-random-OPTForCausalLM, which the gate refuses because OPTLearnedPositionalEmbedding consumes the mask and derives its own positions, while its forward would accept the kwarg: before #1627 cached steps [[9],[2]] [[10],[3]] [[11],[4]] <- per row #1627 as sent cached steps [[9],[9]] [[10],[10]] [[11],[11]] <- pad slots now cached steps None <- OPT derives cached-vs-uncached max |logit diff| goes 7.45e-08 -> 2.98e-01 -> 7.45e-08, so this was a regression the PR introduced and it is now removed. Gate the whole three-way block rather than each branch, so a model that owns its position derivation receives the mask alone, matching the uncached path. Adds tests/integration/model_bridge/test_batched_generate_position_ids.py. The _generate_tokens half of #1627 had no coverage: neither fixed-signature architecture in the suite can reach that site, since LLaDA raises NotImplementedError on generate. A text-level parity test would not catch this either, because greedy argmax absorbs the drift and the decoded strings match in both states, so the tests compare logits and separately assert that no position_ids reaches a refused model. Two of them are red on 15d9553. Refs #1626 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5 <noreply@anthropic.com> --- .../test_batched_generate_position_ids.py | 133 ++++++++++++++++++ .../model_bridge/test_llada_adapter.py | 49 +++++++ .../model_bridge/transformer_bridge.py | 59 +++++--- 3 files changed, 219 insertions(+), 22 deletions(-) create mode 100644 tests/integration/model_bridge/test_batched_generate_position_ids.py diff --git a/tests/integration/model_bridge/test_batched_generate_position_ids.py b/tests/integration/model_bridge/test_batched_generate_position_ids.py new file mode 100644 index 0000000000..a4e6224242 --- /dev/null +++ b/tests/integration/model_bridge/test_batched_generate_position_ids.py @@ -0,0 +1,133 @@ +"""position_ids handling during batched-list generation. + +Batched list input is left-padded internally, so each row's real tokens start at +a different offset. The bridge derives position_ids for that, but only models +that neither reject the kwarg nor derive positions themselves may receive them +(#1626). + +The cached decoding path needs the same gate as the prompt path. Every branch +there supplies position_ids, including a ``total_len - 1`` fallback that counts +pad slots, so gating only the prompt derivation diverts a refused model into the +fallback instead of leaving it alone. + +OPT is the vehicle for the refused case: ``OPTLearnedPositionalEmbedding`` +consumes the attention mask and derives its own positions, so the gate declines +it, while its forward would happily accept the kwarg and use it. +""" + +from __future__ import annotations + +import functools + +import pytest +import torch + +GREEDY = dict(max_new_tokens=4, do_sample=False, verbose=False) +PROMPTS = ["The capital of France is the city of", "Hi"] + + +@pytest.fixture(scope="module") +def opt_bridge(): + """A model the gate refuses. Its positional embedding reads the mask.""" + from transformer_lens.model_bridge import TransformerBridge + + bridge = TransformerBridge.boot_transformers( + "hf-internal-testing/tiny-random-OPTForCausalLM", device="cpu", dtype=torch.float32 + ) + bridge.eval() + return bridge + + +def _stack_logits(output) -> torch.Tensor: + logits = output.logits + return torch.stack(list(logits)) if isinstance(logits, (list, tuple)) else logits + + +def _position_ids_per_step(bridge, use_past_kv_cache: bool) -> list: + """The position_ids each forward actually received, one entry per step.""" + seen: list = [] + original = bridge.original_model.forward + + # functools.wraps so the gate still sees the real signature; a bare + # (*args, **kwargs) spy would look like it accepts position_ids. + @functools.wraps(original) + def _spy(*args, **kwargs): + supplied = kwargs.get("position_ids") + seen.append(None if supplied is None else supplied.tolist()) + return original(*args, **kwargs) + + bridge.original_model.forward = _spy + try: + bridge.generate(list(PROMPTS), use_past_kv_cache=use_past_kv_cache, **GREEDY) + finally: + bridge.original_model.forward = original + return seen + + +def test_gate_refuses_opt(opt_bridge) -> None: + """Guards the premise of the tests below: OPT must be the refused case.""" + assert opt_bridge._accepts_derived_position_ids() is False + + +def test_refused_model_generates_identically_with_and_without_cache(opt_bridge) -> None: + """Cached decoding must not change the answer. + + Compared on logits rather than decoded text on purpose: greedy argmax + absorbs the drift and the strings match even when the positions are wrong. + """ + cached = opt_bridge.generate( + list(PROMPTS), use_past_kv_cache=True, output_logits=True, **GREEDY + ) + uncached = opt_bridge.generate( + list(PROMPTS), use_past_kv_cache=False, output_logits=True, **GREEDY + ) + + torch.testing.assert_close(_stack_logits(cached), _stack_logits(uncached), rtol=0, atol=1e-5) + + +def test_refused_model_receives_no_position_ids_on_cached_steps(opt_bridge) -> None: + """The mechanism, not just the symptom. + + The fallback supplies a per-batch constant, so a coarser check can miss it; + assert the kwarg never reaches a model that derives positions itself. + """ + assert _position_ids_per_step(opt_bridge, use_past_kv_cache=True) == [None] * ( + GREEDY["max_new_tokens"] + ) + + +def test_accepted_model_still_receives_per_row_position_ids(distilgpt2_bridge) -> None: + """The gate must not disarm the models it was never meant to exclude.""" + if distilgpt2_bridge.tokenizer.pad_token_id is None: + distilgpt2_bridge.tokenizer.pad_token = distilgpt2_bridge.tokenizer.eos_token + + seen = _position_ids_per_step(distilgpt2_bridge, use_past_kv_cache=True) + + assert seen[0] is not None, "prompt step must still receive derived positions" + cached_steps = [step for step in seen[1:] if step is not None] + assert len(cached_steps) == len(seen) - 1, "cached steps must still be supplied" + # Row 1 ("Hi") is left-padded, so its position must be strictly lower than + # row 0's. A pad-counting fallback would give both rows the same value. + first_cached = cached_steps[0] + assert first_cached[1][0] < first_cached[0][0], first_cached + + +def test_accepted_model_generates_identically_with_and_without_cache(distilgpt2_bridge) -> None: + """Control for the parity property on a model the gate allows. + + Looser than the OPT case at 1e-3. Cached decoding and full recompute sum in + different orders, which on distilgpt2's logit scale of ~132 shows as 1.4e-04, + or 1e-06 relative. The regression this guards moves logits by ~0.3, so the + margin is still more than two orders of magnitude. + """ + if distilgpt2_bridge.tokenizer.pad_token_id is None: + distilgpt2_bridge.tokenizer.pad_token = distilgpt2_bridge.tokenizer.eos_token + + cached = distilgpt2_bridge.generate( + list(PROMPTS), use_past_kv_cache=True, output_logits=True, **GREEDY + ) + uncached = distilgpt2_bridge.generate( + list(PROMPTS), use_past_kv_cache=False, output_logits=True, **GREEDY + ) + + torch.testing.assert_close(_stack_logits(cached), _stack_logits(uncached), rtol=0, atol=1e-3) diff --git a/tests/integration/model_bridge/test_llada_adapter.py b/tests/integration/model_bridge/test_llada_adapter.py index 9b33b2444c..2078560d96 100644 --- a/tests/integration/model_bridge/test_llada_adapter.py +++ b/tests/integration/model_bridge/test_llada_adapter.py @@ -3,6 +3,7 @@ from __future__ import annotations import copy +import functools import gc import math import weakref @@ -663,6 +664,54 @@ def test_left_padding_does_not_inject_unsupported_position_ids(models: TinyModel torch.testing.assert_close(bridge_logits, reference_logits, rtol=1e-5, atol=1e-6) +def test_batched_list_input_does_not_inject_unsupported_position_ids() -> None: + """Batched list input builds its own attention_mask and position_ids so pad + tokens don't contaminate the forward (#1626). The mask is safe for any model; + the position_ids are not, and this forward takes neither them nor **kwargs. + + A local bridge rather than the module fixture: this needs a tokenizer, and + attaching one to the shared instance would leak into the other tests. The + tokenizer is given a BOS so the path under test is reached independently of + BOS handling elsewhere. + """ + local = _build_models() + tokenizer = _offline_tokenizer() + tokenizer.bos_token = "<bos>" + local.bridge.tokenizer = tokenizer + + seen: dict = {} + original = local.bridge.original_model.forward + + # functools.wraps so inspect.signature() still resolves to the real forward: + # the gate reads that signature, and a bare (*args, **kwargs) spy would look + # like it accepts position_ids and defeat the check under test. + @functools.wraps(original) + def _spy(*args, **kwargs): + seen.clear() + seen.update(kwargs) + return original(*args, **kwargs) + + local.bridge.original_model.forward = _spy + try: + with torch.inference_mode(): + logits = local.bridge(["token_5 token_7 token_9", "token_5"], return_type="logits") + batched = dict(seen) + with torch.inference_mode(): + local.bridge("token_5 token_7 token_9", return_type="logits") + unbatched = dict(seen) + finally: + local.bridge.original_model.forward = original + + assert logits.shape[0] == 2 + assert "position_ids" not in batched + # The mask is still supplied — withholding it would reintroduce the padding + # contamination this branch exists to prevent. + assert "attention_mask" in batched + # Control: a single unbatched string never reached this branch, so the gate + # must not have changed anything for it either. + assert "position_ids" not in unbatched + + def test_run_with_cache_exposes_hooks_without_hf_output_attentions( models: TinyModels, ) -> None: diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index 85e0f1d512..73445ba1df 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -1777,7 +1777,11 @@ def forward( ).to(self.cfg.device) finally: self.tokenizer.padding_side = _prev_side - if "position_ids" not in kwargs: + # Gated on the target for the same reason the derivation below is: + # a fixed-signature forward raises TypeError on the kwarg, and a + # model that owns its own position derivation is overridden by it + # (#1626). + if "position_ids" not in kwargs and self._accepts_derived_position_ids(): position_ids = attention_mask.long().cumsum(-1) - 1 position_ids.masked_fill_(attention_mask == 0, 1) kwargs["position_ids"] = position_ids @@ -2185,9 +2189,12 @@ def _generate_tokens( ).to(self.cfg.device) self.tokenizer.padding_side = _prev_side forward_kwargs["attention_mask"] = attn_mask - position_ids = attn_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attn_mask == 0, 1) - forward_kwargs["position_ids"] = position_ids + # Same target gate as the forward() path: the mask is safe + # for every model, the derived positions are not (#1626). + if self._accepts_derived_position_ids(): + position_ids = attn_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attn_mask == 0, 1) + forward_kwargs["position_ids"] = position_ids if gen_step_idx == 0: if pixel_values is not None: forward_kwargs["pixel_values"] = pixel_values @@ -2241,24 +2248,32 @@ def _generate_tokens( dtype=torch.long, device=device, ) - if "position_ids" in forward_kwargs: - forward_kwargs["position_ids"] = forward_kwargs["position_ids"][ - :, -1: - ] - elif running_attention_mask is not None: - # total_len - 1 counts pad slots, so it is wrong - # for a left-padded prompt. Derive the new token's - # position from the mask instead. - forward_kwargs["position_ids"] = utils.get_offset_position_ids( - 0, running_attention_mask.long() - )[:, -1:] - else: - forward_kwargs["position_ids"] = torch.full( - (batch_size, 1), - total_len - 1, - dtype=torch.long, - device=device, - ) + # Gated as a whole (#1626): every branch below supplies + # position_ids, so gating only the prompt derivation + # above would divert a refused model into the + # total_len - 1 fallback, which counts pad slots and is + # wrong per row for a left-padded batch. A model that + # owns its position derivation gets the mask alone, + # matching the uncached path. + if self._accepts_derived_position_ids(): + if "position_ids" in forward_kwargs: + forward_kwargs["position_ids"] = forward_kwargs["position_ids"][ + :, -1: + ] + elif running_attention_mask is not None: + # total_len - 1 counts pad slots, so it is wrong + # for a left-padded prompt. Derive the new token's + # position from the mask instead. + forward_kwargs["position_ids"] = utils.get_offset_position_ids( + 0, running_attention_mask.long() + )[:, -1:] + else: + forward_kwargs["position_ids"] = torch.full( + (batch_size, 1), + total_len - 1, + dtype=torch.long, + device=device, + ) logits = self( current_tokens[:, -1:], return_type="logits", From cb34fbfc351a5bd9c2aa7f52106c55f131c45bd8 Mon Sep 17 00:00:00 2001 From: emerardd <113128214+emerardd@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:58:13 +0800 Subject: [PATCH 58/87] Fix batchless accumulated residual normalization (#1678) --- tests/unit/test_activation_cache.py | 63 +++++++++++++++++++++++++++++ transformer_lens/ActivationCache.py | 11 ++--- 2 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 tests/unit/test_activation_cache.py diff --git a/tests/unit/test_activation_cache.py b/tests/unit/test_activation_cache.py new file mode 100644 index 0000000000..9debaf4a30 --- /dev/null +++ b/tests/unit/test_activation_cache.py @@ -0,0 +1,63 @@ +import pytest +import torch + +from transformer_lens import ActivationCache +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge import TransformerBridge + + +@pytest.fixture(scope="module", params=["LN", "RMS"]) +def activation_cache(request: pytest.FixtureRequest) -> ActivationCache: + cfg = TransformerBridgeConfig( + n_layers=2, + d_model=16, + n_ctx=8, + d_head=4, + n_heads=4, + d_vocab=32, + act_fn="gelu", + normalization_type=request.param, + ) + with torch.random.fork_rng(devices=[]): + torch.manual_seed(0) + model = TransformerBridge.boot_native(cfg) + tokens = torch.tensor( + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + ] + ) + _, cache = model.run_with_cache(tokens) + return cache + + +@pytest.mark.parametrize("layer", [1, -1], ids=["cached-scale", "recomputed-final-ln"]) +@pytest.mark.parametrize( + "pos_slice", [None, (1, 3), -1], ids=["all-positions", "position-slice", "scalar-position"] +) +@pytest.mark.parametrize("apply_ln", [False, True], ids=["raw", "normalized"]) +def test_batchless_accumulated_resid_matches_batched_row( + activation_cache: ActivationCache, + layer: int, + pos_slice: tuple[int, int] | int | None, + apply_ln: bool, +) -> None: + batch_index = 1 + batched = activation_cache.accumulated_resid( + layer=layer, + pos_slice=pos_slice, + apply_ln=apply_ln, + ) + + batchless_cache = activation_cache.apply_slice_to_batch_dim(batch_index) + assert not batchless_cache.has_batch_dim + batchless = batchless_cache.accumulated_resid( + layer=layer, + pos_slice=pos_slice, + apply_ln=apply_ln, + ) + + expected = batched[:, batch_index] + assert batchless.shape == expected.shape + torch.testing.assert_close(batchless, expected) diff --git a/transformer_lens/ActivationCache.py b/transformer_lens/ActivationCache.py index 41cb22d6ec..53da85cb4e 100644 --- a/transformer_lens/ActivationCache.py +++ b/transformer_lens/ActivationCache.py @@ -521,6 +521,7 @@ def accumulated_resid( layer, pos_slice=pos_slice, mlp_input=mlp_input, + has_batch_dim=self.has_batch_dim, recompute_ln=recompute_ln, ) if return_labels: @@ -1402,17 +1403,17 @@ def apply_ln_to_stack( # Logit lens: apply final layer norm to each component with recomputed statistics if recompute_ln and layer == self.model.cfg.n_layers and hasattr(self.model, "ln_final"): ln_final = self.model.ln_final - had_pos_dim = residual_stack.ndim == 4 results = [] for i in range(residual_stack.shape[0]): x = residual_stack[i] - # ln_final expects (batch, pos, d_model); ensure pos dim present + original_shape = x.shape + # ln_final expects (batch, pos, d_model); restore missing structural dimensions + if not has_batch_dim: + x = x.unsqueeze(0) if x.ndim == 2: x = x.unsqueeze(1) out = ln_final(x) - if not had_pos_dim: - out = out.squeeze(1) - results.append(out) + results.append(out.reshape(original_shape)) return torch.stack(results, dim=0) # Center the stack onlny if the model uses LayerNorm From ea11a860c6da3d1f1223a521f8df27d2986a95f2 Mon Sep 17 00:00:00 2001 From: emerardd <113128214+emerardd@users.noreply.github.com> Date: Mon, 17 Aug 2026 21:01:14 +0800 Subject: [PATCH 59/87] Fix TransformerBridge adapter traversal coverage (#1671) * Fix TransformerBridge adapter traversal coverage * Address TransformerBridge traversal review feedback --- .../test_parent_module_traversal.py | 397 ++++++++++++++++++ .../test_ast_adapter.py | 1 + .../test_bert_adapter.py | 57 ++- .../test_vit_adapter.py | 9 + .../model_bridge/component_setup.py | 120 ++++++ .../supported_architectures/ast.py | 5 + .../supported_architectures/bert.py | 18 +- .../supported_architectures/vit.py | 2 + .../model_bridge/transformer_bridge.py | 15 +- 9 files changed, 613 insertions(+), 11 deletions(-) create mode 100644 tests/integration/model_bridge/test_parent_module_traversal.py diff --git a/tests/integration/model_bridge/test_parent_module_traversal.py b/tests/integration/model_bridge/test_parent_module_traversal.py new file mode 100644 index 0000000000..5f60c79305 --- /dev/null +++ b/tests/integration/model_bridge/test_parent_module_traversal.py @@ -0,0 +1,397 @@ +"""Download-free parent traversal coverage across Bridge architecture shapes.""" + +from dataclasses import dataclass +from typing import Any, Callable + +import pytest +import torch +from torch import nn +from transformers import ( + ASTConfig, + ASTForAudioClassification, + BartConfig, + BartForConditionalGeneration, + BertConfig, + BertForMaskedLM, + BertForNextSentencePrediction, + BertForPreTraining, + BloomConfig, + BloomForCausalLM, + GPT2Config, + GPT2LMHeadModel, + GPTNeoXConfig, + GPTNeoXForCausalLM, + HubertConfig, + HubertForCTC, + LlamaConfig, + LlamaForCausalLM, + MistralConfig, + MistralForCausalLM, + MixtralConfig, + MixtralForCausalLM, + T5Config, + T5ForConditionalGeneration, + ViTConfig, + ViTForImageClassification, + ViTModel, +) + +from transformer_lens.model_bridge.sources import build_bridge_from_module + + +@dataclass(frozen=True) +class ArchitectureCase: + name: str + model_type: type[nn.Module] + config_factory: Callable[[], Any] + architecture: str + + +def _bert_config() -> BertConfig: + return BertConfig( + vocab_size=32, + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=4, + intermediate_size=32, + max_position_embeddings=16, + ) + + +def _vit_config() -> ViTConfig: + return ViTConfig( + image_size=16, + patch_size=4, + num_channels=3, + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=4, + intermediate_size=32, + num_labels=3, + ) + + +def _bart_config() -> BartConfig: + return BartConfig( + vocab_size=32, + d_model=16, + encoder_layers=1, + decoder_layers=1, + encoder_attention_heads=4, + decoder_attention_heads=4, + encoder_ffn_dim=32, + decoder_ffn_dim=32, + max_position_embeddings=16, + ) + + +def _hubert_config() -> HubertConfig: + return HubertConfig( + vocab_size=32, + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=4, + intermediate_size=32, + conv_dim=(8,), + conv_stride=(2,), + conv_kernel=(3,), + num_conv_pos_embeddings=4, + num_conv_pos_embedding_groups=2, + ) + + +ARCHITECTURE_CASES = ( + ArchitectureCase( + "gpt2-joint-qkv", + GPT2LMHeadModel, + lambda: GPT2Config( + vocab_size=32, + n_positions=16, + n_ctx=16, + n_embd=16, + n_layer=1, + n_head=4, + n_inner=32, + ), + "GPT2LMHeadModel", + ), + ArchitectureCase( + "bloom-joint-qkv", + BloomForCausalLM, + lambda: BloomConfig(vocab_size=32, hidden_size=16, n_layer=1, n_head=4), + "BloomForCausalLM", + ), + ArchitectureCase( + "gpt-neox-rotary", + GPTNeoXForCausalLM, + lambda: GPTNeoXConfig( + vocab_size=32, + hidden_size=16, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=4, + max_position_embeddings=16, + ), + "GPTNeoXForCausalLM", + ), + ArchitectureCase( + "llama-split-qkv-rope", + LlamaForCausalLM, + lambda: LlamaConfig( + vocab_size=32, + hidden_size=16, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=4, + max_position_embeddings=16, + ), + "LlamaForCausalLM", + ), + ArchitectureCase( + "mistral-gqa", + MistralForCausalLM, + lambda: MistralConfig( + vocab_size=32, + hidden_size=16, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=2, + max_position_embeddings=16, + ), + "MistralForCausalLM", + ), + ArchitectureCase("bert-mlm", BertForMaskedLM, _bert_config, "BertForMaskedLM"), + ArchitectureCase( + "bert-nsp", + BertForNextSentencePrediction, + _bert_config, + "BertForMaskedLM", + ), + ArchitectureCase( + "bert-mlm-nsp", + BertForPreTraining, + _bert_config, + "BertForMaskedLM", + ), + ArchitectureCase( + "t5-encoder-decoder", + T5ForConditionalGeneration, + lambda: T5Config( + vocab_size=32, + d_model=16, + d_kv=4, + d_ff=32, + num_layers=1, + num_decoder_layers=1, + num_heads=4, + ), + "T5ForConditionalGeneration", + ), + ArchitectureCase( + "bart-encoder-decoder", + BartForConditionalGeneration, + _bart_config, + "BartForConditionalGeneration", + ), + ArchitectureCase( + "mixtral-moe", + MixtralForCausalLM, + lambda: MixtralConfig( + vocab_size=32, + hidden_size=16, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=2, + num_local_experts=2, + num_experts_per_tok=1, + max_position_embeddings=16, + ), + "MixtralForCausalLM", + ), + ArchitectureCase( + "vit-vision", + ViTForImageClassification, + _vit_config, + "ViTForImageClassification", + ), + ArchitectureCase("vit-bare-pooler", ViTModel, _vit_config, "ViTModel"), + ArchitectureCase( + "hubert-audio", + HubertForCTC, + _hubert_config, + "HubertForCTC", + ), + ArchitectureCase( + "ast-audio-classifier", + ASTForAudioClassification, + lambda: ASTConfig( + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=4, + intermediate_size=32, + patch_size=4, + frequency_stride=4, + time_stride=4, + max_length=16, + num_mel_bins=16, + ), + "ASTForAudioClassification", + ), +) + +ARCHITECTURE_CASE_BY_NAME = {case.name: case for case in ARCHITECTURE_CASES} + + +def _named_identities(named_values: Any) -> dict[int, str]: + return {id(value): name for name, value in named_values} + + +def _assert_same_identities(expected: dict[int, str], actual: dict[int, str]) -> None: + missing = [expected[identity] for identity in expected.keys() - actual.keys()] + unexpected = [actual[identity] for identity in actual.keys() - expected.keys()] + assert actual.keys() == expected.keys(), f"missing={missing}, unexpected={unexpected}" + + +@pytest.mark.parametrize("case", ARCHITECTURE_CASES, ids=lambda case: case.name) +def test_parent_and_direct_traversal_have_identical_state(case: ArchitectureCase) -> None: + config = case.config_factory() + model = case.model_type(config).eval() + bridge = build_bridge_from_module( + model, + case.architecture, + hf_config=config, + dtype=torch.float32, + device="cpu", + model_name=f"tiny-{case.name}", + ) + parent = nn.Module() + parent.add_module("bridge", bridge) + + source_parameters = _named_identities(bridge.original_model.named_parameters()) + direct_parameters = _named_identities(bridge.named_parameters()) + parent_parameters = _named_identities(parent.named_parameters()) + source_buffers = _named_identities(bridge.original_model.named_buffers()) + direct_buffers = _named_identities(bridge.named_buffers()) + parent_buffers = _named_identities(parent.named_buffers()) + + _assert_same_identities(source_parameters, direct_parameters) + _assert_same_identities(direct_parameters, parent_parameters) + _assert_same_identities(source_buffers, direct_buffers) + _assert_same_identities(direct_buffers, parent_buffers) + + +def test_parent_dtype_conversion_updates_container_owned_state() -> None: + bart_config = _bart_config() + bridge = build_bridge_from_module( + BartForConditionalGeneration(bart_config), + "BartForConditionalGeneration", + hf_config=bart_config, + dtype=torch.float32, + device="cpu", + model_name="tiny-bart-container-buffer", + ) + parent = nn.Module() + parent.add_module("bridge", bridge) + + parent.to(torch.float64) + + assert bridge.original_model.final_logits_bias.dtype == torch.float64 + assert id(bridge.original_model.final_logits_bias) in { + id(buffer) for buffer in parent.buffers() + } + + +def test_parent_assign_load_updates_container_owned_state() -> None: + bart_config = _bart_config() + bridge = build_bridge_from_module( + BartForConditionalGeneration(bart_config), + "BartForConditionalGeneration", + hf_config=bart_config, + dtype=torch.float32, + device="cpu", + model_name="tiny-bart-container-buffer-load", + ) + parent = nn.Module() + parent.add_module("bridge", bridge) + state = parent.state_dict() + buffer_key = "bridge._container_state_owners.final_logits_bias" + state[buffer_key] = torch.ones_like(state[buffer_key]) + + parent.load_state_dict(state, strict=True, assign=True) + + assert torch.equal(bridge.original_model.final_logits_bias, torch.ones_like(state[buffer_key])) + assert id(bridge.original_model.final_logits_bias) in { + id(buffer) for buffer in parent.buffers() + } + + +@pytest.mark.parametrize( + ("case_name", "container_path", "state_name", "state_key"), + ( + ("bart-encoder-decoder", "", "final_logits_bias", "final_logits_bias"), + ("hubert-audio", "hubert", "masked_spec_embed", "hubert.masked_spec_embed"), + ), +) +def test_direct_assign_load_stays_current_after_apply( + case_name: str, container_path: str, state_name: str, state_key: str +) -> None: + case = ARCHITECTURE_CASE_BY_NAME[case_name] + config = case.config_factory() + bridge = build_bridge_from_module( + case.model_type(config), + case.architecture, + hf_config=config, + dtype=torch.float32, + device="cpu", + model_name=f"tiny-{case.name}-direct-assign", + ) + original_container = ( + bridge.original_model.get_submodule(container_path) + if container_path + else bridge.original_model + ) + owner_container = ( + bridge._container_state_owners.get_submodule(container_path) + if container_path + else bridge._container_state_owners + ) + replacement = torch.full_like(getattr(original_container, state_name), 7) + + bridge.load_state_dict({state_key: replacement}, strict=False, assign=True) + + assert getattr(owner_container, state_name) is getattr(original_container, state_name) + bridge.cpu() + assert torch.equal(getattr(original_container, state_name), replacement) + + +@pytest.mark.parametrize( + ("case_name", "key_fragment", "expected_keys"), + ( + ("bert-nsp", "pooler", {"pooler.weight", "pooler.bias"}), + ("vit-bare-pooler", "pooler", {"pooler.weight", "pooler.bias"}), + ( + "ast-audio-classifier", + "classifier", + {"classifier_ln.weight", "classifier_ln.bias"}, + ), + ), +) +def test_task_head_state_dict_keys_are_not_reexpanded( + case_name: str, key_fragment: str, expected_keys: set[str] +) -> None: + case = ARCHITECTURE_CASE_BY_NAME[case_name] + config = case.config_factory() + bridge = build_bridge_from_module( + case.model_type(config), + case.architecture, + hf_config=config, + dtype=torch.float32, + device="cpu", + model_name=f"tiny-{case.name}-state-dict-keys", + ) + + actual_keys = {key for key in bridge.state_dict() if key_fragment in key} + assert actual_keys == expected_keys diff --git a/tests/unit/model_bridge/supported_architectures/test_ast_adapter.py b/tests/unit/model_bridge/supported_architectures/test_ast_adapter.py index 2bb395ea95..a2fe7f910e 100644 --- a/tests/unit/model_bridge/supported_architectures/test_ast_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_ast_adapter.py @@ -68,6 +68,7 @@ def test_classification_prefix_rebinding(self, hf_config, tl_config): adapter.prepare_model(model) assert adapter.component_mapping["blocks"].name == "audio_spectrogram_transformer.layers" + assert adapter.component_mapping["classifier_ln"].name == "classifier.layernorm" assert adapter.component_mapping["unembed"].name == "classifier.dense" assert adapter.cfg.d_vocab_out == 2 diff --git a/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py b/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py index 06ce437333..948d684b00 100644 --- a/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py @@ -8,9 +8,10 @@ - Anti-drift config flags """ +from types import SimpleNamespace + import pytest -from transformer_lens.config import TransformerBridgeConfig from transformer_lens.config.transformer_bridge_config import TransformerBridgeConfig from transformer_lens.conversion_utils.conversion_steps import RearrangeTensorConversion from transformer_lens.conversion_utils.param_processing_conversion import ( @@ -76,7 +77,9 @@ def test_top_level_keys(self, adapter: BertArchitectureAdapter) -> None: "embed", "token_type_embed", "pos_embed", + "embed_ln", "blocks", + "mlm_head", "ln_final", "unembed", } @@ -86,7 +89,9 @@ def test_bridge_types(self, adapter: BertArchitectureAdapter) -> None: assert isinstance(mapping["embed"], EmbeddingBridge) assert isinstance(mapping["token_type_embed"], EmbeddingBridge) assert isinstance(mapping["pos_embed"], PosEmbedBridge) + assert isinstance(mapping["embed_ln"], NormalizationBridge) assert isinstance(mapping["blocks"], BlockBridge) + assert isinstance(mapping["mlm_head"], LinearBridge) assert isinstance(mapping["ln_final"], NormalizationBridge) assert isinstance(mapping["unembed"], UnembeddingBridge) @@ -95,18 +100,28 @@ def test_top_level_hf_paths(self, adapter: BertArchitectureAdapter) -> None: assert mapping["embed"].name == "bert.embeddings.word_embeddings" assert mapping["token_type_embed"].name == "bert.embeddings.token_type_embeddings" assert mapping["pos_embed"].name == "bert.embeddings.position_embeddings" + assert mapping["embed_ln"].name == "bert.embeddings.LayerNorm" assert mapping["blocks"].name == "bert.encoder.layer" + assert mapping["mlm_head"].name == "cls.predictions.transform.dense" assert mapping["ln_final"].name == "cls.predictions.transform.LayerNorm" assert mapping["unembed"].name == "cls.predictions.decoder" def test_token_type_embedding_is_cached_with_hf_output(self) -> None: import torch - from transformers import BertForMaskedLM + from transformers import BertConfig, BertForMaskedLM from transformer_lens.model_bridge.sources import build_bridge_from_module - hf_model = BertForMaskedLM.from_pretrained("bert-base-cased").eval() - input_ids = torch.tensor([[101, 7592, 102, 2088, 102]]) + hf_config = BertConfig( + vocab_size=32, + hidden_size=16, + num_hidden_layers=1, + num_attention_heads=4, + intermediate_size=32, + max_position_embeddings=16, + ) + hf_model = BertForMaskedLM(hf_config).eval() + input_ids = torch.tensor([[1, 2, 3, 4, 5]]) token_type_ids = torch.tensor([[0, 0, 0, 1, 1]]) with torch.no_grad(): expected = hf_model.bert.embeddings.token_type_embeddings(token_type_ids).clone() @@ -114,10 +129,10 @@ def test_token_type_embedding_is_cached_with_hf_output(self) -> None: bridge = build_bridge_from_module( hf_model, "BertForMaskedLM", - hf_config=hf_model.config, + hf_config=hf_config, dtype=torch.float32, device="cpu", - model_name="bert-base-cased", + model_name="tiny-bert", ) with torch.no_grad(): _, cache = bridge.run_with_cache( @@ -173,6 +188,36 @@ def test_mlp_submodule_hf_paths(self, adapter: BertArchitectureAdapter) -> None: assert mlp.submodules["out"].name == "output.dense" +class TestBertTaskHeadMappings: + def test_nsp_only_model_uses_hooked_encoder_names(self) -> None: + adapter = BertArchitectureAdapter(_make_cfg()) + hf_model = SimpleNamespace( + bert=SimpleNamespace(pooler=object()), + cls=SimpleNamespace(seq_relationship=object()), + ) + + adapter.prepare_model(hf_model) + + assert adapter.components["pooler"].name == "bert.pooler.dense" + assert adapter.components["unembed"].name == "cls.seq_relationship" + assert "mlm_head" not in adapter.components + assert "ln_final" not in adapter.components + + def test_combined_mlm_nsp_model_registers_both_heads(self) -> None: + adapter = BertArchitectureAdapter(_make_cfg()) + hf_model = SimpleNamespace( + bert=SimpleNamespace(pooler=object()), + cls=SimpleNamespace(predictions=object(), seq_relationship=object()), + ) + + adapter.prepare_model(hf_model) + + assert adapter.components["pooler"].name == "bert.pooler.dense" + assert adapter.components["mlm_head"].name == "cls.predictions.transform.dense" + assert adapter.components["nsp_head"].name == "cls.seq_relationship" + assert adapter.components["unembed"].name == "cls.predictions.decoder" + + # --------------------------------------------------------------------------- # Anti-drift config flags # --------------------------------------------------------------------------- diff --git a/tests/unit/model_bridge/supported_architectures/test_vit_adapter.py b/tests/unit/model_bridge/supported_architectures/test_vit_adapter.py index cdf99ae1a3..fdf7541279 100644 --- a/tests/unit/model_bridge/supported_architectures/test_vit_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_vit_adapter.py @@ -279,6 +279,9 @@ def _bare_model(self) -> object: """No 'vit'/'deit'/'classifier' attribute — mimics bare ViTModel/DeiTModel.""" return SimpleNamespace() + def _bare_model_with_pooler(self) -> object: + return SimpleNamespace(pooler=SimpleNamespace(dense=SimpleNamespace())) + def _vit_for_classification(self) -> object: return SimpleNamespace(vit=SimpleNamespace(), classifier=SimpleNamespace()) @@ -305,6 +308,12 @@ def test_bare_model_has_no_unembed(self, adapter: ViTArchitectureAdapter) -> Non adapter.prepare_model(self._bare_model()) assert "unembed" not in adapter.component_mapping + def test_bare_model_maps_pooler_without_root_name_collision( + self, adapter: ViTArchitectureAdapter + ) -> None: + adapter.prepare_model(self._bare_model_with_pooler()) + assert adapter.component_mapping["pooler"].name == "pooler.dense" + def test_bare_model_does_not_require_encoder_attribute( self, adapter: ViTArchitectureAdapter ) -> None: diff --git a/transformer_lens/model_bridge/component_setup.py b/transformer_lens/model_bridge/component_setup.py index ca0181addf..9064e8e09e 100644 --- a/transformer_lens/model_bridge/component_setup.py +++ b/transformer_lens/model_bridge/component_setup.py @@ -20,6 +20,63 @@ pass +class _ContainerStateOwner(nn.Module): + """Registered view of state owned directly by an unwrapped container.""" + + def __init__(self, original_container: nn.Module) -> None: + super().__init__() + self.__dict__["_original_container"] = original_container + + def _sync_original_container(self) -> None: + original_container = self.__dict__["_original_container"] + original_container._parameters.update(self._parameters) + original_container._buffers.update(self._buffers) + + def _refresh_from_original_container(self) -> None: + original_container = self.__dict__["_original_container"] + for name in self._parameters: + self._parameters[name] = original_container._parameters[name] + for name in self._buffers: + self._buffers[name] = original_container._buffers[name] + + def _apply(self, fn: Any, recurse: bool = True) -> "_ContainerStateOwner": + self._refresh_from_original_container() + super()._apply(fn, recurse=recurse) + self._sync_original_container() + return self + + def _load_from_state_dict( + self, + state_dict: dict[str, Any], + prefix: str, + local_metadata: dict[str, Any], + strict: bool, + missing_keys: list[str], + unexpected_keys: list[str], + error_msgs: list[str], + ) -> None: + super()._load_from_state_dict( + state_dict, + prefix, + local_metadata, + strict, + missing_keys, + unexpected_keys, + error_msgs, + ) + self._sync_original_container() + + +def refresh_container_state_owners(bridge_module: nn.Module) -> None: + """Refresh registered container state from the original model tree.""" + root_owner = bridge_module._modules.get("_container_state_owners") + if not isinstance(root_owner, _ContainerStateOwner): + return + for owner in root_owner.modules(): + if isinstance(owner, _ContainerStateOwner): + owner._refresh_from_original_container() + + def replace_remote_component( replacement_component: nn.Module, remote_path: str, remote_model: RemoteModel ) -> None: @@ -56,6 +113,69 @@ def set_original_components( """ component_mapping = architecture_adapter.get_component_mapping() setup_components(component_mapping, bridge_module, architecture_adapter, original_model) + if isinstance(original_model, nn.Module): + _register_unowned_container_state(bridge_module, original_model) + + +def _register_unowned_container_state(bridge_module: nn.Module, original_model: nn.Module) -> None: + """Make direct container parameters and buffers reachable through the Bridge tree.""" + registered_parameter_ids = { + id(parameter) + for module in bridge_module.modules() + for parameter in module._parameters.values() + if parameter is not None + } + registered_buffer_ids = { + id(buffer) + for module in bridge_module.modules() + for buffer in module._buffers.values() + if buffer is not None + } + missing_parameters: list[tuple[str, nn.Module, str, nn.Parameter]] = [] + missing_buffers: list[tuple[str, nn.Module, str, Any]] = [] + + for module_path, module in original_model.named_modules(): + if any(child is not None for child in module._modules.values()): + for parameter_name, parameter in module._parameters.items(): + if parameter is not None and id(parameter) not in registered_parameter_ids: + missing_parameters.append((module_path, module, parameter_name, parameter)) + registered_parameter_ids.add(id(parameter)) + for buffer_name, buffer in module._buffers.items(): + if buffer is not None and id(buffer) not in registered_buffer_ids: + missing_buffers.append((module_path, module, buffer_name, buffer)) + registered_buffer_ids.add(id(buffer)) + + if not missing_parameters and not missing_buffers: + return + + owner_by_path: dict[str, _ContainerStateOwner] = {"": _ContainerStateOwner(original_model)} + root_owner = owner_by_path[""] + original_modules = dict(original_model.named_modules()) + + def get_owner(module_path: str) -> _ContainerStateOwner: + current_path = "" + current_owner = root_owner + for path_part in module_path.split(".") if module_path else (): + child_path = f"{current_path}.{path_part}" if current_path else path_part + if child_path not in owner_by_path: + child_owner = _ContainerStateOwner(original_modules[child_path]) + current_owner.add_module(path_part, child_owner) + owner_by_path[child_path] = child_owner + current_owner = owner_by_path[child_path] + current_path = child_path + return current_owner + + for module_path, _, parameter_name, parameter in missing_parameters: + get_owner(module_path).register_parameter(parameter_name, parameter) + + for module_path, module, buffer_name, buffer in missing_buffers: + get_owner(module_path).register_buffer( + buffer_name, + buffer, + persistent=buffer_name not in module._non_persistent_buffers_set, + ) + + bridge_module.add_module("_container_state_owners", root_owner) def _wire_symbolic_hooks(symbolic: GeneralizedComponent) -> None: diff --git a/transformer_lens/model_bridge/supported_architectures/ast.py b/transformer_lens/model_bridge/supported_architectures/ast.py index 78bde24b9d..5ecd7c7134 100644 --- a/transformer_lens/model_bridge/supported_architectures/ast.py +++ b/transformer_lens/model_bridge/supported_architectures/ast.py @@ -128,6 +128,11 @@ def prepare_model(self, hf_model: Any) -> None: and hasattr(hf_model, "classifier") and hasattr(hf_model.classifier, "dense") ): + self.component_mapping["classifier_ln"] = NormalizationBridge( + name="classifier.layernorm", + config=self.cfg, + use_native_layernorm_autograd=True, + ) self.component_mapping["unembed"] = UnembeddingBridge(name="classifier.dense") self.cfg.d_vocab = num_labels self.cfg.d_vocab_out = num_labels diff --git a/transformer_lens/model_bridge/supported_architectures/bert.py b/transformer_lens/model_bridge/supported_architectures/bert.py index 358fd38ec3..f71a237d6a 100644 --- a/transformer_lens/model_bridge/supported_architectures/bert.py +++ b/transformer_lens/model_bridge/supported_architectures/bert.py @@ -89,6 +89,11 @@ def __init__(self, cfg: Any) -> None: "embed": EmbeddingBridge(name="bert.embeddings.word_embeddings"), "token_type_embed": EmbeddingBridge(name="bert.embeddings.token_type_embeddings"), "pos_embed": PosEmbedBridge(name="bert.embeddings.position_embeddings"), + "embed_ln": NormalizationBridge( + name="bert.embeddings.LayerNorm", + config=self.cfg, + use_native_layernorm_autograd=True, + ), "blocks": BlockBridge( name="bert.encoder.layer", # BERT has no single MLP module (intermediate.dense and output.dense @@ -130,6 +135,7 @@ def __init__(self, cfg: Any) -> None: ), }, ), + "mlm_head": LinearBridge(name="cls.predictions.transform.dense"), "unembed": UnembeddingBridge(name="cls.predictions.decoder"), "ln_final": NormalizationBridge( name="cls.predictions.transform.LayerNorm", @@ -145,7 +151,15 @@ def prepare_model(self, hf_model: Any) -> None: BertForNextSentencePrediction has cls.seq_relationship (NSP head) and no MLM-specific LayerNorm. """ - if hasattr(hf_model, "cls") and hasattr(hf_model.cls, "seq_relationship"): - # NSP model — swap head components + if getattr(getattr(hf_model, "bert", None), "pooler", None) is not None: + self.components["pooler"] = LinearBridge(name="bert.pooler.dense") + + has_predictions = hasattr(getattr(hf_model, "cls", None), "predictions") + has_nsp_head = hasattr(getattr(hf_model, "cls", None), "seq_relationship") + if has_nsp_head and has_predictions: + self.components["nsp_head"] = LinearBridge(name="cls.seq_relationship") + elif has_nsp_head: + # NSP-only model — swap head components. self.components["unembed"] = UnembeddingBridge(name="cls.seq_relationship") + self.components.pop("mlm_head", None) self.components.pop("ln_final", None) diff --git a/transformer_lens/model_bridge/supported_architectures/vit.py b/transformer_lens/model_bridge/supported_architectures/vit.py index 1aa5394b0a..eef75a916c 100644 --- a/transformer_lens/model_bridge/supported_architectures/vit.py +++ b/transformer_lens/model_bridge/supported_architectures/vit.py @@ -204,3 +204,5 @@ def prepare_model(self, hf_model: Any) -> None: self.component_mapping = self._build_component_mapping( prefix=prefix, with_classifier=with_classifier ) + if not with_classifier and getattr(hf_model, "pooler", None) is not None: + self.component_mapping["pooler"] = LinearBridge(name="pooler.dense") diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index 73445ba1df..9dc3b9a847 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -37,7 +37,10 @@ from transformer_lens.hook_points import HookIntrospectionMixin, HookPoint from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter from transformer_lens.model_bridge.bridge_core import _BLOCK_LIST_ATTRS, BridgeCore -from transformer_lens.model_bridge.component_setup import set_original_components +from transformer_lens.model_bridge.component_setup import ( + refresh_container_state_owners, + set_original_components, +) from transformer_lens.model_bridge.composition_scores import CompositionScores from transformer_lens.model_bridge.driver_protocol import ( TensorLike, @@ -405,6 +408,10 @@ def __getattr__(self, name: str) -> Any: # Use __dict__ directly to avoid recursion if "_modules" in self.__dict__ and name in self.__dict__["_modules"]: # type: ignore[arg-type] return self.__dict__["_modules"][name] + adapter = self.__dict__.get("adapter") + component_mapping = getattr(adapter, "component_mapping", None) + if component_mapping is not None and name in component_mapping: + raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") if "original_model" in self.__dict__ and self.__dict__["original_model"] is not None: try: name_split = name.split(".") @@ -3830,8 +3837,8 @@ def _normalize_bridge_key_to_hf(self, key: str) -> str: block_list_names = {"blocks", "L_blocks", "H_blocks", "encoder_blocks", "decoder_blocks"} for tl_name, component in component_mapping.items(): if component.name and tl_name not in block_list_names: - # Skip if TL name is already a suffix of the HF path (avoids doubling). - if tl_name != component.name and not component.name.endswith("." + tl_name): + # Skip if TL name is already a segment of its HF path (avoids doubling). + if tl_name != component.name and tl_name not in component.name.split("."): attr_to_hf[tl_name] = component.name # Map block-level components (ln1, ln2, attn, mlp) for all block lists @@ -4008,6 +4015,8 @@ def load_state_dict(self, state_dict, strict=True, assign=False): ) result = self.original_model.load_state_dict(mapped_state_dict, strict=False, assign=assign) + if assign: + refresh_container_state_owners(self) return type(result)(missing_keys=missing_keys, unexpected_keys=unexpected_keys) def get_params(self): From 0908c986e39e157b6b01c5bb828817a6858da58c Mon Sep 17 00:00:00 2001 From: jlarson4 <jonahalarson@comcast.net> Date: Thu, 20 Aug 2026 09:40:49 -0500 Subject: [PATCH 60/87] additional merge fixes --- .../model_bridge/test_hook_set_enforcement.py | 6 ++++- transformer_lens/model_bridge/bridge_core.py | 27 +++++++++---------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/tests/unit/model_bridge/test_hook_set_enforcement.py b/tests/unit/model_bridge/test_hook_set_enforcement.py index e4d52794b3..888413be99 100644 --- a/tests/unit/model_bridge/test_hook_set_enforcement.py +++ b/tests/unit/model_bridge/test_hook_set_enforcement.py @@ -106,7 +106,11 @@ def test_add_hook_accepts_registry_alias(self): ( name for name, hp in bridge.hook_dict.items() - if hp.name is not None and hp.name != name + # Gated-off names raise by design (see test_gated_hooks) — this + # test's subject is alias resolution, so pick an ungated one. + if hp.name is not None + and hp.name != name + and bridge._gated_hook_reason(name) is None ), None, ) diff --git a/transformer_lens/model_bridge/bridge_core.py b/transformer_lens/model_bridge/bridge_core.py index e6b5b192fb..6964940892 100644 --- a/transformer_lens/model_bridge/bridge_core.py +++ b/transformer_lens/model_bridge/bridge_core.py @@ -781,31 +781,18 @@ def add_hook( if callable(name) and not isinstance(name, str): hook_dict = self.hook_dict seen_hooks: set = set() - gated_names_skipped: List[str] = [] for hook_name, hook_point in hook_dict.items(): if name(hook_name): hook_id = id(hook_point) if hook_id in seen_hooks: continue seen_hooks.add(hook_id) - # A filter was not necessarily targeting a gated name on - # purpose — skip with a warning instead of raising. - if self._gated_hook_reason(hook_name) is not None: - gated_names_skipped.append(hook_name) - continue self._add_fn_to_hook_point(hook_point, hook_name, hook_fn, dir, is_permanent) - if gated_names_skipped: - warnings.warn( - f"add_hook: skipped {len(gated_names_skipped)} gated-off hook name(s) " - f"that will never fire: {gated_names_skipped}. Call the relevant " - "set_use_*(True) setter first to enable them.", - stacklevel=2, - ) return # An explicitly named gated-off hook point is a caller error: the hook - # would silently never fire. Raise naming the setter to call - # (mirrors HookedTransformer.check_hooks_to_add, with a clearer error). + # would silently never fire. Raise naming the setter to call (filters + # above add freely — they were not necessarily targeting gated names). reason = self._gated_hook_reason(name) if reason is not None: raise ValueError( @@ -1223,6 +1210,16 @@ def wrapped_hook_fn(tensor, hook, _orig_fn=original_hook_fn): hook_fn = wrapped_hook_fn if isinstance(hook_name_or_filter, str): + # An explicitly named gated-off hook point is a caller error: + # the hook would silently never fire. Raise naming the setter + # (a filter matching a gated name is left alone — it was not + # necessarily targeting that name on purpose). + reason = self._gated_hook_reason(hook_name_or_filter) + if reason is not None: + raise ValueError( + f"Cannot add hook {hook_name_or_filter} because {reason} is False. " + f"Call set_{reason}(True) first." + ) actual_hook_name, hook_point = self._resolve_hook_point( hook_name_or_filter, aliases, hook_dict ) From c3b534711c8ad8e7a13fd4ac7c84746a96a22968 Mon Sep 17 00:00:00 2001 From: jlarson4 <jonahalarson@comcast.net> Date: Thu, 20 Aug 2026 10:15:35 -0500 Subject: [PATCH 61/87] Additional fixes to gated hook points --- transformer_lens/model_bridge/bridge_core.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/transformer_lens/model_bridge/bridge_core.py b/transformer_lens/model_bridge/bridge_core.py index 6964940892..6233262c86 100644 --- a/transformer_lens/model_bridge/bridge_core.py +++ b/transformer_lens/model_bridge/bridge_core.py @@ -974,10 +974,14 @@ def add_caching_hooks( remove_batch_dim=remove_batch_dim, cache=cache, ) + # Gated-off hook points never fire, so caching them is a no-op; skip + # them rather than trip add_hook's explicit-name guard when enumerating. for name, hook_fn in fwd_hooks: - self.add_hook(name, hook_fn, dir="fwd") + if self._gated_hook_reason(name) is None: + self.add_hook(name, hook_fn, dir="fwd") for name, hook_fn in bwd_hooks: - self.add_hook(name, hook_fn, dir="bwd") + if self._gated_hook_reason(name) is None: + self.add_hook(name, hook_fn, dir="bwd") return cache def cache_all( From 6a221d9ed042986aab765866e8503b11a81db92d Mon Sep 17 00:00:00 2001 From: Jonah Larson <jonahalarson@comcast.net> Date: Wed, 26 Aug 2026 12:47:34 -0500 Subject: [PATCH 62/87] Bug/bridge hook gating followups (#1724) * improving the final hookedTransformer deprecation items * Clean up unnecessary assertions * Fixing CI failures * tolerance fix --- .../test_attention_score_sentinel.py | 18 +- .../model_bridge/test_audio_frame_entry.py | 100 ++++++++ .../model_bridge/test_bert_pooler_hook.py | 69 +++++ .../test_encdec_weight_stacking_parity.py | 41 +++ .../test_nsp_sentence_pair_helper.py | 82 ++++++ .../test_parent_module_traversal.py | 6 +- .../model_bridge/test_qwen2_moe_bridge.py | 10 + .../test_arcee_adapter.py | 3 - .../test_bert_adapter.py | 10 +- .../test_falcon_h1_adapter.py | 3 - .../test_gpt_oss_adapter.py | 10 +- .../test_granite_adapter.py | 3 - .../test_granite_moe_adapter.py | 3 - .../test_moe_routing_hooks.py | 159 ++++++++++++ .../test_bridge_weight_properties.py | 17 +- .../test_encdec_weight_stacking.py | 136 ++++++++++ tests/unit/model_bridge/test_gated_hooks.py | 62 +++++ tests/unit/model_bridge/test_gpt_oss_moe.py | 58 +++++ .../test_stack_block_params_hybrid.py | 18 +- transformer_lens/benchmarks/utils.py | 2 - transformer_lens/model_bridge/bridge_core.py | 20 +- .../generalized_components/__init__.py | 4 + .../generalized_components/moe.py | 118 ++++++++- .../generalized_components/pooler.py | 22 ++ .../supported_architectures/bert.py | 10 +- .../supported_architectures/gpt_oss.py | 8 +- .../supported_architectures/granite.py | 6 +- .../supported_architectures/jetmoe.py | 14 +- .../model_bridge/transformer_bridge.py | 235 +++++++++++++++++- 29 files changed, 1196 insertions(+), 51 deletions(-) create mode 100644 tests/integration/model_bridge/test_audio_frame_entry.py create mode 100644 tests/integration/model_bridge/test_bert_pooler_hook.py create mode 100644 tests/integration/model_bridge/test_encdec_weight_stacking_parity.py create mode 100644 tests/integration/model_bridge/test_nsp_sentence_pair_helper.py create mode 100644 tests/unit/model_bridge/supported_architectures/test_moe_routing_hooks.py create mode 100644 tests/unit/model_bridge/test_encdec_weight_stacking.py create mode 100644 transformer_lens/model_bridge/generalized_components/pooler.py diff --git a/tests/integration/model_bridge/test_attention_score_sentinel.py b/tests/integration/model_bridge/test_attention_score_sentinel.py index 424198df58..dce95b060f 100644 --- a/tests/integration/model_bridge/test_attention_score_sentinel.py +++ b/tests/integration/model_bridge/test_attention_score_sentinel.py @@ -7,19 +7,27 @@ def test_gpt2_compatibility_scores_use_negative_infinity( - gpt2_bridge_compat, gpt2_hooked_processed + gpt2_bridge_compat, gpt2_goldens_processed ) -> None: - """GPT-2's direct HF mask is normalized before the compatibility hook.""" - tokens = gpt2_hooked_processed.to_tokens("The capital of France is") + """GPT-2's direct HF mask is normalized before the compatibility hook. + + Anchored on the frozen HookedTransformer goldens rather than a live + HookedTransformer, matching the rest of the compatibility suite. + """ + golden = gpt2_goldens_processed + tokens = golden.scalars["short_prompt"] _, bridge_cache = gpt2_bridge_compat.run_with_cache(tokens, names_filter=[SCORES]) - _, hooked_cache = gpt2_hooked_processed.run_with_cache(tokens, names_filter=[SCORES]) + hooked_cache = golden.tensors("activations") bridge_scores, hooked_scores = bridge_cache[SCORES], hooked_cache[SCORES] causal_mask = torch.isneginf(hooked_scores) assert causal_mask.any() assert torch.isneginf(bridge_scores[causal_mask]).all() + # The goldens were captured on different hardware, so the unmasked scores + # agree to fp32 accumulation noise rather than bit-exactly. Same tolerance + # the sibling golden comparison uses for this hook. torch.testing.assert_close( - bridge_scores[~causal_mask], hooked_scores[~causal_mask], rtol=0, atol=0 + bridge_scores[~causal_mask], hooked_scores[~causal_mask], rtol=1e-4, atol=1e-4 ) diff --git a/tests/integration/model_bridge/test_audio_frame_entry.py b/tests/integration/model_bridge/test_audio_frame_entry.py new file mode 100644 index 0000000000..c9f199ef96 --- /dev/null +++ b/tests/integration/model_bridge/test_audio_frame_entry.py @@ -0,0 +1,100 @@ +"""Audio frame entry: run the encoder from precomputed frames. + +Mirrors HookedAudioEncoder.encoder_output, the audio-path analogue of +start_at_layer. Deletion evidence for that method: without it the bridge can +only enter at the waveform, so injecting frames means re-running the conv front +end. start_at_layer stays refused for audio — this is a separate entry point. +""" + +from __future__ import annotations + +import math + +import numpy as np +import pytest +import torch + +from transformer_lens.model_bridge.bridge import TransformerBridge + +MODEL = "facebook/hubert-base-ls960" +SAMPLE_RATE = 16000 +FRAMES_HOOK = "feat_proj.hook_out" + + +@pytest.fixture(scope="module") +def audio_bridge() -> TransformerBridge: + return TransformerBridge.boot_transformers(MODEL, device="cpu") + + +@pytest.fixture(scope="module") +def waveform() -> torch.Tensor: + t = np.linspace(0, 1.0, SAMPLE_RATE, endpoint=False, dtype=np.float32) + return torch.tensor(0.1 * np.sin(2 * math.pi * 440.0 * t))[None, :] + + +@pytest.fixture(scope="module") +def full_run(audio_bridge, waveform): + last = f"blocks.{audio_bridge.cfg.n_layers - 1}.hook_out" + _, cache = audio_bridge.run_with_cache( + waveform, names_filter=[FRAMES_HOOK, "blocks.0.hook_out", last] + ) + return cache, last + + +def test_frame_entry_matches_the_full_waveform_run(audio_bridge, full_run): + """Re-entering at the frames reproduces the encoder output exactly.""" + cache, last = full_run + resid = audio_bridge.encoder_output(cache[FRAMES_HOOK]) + torch.testing.assert_close(resid, cache[last], atol=0.0, rtol=0.0) + + +def test_hooks_fire_from_frame_entry(audio_bridge, full_run): + """Block hooks fire on the frame path, so caching composes with it.""" + cache, last = full_run + wanted = {"blocks.0.hook_out", last} + cached, fwd_hooks, _ = audio_bridge.get_caching_hooks(names_filter=lambda name: name in wanted) + with audio_bridge.hooks(fwd_hooks=fwd_hooks): + audio_bridge.encoder_output(cache[FRAMES_HOOK]) + + assert set(cached) == wanted + for name in wanted: + torch.testing.assert_close(cached[name], cache[name], atol=0.0, rtol=0.0) + + +def test_padding_mask_changes_the_encoding(audio_bridge, full_run): + """The mask is applied, not ignored.""" + cache, last = full_run + frames = cache[FRAMES_HOOK] + mask = torch.ones(frames.shape[:2], dtype=torch.long) + mask[:, -10:] = 0 + + masked = audio_bridge.encoder_output(frames, one_zero_attention_mask=mask) + assert not torch.allclose(masked, cache[last]) + + +def test_waveform_shaped_input_is_rejected(audio_bridge, waveform): + """A 2D waveform is not frames; say so instead of silently mis-running.""" + with pytest.raises(ValueError, match=r"\[batch, frames, d_model\]"): + audio_bridge.encoder_output(waveform) + + +def test_start_at_layer_remains_refused_for_audio(audio_bridge, waveform): + """Frame entry is a separate API; the residual-injection guard is untouched.""" + with pytest.raises(NotImplementedError, match="audio models"): + audio_bridge(waveform, start_at_layer=1) + + +def test_text_models_reject_the_audio_frame_entry(): + """Non-audio bridges have no conv-frame stage to re-enter.""" + bridge = TransformerBridge.boot_transformers("gpt2", device="cpu") + with pytest.raises(NotImplementedError, match="not an audio model"): + bridge.encoder_output(torch.zeros(1, 4, bridge.cfg.d_model)) + + +def test_spectrogram_encoders_reject_the_frame_entry(): + """AST has no conv feature extractor, so there is no frame stage to bypass.""" + bridge = TransformerBridge.boot_transformers( + "MIT/ast-finetuned-audioset-10-10-0.4593", device="cpu" + ) + with pytest.raises(NotImplementedError, match="convolutional front end"): + bridge.encoder_output(torch.zeros(1, 4, bridge.cfg.d_model)) diff --git a/tests/integration/model_bridge/test_bert_pooler_hook.py b/tests/integration/model_bridge/test_bert_pooler_hook.py new file mode 100644 index 0000000000..afef28fed7 --- /dev/null +++ b/tests/integration/model_bridge/test_bert_pooler_hook.py @@ -0,0 +1,69 @@ +"""The BERT [CLS] pooler is observable through a named bridge hook. + +Mirrors HookedEncoder's BertPooler, whose hook_pooler_out carries the +post-tanh pooled [CLS]. Deletion evidence for that component: without a named +bridge hook the pooled [CLS] is only reachable coincidentally, via the NSP +head's unembed.hook_in. +""" + +from __future__ import annotations + +import pytest +import torch +from transformers import BertForMaskedLM, BertForNextSentencePrediction + +from transformer_lens.model_bridge.bridge import TransformerBridge + +MODEL = "google-bert/bert-base-cased" + + +@pytest.fixture(scope="module") +def nsp_bridge() -> TransformerBridge: + return TransformerBridge.boot_transformers( + MODEL, device="cpu", model_class=BertForNextSentencePrediction + ) + + +def _tokens(bridge: TransformerBridge) -> torch.Tensor: + return bridge.tokenizer("Hello there my friend.", return_tensors="pt")["input_ids"] + + +def test_pooler_hook_matches_huggingfaces_own_pooler(nsp_bridge): + """hook_out is the pooled [CLS], checked against HF's pooler directly.""" + tokens = _tokens(nsp_bridge) + _, cache = nsp_bridge.run_with_cache(tokens) + + hf = nsp_bridge.original_model + with torch.no_grad(): + expected = hf.bert.pooler(hf.bert(tokens).last_hidden_state) + + torch.testing.assert_close(cache["pooler.hook_out"], expected, atol=0.0, rtol=0.0) + + +def test_pooler_hook_is_post_activation(nsp_bridge): + """HookedEncoder fires hook_pooler_out after tanh; the projection is separate.""" + tokens = _tokens(nsp_bridge) + _, cache = nsp_bridge.run_with_cache(tokens) + + pre_activation = cache["pooler.dense.hook_out"] + pooled = cache["pooler.hook_out"] + assert not torch.allclose(pre_activation, pooled) + torch.testing.assert_close(torch.tanh(pre_activation), pooled) + + +def test_hooked_encoder_hook_name_is_aliased(nsp_bridge): + """Code migrated from HookedEncoder asks for hook_pooler_out.""" + tokens = _tokens(nsp_bridge) + _, cache = nsp_bridge.run_with_cache(tokens) + + assert torch.equal(cache["pooler.hook_pooler_out"], cache["pooler.hook_out"]) + + +def test_masked_lm_checkpoint_without_a_pooler_still_boots(): + """BertForMaskedLM leaves bert.pooler as None; the mapping must skip it.""" + bridge = TransformerBridge.boot_transformers(MODEL, device="cpu", model_class=BertForMaskedLM) + tokens = _tokens(bridge) + logits, cache = bridge.run_with_cache(tokens) + + assert logits.shape[0] == 1 + assert not [name for name in cache if "pooler" in name] diff --git a/tests/integration/model_bridge/test_encdec_weight_stacking_parity.py b/tests/integration/model_bridge/test_encdec_weight_stacking_parity.py new file mode 100644 index 0000000000..a84b3d00e2 --- /dev/null +++ b/tests/integration/model_bridge/test_encdec_weight_stacking_parity.py @@ -0,0 +1,41 @@ +"""Stacked enc-dec weights on the bridge match HookedEncoderDecoder. + +Deletion evidence for HookedEncoderDecoder's stacked-weight properties: the +bridge must produce the same tensors over chain(encoder, decoder), and the same +head labels, before the legacy class can go. +""" + +from __future__ import annotations + +import pytest +import torch + +from transformer_lens import HookedEncoderDecoder +from transformer_lens.model_bridge.bridge import TransformerBridge + +MODEL = "google-t5/t5-small" +STACKED = ["W_Q", "W_K", "W_V", "W_O", "W_in", "W_out"] + + +@pytest.fixture(scope="module") +def hooked() -> HookedEncoderDecoder: + return HookedEncoderDecoder.from_pretrained(MODEL, device="cpu") + + +@pytest.fixture(scope="module") +def bridge() -> TransformerBridge: + return TransformerBridge.boot_transformers(MODEL, device="cpu") + + +@pytest.mark.parametrize("name", STACKED) +def test_stacked_weights_match_hooked_encoder_decoder(name, hooked, bridge): + """HookedEncoderDecoder does no weight processing, so these are directly comparable.""" + expected = getattr(hooked, name) + actual = getattr(bridge, name) + assert actual.shape == expected.shape + torch.testing.assert_close(actual, expected, atol=0.0, rtol=0.0) + + +def test_head_labels_match_hooked_encoder_decoder(hooked, bridge): + """all_head_labels is a property on the bridge; HT exposes it as a method.""" + assert bridge.all_head_labels == hooked.all_head_labels() diff --git a/tests/integration/model_bridge/test_nsp_sentence_pair_helper.py b/tests/integration/model_bridge/test_nsp_sentence_pair_helper.py new file mode 100644 index 0000000000..fa364707b8 --- /dev/null +++ b/tests/integration/model_bridge/test_nsp_sentence_pair_helper.py @@ -0,0 +1,82 @@ +"""Next-sentence prediction from strings on the bridge. + +Mirrors BertNextSentencePrediction's string interface, which cannot be adapted +onto a bridge (its forward reaches for encoder_output/pooler/nsp_head). This +helper is where that ergonomics survives the Hooked* removal. +""" + +from __future__ import annotations + +import pytest +import torch +from transformers import AutoTokenizer, BertForNextSentencePrediction + +from transformer_lens.model_bridge.bridge import TransformerBridge + +MODEL = "google-bert/bert-base-cased" +SENTENCE_A = "A man walked into a grocery store." +SEQUENTIAL_B = "He bought an apple." +UNRELATED_B = "The Eiffel Tower is in Paris." + + +@pytest.fixture(scope="module") +def nsp_bridge() -> TransformerBridge: + bridge = TransformerBridge.boot_transformers( + MODEL, device="cpu", model_class=BertForNextSentencePrediction + ) + bridge.enable_compatibility_mode() + return bridge + + +@pytest.fixture(scope="module") +def hf_tokenizer(): + return AutoTokenizer.from_pretrained(MODEL) + + +def test_pair_tokenization_matches_huggingface(nsp_bridge, hf_tokenizer): + """[CLS] a [SEP] b [SEP] with segment ids, identical to tokenizer(a, b).""" + tokens = nsp_bridge.to_sentence_pair_tokens(SENTENCE_A, SEQUENTIAL_B) + expected = hf_tokenizer(SENTENCE_A, SEQUENTIAL_B, return_tensors="pt") + + assert torch.equal(tokens["input_ids"], expected["input_ids"]) + assert torch.equal(tokens["token_type_ids"], expected["token_type_ids"]) + assert tokens["token_type_ids"].unique().tolist() == [0, 1] + + +def test_logits_match_a_direct_huggingface_nsp_forward(nsp_bridge, hf_tokenizer): + encodings = hf_tokenizer(SENTENCE_A, SEQUENTIAL_B, return_tensors="pt") + with torch.no_grad(): + expected = nsp_bridge.original_model(**encodings).logits + + actual = nsp_bridge.predict_next_sentence(SENTENCE_A, SEQUENTIAL_B, return_type="logits") + torch.testing.assert_close(actual, expected, atol=0.0, rtol=0.0) + + +def test_predictions_distinguish_sequential_from_unrelated(nsp_bridge): + assert nsp_bridge.predict_next_sentence(SENTENCE_A, SEQUENTIAL_B) == ( + "The sentences are sequential" + ) + assert nsp_bridge.predict_next_sentence(SENTENCE_A, UNRELATED_B) == ( + "The sentences are NOT sequential" + ) + + +def test_segment_ids_are_load_bearing(nsp_bridge): + """Dropping token_type_ids collapses the NSP margin — why the helper owns them.""" + tokens = nsp_bridge.to_sentence_pair_tokens(SENTENCE_A, SEQUENTIAL_B) + with_segments = nsp_bridge.predict_next_sentence(SENTENCE_A, SEQUENTIAL_B, return_type="logits") + without_segments = nsp_bridge( + tokens["input_ids"], + attention_mask=tokens["attention_mask"], + return_type="logits", + ) + + margin = lambda logits: float((logits[0, 0] - logits[0, 1]).abs()) + assert margin(with_segments) > 2 * margin(without_segments) + + +def test_helper_rejects_a_model_without_an_nsp_head(): + """An MLM-headed bridge has no 2-class output; say so instead of decoding noise.""" + bridge = TransformerBridge.boot_transformers(MODEL, device="cpu") + with pytest.raises(ValueError, match="next-sentence-prediction head"): + bridge.predict_next_sentence(SENTENCE_A, SEQUENTIAL_B) diff --git a/tests/integration/model_bridge/test_parent_module_traversal.py b/tests/integration/model_bridge/test_parent_module_traversal.py index 5f60c79305..3d80d25d10 100644 --- a/tests/integration/model_bridge/test_parent_module_traversal.py +++ b/tests/integration/model_bridge/test_parent_module_traversal.py @@ -370,7 +370,11 @@ def test_direct_assign_load_stays_current_after_apply( @pytest.mark.parametrize( ("case_name", "key_fragment", "expected_keys"), ( - ("bert-nsp", "pooler", {"pooler.weight", "pooler.bias"}), + # BERT wraps the pooler module (not its inner Linear) so hook_out carries + # the post-tanh pooled [CLS], which nests the weights one level deeper. + # Still two keys, so nothing is re-expanded — only renamed. ViT wraps + # pooler.dense and keeps the flat names. + ("bert-nsp", "pooler", {"pooler.dense.weight", "pooler.dense.bias"}), ("vit-bare-pooler", "pooler", {"pooler.weight", "pooler.bias"}), ( "ast-audio-classifier", diff --git a/tests/integration/model_bridge/test_qwen2_moe_bridge.py b/tests/integration/model_bridge/test_qwen2_moe_bridge.py index 05ee6a3519..2f2ca75fb7 100644 --- a/tests/integration/model_bridge/test_qwen2_moe_bridge.py +++ b/tests/integration/model_bridge/test_qwen2_moe_bridge.py @@ -117,3 +117,13 @@ def test_run_with_cache_captures_moe_hooks(self) -> None: router_scores_key = f"blocks.{layer_idx}.mlp.hook_router_scores" assert router_scores_key not in cache + + # Routing observables mirror HookedTransformer: weights at full + # expert width, indices at top-k. + weights_key = f"blocks.{layer_idx}.mlp.gate.hook_expert_weights" + assert weights_key in cache, f"Missing cache key: {weights_key}" + assert cache[weights_key].shape == (flat_tokens, num_experts) + + indices_key = f"blocks.{layer_idx}.mlp.gate.hook_expert_indices" + assert indices_key in cache, f"Missing cache key: {indices_key}" + assert cache[indices_key].shape == (flat_tokens, bridge.cfg.experts_per_token) diff --git a/tests/unit/model_bridge/supported_architectures/test_arcee_adapter.py b/tests/unit/model_bridge/supported_architectures/test_arcee_adapter.py index 9775e11144..55168218ed 100644 --- a/tests/unit/model_bridge/supported_architectures/test_arcee_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_arcee_adapter.py @@ -97,9 +97,6 @@ def test_attn_not_only_and_eager(self, adapter: ArceeArchitectureAdapter) -> Non assert adapter.cfg.attn_only is False assert adapter.cfg.attn_implementation == "eager" - def test_gqa_propagated(self, adapter: ArceeArchitectureAdapter) -> None: - assert adapter.cfg.n_key_value_heads == 4 - class TestArceeAdapterComponentMapping: """Component-mapping structure and HF module names. Key contrasts with Llama: diff --git a/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py b/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py index 948d684b00..28083a3d4d 100644 --- a/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_bert_adapter.py @@ -198,7 +198,10 @@ def test_nsp_only_model_uses_hooked_encoder_names(self) -> None: adapter.prepare_model(hf_model) - assert adapter.components["pooler"].name == "bert.pooler.dense" + # The pooler module itself is wrapped so hook_out is the post-tanh + # pooled [CLS]; the projection stays hookable underneath. + assert adapter.components["pooler"].name == "bert.pooler" + assert adapter.components["pooler"].submodules["dense"].name == "dense" assert adapter.components["unembed"].name == "cls.seq_relationship" assert "mlm_head" not in adapter.components assert "ln_final" not in adapter.components @@ -212,7 +215,10 @@ def test_combined_mlm_nsp_model_registers_both_heads(self) -> None: adapter.prepare_model(hf_model) - assert adapter.components["pooler"].name == "bert.pooler.dense" + # The pooler module itself is wrapped so hook_out is the post-tanh + # pooled [CLS]; the projection stays hookable underneath. + assert adapter.components["pooler"].name == "bert.pooler" + assert adapter.components["pooler"].submodules["dense"].name == "dense" assert adapter.components["mlm_head"].name == "cls.predictions.transform.dense" assert adapter.components["nsp_head"].name == "cls.seq_relationship" assert adapter.components["unembed"].name == "cls.predictions.decoder" diff --git a/tests/unit/model_bridge/supported_architectures/test_falcon_h1_adapter.py b/tests/unit/model_bridge/supported_architectures/test_falcon_h1_adapter.py index ede9e019ca..319f48567d 100644 --- a/tests/unit/model_bridge/supported_architectures/test_falcon_h1_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_falcon_h1_adapter.py @@ -123,9 +123,6 @@ def test_not_stateful(self, adapter: FalconH1ArchitectureAdapter) -> None: def test_eps_attr_variance_epsilon(self, adapter: FalconH1ArchitectureAdapter) -> None: assert adapter.cfg.eps_attr == "variance_epsilon" - def test_n_key_value_heads_propagated(self, adapter: FalconH1ArchitectureAdapter) -> None: - assert adapter.cfg.n_key_value_heads == 2 - def test_mamba_intermediate_size_propagated(self, adapter: FalconH1ArchitectureAdapter) -> None: # mamba_d_ssm is the inner SSM width directly. assert getattr(adapter.cfg, "mamba_intermediate_size", None) == 32 diff --git a/tests/unit/model_bridge/supported_architectures/test_gpt_oss_adapter.py b/tests/unit/model_bridge/supported_architectures/test_gpt_oss_adapter.py index 212a6ffebc..054fb3615b 100644 --- a/tests/unit/model_bridge/supported_architectures/test_gpt_oss_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_gpt_oss_adapter.py @@ -32,6 +32,7 @@ GatedMLPBridge, LinearBridge, MoEBridge, + MoERouterBridge, PositionEmbeddingsAttentionBridge, RMSNormalizationBridge, RotaryEmbeddingBridge, @@ -222,11 +223,12 @@ def test_ln1_ln2_are_rms_norm_bridges(self, adapter: GPTOSSArchitectureAdapter) assert isinstance(subs["ln1"], RMSNormalizationBridge) assert isinstance(subs["ln2"], RMSNormalizationBridge) - def test_mlp_has_no_submodules(self, adapter: GPTOSSArchitectureAdapter) -> None: - """GPT-OSS exposes no router submodule on the MoE block, the entire MoE module - is wrapped opaquely by MoEBridge.""" + def test_mlp_exposes_a_hookable_router(self, adapter: GPTOSSArchitectureAdapter) -> None: + """The MoE block wraps GPT-OSS's router so the routing observables are hookable.""" mlp = _mapping(adapter)["blocks"].submodules["mlp"] - assert mlp.submodules == {} + assert set(mlp.submodules) == {"router"} + assert isinstance(mlp.submodules["router"], MoERouterBridge) + assert mlp.submodules["router"].name == "router" def test_hf_module_paths(self, adapter: GPTOSSArchitectureAdapter) -> None: mapping = _mapping(adapter) diff --git a/tests/unit/model_bridge/supported_architectures/test_granite_adapter.py b/tests/unit/model_bridge/supported_architectures/test_granite_adapter.py index 41e617cd91..538167664f 100644 --- a/tests/unit/model_bridge/supported_architectures/test_granite_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_granite_adapter.py @@ -98,9 +98,6 @@ def test_default_prepend_bos_false(self, adapter: GraniteArchitectureAdapter) -> """Granite models do not prepend BOS by default.""" assert adapter.cfg.default_prepend_bos is False - def test_n_key_value_heads_propagated(self, adapter: GraniteArchitectureAdapter) -> None: - assert adapter.cfg.n_key_value_heads == N_KV_HEADS - # --------------------------------------------------------------------------- # Component mapping tests — dense Granite diff --git a/tests/unit/model_bridge/supported_architectures/test_granite_moe_adapter.py b/tests/unit/model_bridge/supported_architectures/test_granite_moe_adapter.py index 1bd89388ed..0a8e01348b 100644 --- a/tests/unit/model_bridge/supported_architectures/test_granite_moe_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_granite_moe_adapter.py @@ -119,9 +119,6 @@ def test_uses_rms_norm(self, adapter: GraniteMoeArchitectureAdapter) -> None: def test_default_prepend_bos_false(self, adapter: GraniteMoeArchitectureAdapter) -> None: assert adapter.cfg.default_prepend_bos is False - def test_n_key_value_heads_propagated(self, adapter: GraniteMoeArchitectureAdapter) -> None: - assert adapter.cfg.n_key_value_heads == N_KV_HEADS - # --------------------------------------------------------------------------- # Component mapping tests diff --git a/tests/unit/model_bridge/supported_architectures/test_moe_routing_hooks.py b/tests/unit/model_bridge/supported_architectures/test_moe_routing_hooks.py new file mode 100644 index 0000000000..5ea3ab54a2 --- /dev/null +++ b/tests/unit/model_bridge/supported_architectures/test_moe_routing_hooks.py @@ -0,0 +1,159 @@ +"""MoE routing observables on the bridge (hook_expert_weights / hook_expert_indices). + +Mirrors HookedTransformer's MoE routing hooks: weights are exposed in HT's +``[tokens, num_experts]`` layout, indices as ``[tokens, top_k]``, on both MoE +families (5.13 ``TopKRouter`` blocks and GPT-OSS). +""" + +from __future__ import annotations + +import copy + +import torch +from transformers import Qwen2MoeConfig, Qwen2MoeForCausalLM + +from transformer_lens.model_bridge.bridge import TransformerBridge +from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_from_module, +) + +NUM_EXPERTS = 4 +TOP_K = 2 +WEIGHTS = "blocks.0.mlp.gate.hook_expert_weights" +INDICES = "blocks.0.mlp.gate.hook_expert_indices" + + +def _tiny_bridge() -> TransformerBridge: + torch.manual_seed(0) + cfg = Qwen2MoeConfig( + vocab_size=128, + hidden_size=64, + intermediate_size=96, + moe_intermediate_size=32, + shared_expert_intermediate_size=96, + num_hidden_layers=2, + num_attention_heads=4, + num_key_value_heads=2, + num_experts=NUM_EXPERTS, + num_experts_per_tok=TOP_K, + max_position_embeddings=64, + decoder_sparse_step=1, + mlp_only_layers=[], + ) + cfg._attn_implementation = "eager" + model = Qwen2MoeForCausalLM(cfg).eval() + bridge = build_bridge_from_module( + model, + "Qwen2MoeForCausalLM", + hf_config=copy.deepcopy(cfg), + tokenizer=None, + device="cpu", + ).eval() + bridge.adapter.setup_component_testing(model, bridge_model=bridge) + return bridge + + +def _tokens() -> torch.Tensor: + return torch.randint(0, 128, (1, 6)) + + +def test_routing_hooks_are_cached_with_hooked_transformer_shapes(): + """Weights arrive at HT's [tokens, num_experts]; indices at [tokens, top_k].""" + bridge, tokens = _tiny_bridge(), _tokens() + _, cache = bridge.run_with_cache(tokens) + + assert cache[WEIGHTS].shape == (tokens.numel(), NUM_EXPERTS) + assert cache[INDICES].shape == (tokens.numel(), TOP_K) + + +def test_routing_hooks_are_observe_only(): + """Firing the hooks does not perturb the forward pass.""" + bridge, tokens = _tiny_bridge(), _tokens() + baseline = bridge(tokens) + cached_logits, _ = bridge.run_with_cache(tokens) + + assert torch.equal(baseline, cached_logits) + + +def test_expanded_weights_carry_only_the_selected_experts(): + """The scatter puts each top-k score at its expert id and zeroes the rest.""" + bridge, tokens = _tiny_bridge(), _tokens() + _, cache = bridge.run_with_cache(tokens) + + weights, indices = cache[WEIGHTS], cache[INDICES] + assert (weights != 0).sum(-1).eq(TOP_K).all() + gathered = weights.gather(-1, indices.long()) + assert torch.equal( + gathered.sort(dim=-1).values, weights.topk(TOP_K, dim=-1).values.sort(dim=-1).values + ) + + +def test_editing_expert_weights_reaches_the_model(): + """An edit in the weights hook changes the output — not a dead hook.""" + bridge, tokens = _tiny_bridge(), _tokens() + baseline = bridge(tokens) + + bridge.add_hook(WEIGHTS, lambda t, hook=None: torch.zeros_like(t)) + edited = bridge(tokens) + bridge.reset_hooks() + + assert not torch.equal(baseline, edited) + + +def test_editing_expert_indices_reroutes(): + """An edit in the indices hook re-routes tokens to different experts.""" + bridge, tokens = _tiny_bridge(), _tokens() + baseline = bridge(tokens) + + bridge.add_hook(INDICES, lambda t, hook=None: torch.zeros_like(t)) + edited = bridge(tokens) + bridge.reset_hooks() + + assert not torch.equal(baseline, edited) + + +def test_hooked_transformer_names_alias_onto_the_router(): + """HT places these on the MoE block itself; migrated code must find them there.""" + bridge, tokens = _tiny_bridge(), _tokens() + _, cache = bridge.run_with_cache(tokens) + + assert torch.equal(cache["blocks.0.mlp.hook_expert_weights"], cache[WEIGHTS]) + assert torch.equal(cache["blocks.0.mlp.hook_expert_indices"], cache[INDICES]) + + +class _StubRouter(torch.nn.Module): + """Router returning a fixed (logits, top-k weights, top-k indices) tuple.""" + + def __init__(self, logits, weights, indices): + super().__init__() + self._out = (logits, weights, indices) + + def forward(self, hidden_states): # noqa: ARG002 - fixed output by design + return self._out + + +def test_rerouting_picks_up_the_weight_at_the_newly_selected_expert(): + """The gather runs after the indices hook, as HookedTransformer does. + + Re-routing a token to a different expert must pick up the weight sitting at + that expert (zero when it was not originally selected), not carry the old + expert's weight over to the new one. + """ + from transformer_lens.model_bridge.generalized_components.moe import MoERouterBridge + + logits = torch.zeros(1, NUM_EXPERTS) + weights = torch.tensor([[0.7, 0.3]]) + indices = torch.tensor([[1, 2]]) + + router = MoERouterBridge(name="router") + router.set_original_component(_StubRouter(logits, weights, indices)) + # Route both slots to expert 0, which held no weight in the original top-k. + router.hook_expert_indices.add_hook(lambda t, hook=None: torch.zeros_like(t)) + + _, out_weights, out_indices = router(torch.zeros(1, 8)) + + assert torch.equal(out_indices, torch.zeros_like(indices)) + assert torch.equal(out_weights, torch.zeros_like(weights)), ( + "weights must be gathered at the edited indices, so a re-route to an " + f"unselected expert yields 0; got {out_weights.tolist()}" + ) diff --git a/tests/unit/model_bridge/test_bridge_weight_properties.py b/tests/unit/model_bridge/test_bridge_weight_properties.py index c81c2ca550..669a954ff2 100644 --- a/tests/unit/model_bridge/test_bridge_weight_properties.py +++ b/tests/unit/model_bridge/test_bridge_weight_properties.py @@ -8,6 +8,12 @@ ) from transformer_lens.model_bridge.transformer_bridge import TransformerBridge +_BORROWED_HELPERS = ( + "_enumerate_blocks", + "_resolve_submodule_name", + "_rewrite_submodule_path", +) + class TestReshapeBias: """Tests for AttentionBridge._reshape_bias().""" @@ -67,7 +73,10 @@ def __init__(self, bias): class _FakeBridge: def __init__(self, biases): - self.blocks = [TestStackBlockParams._Block(b) for b in biases] + self.blocks = torch.nn.ModuleList([TestStackBlockParams._Block(b) for b in biases]) + # _stack_block_params walks the registered block lists, so the stub + # exposes the same _modules shape a real bridge does. + self._modules = {"blocks": self.blocks} class Cfg: n_devices = 1 @@ -75,6 +84,12 @@ class Cfg: self.cfg = Cfg() + def __getattr__(self, name): + """Borrow the real block-walking helpers rather than reimplementing them.""" + if name in _BORROWED_HELPERS: + return getattr(TransformerBridge, name).__get__(self) + raise AttributeError(name) + def test_stacks_present_params(self): fake = self._FakeBridge([torch.ones(2, 3), torch.zeros(2, 3)]) stacked = TransformerBridge._stack_block_params(fake, "attn.b_Q") diff --git a/tests/unit/model_bridge/test_encdec_weight_stacking.py b/tests/unit/model_bridge/test_encdec_weight_stacking.py new file mode 100644 index 0000000000..995bf989fb --- /dev/null +++ b/tests/unit/model_bridge/test_encdec_weight_stacking.py @@ -0,0 +1,136 @@ +"""Stacked weight properties and head labels on encoder-decoder bridges. + +The stacking helpers assumed a single top-level ``blocks``; encoder-decoder +adapters register ``encoder_blocks``/``decoder_blocks`` instead, so every +stacked property raised AttributeError and ``all_head_labels`` silently named +only half the heads. Mirrors ``HookedEncoderDecoder``, which stacks +self-attention over ``chain(encoder, decoder)`` and omits cross-attention. +""" + +from __future__ import annotations + +import copy + +import pytest +import torch +from transformers import T5Config, T5ForConditionalGeneration + +from transformer_lens.model_bridge.bridge import TransformerBridge +from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_from_module, +) + +N_LAYERS = 2 +N_HEADS = 4 +D_MODEL = 32 +D_FF = 64 + + +@pytest.fixture(scope="module") +def t5_bridge() -> TransformerBridge: + torch.manual_seed(0) + cfg = T5Config( + vocab_size=128, + d_model=D_MODEL, + d_kv=D_MODEL // N_HEADS, + d_ff=D_FF, + num_layers=N_LAYERS, + num_decoder_layers=N_LAYERS, + num_heads=N_HEADS, + ) + model = T5ForConditionalGeneration(cfg).eval() + return build_bridge_from_module( + model, + "T5ForConditionalGeneration", + hf_config=copy.deepcopy(cfg), + tokenizer=None, + device="cpu", + ).eval() + + +def test_attention_weights_stack_over_encoder_then_decoder(t5_bridge): + """W_Q/K/V/O span encoder + decoder layers, not just one stack.""" + total_layers = 2 * N_LAYERS + d_head = D_MODEL // N_HEADS + assert t5_bridge.W_Q.shape == (total_layers, N_HEADS, D_MODEL, d_head) + assert t5_bridge.W_K.shape == (total_layers, N_HEADS, D_MODEL, d_head) + assert t5_bridge.W_V.shape == (total_layers, N_HEADS, D_MODEL, d_head) + assert t5_bridge.W_O.shape == (total_layers, N_HEADS, d_head, D_MODEL) + + +def test_mlp_weights_stack_over_encoder_then_decoder(t5_bridge): + total_layers = 2 * N_LAYERS + assert t5_bridge.W_in.shape == (total_layers, D_MODEL, D_FF) + assert t5_bridge.W_out.shape == (total_layers, D_FF, D_MODEL) + + +def test_factored_circuits_are_available(t5_bridge): + """QK/OV build on the stacked weights, so they follow for free.""" + total_layers = 2 * N_LAYERS + assert t5_bridge.QK.A.shape[0] == total_layers + assert t5_bridge.OV.A.shape[0] == total_layers + + +def test_blocks_with_spans_both_stacks(t5_bridge): + """'attn' means this block's self-attention: encoder attn + decoder self_attn.""" + matching = t5_bridge.blocks_with("attn") + assert [idx for idx, _ in matching] == list(range(2 * N_LAYERS)) + + +def test_stack_params_for_reports_encoder_and_decoder_layers(t5_bridge): + indices, stacked = t5_bridge.stack_params_for("attn", "attn.W_Q") + assert indices == list(range(2 * N_LAYERS)) + assert stacked.shape[0] == 2 * N_LAYERS + + +def test_all_head_labels_uses_the_encoder_decoder_scheme(t5_bridge): + """EL/DL labels name every head; a plain L{l}H{h} list named only half.""" + labels = t5_bridge.all_head_labels + assert len(labels) == 2 * N_LAYERS * N_HEADS + assert labels[0] == "EL0H0" + assert labels[-1] == f"DL{N_LAYERS - 1}H{N_HEADS - 1}" + assert sum(label.startswith("EL") for label in labels) == N_LAYERS * N_HEADS + + +def test_attn_head_labels_cover_both_stacks(t5_bridge): + """Derives from composition_layer_indices, which routes through blocks_with.""" + assert len(t5_bridge.attn_head_labels) == 2 * N_LAYERS * N_HEADS + + +def test_cross_attention_is_excluded_from_stacking(t5_bridge): + """Decoder blocks also carry cross_attn; HookedEncoderDecoder omits it and so do we.""" + decoder_block = t5_bridge.decoder_blocks[0] + assert "cross_attn" in decoder_block._modules, "fixture should have cross-attention" + assert t5_bridge.W_Q.shape[0] == 2 * N_LAYERS + + +def test_labels_follow_actual_block_counts_when_the_stacks_differ(): + """Asymmetric encoder/decoder depths: labels come from the real block lists. + + cfg.n_layers cannot describe both stacks at once, so deriving labels from it + would mislabel every decoder head on a lopsided model. + """ + torch.manual_seed(0) + encoder_layers, decoder_layers = 3, 1 + cfg = T5Config( + vocab_size=128, + d_model=D_MODEL, + d_kv=D_MODEL // N_HEADS, + d_ff=D_FF, + num_layers=encoder_layers, + num_decoder_layers=decoder_layers, + num_heads=N_HEADS, + ) + model = T5ForConditionalGeneration(cfg).eval() + bridge = build_bridge_from_module( + model, + "T5ForConditionalGeneration", + hf_config=copy.deepcopy(cfg), + tokenizer=None, + device="cpu", + ).eval() + + labels = bridge.all_head_labels + assert sum(label.startswith("EL") for label in labels) == encoder_layers * N_HEADS + assert sum(label.startswith("DL") for label in labels) == decoder_layers * N_HEADS + assert bridge.W_Q.shape[0] == encoder_layers + decoder_layers diff --git a/tests/unit/model_bridge/test_gated_hooks.py b/tests/unit/model_bridge/test_gated_hooks.py index 4e212eb0af..2f4eb058c6 100644 --- a/tests/unit/model_bridge/test_gated_hooks.py +++ b/tests/unit/model_bridge/test_gated_hooks.py @@ -88,3 +88,65 @@ def test_run_with_cache_warns_on_fully_gated_names_filter(): assert any("gated-off" in str(w.message) for w in caught), ( "Expected a warning naming the gated-off hook, got: " f"{[str(w.message) for w in caught]}" ) + + +def test_add_hook_callable_filter_warns_and_skips_gated_points(): + """A callable filter matching only gated-off points attaches nothing and warns, + rather than leaving dead hooks that never fire.""" + bridge = TransformerBridge.boot_native(_cfg()) + tokens = torch.randint(0, 16, (1, 8)) + + fired = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + bridge.add_hook( + lambda name: name.endswith("hook_mlp_in"), + lambda t, hook=None: fired.append(1) or t, + ) + bridge(tokens, return_type="logits") + + # Not merely "did not fire" — a gated point never fires even when a dead + # hook is attached, so assert nothing was attached in the first place. + assert not bridge.hook_dict["blocks.0.hook_mlp_in"].fwd_hooks + assert not fired, "Gated-off hook fired; the filter should have skipped it" + assert any("gated-off" in str(w.message) for w in caught), ( + "Expected a warning naming the skipped gated-off hook, got: " + f"{[str(w.message) for w in caught]}" + ) + + +def test_add_hook_callable_filter_attaches_once_flag_enabled(): + """The same filter attaches and fires — silently — once the setter enables the flag.""" + bridge = TransformerBridge.boot_native(_cfg()) + bridge.set_use_hook_mlp_in(True) + tokens = torch.randint(0, 16, (1, 8)) + + fired = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + bridge.add_hook( + lambda name: name.endswith("hook_mlp_in"), + lambda t, hook=None: fired.append(1) or t, + ) + bridge(tokens, return_type="logits") + + assert fired, "Hook did not fire after enabling use_hook_mlp_in via the setter" + assert not [w for w in caught if "gated-off" in str(w.message)] + + +def test_add_hook_callable_filter_leaves_ungated_points_alone(): + """Control: a filter over an ungated point attaches and fires without warning.""" + bridge = TransformerBridge.boot_native(_cfg()) + tokens = torch.randint(0, 16, (1, 8)) + + fired = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + bridge.add_hook( + lambda name: name == "blocks.0.hook_resid_post", + lambda t, hook=None: fired.append(1) or t, + ) + bridge(tokens, return_type="logits") + + assert fired, "Ungated hook should still attach and fire" + assert not [w for w in caught if "gated-off" in str(w.message)] diff --git a/tests/unit/model_bridge/test_gpt_oss_moe.py b/tests/unit/model_bridge/test_gpt_oss_moe.py index 81c5bdee1e..773c566914 100644 --- a/tests/unit/model_bridge/test_gpt_oss_moe.py +++ b/tests/unit/model_bridge/test_gpt_oss_moe.py @@ -198,3 +198,61 @@ def test_gpt_oss_run_with_cache_with_random_weights(): # GPT-OSS has 32 experts with top-4 routing, so router_scores is (seq_len, 4) router_scores_0 = cache["blocks.0.mlp.hook_router_scores"] assert router_scores_0.shape == (5, 4) # seq_len=5, num_experts_per_tok=4 + + +def test_gpt_oss_routing_hooks_match_hooked_transformer_construction(): + """GPT-OSS softmaxes after top-k, so the scattered weights are tensor-identical + to HookedTransformer's gpt_oss_moe routing_weights, and the hooks are observe-only.""" + from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer + + from transformer_lens.config import TransformerBridgeConfig + from transformer_lens.model_bridge import TransformerBridge + from transformer_lens.model_bridge.sources._hf_format import ( + map_default_transformer_lens_config, + ) + from transformer_lens.model_bridge.supported_architectures.gpt_oss import ( + GPTOSSArchitectureAdapter, + ) + + torch.manual_seed(0) + config = AutoConfig.from_pretrained("openai/gpt-oss-20b", trust_remote_code=True) + config.num_hidden_layers = 2 + config.hidden_size = 128 + config.intermediate_size = 256 + config.num_attention_heads = 8 + config.num_key_value_heads = 2 + model = AutoModelForCausalLM.from_config(config, trust_remote_code=True) + + tl_config = map_default_transformer_lens_config(config) + bridge = TransformerBridge( + model=model, + adapter=GPTOSSArchitectureAdapter( + TransformerBridgeConfig( + d_model=tl_config.d_model, + d_head=tl_config.d_head, + n_layers=tl_config.n_layers, + n_ctx=tl_config.n_ctx, + architecture="GptOssForCausalLM", + ) + ), + tokenizer=AutoTokenizer.from_pretrained("openai/gpt-oss-20b", trust_remote_code=True), + ) + bridge.enable_compatibility_mode(no_processing=True) + + tokens = torch.randint(0, 1000, (1, 5)) + baseline = bridge(tokens) + cached_logits, cache = bridge.run_with_cache(tokens) + + weights = cache["blocks.0.mlp.router.hook_expert_weights"] + indices = cache["blocks.0.mlp.router.hook_expert_indices"] + scores = cache["blocks.0.mlp.hook_router_scores"] + + assert weights.shape == (5, config.num_local_experts) + assert indices.shape == (5, config.num_experts_per_tok) + assert torch.equal(baseline, cached_logits), "routing hooks must be observe-only" + + # HookedTransformer's gpt_oss_moe.py builds the same tensor by scattering the + # post-top-k softmax back over expert ids. + hooked_construction = torch.zeros_like(weights) + hooked_construction.scatter_(-1, indices.long(), scores) + assert torch.equal(weights, hooked_construction) diff --git a/tests/unit/model_bridge/test_stack_block_params_hybrid.py b/tests/unit/model_bridge/test_stack_block_params_hybrid.py index d1a79987fa..803d431f1a 100644 --- a/tests/unit/model_bridge/test_stack_block_params_hybrid.py +++ b/tests/unit/model_bridge/test_stack_block_params_hybrid.py @@ -35,11 +35,25 @@ def __init__(self, mlp) -> None: self.mlp = mlp # nn.Module assignment registers into _modules +_BORROWED_HELPERS = ( + "_enumerate_blocks", + "_resolve_submodule_name", + "_rewrite_submodule_path", +) + + def _stub(mlps): - return SimpleNamespace( - blocks=[_Block(m) for m in mlps], + blocks = torch.nn.ModuleList([_Block(m) for m in mlps]) + # _stack_block_params walks the registered block lists, so the stub exposes + # the same _modules shape a real bridge does and borrows the real helpers. + stub = SimpleNamespace( + blocks=blocks, + _modules={"blocks": blocks}, cfg=SimpleNamespace(n_devices=1, device=None), ) + for helper in _BORROWED_HELPERS: + setattr(stub, helper, getattr(TransformerBridge, helper).__get__(stub)) + return stub def test_interleaved_model_stacks_dense_layers_only(caplog) -> None: diff --git a/transformer_lens/benchmarks/utils.py b/transformer_lens/benchmarks/utils.py index d3888c4fce..cac7251df8 100644 --- a/transformer_lens/benchmarks/utils.py +++ b/transformer_lens/benchmarks/utils.py @@ -68,8 +68,6 @@ def is_tiny_test_model(model_name: str) -> bool: # modules with per-expert hooks (e.g., blocks.0.mlp.experts.3.hook_pre). "mlp.experts.", "mlp.hook_experts", - "mlp.hook_expert_indices", - "mlp.hook_expert_weights", # Parallel attention+MLP architectures (GPT-J, GPT-NeoX): HF has a single # shared layer norm (ln_1), while HT creates a virtual ln2 that shares weights # with ln1. The Bridge only wraps the actual HF ln_1, so ln2 hooks don't exist. diff --git a/transformer_lens/model_bridge/bridge_core.py b/transformer_lens/model_bridge/bridge_core.py index 6233262c86..08ab2fdc76 100644 --- a/transformer_lens/model_bridge/bridge_core.py +++ b/transformer_lens/model_bridge/bridge_core.py @@ -40,6 +40,9 @@ # Block-list container attributes a bridge may expose. _BLOCK_LIST_ATTRS = ("blocks", "encoder_blocks", "decoder_blocks", "L_blocks", "H_blocks") +# Encoder blocks name self-attention ``attn``; decoder blocks name it ``self_attn`` +# (``cross_attn`` is a separate submodule, deliberately excluded from stacking). +_SELF_ATTENTION_NAMES = {"attn": ("attn", "self_attn")} def build_alias_to_canonical_map(hook_dict: Any, prefix: str = "") -> dict: @@ -781,18 +784,33 @@ def add_hook( if callable(name) and not isinstance(name, str): hook_dict = self.hook_dict seen_hooks: set = set() + gated_names_skipped: List[str] = [] for hook_name, hook_point in hook_dict.items(): if name(hook_name): hook_id = id(hook_point) if hook_id in seen_hooks: continue seen_hooks.add(hook_id) + # A filter is a sweep, not a targeted request, so a gated-off + # match is skipped rather than raised on — but silently + # attaching here would leave a dead hook that never fires, + # which is the failure this warns about. + if self._gated_hook_reason(hook_name) is not None: + gated_names_skipped.append(hook_name) + continue self._add_fn_to_hook_point(hook_point, hook_name, hook_fn, dir, is_permanent) + if gated_names_skipped: + warnings.warn( + f"add_hook: skipped {len(gated_names_skipped)} gated-off hook name(s) " + f"that would never fire: {gated_names_skipped}. Call the relevant " + "set_use_*(True) setter first to enable them.", + stacklevel=2, + ) return # An explicitly named gated-off hook point is a caller error: the hook # would silently never fire. Raise naming the setter to call (filters - # above add freely — they were not necessarily targeting gated names). + # above skip with a warning — they were not necessarily targeting gated names). reason = self._gated_hook_reason(name) if reason is not None: raise ValueError( diff --git a/transformer_lens/model_bridge/generalized_components/__init__.py b/transformer_lens/model_bridge/generalized_components/__init__.py index 5e58a30a31..d4d4476da3 100644 --- a/transformer_lens/model_bridge/generalized_components/__init__.py +++ b/transformer_lens/model_bridge/generalized_components/__init__.py @@ -74,6 +74,9 @@ MLAAttentionBridge, ) from transformer_lens.model_bridge.generalized_components.mlp import MLPBridge +from transformer_lens.model_bridge.generalized_components.pooler import ( + BertPoolerBridge, +) from transformer_lens.model_bridge.generalized_components.moe import ( MoEBridge, MoERouterBridge, @@ -186,6 +189,7 @@ "GatedRMSNormBridge", "MoEBridge", "MoERouterBridge", + "BertPoolerBridge", "PositionEmbeddingsAttentionBridge", "Qwen3_5VisionBlockBridge", "Qwen3_5VisionEncoderBridge", diff --git a/transformer_lens/model_bridge/generalized_components/moe.py b/transformer_lens/model_bridge/generalized_components/moe.py index 19ba06ae68..695f9b93d4 100644 --- a/transformer_lens/model_bridge/generalized_components/moe.py +++ b/transformer_lens/model_bridge/generalized_components/moe.py @@ -91,6 +91,28 @@ def __init__( f"submodules (declared: {sorted(submodules or {})})" ) self._sparse_required = sparse_required + # HookedTransformer exposes the routing observables on the MoE block + # itself; the bridge fires them on the router submodule, whose adapter + # key differs ("gate" on 5.13 SparseMoeBlocks, "router" on GPT-OSS). + # Alias so code migrated from HT finds them under the HT name. + self._router_hook_aliases = self._build_router_hook_aliases(submodules or {}) + self.hook_aliases = {**self.hook_aliases, **self._router_hook_aliases} + + @staticmethod + def _build_router_hook_aliases( + submodules: Mapping[str, GeneralizedComponent], + ) -> Dict[str, str]: + """Map HT's block-level routing hook names onto the router submodule.""" + aliases: Dict[str, str] = {} + for key, component in submodules.items(): + if not isinstance(component, MoERouterBridge): + continue + if component.weights_index is not None: + aliases["hook_expert_weights"] = f"{key}.hook_expert_weights" + if component.indices_index is not None: + aliases["hook_expert_indices"] = f"{key}.hook_expert_indices" + break + return aliases def _binds_dense_projections(self, component: torch.nn.Module) -> bool: """Whether this layer is the dense variant of an interleaved MoE stack. @@ -160,7 +182,7 @@ def set_original_component(self, component: torch.nn.Module) -> None: del self.hook_router_scores else: # Symmetric restore so a rebinding harness cannot leave a chimera. - self.hook_aliases = dict(type(self).hook_aliases) + self.hook_aliases = {**type(self).hook_aliases, **self._router_hook_aliases} self.property_aliases = { key: value for key, value in self.property_aliases.items() @@ -329,11 +351,33 @@ class MoERouterBridge(LinearBridge): 5.13 TopKRouters return ``(router_logits, topk_weights, topk_indices)``; hook_out fires on the logits (element ``logits_index`` — JetMoe puts them last) and the tuple is re-packed so HF's unpacking is undisturbed. + + ``hook_expert_weights`` / ``hook_expert_indices`` mirror the HookedTransformer + MoE routing hooks. HF routers hand back top-k-shaped weights + ``[tokens, top_k]``, so the weights are scattered to HT's + ``[tokens, num_experts]`` before firing and gathered back afterwards — an + unedited round trip returns the values bit-for-bit, and an edit reaches HF. """ - def __init__(self, *args: Any, logits_index: int = 0, **kwargs: Any): + def __init__( + self, + *args: Any, + logits_index: int = 0, + weights_index: Optional[int] = 1, + indices_index: Optional[int] = 2, + **kwargs: Any, + ): super().__init__(*args, **kwargs) self.logits_index = logits_index + self.weights_index = weights_index + self.indices_index = indices_index + # None means this router's tuple has no clean [tokens, top_k] pair + # (JetMoe returns a sorted-expert layout); registering the hook anyway + # would advertise an intervention point that can never fire. + if weights_index is not None: + self.hook_expert_weights = HookPoint() + if indices_index is not None: + self.hook_expert_indices = HookPoint() def forward(self, input: torch.Tensor, *args: Any, **kwargs: Any) -> Any: if self.original_component is None: @@ -344,9 +388,73 @@ def forward(self, input: torch.Tensor, *args: Any, **kwargs: Any) -> Any: output = self.original_component(input, *args, **kwargs) if not isinstance(output, tuple) or len(output) == 0: return self.hook_out(output) - idx = self.logits_index % len(output) - router_logits = self.hook_out(output[idx]) - return output[:idx] + (router_logits,) + output[idx + 1 :] + parts = list(output) + count = len(parts) + logits_at = self.logits_index % count + parts[logits_at] = self.hook_out(parts[logits_at]) + + weights_at = None if self.weights_index is None else self.weights_index % count + indices_at = None if self.indices_index is None else self.indices_index % count + if weights_at is None and indices_at is None: + return tuple(parts) + + indices = None if indices_at is None else parts[indices_at] + expanded = None + if weights_at is not None: + expanded = self._expand_expert_weights(parts[weights_at], indices, parts[logits_at]) + expanded = self.hook_expert_weights(expanded) + if indices_at is not None: + indices = self.hook_expert_indices(indices) + parts[indices_at] = indices + if weights_at is not None: + # Gathered after the indices hook so re-routing picks up the weight + # sitting at the newly selected expert, as HookedTransformer does. + parts[weights_at] = self._collapse_expert_weights(expanded, indices, parts[weights_at]) + return tuple(parts) + + def _expand_expert_weights( + self, + weights: torch.Tensor, + indices: Optional[torch.Tensor], + logits: torch.Tensor, + ) -> torch.Tensor: + """Scatter top-k weights into HT's ``[tokens, num_experts]`` layout.""" + if not self._is_top_k_shaped(weights, indices, logits): + return weights + assert indices is not None + scattered = torch.zeros( + (*weights.shape[:-1], logits.shape[-1]), + dtype=weights.dtype, + device=weights.device, + ) + scattered.scatter_(-1, indices.long(), weights) + return scattered + + def _collapse_expert_weights( + self, + expanded: Optional[torch.Tensor], + indices: Optional[torch.Tensor], + original: torch.Tensor, + ) -> torch.Tensor: + """Gather the expanded weights back to the top-k layout HF expects.""" + if expanded is None or indices is None or expanded.shape == original.shape: + return expanded if expanded is not None else original + return expanded.gather(-1, indices.long()) + + @staticmethod + def _is_top_k_shaped( + weights: torch.Tensor, + indices: Optional[torch.Tensor], + logits: torch.Tensor, + ) -> bool: + """Whether the weights are the top-k slice rather than full expert width.""" + return ( + indices is not None + and isinstance(weights, torch.Tensor) + and isinstance(logits, torch.Tensor) + and weights.shape == indices.shape + and weights.shape[-1] != logits.shape[-1] + ) def set_processed_weights( self, weights: Mapping[str, Optional[torch.Tensor]], verbose: bool = False diff --git a/transformer_lens/model_bridge/generalized_components/pooler.py b/transformer_lens/model_bridge/generalized_components/pooler.py new file mode 100644 index 0000000000..7558875df0 --- /dev/null +++ b/transformer_lens/model_bridge/generalized_components/pooler.py @@ -0,0 +1,22 @@ +"""Pooler bridge component. + +This module contains the bridge component for [CLS] pooling heads. +""" + +from __future__ import annotations + +from transformer_lens.model_bridge.generalized_components.base import ( + GeneralizedComponent, +) + + +class BertPoolerBridge(GeneralizedComponent): + """Bridge component for BERT's [CLS] pooler. + + Wraps the whole pooler, so ``hook_out`` carries the post-tanh pooled + ``[CLS]`` vector rather than the pre-activation projection — the tensor + ``HookedEncoder``'s ``BertPooler`` exposes as ``hook_pooler_out``, which is + aliased here so code migrated from the legacy stack keeps working. + """ + + hook_aliases = {"hook_pooler_out": "hook_out"} diff --git a/transformer_lens/model_bridge/supported_architectures/bert.py b/transformer_lens/model_bridge/supported_architectures/bert.py index f71a237d6a..219ab352d5 100644 --- a/transformer_lens/model_bridge/supported_architectures/bert.py +++ b/transformer_lens/model_bridge/supported_architectures/bert.py @@ -12,6 +12,7 @@ from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter from transformer_lens.model_bridge.generalized_components import ( AttentionBridge, + BertPoolerBridge, BlockBridge, EmbeddingBridge, LinearBridge, @@ -152,7 +153,14 @@ def prepare_model(self, hf_model: Any) -> None: and no MLM-specific LayerNorm. """ if getattr(getattr(hf_model, "bert", None), "pooler", None) is not None: - self.components["pooler"] = LinearBridge(name="bert.pooler.dense") + # Wrap the pooler itself, not its inner Linear: HF applies tanh after + # the projection, so hook_out here is the pooled [CLS] that + # HookedEncoder's BertPooler exposes as hook_pooler_out. The dense + # stays hookable as a submodule for the pre-activation projection. + self.components["pooler"] = BertPoolerBridge( + name="bert.pooler", + submodules={"dense": LinearBridge(name="dense")}, + ) has_predictions = hasattr(getattr(hf_model, "cls", None), "predictions") has_nsp_head = hasattr(getattr(hf_model, "cls", None), "seq_relationship") diff --git a/transformer_lens/model_bridge/supported_architectures/gpt_oss.py b/transformer_lens/model_bridge/supported_architectures/gpt_oss.py index 7195e42bb9..1de8ad1086 100644 --- a/transformer_lens/model_bridge/supported_architectures/gpt_oss.py +++ b/transformer_lens/model_bridge/supported_architectures/gpt_oss.py @@ -8,6 +8,7 @@ EmbeddingBridge, LinearBridge, MoEBridge, + MoERouterBridge, PositionEmbeddingsAttentionBridge, RMSNormalizationBridge, RotaryEmbeddingBridge, @@ -67,7 +68,12 @@ def __init__(self, cfg: Any) -> None: ), # GPT-OSS uses batched MoE experts with router scores # MoEBridge handles the (hidden_states, router_scores) tuple returns - "mlp": MoEBridge(name="mlp", config=self.cfg), + "mlp": MoEBridge( + name="mlp", + config=self.cfg, + submodules={"router": MoERouterBridge(name="router")}, + sparse_required=("router",), + ), }, ), "ln_final": RMSNormalizationBridge( diff --git a/transformer_lens/model_bridge/supported_architectures/granite.py b/transformer_lens/model_bridge/supported_architectures/granite.py index f45e457ea6..eef2b57106 100644 --- a/transformer_lens/model_bridge/supported_architectures/granite.py +++ b/transformer_lens/model_bridge/supported_architectures/granite.py @@ -74,7 +74,11 @@ def _build_moe_bridge(self) -> MoEBridge: return MoEBridge( name="block_sparse_moe", config=self.cfg, - submodules={"gate": MoERouterBridge(name="router", logits_index=-1)}, + submodules={ + "gate": MoERouterBridge( + name="router", logits_index=-1, indices_index=0, weights_index=1 + ) + }, ) def _build_component_mapping(self) -> dict: diff --git a/transformer_lens/model_bridge/supported_architectures/jetmoe.py b/transformer_lens/model_bridge/supported_architectures/jetmoe.py index 3f49f5b6f3..95e82ff767 100644 --- a/transformer_lens/model_bridge/supported_architectures/jetmoe.py +++ b/transformer_lens/model_bridge/supported_architectures/jetmoe.py @@ -77,7 +77,12 @@ def __init__(self, cfg: Any) -> None: name="experts", submodules={ # JetMoeTopKGating puts logits last in its 5-tuple. - "router": MoERouterBridge(name="router", logits_index=-1), + "router": MoERouterBridge( + name="router", + logits_index=-1, + weights_index=None, + indices_index=None, + ), }, ), }, @@ -87,7 +92,12 @@ def __init__(self, cfg: Any) -> None: name="mlp", config=self.cfg, submodules={ - "gate": MoERouterBridge(name="router", logits_index=-1), + "gate": MoERouterBridge( + name="router", + logits_index=-1, + weights_index=None, + indices_index=None, + ), }, ), }, diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index 97ef50940b..64c53ce8fb 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -37,7 +37,11 @@ from transformer_lens.FactoredMatrix import FactoredMatrix from transformer_lens.hook_points import HookIntrospectionMixin, HookPoint from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter -from transformer_lens.model_bridge.bridge_core import _BLOCK_LIST_ATTRS, BridgeCore +from transformer_lens.model_bridge.bridge_core import ( + _BLOCK_LIST_ATTRS, + _SELF_ATTENTION_NAMES, + BridgeCore, +) from transformer_lens.model_bridge.component_setup import ( refresh_container_state_owners, set_original_components, @@ -804,6 +808,153 @@ def to_tokens( tokens = tokens.to(self.cfg.device) return tokens + def encoder_output( + self, + frames: torch.Tensor, + one_zero_attention_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Run the audio encoder from precomputed frames, skipping feature extraction. + + The audio-path analogue of ``start_at_layer``: ``frames`` is + ``[batch, frames, d_model]``, the tensor the conv front end would have + produced (observable at ``feat_proj.hook_out``), so callers can inject or + reuse frames without re-running the waveform convolutions. Positional + convolution and the encoder layer norm are applied first, exactly as the + full path does, then the blocks run. Mirrors + ``HookedAudioEncoder.encoder_output``. + + Hooks on the bridged components fire as usual, so this composes with + ``add_hook`` / ``get_caching_hooks``. + + Args: + frames: ``[batch, frames, d_model]`` precomputed encoder frames. + one_zero_attention_mask: Optional ``[batch, frames]`` mask, 1 for + real frames and 0 for padding. + + Returns: + The residual stream leaving the final block, ``[batch, frames, d_model]``. + """ + self._require_frame_entry_support() + if frames.ndim != 3: + raise ValueError( + "encoder_output expects precomputed frames [batch, frames, d_model]; " + f"got a {frames.ndim}D tensor. Pass a waveform to forward() instead." + ) + frames = frames.to(self.cfg.device) + + resid = frames + self.conv_pos_embed(frames) + resid = self.embed_ln(resid) + + additive_attention_mask = None + if one_zero_attention_mask is not None: + mask = one_zero_attention_mask.to(self.cfg.device) + additive_attention_mask = torch.where( + mask[:, None, None, :] == 0, + torch.tensor(float("-inf"), dtype=resid.dtype, device=resid.device), + torch.tensor(0.0, dtype=resid.dtype, device=resid.device), + ) + + for block in self.blocks: + output = block(resid, attention_mask=additive_attention_mask) + resid = output[0] if isinstance(output, tuple) else output + return resid + + def _require_frame_entry_support(self) -> None: + """Reject models with no conv-frame stage to re-enter.""" + if not getattr(self.cfg, "is_audio_model", False): + raise NotImplementedError( + "encoder_output is an audio-encoder entry point; this bridge is not an " + "audio model. Use start_at_layer for residual re-entry on text models." + ) + missing = [ + name + for name in ("conv_pos_embed", "embed_ln", "blocks") + if self._modules.get(name) is None + ] + if missing: + raise NotImplementedError( + "encoder_output needs a waveform encoder with a convolutional front end " + f"(missing {missing}). Spectrogram encoders such as AST have no " + "precomputed-frame stage to re-enter, so there is nothing to bypass." + ) + + def to_sentence_pair_tokens( + self, + sentence_a: str, + sentence_b: str, + move_to_device: bool = True, + truncate: bool = True, + ) -> Dict[str, torch.Tensor]: + """Pair-tokenize two sentences as ``[CLS] a [SEP] b [SEP]``. + + Returns ``input_ids``, ``token_type_ids`` and ``attention_mask``. The + segment ids are not decorative: without them a BERT NSP head sees both + sentences as one segment and its logits collapse. Mirrors + ``BertNextSentencePrediction.to_tokens``. + + Args: + sentence_a: First sentence of the pair. + sentence_b: Second sentence of the pair. + move_to_device: Move the returned tensors to ``cfg.device``. + truncate: Truncate to the model's context window. + """ + assert self.tokenizer is not None, "Cannot pair-tokenize without a tokenizer" + encodings = self.tokenizer( + sentence_a, + sentence_b, + return_tensors="pt", + padding=True, + truncation=truncate, + max_length=self.cfg.n_ctx if truncate else None, + ) + if "token_type_ids" not in encodings: + raise ValueError( + f"{type(self.tokenizer).__name__} emits no token_type_ids, so it cannot " + "express a sentence pair. Next-sentence prediction needs a " + "segment-aware tokenizer (e.g. BERT's)." + ) + keys = ("input_ids", "token_type_ids", "attention_mask") + tokens = {key: encodings[key] for key in keys if key in encodings} + if move_to_device: + tokens = {key: value.to(self.cfg.device) for key, value in tokens.items()} + return tokens + + def predict_next_sentence( + self, + sentence_a: str, + sentence_b: str, + return_type: Optional[str] = "predictions", + truncate: bool = True, + ) -> Any: + """Run next-sentence prediction over a sentence pair given as strings. + + Owns the ``token_type_ids`` plumbing that a hand-rolled pair forward has + to remember. Requires a bridge booted onto an NSP head — otherwise the + model has no 2-class output to decode. Mirrors + ``BertNextSentencePrediction.forward``. + + Args: + sentence_a: First sentence of the pair. + sentence_b: Second sentence of the pair. + return_type: ``"predictions"`` for the decoded verdict, or + ``"logits"`` for the raw 2-class scores. + truncate: Truncate to the model's context window. + """ + tokens = self.to_sentence_pair_tokens(sentence_a, sentence_b, truncate=truncate) + forward_kwargs: Dict[str, Any] = { + key: value for key, value in tokens.items() if key != "input_ids" + } + logits = self(tokens["input_ids"], return_type="logits", **forward_kwargs) + if logits.shape[-1] != 2: + raise ValueError( + "predict_next_sentence needs a next-sentence-prediction head, but this " + f"bridge produces {logits.shape[-1]} output classes. Boot it with " + "model_class=BertForNextSentencePrediction." + ) + if return_type == "logits": + return logits + return self._finalize_return(return_type, logits, tokens["input_ids"]) + def to_string( self, tokens: Union[List[int], torch.Tensor, np.ndarray] ) -> Union[str, List[str]]: @@ -962,15 +1113,58 @@ def to_single_str_token(self, int_token: int) -> str: return str(token[0]) raise AssertionError("Expected a single string token.") + def _enumerate_blocks(self) -> List[Tuple[int, Any]]: + """(index, block) over every registered block list, encoder before decoder. + + Decoder-only models register a single ``blocks``, so the indices are the + plain layer indices. Encoder-decoder models register ``encoder_blocks`` + and ``decoder_blocks`` instead; those are concatenated into one index + space, matching ``HookedEncoderDecoder``'s ``chain(encoder, decoder)``. + """ + pairs: List[Tuple[int, Any]] = [] + for list_name in _BLOCK_LIST_ATTRS: + block_list = self._modules.get(list_name) + if not isinstance(block_list, nn.ModuleList): + continue + for block in block_list: + pairs.append((len(pairs), block)) + return pairs + + def _resolve_submodule_name(self, block: Any, submodule: str) -> Optional[str]: + """The block's actual name for ``submodule``. + + Encoder blocks name self-attention ``attn`` while decoder blocks name it + ``self_attn``, so a caller asking for ``attn`` means "this block's + self-attention" on either side. Cross-attention is never resolved here: + ``HookedEncoderDecoder`` omits it from stacked weights, and this mirrors + that so the two stacks line up layer for layer. + """ + for candidate in _SELF_ATTENTION_NAMES.get(submodule, (submodule,)): + if candidate in block._modules: + return candidate + return None + + def _rewrite_submodule_path(self, attr_path: str, submodule: str, actual: Optional[str]) -> str: + """Re-point ``attr_path``'s leading segment at the block's actual submodule.""" + if actual is None or actual == submodule: + return attr_path + if attr_path.split(".")[0] != submodule: + return attr_path + return actual + attr_path[len(submodule) :] + def blocks_with(self, submodule: str) -> List[Tuple[int, "GeneralizedComponent"]]: """Return (index, block) pairs for blocks with the named bridged submodule. Checks _modules (not hasattr) so HF-internal attrs don't match. Use instead of assuming blocks[0] is representative on hybrid models. + On encoder-decoder models the indices span encoder then decoder blocks, + and ``"attn"`` matches the decoder's ``self_attn`` too. """ - if not hasattr(self, "blocks"): - return [] - return [(i, block) for i, block in enumerate(self.blocks) if submodule in block._modules] + return [ + (index, block) + for index, block in self._enumerate_blocks() + if self._resolve_submodule_name(block, submodule) is not None + ] def stack_params_for( self, submodule: str, attr_path: str, reshape_fn: Optional[Callable] = None @@ -988,7 +1182,10 @@ def stack_params_for( indices: List[int] = [] weights: List[torch.Tensor] = [] for idx, block in matching: - w = _resolve_attr_path(block, attr_path) + resolved = self._resolve_submodule_name(block, submodule) + w = _resolve_attr_path( + block, self._rewrite_submodule_path(attr_path, submodule, resolved) + ) if w is None: raise AttributeError( f"blocks[{idx}].{attr_path} is None — this checkpoint has no such " @@ -1011,12 +1208,16 @@ def _stack_block_params( AttributeError killed the accessor for the whole model. """ first_attr = attr_path.split(".")[0] + all_blocks = self._enumerate_blocks() matching_blocks: List[Tuple[int, torch.Tensor]] = [] - for i, block in enumerate(self.blocks): - if first_attr not in block._modules: + for i, block in all_blocks: + resolved = self._resolve_submodule_name(block, first_attr) + if resolved is None: continue try: - weight = _resolve_attr_path(block, attr_path) + weight = _resolve_attr_path( + block, self._rewrite_submodule_path(attr_path, first_attr, resolved) + ) except AttributeError: continue if weight is None: @@ -1036,7 +1237,7 @@ def _stack_block_params( f"Use bridge.blocks_with('{first_attr}') to check availability." ) - if len(matching_blocks) < len(self.blocks): + if len(matching_blocks) < len(all_blocks): indices = [i for i, _ in matching_blocks] logging.warning( "Hybrid model: only %d/%d blocks resolve '%s'. Returning stacked tensor " @@ -1044,7 +1245,7 @@ def _stack_block_params( "indices[i], not layer i. For explicit index mapping, use " "bridge.stack_params_for('%s', '%s').", len(matching_blocks), - len(self.blocks), + len(all_blocks), attr_path, indices, first_attr, @@ -1389,7 +1590,19 @@ def layer_types(self) -> List[str]: @property def all_head_labels(self) -> list[str]: - """Human-readable labels for all attention heads, e.g. ['L0H0', 'L0H1', ...].""" + """Human-readable labels for all attention heads, e.g. ['L0H0', 'L0H1', ...]. + + Encoder-decoder models use ``HookedEncoderDecoder``'s ``EL{l}H{h}`` / + ``DL{l}H{h}`` scheme so encoder and decoder heads stay distinguishable; + a plain ``L{l}H{h}`` list would name only half of them. + """ + encoder_blocks = self._modules.get("encoder_blocks") + decoder_blocks = self._modules.get("decoder_blocks") + if isinstance(encoder_blocks, nn.ModuleList) and isinstance(decoder_blocks, nn.ModuleList): + heads = range(self.cfg.n_heads) + return [f"EL{l}H{h}" for l in range(len(encoder_blocks)) for h in heads] + [ + f"DL{l}H{h}" for l in range(len(decoder_blocks)) for h in heads + ] return [f"L{l}H{h}" for l in range(self.cfg.n_layers) for h in range(self.cfg.n_heads)] @property From a169f28496d8933fedbf2a628077ad87e3b3912a Mon Sep 17 00:00:00 2001 From: Nayab_code <147242551+LightWork666@users.noreply.github.com> Date: Mon, 31 Aug 2026 20:23:42 +0530 Subject: [PATCH 63/87] fix(bridge): preserve split-component views under load_state_dict(assign=True) (#1660) * fix(bridge): preserve split-component views under load_state_dict(assign=True) assign=True replaces each target parameter object instead of copying into existing storage, which desyncs view-backed split components (gpt2's q/k/v, gate/up) from the combined weight (c_attn, gate_up_proj) they share storage with -- original_model.state_dict() silently keeps stale data. Route those specific keys through an explicit in-place copy_() instead, and raise a clear error if a shape mismatch makes that unsafe. Fixes #1637 * fix(bridge): route storage-sharing assign=True writes through copy_, not just view-backed ones Generalizes #1660's detection from partial views (torn from tensor size vs. its own storage) to any current tensor that shares underlying storage with another key. This covers the reviewer-flagged gap of writing a combined weight's own raw key directly under assign=True (which orphaned the views sharing its storage), and as a side effect also fixes tied embed/unembed desyncing under assign=True (#1725). Adds a dtype/device guard alongside the existing shape guard, validates every key before applying any copy_ instead of copy-as-it-validates, and excludes meta tensors from the storage-group check -- every meta tensor reports data_ptr()==0, which would otherwise falsely group unrelated offloaded parameters together -- so a meta target takes the ordinary assign=True materialization path instead. Addresses review from jlarson4 and koriyoshi2041 on #1660. --- .../test_state_dict_round_trip.py | 331 ++++++++++++++++++ .../model_bridge/transformer_bridge.py | 98 +++++- 2 files changed, 425 insertions(+), 4 deletions(-) diff --git a/tests/unit/model_bridge/test_state_dict_round_trip.py b/tests/unit/model_bridge/test_state_dict_round_trip.py index b0d707d89f..e76887e186 100644 --- a/tests/unit/model_bridge/test_state_dict_round_trip.py +++ b/tests/unit/model_bridge/test_state_dict_round_trip.py @@ -4,14 +4,23 @@ load_state_dict() only matched raw native parameter names, so a state_dict() -> load_state_dict() round trip silently loaded nothing and strict=True was silently downgraded to strict=False. + +Also covers "copy-split staleness" (#1637): load_state_dict(..., assign=True) +used to replace parameters that share underlying storage with something else +(a split QKV/gate-up component's view into its combined weight, the combined +weight itself, or a tied pair like embed/unembed, #1725) wholesale, desyncing +them from whatever they share storage with. """ from __future__ import annotations import pytest import torch +from transformers import GPT2Config, GPT2LMHeadModel, Phi3Config, Phi3ForCausalLM from transformer_lens.config import TransformerBridgeConfig from transformer_lens.model_bridge import TransformerBridge +from transformer_lens.model_bridge.sources import build_bridge_from_module +from transformer_lens.model_bridge.transformer_bridge import _storage_group_keys def _native_cfg(**overrides) -> TransformerBridgeConfig: @@ -31,6 +40,49 @@ def _native_cfg(**overrides) -> TransformerBridgeConfig: return TransformerBridgeConfig(**base) +def _tiny_gpt2_bridge() -> TransformerBridge: + """Real, randomly-initialized tiny GPT-2 bridge (JointQKVAttentionBridge path). + + Built from a config, not a download -- fast enough for the default test + tier, unlike the boot_transformers("gpt2") tests below. + """ + cfg = GPT2Config( + vocab_size=32, + n_positions=16, + n_embd=16, + n_layer=1, + n_head=2, + n_inner=32, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + ) + torch.manual_seed(0) + return build_bridge_from_module( + GPT2LMHeadModel(cfg), architecture="GPT2LMHeadModel", hf_config=cfg + ) + + +def _tiny_phi3_bridge() -> TransformerBridge: + """Real, randomly-initialized tiny Phi-3 bridge (JointGateUpMLPBridge path).""" + cfg = Phi3Config( + vocab_size=32, + hidden_size=16, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=2, + num_key_value_heads=2, + max_position_embeddings=16, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + ) + torch.manual_seed(0) + return build_bridge_from_module( + Phi3ForCausalLM(cfg), architecture="Phi3ForCausalLM", hf_config=cfg + ) + + def test_native_round_trip_overwrites_params_not_a_noop(): bridge = TransformerBridge.boot_native(_native_cfg()) @@ -143,6 +195,219 @@ def test_native_clean_key_dict_with_partial_aliases_does_not_raise_strict(): ), f"{actual_key} (alias of {clean_key}) did not round-trip" +def test_storage_group_keys_detects_shared_storage(): + """Core detection helper for #1637/#1725: any key whose tensor shares + underlying storage with another key's belongs to the same group, whether + it's the smaller view, the larger owner, or a same-size tied duplicate -- + even though nn.Parameter wrapping doesn't reliably preserve Tensor._base, + so this can't rely on that.""" + combined = torch.nn.Parameter(torch.randn(12, 4)) + view = torch.nn.Parameter(torch.tensor_split(combined, 3, dim=0)[0]) + tied_a = torch.nn.Parameter(torch.randn(4, 4)) + tied_b = torch.nn.Parameter(tied_a.data) + independent = torch.nn.Parameter(torch.randn(4, 4)) + + state_dict = { + "combined": combined, + "view": view, + "tied_a": tied_a, + "tied_b": tied_b, + "independent": independent, + } + grouped = _storage_group_keys(state_dict) + assert grouped == {"combined", "view", "tied_a", "tied_b"} + + +def test_storage_group_keys_excludes_meta_tensors(): + """Every meta tensor reports untyped_storage().data_ptr() == 0 (no real + backing memory), so comparing meta tensors by data pointer would + spuriously group every unrelated offloaded parameter in the model + together. Meta-target keys must never be flagged as a storage group.""" + a = torch.nn.Parameter(torch.empty(4, 4, device="meta")) + b = torch.nn.Parameter(torch.empty(4, 4, device="meta")) + assert a.untyped_storage().data_ptr() == b.untyped_storage().data_ptr() == 0 + + grouped = _storage_group_keys({"a": a, "b": b}) + assert grouped == set() + + +def test_native_assign_true_round_trip_no_split_components(): + """boot_native's components are independent parameters (no split/view or + tied components), so assign=True should take the ordinary passthrough + path and round-trip exactly like assign=False does -- and, unlike + assign=False, actually adopt the incoming tensor objects rather than + copying into the existing ones, since that's the point of assign=True.""" + bridge = TransformerBridge.boot_native(_native_cfg()) + + sd = {k: v.clone() for k, v in bridge.state_dict().items()} + with torch.no_grad(): + for p in bridge.parameters(): + p.zero_() + + bridge.load_state_dict(sd, strict=True, assign=True) + + reloaded = bridge.state_dict() + for key, value in sd.items(): + assert torch.equal(reloaded[key], value), f"{key} did not round-trip under assign=True" + + first_key = next(iter(sd)) + assert ( + bridge.state_dict()[first_key].data_ptr() == sd[first_key].data_ptr() + ), "non-shared key should be assigned (adopt the incoming tensor object), not copied into" + + +def test_tiny_gpt2_assign_true_does_not_leave_combined_weight_stale(): + """Fast (no-download) CI-covered version of the #1637 repro on the QKV + split path: assign=True on a split component must not desync it from the + combined weight (c_attn) it shares storage with.""" + bridge = _tiny_gpt2_bridge() + + sd = {k: v.clone() for k, v in bridge.state_dict().items()} + mutated = dict(sd) + mutated["blocks.0.attn.q.weight"] = sd["blocks.0.attn.q.weight"] + 100.0 + + bridge.load_state_dict(mutated, strict=True, assign=True) + + assert torch.equal( + bridge.blocks[0].attn.q.original_component.weight, mutated["blocks.0.attn.q.weight"] + ) + raw_sd = bridge.original_model.state_dict() + c_attn_w = raw_sd["transformer.h.0.attn._original_component.c_attn._original_component.weight"] + d_model = bridge.cfg.d_model + assert torch.allclose(c_attn_w[:, :d_model].T, mutated["blocks.0.attn.q.weight"]) + + +def test_tiny_phi3_assign_true_does_not_leave_combined_weight_stale(): + """Fast (no-download) CI-covered version of the #1637 repro on the + gate/up split path (JointGateUpMLPBridge).""" + bridge = _tiny_phi3_bridge() + + sd = {k: v.clone() for k, v in bridge.state_dict().items()} + gate_key = next(k for k in sd if k.endswith("mlp.gate.weight")) + mutated = dict(sd) + mutated[gate_key] = sd[gate_key] + 100.0 + + bridge.load_state_dict(mutated, strict=True, assign=True) + + gate_component = bridge + for part in gate_key.split(".")[:-1]: + gate_component = ( + getattr(gate_component, part) if not part.isdigit() else gate_component[int(part)] + ) + assert torch.equal(gate_component.original_component.weight, mutated[gate_key]) + + raw_sd = bridge.original_model.state_dict() + combined_key = next( + k + for k in raw_sd + if "gate_up_proj" in k and k.endswith("weight") and "0" in k.split(".")[:3] + ) + d_mlp = bridge.cfg.d_mlp + combined_w = raw_sd[combined_key] + assert torch.allclose(combined_w[:d_mlp, :], mutated[gate_key]) + + +def test_tiny_gpt2_assign_true_raw_combined_key_does_not_orphan_views(): + """Reviewer-flagged gap (jlarson4, PR #1660): loading the combined + weight's own *raw* key directly (not through a split alias) under + assign=True must not orphan the q/k/v views that share its storage -- + otherwise the bridge's own forward pass would keep reading stale q/k/v + data while original_model.state_dict() shows the new c_attn value.""" + bridge = _tiny_gpt2_bridge() + + raw_sd = bridge.original_model.state_dict() + c_attn_key = next( + k for k in raw_sd if "h.0.attn" in k and k.endswith("c_attn._original_component.weight") + ) + new_c_attn = raw_sd[c_attn_key].clone() + 100.0 + + bridge.load_state_dict({c_attn_key: new_c_attn}, strict=False, assign=True) + + d_model = bridge.cfg.d_model + expected_q = new_c_attn[:, :d_model].T + assert torch.equal( + bridge.blocks[0].attn.q.original_component.weight, expected_q + ), "q view was orphaned by a direct assign=True write to the combined c_attn key" + + +def test_assign_true_dtype_mismatch_raises_clear_error(): + """Reviewer-flagged gap (jlarson4): only shape was guarded; a dtype + mismatch on a storage-shared target would otherwise silently + copy-convert instead of erroring, e.g. producing a silently mixed-dtype + model from one call.""" + bridge = _tiny_gpt2_bridge() + + sd = dict(bridge.state_dict()) + sd["blocks.0.attn.q.weight"] = sd["blocks.0.attn.q.weight"].to(torch.float64) + + with pytest.raises(RuntimeError, match="dtype"): + bridge.load_state_dict(sd, strict=True, assign=True) + + +def test_assign_true_device_mismatch_raises_clear_error(): + """Reviewer-flagged gap (jlarson4): a device mismatch on a storage-shared + target must also be rejected explicitly, not silently accepted.""" + bridge = _tiny_gpt2_bridge() + + sd = dict(bridge.state_dict()) + sd["blocks.0.attn.q.weight"] = sd["blocks.0.attn.q.weight"].to("meta") + + with pytest.raises(RuntimeError, match="device"): + bridge.load_state_dict(sd, strict=True, assign=True) + + +def test_assign_true_meta_target_materializes_via_passthrough(): + """koriyoshi2041's review finding (PR #1660): copy_() onto a meta target + silently no-ops rather than raising, so a naive view-backed branch could + report a successful load while the parameter stays meta forever. Meta + targets are excluded from the storage-group check (see + test_storage_group_keys_excludes_meta_tensors) and instead take the + ordinary assign=True path, which correctly materializes them -- the + standard way to load real weights onto a meta-initialized model.""" + bridge = _tiny_gpt2_bridge() + + current = bridge.original_model.state_dict(keep_vars=True) + q_key = next(k for k in current if "h.0.attn.q" in k and k.endswith("weight")) + q_param = current[q_key] + assert not q_param.is_meta + + # A plain `.data = ...to("meta")` reassignment is rejected by PyTorch + # (incompatible tensor type), so simulate an offloaded/meta parameter by + # replacing the owning submodule's Parameter object outright, the same + # way accelerate's own offload hooks do it. + owner_path = ".".join(q_key.split(".")[:-1]) + owner = bridge.original_model.get_submodule(owner_path) + owner.weight = torch.nn.Parameter(q_param.data.to("meta"), requires_grad=q_param.requires_grad) + assert bridge.original_model.state_dict(keep_vars=True)[q_key].is_meta + + new_value = torch.randn(q_param.shape) + bridge.load_state_dict({"blocks.0.attn.q.weight": new_value}, strict=False, assign=True) + + reloaded = bridge.original_model.state_dict(keep_vars=True)[q_key] + assert not reloaded.is_meta, "meta target was not materialized by assign=True" + assert torch.equal(reloaded, new_value) + + +def test_assign_true_validates_all_keys_before_copying_any(): + """Reviewer-flagged gap (jlarson4): the loop must validate every + storage-shared key before applying any copy_, not copy-as-it-validates -- + otherwise a failure partway through leaves some keys already written and + others untouched instead of the whole call atomically failing.""" + bridge = _tiny_gpt2_bridge() + + sd = dict(bridge.state_dict()) + original_k = sd["blocks.0.attn.k.weight"].clone() + sd["blocks.0.attn.k.weight"] = sd["blocks.0.attn.k.weight"] + 100.0 # valid + sd["blocks.0.attn.q.weight"] = sd["blocks.0.attn.q.weight"][:-1] # shape mismatch + + with pytest.raises(RuntimeError): + bridge.load_state_dict(sd, strict=True, assign=True) + + assert torch.equal( + bridge.blocks[0].attn.k.original_component.weight, original_k + ), "a valid key was written before the whole call raised on a different key's mismatch" + + @pytest.mark.slow def test_boot_transformers_round_trip_matches_forward_pass(): """GPT-2's Conv1D-combined attention makes the bridge's q/k/v components @@ -194,3 +459,69 @@ def test_boot_transformers_clean_key_dict_does_not_raise_strict(): result = bridge.load_state_dict(clean_sd, strict=True) assert result.missing_keys == [] assert result.unexpected_keys == [] + + +@pytest.mark.slow +def test_boot_transformers_assign_true_does_not_leave_combined_weight_stale(): + """#1637 repro: assign=True on a split QKV component used to replace the + parameter object instead of copying into it, breaking the view relationship + with c_attn -- the bridge itself read the new value, but + original_model.state_dict() (what save_pretrained() exports) silently kept + the pre-load data for the combined weight.""" + bridge = TransformerBridge.boot_transformers("gpt2", device="cpu") + + sd = {k: v.clone() for k, v in bridge.state_dict().items()} + mutated = dict(sd) + mutated["blocks.0.attn.q.weight"] = sd["blocks.0.attn.q.weight"] + 100.0 + + bridge.load_state_dict(mutated, strict=True, assign=True) + + assert torch.equal( + bridge.blocks[0].attn.q.original_component.weight, mutated["blocks.0.attn.q.weight"] + ) + + raw_sd = bridge.original_model.state_dict() + c_attn_w = raw_sd["transformer.h.0.attn._original_component.c_attn._original_component.weight"] + d_model = bridge.cfg.d_model + assert torch.allclose(c_attn_w[:, :d_model].T, mutated["blocks.0.attn.q.weight"]) + + +@pytest.mark.slow +def test_boot_transformers_assign_true_shape_mismatch_raises_clear_error(): + """A storage-shared target can only be loaded under assign=True via an + in-place copy, which requires a matching shape -- fail loudly instead of a + confusing error surfacing from deep inside copy_, or silently corrupting data.""" + bridge = TransformerBridge.boot_transformers("gpt2", device="cpu") + + sd = dict(bridge.state_dict()) + sd["blocks.0.attn.q.weight"] = sd["blocks.0.attn.q.weight"][:-1] + + with pytest.raises(RuntimeError, match="shares storage"): + bridge.load_state_dict(sd, strict=True, assign=True) + + +@pytest.mark.slow +def test_boot_transformers_assign_true_tied_embed_unembed_stays_in_sync(): + """#1725 repro, fixed as a consequence of the same storage-group + generalization: gpt2 ties embed/unembed weights (separate nn.Parameter + objects sharing storage, confirmed via data_ptr equality). Loading only + embed.weight under assign=True must not leave unembed reading stale + pre-load data -- previously this broke the bridge's own forward pass, + not just the exported checkpoint.""" + bridge = TransformerBridge.boot_transformers("gpt2", device="cpu") + bridge.eval() + tokens = torch.randint(0, 1000, (1, 6)) + with torch.no_grad(): + logits_before = bridge(tokens).clone() + + sd = {k: v.clone() for k, v in bridge.state_dict().items()} + new_embed = sd["embed.weight"] + 3.0 + bridge.load_state_dict({"embed.weight": new_embed}, strict=False, assign=True) + + assert torch.equal(bridge.unembed.original_component.weight, new_embed) + + with torch.no_grad(): + logits_after = bridge(tokens).clone() + assert not torch.allclose( + logits_before, logits_after + ), "loading a new embedding matrix should change the forward pass output" diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index 64c53ce8fb..77ac1dd7c0 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -81,6 +81,32 @@ def _resolve_attr_path(obj: nn.Module, attr_path: str) -> Optional[torch.Tensor] return cast(torch.Tensor, result) +def _storage_group_keys(state_dict: dict[str, torch.Tensor]) -> set[str]: + """Keys whose current tensor shares underlying storage with another key's. + + Covers every direction of the desync #1637 reported: a partial view onto + a combined weight (e.g. a split QKV/gate-up component's ``torch.tensor_split`` + view into ``c_attn``/``gate_up_proj``), the combined weight itself (writing + it directly would silently orphan the views that share its storage), and a + same-size tied pair (e.g. tied embed/unembed weights, #1725) -- none of + which reliably show up via ``Tensor._base``, since wrapping in + ``nn.Parameter`` doesn't preserve that tracking here. + + Meta tensors are excluded from the comparison: every meta tensor reports + ``untyped_storage().data_ptr() == 0`` (it has no real backing memory), so + comparing meta tensors by data pointer would spuriously group every + unrelated offloaded parameter in the model together. A key whose current + target is meta falls through to ordinary ``assign=True`` handling instead + (the standard way to materialize a meta tensor from a real one). + """ + by_storage: dict[int, list[str]] = {} + for key, tensor in state_dict.items(): + if tensor.is_meta: + continue + by_storage.setdefault(tensor.untyped_storage().data_ptr(), []).append(key) + return {key for keys in by_storage.values() if len(keys) > 1 for key in keys} + + class TransformerBridge(BridgeCore, HookIntrospectionMixin, nn.Module): """Torch-backed bridge: HF, vLLM-via-torch, anything that wraps an ``nn.Module``. @@ -4341,7 +4367,7 @@ def load_state_dict(self, state_dict, strict=True, assign=False): Returns: NamedTuple with missing_keys and unexpected_keys fields """ - current_state_dict = self.original_model.state_dict() + current_state_dict = self.original_model.state_dict(keep_vars=True) clean_to_actual = {} for actual_key in current_state_dict.keys(): if actual_key != "_original_component": @@ -4392,9 +4418,73 @@ def load_state_dict(self, state_dict, strict=True, assign=False): ) ) - result = self.original_model.load_state_dict(mapped_state_dict, strict=False, assign=assign) - if assign: - refresh_container_state_owners(self) + if not assign: + result = self.original_model.load_state_dict( + mapped_state_dict, strict=False, assign=False + ) + return type(result)(missing_keys=missing_keys, unexpected_keys=unexpected_keys) + + # assign=True normally makes nn.Module.load_state_dict *replace* each + # target parameter/buffer with the incoming tensor rather than copying + # into existing storage. Any key whose current tensor shares storage + # with another key -- a split QKV/gate-up component's view into a + # combined weight, the combined weight itself (writing it directly + # would silently orphan the views), or a tied pair like embed/unembed + # -- would desync from whatever it shares storage with: the bridge + # might keep reading a stale value, or a live view not even part of + # this load would silently keep the pre-load data while + # original_model.state_dict() (what save_pretrained() exports) shows + # the new one. Route every key in a storage-sharing group through an + # explicit in-place .data.copy_() instead, regardless of the caller's + # assign=True, so those relationships survive. + shared_keys = _storage_group_keys(current_state_dict) + + copy_items: dict[str, tuple[torch.Tensor, torch.Tensor]] = {} + passthrough_items = {} + errors = [] + for key, value in mapped_state_dict.items(): + target = current_state_dict.get(key) + if target is None or key not in shared_keys: + passthrough_items[key] = value + continue + problems = [] + if target.is_meta: + # copy_() onto a meta tensor silently no-ops rather than + # raising, so a meta target would otherwise report a + # successful load while never actually materializing. + problems.append( + "the current parameter is a meta tensor; an in-place copy " + "can't materialize it, and this key shares storage with " + "another parameter so ordinary assign=True replacement " + "isn't safe here either" + ) + else: + if tuple(target.shape) != tuple(value.shape): + problems.append(f"shape {tuple(value.shape)} != expected {tuple(target.shape)}") + if target.dtype != value.dtype: + problems.append(f"dtype {value.dtype} != expected {target.dtype}") + if target.device != value.device: + problems.append(f"device {value.device} != expected {target.device}") + if problems: + errors.append(f'"{key}": ' + "; ".join(problems)) + else: + copy_items[key] = (target, value) + + if errors: + raise RuntimeError( + "Cannot load the following key(s) with assign=True: each shares " + "storage with another parameter/buffer (e.g. a split QKV/" + "gate-up component's view into a combined weight, or a tied " + "pair like embed/unembed), so it can only be loaded via an " + "in-place copy, which requires an exact match.\n\t" + "\n\t".join(errors) + ) + + for target, value in copy_items.values(): + with torch.no_grad(): + target.data.copy_(value) + + result = self.original_model.load_state_dict(passthrough_items, strict=False, assign=True) + refresh_container_state_owners(self) return type(result)(missing_keys=missing_keys, unexpected_keys=unexpected_keys) def get_params(self): From 7c1e2753a17d756a291079183f895378577dce84 Mon Sep 17 00:00:00 2001 From: Jonah Larson <jonahalarson@comcast.net> Date: Tue, 1 Sep 2026 19:23:34 -0500 Subject: [PATCH 64/87] Fixes to moe and composition scores (#1737) --- .../test_moe_routing_hooks.py | 44 +++++++++++++++++++ .../test_encdec_weight_stacking.py | 17 +++++-- .../generalized_components/moe.py | 25 +++++++++-- .../model_bridge/transformer_bridge.py | 12 +++++ 4 files changed, 92 insertions(+), 6 deletions(-) diff --git a/tests/unit/model_bridge/supported_architectures/test_moe_routing_hooks.py b/tests/unit/model_bridge/supported_architectures/test_moe_routing_hooks.py index 5ea3ab54a2..d879d6956c 100644 --- a/tests/unit/model_bridge/supported_architectures/test_moe_routing_hooks.py +++ b/tests/unit/model_bridge/supported_architectures/test_moe_routing_hooks.py @@ -157,3 +157,47 @@ def test_rerouting_picks_up_the_weight_at_the_newly_selected_expert(): "weights must be gathered at the edited indices, so a re-route to an " f"unselected expert yields 0; got {out_weights.tolist()}" ) + + +def test_boosting_a_suppressed_expert_reroutes_the_selection(): + """A weight edit outside the top-k re-derives the selection from the edited + tensor — HT's pre-top-k contract — instead of being discarded by the gather. + """ + from transformer_lens.model_bridge.generalized_components.moe import MoERouterBridge + + logits = torch.zeros(1, NUM_EXPERTS) + weights = torch.tensor([[0.7, 0.3]]) + indices = torch.tensor([[1, 2]]) + + router = MoERouterBridge(name="router") + router.set_original_component(_StubRouter(logits, weights, indices)) + + def boost_expert_zero(t, hook=None): + t = t.clone() + t[:, 0] = 5.0 # expert 0 held no weight in the original top-k + return t + + router.hook_expert_weights.add_hook(boost_expert_zero) + _, out_weights, out_indices = router(torch.zeros(1, 8)) + + assert out_indices[0].tolist() == [0, 1], out_indices + assert torch.allclose(out_weights, torch.tensor([[5.0, 0.7]])), out_weights + + +def test_suppressed_expert_boost_reaches_the_model_output(): + """End-to-end: the edit changes logits, where the pre-fix gather no-opped it.""" + bridge = _tiny_bridge() + tokens = torch.tensor([[3, 17, 42, 9]]) + + with torch.no_grad(): + base = bridge(tokens, return_type="logits") + + def boost_all_nonselected(t, hook=None): + t = t.clone() + t[t == 0.0] = 100.0 + return t + + with torch.no_grad(): + edited = bridge.run_with_hooks(tokens, fwd_hooks=[(WEIGHTS, boost_all_nonselected)]) + + assert not torch.allclose(edited, base), "off-top-k weight edit must reach the model" diff --git a/tests/unit/model_bridge/test_encdec_weight_stacking.py b/tests/unit/model_bridge/test_encdec_weight_stacking.py index 995bf989fb..60dc6a0a4d 100644 --- a/tests/unit/model_bridge/test_encdec_weight_stacking.py +++ b/tests/unit/model_bridge/test_encdec_weight_stacking.py @@ -92,9 +92,20 @@ def test_all_head_labels_uses_the_encoder_decoder_scheme(t5_bridge): assert sum(label.startswith("EL") for label in labels) == N_LAYERS * N_HEADS -def test_attn_head_labels_cover_both_stacks(t5_bridge): - """Derives from composition_layer_indices, which routes through blocks_with.""" - assert len(t5_bridge.attn_head_labels) == 2 * N_LAYERS * N_HEADS +def test_composition_surfaces_refuse_encoder_decoder(t5_bridge): + """Composition scores live in one residual stream; enc-dec models have two. + + The labels must refuse alongside the scores — advertising 2*L*H heads for + dims that all_composition_scores can never produce is worse than raising. + """ + import pytest + + with pytest.raises(NotImplementedError, match="residual streams"): + _ = t5_bridge.attn_head_labels + with pytest.raises(NotImplementedError, match="residual streams"): + t5_bridge.composition_layer_indices() + with pytest.raises(NotImplementedError, match="residual streams"): + t5_bridge.all_composition_scores("Q") def test_cross_attention_is_excluded_from_stacking(t5_bridge): diff --git a/transformer_lens/model_bridge/generalized_components/moe.py b/transformer_lens/model_bridge/generalized_components/moe.py index 695f9b93d4..29c74f5216 100644 --- a/transformer_lens/model_bridge/generalized_components/moe.py +++ b/transformer_lens/model_bridge/generalized_components/moe.py @@ -356,7 +356,11 @@ class MoERouterBridge(LinearBridge): MoE routing hooks. HF routers hand back top-k-shaped weights ``[tokens, top_k]``, so the weights are scattered to HT's ``[tokens, num_experts]`` before firing and gathered back afterwards — an - unedited round trip returns the values bit-for-bit, and an edit reaches HF. + unedited round trip returns the values bit-for-bit. Any weight edit + re-derives the top-k selection from the edited tensor, so boosting a + suppressed expert re-routes the token (HT's pre-top-k contract); unlike + HT's mixtral component, edited weights are used as-is with no + renormalization after the hook. """ def __init__( @@ -401,8 +405,23 @@ def forward(self, input: torch.Tensor, *args: Any, **kwargs: Any) -> Any: indices = None if indices_at is None else parts[indices_at] expanded = None if weights_at is not None: - expanded = self._expand_expert_weights(parts[weights_at], indices, parts[logits_at]) - expanded = self.hook_expert_weights(expanded) + scattered = self._expand_expert_weights(parts[weights_at], indices, parts[logits_at]) + expanded = self.hook_expert_weights(scattered) + if ( + indices is not None + and expanded.shape != parts[weights_at].shape + and not torch.equal(expanded, scattered) + ): + # An edit outside the current top-k would otherwise be discarded + # by the gather below (those columns have no downstream reader in + # the [tokens, top_k] layout). Re-derive the selection from the + # edited tensor so boosting a suppressed expert re-routes, as it + # does on HookedTransformer's pre-top-k hook. The edited values + # are used as-is — no per-arch renormalization is re-applied. + _, new_indices = torch.topk(expanded, indices.shape[-1], dim=-1) + indices = new_indices.to(indices.dtype) + if indices_at is not None: + parts[indices_at] = indices if indices_at is not None: indices = self.hook_expert_indices(indices) parts[indices_at] = indices diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index 105a3ac69b..485dc321f3 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -1540,6 +1540,7 @@ def all_composition_scores(self, mode: str) -> CompositionScores: On hybrid models, only attention layers are included; layer_indices maps tensor position i to original layer number. """ + self._reject_encoder_decoder_composition() attn_blocks = self.blocks_with("attn") if not attn_blocks: raise ValueError("No attention layers found — cannot compute composition scores.") @@ -1591,8 +1592,18 @@ def _stack(attr_path: str, reshape_fn: Optional[Callable] = None) -> torch.Tenso labels = [f"L{l}H{h}" for l in indices for h in range(self.cfg.n_heads)] return CompositionScores(scores=scores, layer_indices=indices, head_labels=labels) + def _reject_encoder_decoder_composition(self) -> None: + """Composition scores live in one residual stream; enc-dec models have two.""" + if any(hasattr(self, a) for a in ("encoder_blocks", "decoder_blocks")): + raise NotImplementedError( + "Composition scores are not defined across an encoder-decoder " + "model's two residual streams (HookedTransformer never " + "supported them there either)." + ) + def composition_layer_indices(self) -> List[int]: """Original layer indices for attention layers (maps composition score positions).""" + self._reject_encoder_decoder_composition() return [idx for idx, _ in self.blocks_with("attn")] def block_hooks(self, layer_idx: int) -> List[str]: @@ -1641,6 +1652,7 @@ def all_head_labels(self) -> list[str]: @property def attn_head_labels(self) -> list[str]: """Head labels for attention layers only — matches all_composition_scores() dims.""" + self._reject_encoder_decoder_composition() return [ f"L{l}H{h}" for l in self.composition_layer_indices() for h in range(self.cfg.n_heads) ] From 675f280adf7327cf8042e79154e00d289f0da531 Mon Sep 17 00:00:00 2001 From: jlarson4 <jonahalarson@comcast.net> Date: Tue, 1 Sep 2026 20:07:06 -0500 Subject: [PATCH 65/87] Fixing audio issue on dev-4.x --- .../model_bridge/test_audio_frame_entry.py | 43 +++++++++++++++---- .../model_bridge/transformer_bridge.py | 8 ++++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/tests/integration/model_bridge/test_audio_frame_entry.py b/tests/integration/model_bridge/test_audio_frame_entry.py index c9f199ef96..9c4bd3fb13 100644 --- a/tests/integration/model_bridge/test_audio_frame_entry.py +++ b/tests/integration/model_bridge/test_audio_frame_entry.py @@ -61,15 +61,40 @@ def test_hooks_fire_from_frame_entry(audio_bridge, full_run): torch.testing.assert_close(cached[name], cache[name], atol=0.0, rtol=0.0) -def test_padding_mask_changes_the_encoding(audio_bridge, full_run): - """The mask is applied, not ignored.""" - cache, last = full_run - frames = cache[FRAMES_HOOK] - mask = torch.ones(frames.shape[:2], dtype=torch.long) - mask[:, -10:] = 0 - - masked = audio_bridge.encoder_output(frames, one_zero_attention_mask=mask) - assert not torch.allclose(masked, cache[last]) +def test_masked_frame_entry_matches_hf_encoder(audio_bridge, waveform): + """Under a padding mask, frame entry must match HF's encoder on real frames. + + HF zeroes pad frames before pos_conv_embed; without that, the kernel-128 + conv smears pad content into real frames (~62% relative error here). + Frames come from an unmasked pass — HF mutates hidden_states in place, so + masked-run frames are already pre-zeroed and would make this comparison + self-fulfilling. "Changes the output" is not asserted anywhere: a wrong + mask also changes the output. + """ + hf = audio_bridge.original_model + padded = torch.cat([waveform, torch.zeros(1, 4000)], dim=1) + sample_mask = torch.cat([torch.ones_like(waveform), torch.zeros(1, 4000)], dim=1).long() + + with torch.no_grad(): + ref = hf(padded, attention_mask=sample_mask).last_hidden_state + feats = hf.feature_extractor(padded).transpose(1, 2) + frames = hf.feature_projection(feats) + frame_mask = hf._get_feature_vector_attention_mask(frames.shape[1], sample_mask) + out = audio_bridge.encoder_output(frames, one_zero_attention_mask=frame_mask.long()) + + real = frame_mask[0].bool() + assert not real.all(), "padding produced no masked frames; test setup is broken" + torch.testing.assert_close(out[:, real], ref[:, real], atol=1e-4, rtol=1e-4) + + +def test_masked_frame_entry_leaves_caller_frames_untouched(audio_bridge): + """masked_fill, not HF's in-place write: the caller's tensor survives.""" + frames = torch.randn(1, 8, audio_bridge.cfg.d_model) + keep = frames.clone() + mask = torch.tensor([[1, 1, 1, 1, 0, 0, 0, 0]]) + with torch.no_grad(): + audio_bridge.encoder_output(frames, one_zero_attention_mask=mask) + assert torch.equal(frames, keep) def test_waveform_shaped_input_is_rejected(audio_bridge, waveform): diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index 485dc321f3..531abb9ce8 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -875,6 +875,14 @@ def encoder_output( ) frames = frames.to(self.cfg.device) + if one_zero_attention_mask is not None: + # HF zeroes pad frames before the positional conv (kernel 128 smears + # pad content into real frames otherwise); masked_fill, not HF's + # in-place write, so the caller's tensor survives. + frames = frames.masked_fill( + ~one_zero_attention_mask.to(frames.device).bool().unsqueeze(-1), 0.0 + ) + resid = frames + self.conv_pos_embed(frames) resid = self.embed_ln(resid) From a96ead8193fff9cd3cc340065b322de3e90d9296 Mon Sep 17 00:00:00 2001 From: Jonah Larson <jonahalarson@comcast.net> Date: Wed, 2 Sep 2026 15:46:37 -0500 Subject: [PATCH 66/87] Final deprecation prep (#1740) * wrappign up final pre-deprecation items * Eliminating Hooked* classes from test suites * Updating goldens revision for latest data * Final demo updates * Codified hookedrootmodule * prep for removal of Hooked* system * Format cleanup * test fix --- demos/Activation_Patching_in_TL_Demo.ipynb | 401 ++---------------- demos/BERT.ipynb | 17 +- demos/Colab_Compatibility.ipynb | 103 +++-- demos/Exploratory_Analysis_Demo.ipynb | 5 +- demos/GPT_OSS_Demo.ipynb | 270 ++---------- demos/LIT_Integration_Demo.ipynb | 2 +- demos/Main_Demo.ipynb | 9 +- demos/SVD_Interpreter_Demo.ipynb | 24 +- demos/direct_path_patching_ioi.ipynb | 32 +- docs/source/content/migrating_to_v3.md | 36 +- pyproject.toml | 4 +- scripts/capture_ht_goldens.py | 15 + scripts/capture_tl_checkpoint_fixtures.py | 66 +++ .../tl_checkpoints/attn_only/checkpoint.pt | Bin 0 -> 51189 bytes .../tl_checkpoints/attn_only/meta.json | 13 + .../tl_checkpoints/attn_only/reference.pt | Bin 0 -> 2165 bytes .../tl_checkpoints/default/checkpoint.pt | Bin 0 -> 89525 bytes .../fixtures/tl_checkpoints/default/meta.json | 12 + .../tl_checkpoints/default/reference.pt | Bin 0 -> 38053 bytes .../tl_checkpoints/gated_rms/checkpoint.pt | Bin 0 -> 104501 bytes .../tl_checkpoints/gated_rms/meta.json | 13 + .../tl_checkpoints/gated_rms/reference.pt | Bin 0 -> 2165 bytes .../fixtures/tl_checkpoints/gqa/checkpoint.pt | Bin 0 -> 81077 bytes tests/fixtures/tl_checkpoints/gqa/meta.json | 13 + .../fixtures/tl_checkpoints/gqa/reference.pt | Bin 0 -> 2165 bytes .../tl_checkpoints/lnpre/checkpoint.pt | Bin 0 -> 85365 bytes tests/fixtures/tl_checkpoints/lnpre/meta.json | 12 + .../tl_checkpoints/lnpre/reference.pt | Bin 0 -> 2165 bytes tests/goldens.py | 7 +- .../model_bridge/test_bert_weight_surface.py | 196 +++++++++ .../test_bridge_run_with_cache_incl_bwd.py | 67 +-- .../test_encdec_weight_stacking_parity.py | 68 ++- .../test_nsp_sentence_pair_helper.py | 34 +- .../test_positional_weight_accessors.py | 156 +++++++ .../model_bridge/test_tl_legacy_loader.py | 98 +++++ .../model_bridge/test_wav2vec2_bridge.py | 53 +++ .../test_wav2vec2_adapter.py | 127 ++++++ tests/unit/model_bridge/test_boot_native.py | 77 +++- tests/unit/model_bridge/test_gated_hooks.py | 160 +++++++ .../test_tl_checkpoint_conversion.py | 210 +++------ tests/unit/test_deprecation_warnings.py | 55 ++- transformer_lens/HookedAudioEncoder.py | 24 +- transformer_lens/HookedEncoder.py | 27 +- transformer_lens/HookedEncoderDecoder.py | 24 +- transformer_lens/HookedRootModule.py | 17 +- transformer_lens/HookedTransformer.py | 99 +++-- transformer_lens/SVDInterpreter.py | 20 +- transformer_lens/__init__.py | 78 +++- .../factories/architecture_adapter_factory.py | 4 + transformer_lens/hook_points.py | 2 +- transformer_lens/lit/model.py | 5 +- transformer_lens/model_bridge/bridge_core.py | 102 +++-- .../generalized_components/attention.py | 9 +- .../model_bridge/sources/__init__.py | 2 + .../model_bridge/sources/native/model.py | 31 +- .../model_bridge/sources/tl_legacy.py | 226 ++++++++++ .../sources/transformers/helpers.py | 7 +- .../sources/transformers/source.py | 15 + .../supported_architectures/__init__.py | 4 + .../supported_architectures/native.py | 16 +- .../supported_architectures/wav2vec2.py | 39 ++ .../model_bridge/transformer_bridge.py | 74 +++- transformer_lens/model_protocol.py | 25 ++ .../tools/model_registry/__init__.py | 6 + .../tools/model_registry/checkpoints.py | 15 + transformer_lens/tools/training.py | 14 +- transformer_lens/train.py | 6 +- transformer_lens/utilities/architectures.py | 5 + 68 files changed, 2226 insertions(+), 1025 deletions(-) create mode 100644 scripts/capture_tl_checkpoint_fixtures.py create mode 100644 tests/fixtures/tl_checkpoints/attn_only/checkpoint.pt create mode 100644 tests/fixtures/tl_checkpoints/attn_only/meta.json create mode 100644 tests/fixtures/tl_checkpoints/attn_only/reference.pt create mode 100644 tests/fixtures/tl_checkpoints/default/checkpoint.pt create mode 100644 tests/fixtures/tl_checkpoints/default/meta.json create mode 100644 tests/fixtures/tl_checkpoints/default/reference.pt create mode 100644 tests/fixtures/tl_checkpoints/gated_rms/checkpoint.pt create mode 100644 tests/fixtures/tl_checkpoints/gated_rms/meta.json create mode 100644 tests/fixtures/tl_checkpoints/gated_rms/reference.pt create mode 100644 tests/fixtures/tl_checkpoints/gqa/checkpoint.pt create mode 100644 tests/fixtures/tl_checkpoints/gqa/meta.json create mode 100644 tests/fixtures/tl_checkpoints/gqa/reference.pt create mode 100644 tests/fixtures/tl_checkpoints/lnpre/checkpoint.pt create mode 100644 tests/fixtures/tl_checkpoints/lnpre/meta.json create mode 100644 tests/fixtures/tl_checkpoints/lnpre/reference.pt create mode 100644 tests/integration/model_bridge/test_bert_weight_surface.py create mode 100644 tests/integration/model_bridge/test_positional_weight_accessors.py create mode 100644 tests/integration/model_bridge/test_tl_legacy_loader.py create mode 100644 tests/integration/model_bridge/test_wav2vec2_bridge.py create mode 100644 tests/unit/model_bridge/supported_architectures/test_wav2vec2_adapter.py create mode 100644 transformer_lens/model_bridge/sources/tl_legacy.py create mode 100644 transformer_lens/model_bridge/supported_architectures/wav2vec2.py diff --git a/demos/Activation_Patching_in_TL_Demo.ipynb b/demos/Activation_Patching_in_TL_Demo.ipynb index cd322e64ce..8473271821 100644 --- a/demos/Activation_Patching_in_TL_Demo.ipynb +++ b/demos/Activation_Patching_in_TL_Demo.ipynb @@ -74,7 +74,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -90,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -121,7 +121,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -140,7 +140,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -157,7 +157,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -173,7 +173,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -196,7 +196,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fe942617d9f4430586031676e4597f95", + "model_id": "1167c15166df4240b2cbe90008982dc3", "version_major": 2, "version_minor": 0 }, @@ -206,13 +206,6 @@ }, "metadata": {}, "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n" - ] } ], "source": [ @@ -326,7 +319,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -359,7 +352,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "68747081c29149e0821733da286e3d9d", + "model_id": "b7d1da07be404b4fb8702909a3881aca", "version_major": 2, "version_minor": 0 }, @@ -369,45 +362,6 @@ }, "metadata": {}, "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "<html>\n", - "<head><meta charset=\"utf-8\" /></head>\n", - "<body>\n", - " <div> <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script> <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n", - " <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-3.0.1.min.js\" integrity=\"sha256-oy6Be7Eh6eiQFs5M7oXuPxxm9qbJXEtTpfSI93dW16Q=\" crossorigin=\"anonymous\"></script> <div id=\"c51d84d8-084e-4d78-9bc5-e5db33bbcf16\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div> <script type=\"text/javascript\"> window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById(\"c51d84d8-084e-4d78-9bc5-e5db33bbcf16\")) { Plotly.newPlot( \"c51d84d8-084e-4d78-9bc5-e5db33bbcf16\", [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e 0\",\"When 1\",\" John 2\",\" and 3\",\" Mary 4\",\" went 5\",\" to 6\",\" the 7\",\" shops 8\",\", 9\",\" John 10\",\" gave 11\",\" the 12\",\" bag 13\",\" to 14\"],\"z\":{\"dtype\":\"f4\",\"bdata\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFMVgD\\u002ftVYK53OcWN3whv7mxRUu4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHMigD9Yq+C3QR6vt5zS8Lnr2xu6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALwIgD+ov2M6vlAHOqEmgrru4mK6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEyyfj9X67E73IfSOk0LIro\\u002f5NW6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGuydz99YQA9o0U6OxlFobqhNYG6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGqvdz909v08VqHpOmAT\\u002f7nHhSm6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAK4+bD+tUFI91\\u002fmaO7LwdDpPpYs8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHoTKD+Tb8M8iWoaO3D0ibepK6M+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJuo3zw9uAA9K67uOmzRUTo6OnA\\u002fAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFjj27z64Ks8dhKkOkN4qTk6noA\\u002fAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGpeurvVr2k8bED\\u002fORaevbjbzX0\\u002f\",\"shape\":\"12, 15\"},\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"x: %{x}\\u003cbr\\u003ey: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}], {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermap\":[{\"type\":\"scattermap\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"resid_pre Activation Patching\"}}, {\"responsive\": true} ).then(function(){\n", - " \n", - "var gd = document.getElementById('c51d84d8-084e-4d78-9bc5-e5db33bbcf16');\n", - "var x = new MutationObserver(function (mutations, observer) {{\n", - " var display = window.getComputedStyle(gd).display;\n", - " if (!display || display === 'none') {{\n", - " console.log([gd, 'removed!']);\n", - " Plotly.purge(gd);\n", - " observer.disconnect();\n", - " }}\n", - "}});\n", - "\n", - "// Listen for the removal of the full notebook cells\n", - "var notebookContainer = gd.closest('#notebook-container');\n", - "if (notebookContainer) {{\n", - " x.observe(notebookContainer, {childList: true});\n", - "}}\n", - "\n", - "// Listen for the clearing of the current output cell\n", - "var outputEl = gd.closest('.output');\n", - "if (outputEl) {{\n", - " x.observe(outputEl, {childList: true});\n", - "}}\n", - "\n", - " }) }; </script> </div>\n", - "</body>\n", - "</html>" - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ @@ -435,7 +389,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0191b6a12fd6425ba35a8c9fdbe0b1bb", + "model_id": "0919e4d9414a4067bffb3a83cabf2302", "version_major": 2, "version_minor": 0 }, @@ -445,45 +399,6 @@ }, "metadata": {}, "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "<html>\n", - "<head><meta charset=\"utf-8\" /></head>\n", - "<body>\n", - " <div> <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script> <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n", - " <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-3.0.1.min.js\" integrity=\"sha256-oy6Be7Eh6eiQFs5M7oXuPxxm9qbJXEtTpfSI93dW16Q=\" crossorigin=\"anonymous\"></script> <div id=\"37e832b5-648e-450c-bccb-018b6ce28505\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div> <script type=\"text/javascript\"> window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById(\"37e832b5-648e-450c-bccb-018b6ce28505\")) { Plotly.newPlot( \"37e832b5-648e-450c-bccb-018b6ce28505\", [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":{\"dtype\":\"f4\",\"bdata\":\"N8t4Om8VhDy9MvM6SWZhO7\\u002fyILwMMjU8pjGFuyrzzrpTWZ66qO96Owh+i7vOwJW6iSqNunCFxrmBogI2D4OIuSIpFLnoK3s7WrTguUR2u7pZuHG6kPY1Oo5c4zkyC2q7IisCuhpyUzoROw46EmgLug+hO7q0HYi6N8t4uuRPtbhTGZE5DCIKO5q4TrkI99a68RruPaAXgzmMUcC6eBzQuVBXmzwaQUW5QAY3PIdBrroz5kC69tueuYdwGblvlmi5Yk3YupGUmTlzKry6Mqj8PPKE87siKZS51SzFO5JXNLwV8aW6l1HDOubpPLmZ+0A7c\\u002f2JuxIQQjvpEAY7Z66vOk+1n7oc368++o8TOrp6A7nQ1ai7OAuFPE0BjDyKzIi7CuwiPVV6eTxj+oW6cs5TOt60obsgIQu73t1pvLgwtDpeecI6i2UFPokssLn3SKk6ji3DOSbdnzzfPWo5xHb+PXDF07mrvvq77pKqOm1ek7oqGvS7H9tEPjCfVrvEngO6ZiGEuvRSBDiDTWG6KYJKPOqu07k42o27qBmjPpa1GDsr0Qo7QZ4UOZFNjj6cDLw7CwTBO98dfroXhxU8yV6pPIaRF70ztGk8f8VFvYL3bj0awSo66\\u002feGPt8uATo+Diu7U4GrPTFgqTwrTnW7tcgxPO+ej7q2xvg5MFhFPQC39b7pv0A5ilpCPL1jeT0Psl06PpiuO+1NPLzwdui9Wm3PO8\\u002fcpTnsF5i6xbW5vLxlhTtzvO87PboIvAhtjr5KMm47\",\"shape\":\"12, 12\"},\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"x: %{x}\\u003cbr\\u003ey: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}], {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermap\":[{\"type\":\"scattermap\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"attn_head_out Activation Patching (All Pos)\"}}, {\"responsive\": true} ).then(function(){\n", - " \n", - "var gd = document.getElementById('37e832b5-648e-450c-bccb-018b6ce28505');\n", - "var x = new MutationObserver(function (mutations, observer) {{\n", - " var display = window.getComputedStyle(gd).display;\n", - " if (!display || display === 'none') {{\n", - " console.log([gd, 'removed!']);\n", - " Plotly.purge(gd);\n", - " observer.disconnect();\n", - " }}\n", - "}});\n", - "\n", - "// Listen for the removal of the full notebook cells\n", - "var notebookContainer = gd.closest('#notebook-container');\n", - "if (notebookContainer) {{\n", - " x.observe(notebookContainer, {childList: true});\n", - "}}\n", - "\n", - "// Listen for the clearing of the current output cell\n", - "var outputEl = gd.closest('.output');\n", - "if (outputEl) {{\n", - " x.observe(outputEl, {childList: true});\n", - "}}\n", - "\n", - " }) }; </script> </div>\n", - "</body>\n", - "</html>" - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ @@ -511,7 +426,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "237157b3e7ca4a69b8ab320584fad22b", + "model_id": "3c04db590dcf450494d67ebc67971177", "version_major": 2, "version_minor": 0 }, @@ -521,45 +436,6 @@ }, "metadata": {}, "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "<html>\n", - "<head><meta charset=\"utf-8\" /></head>\n", - "<body>\n", - " <div> <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script> <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n", - " <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-3.0.1.min.js\" integrity=\"sha256-oy6Be7Eh6eiQFs5M7oXuPxxm9qbJXEtTpfSI93dW16Q=\" crossorigin=\"anonymous\"></script> <div id=\"c9f44f79-4dfb-42c4-b8a1-10d901511464\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div> <script type=\"text/javascript\"> window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById(\"c9f44f79-4dfb-42c4-b8a1-10d901511464\")) { Plotly.newPlot( \"c9f44f79-4dfb-42c4-b8a1-10d901511464\", [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e 0\",\"When 1\",\" John 2\",\" and 3\",\" Mary 4\",\" went 5\",\" to 6\",\" the 7\",\" shops 8\",\", 9\",\" John 10\",\" gave 11\",\" the 12\",\" bag 13\",\" to 14\"],\"y\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"z\":{\"dtype\":\"f4\",\"bdata\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALSKfDp+TBk4dsIguNUqojVMDzO3AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPAehDwTJhCzdCiZtR05WLaYL7S0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACwd5zp5GHQ4O2G6N8iwOLanMoM3AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKbFYjt0KJk13OeWtuKkC7btlY+3AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANrAH7x+O2E5yzmet+ouhLkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJdaNTwdOVi0BqymthMmkLNKZIm2AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPQucbtZeGS5RKeUtrcT4rj7y8S4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABt9drqFeAa6IjqBNydMNrnW5u03AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEswlrrc55a24qSLNh71hLgTJpCzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACdqaTsxX5Q4pLo\\u002fOPVsJjnkTzW3AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACPajruYL7S3rrxlNqqI1jhWALc3AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC6XmLqXYjC4gMRGNwrx1zdtjeg3AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAF5YqrrkPpM4aGp7OBMmxbhVIkY5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB51VDn+Qwg46gwqud89arXqLu65AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMz16Th\\u002fCLC4\\u002fkOItwNWCDlfRhG5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIBE4bkWfOM36HIiOFmJHDdtjbM4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEBAPrlUd1G2FPP9NwhXUDisMxa4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC2Jejup3Sy3aVlZN4u1SjgdKDa4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAG1vALqjD5a4ylvityZuEDlMD7M3AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAO0ml7pJhs23dCiZNQGr\\u002fTfwTZW5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACKJd7pvFs42labON7SK\\u002fLgsK+84AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOuqDTrvQLk47LdTOBxK+rfwDT04AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKUHqTmOHCE5siMHN5aECjjMBqK4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAhVYrtvFk437LdTtigZBbkasHI3AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC3aP7rog4+4DhRFuB0Xfjlyfdk4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJ\\u002fqujqCgKi4HSi2N1kntbm2Rl65AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFLdlDo1Apy5K15rOJrJu7fJH7G5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFlnd7pXb685ZEekOYDExjehQny5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGcQF7rvQDk4uPGdN1sSN7l+O+G3AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA42n7lOh6u5BCMMuTqDFLnjcQ+5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHAF4bnQS5646JSxuIBVGbn1\\u002fWK5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGCT5Lnw6y05EyYQNzqDFDiriCE5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJdRDjkJAnq3Fo0buJI\\u002feLnKW+I5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAasWzpSPY46ULI6OcR8qbgyCvM4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACG+LLruYn24\\u002fXbuOESWXDkzxlQ5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMckubp+O2G4eFyot3Nbf7h\\u002fGdK4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKZj7j1Ellw4gaICtpP\\u002fgLnst9O2AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPwpGzqHgTs43oGeuFzOTbhFUom5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAE28lbqnw3Q5QGJNuDUTiblTGXu5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGVWI7qF+KA4wEgaOFOqmDhI2yM4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMW1mzyEGrC4plRHN0p1q7fzYyY4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADPo47keBly5SzH3NyMHhTiDq7c5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHO87zsg8GY7wmLxOUtCLzhXXo25AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPHHZbpKU1E4fX+VOIQasLjhKAK6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFtBV7qmdtY4SYbNN06YGLmNT9I4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALE0Kbr0MCo4txNiOAGapjnfTqK4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM2kBbqGkl04iQohN797SzkrXus4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAG5at7nQKY842W9TuBdan7dw9Ak5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEKy7oV0Tm30BjXtjwdnLg1ggG3AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJkNJTn0H3I43OeWNsPAkrg2Phg5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAvN2rokVFg5iQqhNuRPNbiF1pE4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADVc+TvVI608icrIOnIOYTnKrKe5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFpMt7tYq+C6qCEWuUPJI7loDAY5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOjFCromTOs4co6RNyWhdriTrjs5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJJXtDvEHDA6n8rOuP\\u002f\\u002fU7k8HRw5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP5n9rt6Vee68Yl7uoxxrLif6jq6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADJ5a7pY67i5wSZAOLV5WrdBHi+4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJ3wBDrJjl442ysAOWidjThLMUI6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABqw8rjxSQS51Soit25rJDe94Xg4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOb6Xrc46fY6aN0aOoSJKDlyn7M5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOaCiruuvGW2yLC4NpFyv7aYL7Q3AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIskQztvFs62+TG9NUpkibbtlQ+3AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKZUfDlR\\u002f9g4yEELOQeKFzl43bk6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMMhgzoHaHI4Ny2rN7gCCzlh8QU5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJdznTjzY6a4n6h0uHHBd7f45ZW6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEatrz7EDeY5D\\u002fIANyte67cJE7K4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAErTtjlswGQ5GBZrOLwUi7f+VCq4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGZhxrm94Xi3WjTGNqtmEraGY4g5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAH0Vs7vkvi25rrzltZaVLDihwuE5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJMTjzzVC626yyjmuO2mMTjlLSY5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAH8mejzFeUQ7rQCaOTwuPrmVps66AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABGMcruHgYa6vBT1uFmJHDiHASE6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACoavzuREdc8e+NUO9lv0zrWt806AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANWq8TmRg2E4SYbNN1R30TdNxm88AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANAYVzdB7465+9xmuJWmzrf8xzO6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPE4FznuYv04UDKguPRSBLklQUg6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJzOo7tLMfc3qP\\u002fwNiWhdjehQnw2AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHUEUbpY2pa6l2JlufUO0LhKUxw5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACRZULwNpkO7Ukx3uTthOjg2jdk6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFRGDjteaNU4kXI\\u002fuJQMErnEXD26AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABvOOzpAc4W4SNujOJMd6Tb1zkI6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGy5BT6Go383DWmbthZ8Y7YxvZ+5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACUQhbndkvU4EyYQN7IjB7cYllC5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANzFcTr0UgQ3eUsGOLwUizfs96s5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHFjAjqyI4e4zdOlt6qI1jfipIu4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALhdqjwGzoC4hqP\\u002fNnlLhrdGAJi6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIXWxjm2Vxa3pamdtXHB97W9cha5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGAFDbus7uM7IVxFOuXtTToDwPA9AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM6PvLnzdMi32ID1NpXIqLhKZHM4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFIdIrp71FW5DwOjuJgeR7r5wsS7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANhzZLpZZ0I5Ny2ruHhtyjjC5QY7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOCsrbmMcWG5dtNCuN5f+bjEzdi5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAN9fDzk\\u002flcm5kXK\\u002fuKJCkrlm8N+7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEaDDq1gaA7l2KwuGBTojkAUD4+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAG0cTbt8kDc3WjRGtqFCfDRfNSS5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAL4\\u002fTzkoCJi4i7XKNhMmELXN4iS6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJ+odDmqux25QR6vt\\u002fkxvbblrYu6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADWCgTh75Y22TA+zNk263DXipAu4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGRYkblpaka5cPQJuLjxHbiOC7S5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPB8gDlJ5hG6yxdEuDKbRbn2zVI8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJZz0jhs0Ry4DWkbuLcC9bjkYKK5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADrD1jmfuSy5iE50ObhCmDm0Pam7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPBcFLoa4RY8CQJ6Nwg1djgnMZ8+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACIpfjkTJpCzgbOkN0SW3DgC\\u002fwA7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACte67ixVoM4ji2Ot2Ulyrd+BBE7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEBizbhpahE5i7XKNVYANzjVGWo4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJ3+9rpGFxI8zDX3OQo+djoWlok+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMa4JTn221M4mC+0tBC\\u002fbrit\\u002fLY7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACCxG7mLtUo3YCQ3OCw8p7fhaMQ7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHGwVTgdOdg1EyaQNBMmELTi9YW6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHyhWTnkz085SmQJt7qLJbdD+A48AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANfmAzngCgQ5kMfKuLp6ODkO2qY8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABJIVLjugEa6z20tt50O7ber9hO9AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJ+54Th\\u002fd9057nOAOCJLIzdTh148AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADthure9g4O51SqiNtznFrcFikW9AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABI3Z7m+ntE6N5wjOaeyHTkql2U9AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANoafbl200K5Yr6+uCP2zLd3gZw6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAF2siTir8JU7kNiCOEpkCbctaYQ+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMYnU7iLtco2B5sEOG84qDg3S945AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJxBaTiKxuy3e+UNtn47YbbPfCy7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAE2c3rneiRk7CFdQOIeBO7jrUKc9AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJpJVjkCuKQ53z1qtg1pG7bD26Q8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIhOv7nzwea583RIt284KLiNuD27AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOD5y7h5GPQ48cmetrs2TzYIljE8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJDHlTc3LSu2cPSJN2euL7fVi5K6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAO5iEzgK4AA5EyYQsw\\u002fyADeME6E5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIajf7jVDFk6cp+zN2YDBrf4IkI9AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHvDM7lMkI+7BqwmOHKfszer7PO+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMLzQ7hJhk23EyaQs\\u002fkxvbV\\u002fiH85AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJXZFbllFCg4EyaQNDIsmDgukj48AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHQoGTc5du06SZcFOXQombdpvnA9AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANsayDgnKlw3h3DOOCP2zDfODyI6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+mK47AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADtTTy8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADwdui9AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABabc87AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADP3KU5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADsF5i6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADFtbm8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC8ZYU7AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABzvO87AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9ugi8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIbY6+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABKMm47\",\"shape\":\"144, 15\"},\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"x: %{x}\\u003cbr\\u003ey: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}], {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermap\":[{\"type\":\"scattermap\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Pos\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"title\":{\"text\":\"Head Label\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"attn_head_out Activation Patching By Pos\"}}, {\"responsive\": true} ).then(function(){\n", - " \n", - "var gd = document.getElementById('c9f44f79-4dfb-42c4-b8a1-10d901511464');\n", - "var x = new MutationObserver(function (mutations, observer) {{\n", - " var display = window.getComputedStyle(gd).display;\n", - " if (!display || display === 'none') {{\n", - " console.log([gd, 'removed!']);\n", - " Plotly.purge(gd);\n", - " observer.disconnect();\n", - " }}\n", - "}});\n", - "\n", - "// Listen for the removal of the full notebook cells\n", - "var notebookContainer = gd.closest('#notebook-container');\n", - "if (notebookContainer) {{\n", - " x.observe(notebookContainer, {childList: true});\n", - "}}\n", - "\n", - "// Listen for the clearing of the current output cell\n", - "var outputEl = gd.closest('.output');\n", - "if (outputEl) {{\n", - " x.observe(outputEl, {childList: true});\n", - "}}\n", - "\n", - " }) }; </script> </div>\n", - "</body>\n", - "</html>" - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ @@ -594,7 +470,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "440041cdc83b4bc3b1f991cefa1df6f4", + "model_id": "7692b6e4524d412d90e2d7d2453ee96a", "version_major": 2, "version_minor": 0 }, @@ -608,7 +484,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "05b1394ee4a54d8e94050197279d2a11", + "model_id": "df2c11bc521e4a38b256ea717f0de5a1", "version_major": 2, "version_minor": 0 }, @@ -622,7 +498,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4f5b66701221464580ccc06f25d257e0", + "model_id": "fc053cf272dd4ef284c1b3199f008b02", "version_major": 2, "version_minor": 0 }, @@ -632,45 +508,6 @@ }, "metadata": {}, "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "<html>\n", - "<head><meta charset=\"utf-8\" /></head>\n", - "<body>\n", - " <div> <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script> <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n", - " <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-3.0.1.min.js\" integrity=\"sha256-oy6Be7Eh6eiQFs5M7oXuPxxm9qbJXEtTpfSI93dW16Q=\" crossorigin=\"anonymous\"></script> <div id=\"73cfc006-eeb9-4d45-a068-81e483519980\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div> <script type=\"text/javascript\"> window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById(\"73cfc006-eeb9-4d45-a068-81e483519980\")) { Plotly.newPlot( \"73cfc006-eeb9-4d45-a068-81e483519980\", [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":{\"dtype\":\"f4\",\"bdata\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFMVgD\\u002ftVYK53OcWN3whv7mxRUu4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHMigD9Yq+C3QR6vt5zS8Lnr2xu6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALwIgD+ov2M6vlAHOqEmgrru4mK6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEyyfj9X67E73IfSOk0LIro\\u002f5NW6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGuydz99YQA9o0U6OxlFobqhNYG6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGqvdz909v08VqHpOmAT\\u002f7nHhSm6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAK4+bD+tUFI91\\u002fmaO7LwdDpPpYs8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHoTKD+Tb8M8iWoaO3D0ibepK6M+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJuo3zw9uAA9K67uOmzRUTo6OnA\\u002fAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFjj27z64Ks8dhKkOkN4qTk6noA\\u002fAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGpeurvVr2k8bED\\u002fORaevbjbzX0\\u002f\",\"shape\":\"12, 15\"},\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"x: %{x}\\u003cbr\\u003ey: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"},{\"coloraxis\":\"coloraxis\",\"name\":\"1\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":{\"dtype\":\"f4\",\"bdata\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIg7ET3tVYK53OcWN3whv7mxRUu4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAmUQ7vzY6Y4evavN+oMqjjEPBy6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADKb+rrkry46odPOOU7HOLrJDo+5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALlXHj6DNHk7mroHOl9G+7hVBBO6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFspsbv1a6A8hyKEOj6Vfrn1DlA6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANGtD7bG4o6HzFrOdRuCzmyAa04AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACN59T2IB648OccyOzDQrzpYPZM8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHoVWjzDkjw86ozEOSTlKriMX5g+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPa5xLoYJo48fkwZOkyvhDrz2RE\\u002fAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJrJBrkaes47Ut0UOUztozlHBYs+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGGgdboP8rU4ASuuOXQomTXmm0W+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC89s++\",\"shape\":\"12, 15\"},\"type\":\"heatmap\",\"xaxis\":\"x2\",\"yaxis\":\"y2\",\"hovertemplate\":\"x: %{x}\\u003cbr\\u003ey: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"},{\"coloraxis\":\"coloraxis\",\"name\":\"2\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":{\"dtype\":\"f4\",\"bdata\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFrNWT+M8ZG5U6qYuJ5s+LmLpCg4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKU9ETw0tWc5i0YdOSP2TLiYXp85AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKP+XTyMYHQ4HrWsufNUJ7oHqEo6AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACk1\\u002fzpP1gI6Efs1OdxnMTlxsNU5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEQXorwrR4c7xI1LuO2mMbkRik86AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMOuxT1vx\\u002fa6\\u002fJj9uaYUOjr6\\u002fkC5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAI8+gryZy166LF4BOeou7jctLO27AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKa+KTyMgk47xC8LOnHBdzmmNws8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAExVTbzTobw3fnu5uTWCtjgKNbG8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAE9Ur7lTSlQ6eUuGN9\\u002fOBzktiQE9AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC4msjrlC0y54cbPuGMcFbkFous8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADTg6c8\",\"shape\":\"12, 15\"},\"type\":\"heatmap\",\"xaxis\":\"x3\",\"yaxis\":\"y3\",\"hovertemplate\":\"x: %{x}\\u003cbr\\u003ey: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}], {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermap\":[{\"type\":\"scattermap\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,0.31999999999999995],\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"title\":{\"text\":\"Layer\"}},\"xaxis2\":{\"anchor\":\"y2\",\"domain\":[0.33999999999999997,0.6599999999999999],\"matches\":\"x\",\"title\":{\"text\":\"Position\"}},\"yaxis2\":{\"anchor\":\"x2\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis3\":{\"anchor\":\"y3\",\"domain\":[0.6799999999999999,0.9999999999999999],\"matches\":\"x\",\"title\":{\"text\":\"Position\"}},\"yaxis3\":{\"anchor\":\"x3\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"annotations\":[{\"font\":{},\"showarrow\":false,\"text\":\"Residual Stream\",\"x\":0.15999999999999998,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Attn Output\",\"x\":0.49999999999999994,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"MLP Output\",\"x\":0.8399999999999999,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"}],\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1,\"cmax\":1},\"title\":{\"text\":\"Activation Patching Per Block\"}}, {\"responsive\": true} ).then(function(){\n", - " \n", - "var gd = document.getElementById('73cfc006-eeb9-4d45-a068-81e483519980');\n", - "var x = new MutationObserver(function (mutations, observer) {{\n", - " var display = window.getComputedStyle(gd).display;\n", - " if (!display || display === 'none') {{\n", - " console.log([gd, 'removed!']);\n", - " Plotly.purge(gd);\n", - " observer.disconnect();\n", - " }}\n", - "}});\n", - "\n", - "// Listen for the removal of the full notebook cells\n", - "var notebookContainer = gd.closest('#notebook-container');\n", - "if (notebookContainer) {{\n", - " x.observe(notebookContainer, {childList: true});\n", - "}}\n", - "\n", - "// Listen for the clearing of the current output cell\n", - "var outputEl = gd.closest('.output');\n", - "if (outputEl) {{\n", - " x.observe(outputEl, {childList: true});\n", - "}}\n", - "\n", - " }) }; </script> </div>\n", - "</body>\n", - "</html>" - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ @@ -695,7 +532,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fd46410602d648ecb4d8e5c603f599c7", + "model_id": "6b97cbf0e1884783860200445f6a8be6", "version_major": 2, "version_minor": 0 }, @@ -709,7 +546,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "898ce7b338ec42b692974fb4af3d3d3a", + "model_id": "776aec1dddb44e78882766725a357e60", "version_major": 2, "version_minor": 0 }, @@ -723,7 +560,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a202d7fa0e894969b6093823c3a07900", + "model_id": "99c94f69f5134f3e82822f0d976fabaf", "version_major": 2, "version_minor": 0 }, @@ -737,7 +574,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "07f6b5419b414d739882be1135a0ac26", + "model_id": "694b233299314a0aa379295b1160f8c4", "version_major": 2, "version_minor": 0 }, @@ -751,7 +588,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "487f335aa1fb4f0a93c04944fcae2918", + "model_id": "45902f9c36a44da7b293812306028a59", "version_major": 2, "version_minor": 0 }, @@ -761,45 +598,6 @@ }, "metadata": {}, "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "<html>\n", - "<head><meta charset=\"utf-8\" /></head>\n", - "<body>\n", - " <div> <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script> <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n", - " <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-3.0.1.min.js\" integrity=\"sha256-oy6Be7Eh6eiQFs5M7oXuPxxm9qbJXEtTpfSI93dW16Q=\" crossorigin=\"anonymous\"></script> <div id=\"88a69602-b818-412d-9135-7b4ad5b740fe\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div> <script type=\"text/javascript\"> window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById(\"88a69602-b818-412d-9135-7b4ad5b740fe\")) { Plotly.newPlot( \"88a69602-b818-412d-9135-7b4ad5b740fe\", [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":{\"dtype\":\"f4\",\"bdata\":\"N8t4Om8VhDy9MvM6SWZhO7\\u002fyILwMMjU8pjGFuyrzzrpTWZ66qO96Owh+i7vOwJW6iSqNunCFxrmBogI2D4OIuSIpFLnoK3s7WrTguUR2u7pZuHG6kPY1Oo5c4zkyC2q7IisCuhpyUzoROw46EmgLug+hO7q0HYi6N8t4uuRPtbhTGZE5DCIKO5q4TrkI99a68RruPaAXgzmMUcC6eBzQuVBXmzwaQUW5QAY3PIdBrroz5kC69tueuYdwGblvlmi5Yk3YupGUmTlzKry6Mqj8PPKE87siKZS51SzFO5JXNLwV8aW6l1HDOubpPLmZ+0A7c\\u002f2JuxIQQjvpEAY7Z66vOk+1n7oc368++o8TOrp6A7nQ1ai7OAuFPE0BjDyKzIi7CuwiPVV6eTxj+oW6cs5TOt60obsgIQu73t1pvLgwtDpeecI6i2UFPokssLn3SKk6ji3DOSbdnzzfPWo5xHb+PXDF07mrvvq77pKqOm1ek7oqGvS7H9tEPjCfVrvEngO6ZiGEuvRSBDiDTWG6KYJKPOqu07k42o27qBmjPpa1GDsr0Qo7QZ4UOZFNjj6cDLw7CwTBO98dfroXhxU8yV6pPIaRF70ztGk8f8VFvYL3bj0awSo66\\u002feGPt8uATo+Diu7U4GrPTFgqTwrTnW7tcgxPO+ej7q2xvg5MFhFPQC39b7pv0A5ilpCPL1jeT0Psl06PpiuO+1NPLzwdui9Wm3PO8\\u002fcpTnsF5i6xbW5vLxlhTtzvO87PboIvAhtjr5KMm47\",\"shape\":\"12, 12\"},\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"x: %{x}\\u003cbr\\u003ey: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"},{\"coloraxis\":\"coloraxis\",\"name\":\"1\",\"z\":{\"dtype\":\"f4\",\"bdata\":\"GRYBOjhOMzy8Y8w65VsaPCIKn7sxIig8E81Pu6wzy7k3rZC6rSAGOsnfWLqXglG6a9SBOnX1nLmO+vs4RIW6ueOC5rjSjOU7I1i0uSCPwbq\\u002fDJ66l\\u002fOCOQDejzjaDQK79d4DuxB4KDvm6Qe6gwyoOjWCazmvmlY7VbNNOfY5Xzm7WCk4BcpSPPx6lTkgPvy6oZ\\u002f2PbKjobmGI2U6KoB6uRoY5zwC2Qc793eUOk28yjvqnTE6bK9COF6q0Lp4LYi5aB2oupUItjq07OO6P0TPuW2t1LrMhnG6jowQuypi\\u002fLnLF0S6rcB2OmK+ibnVO8Q6o96Hu5LRQTv8GOM5VGgdOh\\u002fCCLo2468+srDnuSZ\\u002fsrhwY7e7FBPqPFS4czytURS7dgSculPqJTp3QJi6rCLeOShIpbsiSTU7RVI+uQH6nzp8IYo6W3MFPuKki7ed8IQ6wy\\u002fAOd+\\u002fpjwWXrA5lwHAuqqqMLku5xu6daRXOoLxDrop5r04+bHXub0iSLvp4Ro5Rh\\u002f3ObprBDrZH9A6KcvJPNsJW7jBd4W60V8zPH39DLqg1fE6ct9AOjwX4ry3Xxg8YinUOwgWzLqg8Ec8cLO6OjHz2rzSUyc7vSEzvX6UVrw1+5c7UgqPPqZ2VjmnFIW5YP7IPfYU1TyqAmQ889KAPLylEjozaJS5Ls5VPRJLA79\\u002fmTc570CEu9unjT12AyU7G73OOmjQEDxWna+9EZzouw0Fe7oaMka7KybZu3B0pDmnIcu4WoTJu7S5rL5rghC7\",\"shape\":\"12, 12\"},\"type\":\"heatmap\",\"xaxis\":\"x2\",\"yaxis\":\"y2\",\"hovertemplate\":\"x: %{x}\\u003cbr\\u003ey: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"},{\"coloraxis\":\"coloraxis\",\"name\":\"2\",\"z\":{\"dtype\":\"f4\",\"bdata\":\"5M9POcfUALoMC0U6zWPrO69alDoZBRS5CsAUOtQOEruzzjC3XzabulrlbruF1hG5tyTPOOaaMLqeDoM4WInROT7IEDl8Mqw6TocrOeb6Xri8FIs3wTetOat3NDjVDFk706CQupcRAbpBvmq63AV\\u002fOsKiFDqvvDA7QusyOSuAxTjUTGY41MY+PIu1yrgsK2+5yqq5uiP2zLmk\\u002fDo6WSc1uqpI\\u002fjutAE+5n+q6OjiQzDurtww6wuIhuFo0xjVMvji6oWQhuS+jsjm2tdY5hqN\\u002futjzSToz10E47Ni2u4ajlTlfZ946arcvOY\\u002fYt7iwJ0065E81NxMmELWc0oY5h9\\u002f7uQf5RLnzhYA4mfxsuUwPMzkkVNg5ldshPfJE5jpqJl25F9q5uaqqMLuky2E4dnNJuraXIzpFg5e6Brjuuw4lZ7isBOC5DJwXOSgZurcP8gA3Bc7qN7B4xzjHBY+3t5N8OTwM5LjnpZ45NKT6OID3jbiQtt04\\u002fX60O3QombUc7AS6oOgXOr3hjrj3qFe5mpdrO+8e37i94Xg4pLq\\u002fOAjXgDnvL5c5C76mOdrL0rzaTQ+4met\\u002fuJsF7Tmcki46b\\u002fYsOsV6BjpOJ7I6avYQuz1gFbyblnQ5194IPG5rWbm6C8C5Lq++OxB\\u002frDpji426Zw6pOqUHqTlonQ04\\u002fJiTOQIsBr5inOQ43O4au1ZRsTrX1cu4dMoNuvwp0Do335y7sf13u2OLwjgP8jW51ljLuubYz7gsvHY6dXXsuQCv6L3ZUSC6\",\"shape\":\"12, 12\"},\"type\":\"heatmap\",\"xaxis\":\"x3\",\"yaxis\":\"y3\",\"hovertemplate\":\"x: %{x}\\u003cbr\\u003ey: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"},{\"coloraxis\":\"coloraxis\",\"name\":\"3\",\"z\":{\"dtype\":\"f4\",\"bdata\":\"A0VQuZn9rjui4Co6m65lOyAvE7zFK2Q78PphukpkCbhXrwc6VzzoOTILgLs7v0W5+NPmudor6rlxwY24AniXOIdwGbglQf26mxYluaFCfDcRDFg5r7ywOS1JuDk\\u002flRS6P4RcOWLcPLpsQP85Z78cugE8G7qytA667kBuuWRY+7kF31c5OBqbOn1du7lJpm66uziIOmhq+7gYJ6O4U4g+uhy7q7qfaxe77QU0PHGwVbi7aRY54IoeOYukqLg3Lau3NuAMuk4pVbrsyAu55ML6PMp367skwxs58uWuOjoiO7xrxAu6gm8GOjK5wzn6tUE7NYIBN6685TZ0eNE66p2xOY\\u002fppLpinGS4XA4mOvrgjbnw\\u002fJo4Z6\\u002fbuz81UDtcn+K6XzJHPSfrejxTiPO5RVLzuFzOTbh4jYG7ZFWOvIDmILmFB6A6x3Q8uZaz37kzeQE54dfxt\\u002fWOtbonO\\u002f64G2oIPp79f7k3r++7LtauOQVOULraDQK8hGpVPvyYSLlAwFi58zQGu5tFxbkhPFm7dFoevP+Qprlfdqi7PmSiPhZOOjv54Pc5KyCXuoibiz4aIlC74HnmuT9zbzhELQe7A2GkPO\\u002fwALxAZTk8HWBIO4dqjD3vDjS7NwQYvN7yBDoGW6y6DklUvCWKR7sU0pK8idKOuwYNl7okJQM6cwNMuz1xoDxWADc4IH5UPP4z\\u002fLtVxG+6Ll+GO7Oap7xr8hK9vBxwPG84XTriVek6tM2TvKmbZjuHOAc8I3bnupTnND2yJ9Q7\",\"shape\":\"12, 12\"},\"type\":\"heatmap\",\"xaxis\":\"x4\",\"yaxis\":\"y4\",\"hovertemplate\":\"x: %{x}\\u003cbr\\u003ey: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"},{\"coloraxis\":\"coloraxis\",\"name\":\"4\",\"z\":{\"dtype\":\"f4\",\"bdata\":\"wIgnOklIrjta1Jc6HEp6uE3Mi7qXcqY7+PRJu9d\\u002fBrti3Ly61udkOzE\\u002fKLuoDnC6plRHusIEMTlzW\\u002f84WQm3uTqDfjfaHqQ7KdWbuSNXvbovFZC6l1H4ObSsVji\\u002fCmW7Jt09ujn05DrfTiK5\\u002fXZuONLDy7gyOd65A1RPuuuMjzl71NU4eFzdOfHJHjmECUO6UKnqPa8rXjk68na6g03hOfsypDzP7gk7VUSgOHFSyrqPp166CYQYumRY+zhbI6S3Z8CTul5Kojone6G6OhQcurixRbo0Jhm697CdOxqwPTndgwy6cAVhOmzRnLmLtUq3hvyKu72CQTsm3Qg6xvZ5OhrBKjly4K8+NMafuTml2DjYrq27UrXLPH+oazwdkhi7t4Mcu1thDrv2ufm5D2HjOa0smbtYmvM6AvEtu19WvDrz0p45dY8FPiMHujjDUZo6qS6nOS+DqDwQ0KY5fqoku092ibkpVYG5iO4QOjYP4rkxH4c5M48EPGAiSbsmDN65m\\u002fR\\u002fOmsEzjkZ5OU6yBq5PGHxOrhS3RQ5iwwdPKdDpbljOsg6b\\u002fiaOtCDMLzschk8B+nOO6Nvj7ogEz88qqqwOi8X77y7N3s7AJs0vWzfpLv9BZ472qSNPm5aArhTey26PFvDPXNezTwzSmk8+r1xPMVazzhvJ7u43jFQPTdOAb+3JBo5NPPRuqr3jD3pghg7uO+vOpbKHTxWQp+9\\u002fUH9u6ghFrpPxkG7IyCiuw2Y8DlaBSa6fqPVu0YXpL6uDiK7\",\"shape\":\"12, 12\"},\"type\":\"heatmap\",\"xaxis\":\"x5\",\"yaxis\":\"y5\",\"hovertemplate\":\"x: %{x}\\u003cbr\\u003ey: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}], {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermap\":[{\"type\":\"scattermap\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,0.18400000000000002],\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"title\":{\"text\":\"Layer\"}},\"xaxis2\":{\"anchor\":\"y2\",\"domain\":[0.20400000000000001,0.388],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis2\":{\"anchor\":\"x2\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis3\":{\"anchor\":\"y3\",\"domain\":[0.40800000000000003,0.5920000000000001],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis3\":{\"anchor\":\"x3\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis4\":{\"anchor\":\"y4\",\"domain\":[0.6120000000000001,0.7960000000000002],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis4\":{\"anchor\":\"x4\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis5\":{\"anchor\":\"y5\",\"domain\":[0.8160000000000001,1.0],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis5\":{\"anchor\":\"x5\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"annotations\":[{\"font\":{},\"showarrow\":false,\"text\":\"Output\",\"x\":0.09200000000000001,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Query\",\"x\":0.29600000000000004,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Key\",\"x\":0.5,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Value\",\"x\":0.7040000000000002,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Pattern\",\"x\":0.908,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"}],\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1,\"cmax\":1},\"title\":{\"text\":\"Activation Patching Per Head (All Pos)\"}}, {\"responsive\": true} ).then(function(){\n", - " \n", - "var gd = document.getElementById('88a69602-b818-412d-9135-7b4ad5b740fe');\n", - "var x = new MutationObserver(function (mutations, observer) {{\n", - " var display = window.getComputedStyle(gd).display;\n", - " if (!display || display === 'none') {{\n", - " console.log([gd, 'removed!']);\n", - " Plotly.purge(gd);\n", - " observer.disconnect();\n", - " }}\n", - "}});\n", - "\n", - "// Listen for the removal of the full notebook cells\n", - "var notebookContainer = gd.closest('#notebook-container');\n", - "if (notebookContainer) {{\n", - " x.observe(notebookContainer, {childList: true});\n", - "}}\n", - "\n", - "// Listen for the clearing of the current output cell\n", - "var outputEl = gd.closest('.output');\n", - "if (outputEl) {{\n", - " x.observe(outputEl, {childList: true});\n", - "}}\n", - "\n", - " }) }; </script> </div>\n", - "</body>\n", - "</html>" - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ @@ -848,11 +646,9 @@ "outputs": [], "source": [ "# NBVAL_IGNORE_OUTPUT\n", - "# attn-only-2l is a custom TransformerLens model (not a standard HF architecture),\n", - "# so we load it via HookedTransformer rather than TransformerBridge.\n", - "from transformer_lens import HookedTransformer\n", - "\n", - "attn_only = HookedTransformer.from_pretrained(\"attn-only-2l\")\n", + "# attn-only-2l is a legacy TransformerLens-format repo (not a standard HF\n", + "# architecture); boot_tl_legacy loads it onto the bridge's native backend.\n", + "attn_only = TransformerBridge.boot_tl_legacy(\"NeelNanda/Attn_Only_2L512W_C4_Code\")\n", "batch = 4\n", "seq_len = 20\n", "rand_tokens_A = torch.randint(100, 10000, (batch, seq_len)).to(attn_only.cfg.device)\n", @@ -865,7 +661,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -939,7 +735,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -950,13 +746,13 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1a2fbc805d4549af9edf95b36f71b3d7", + "model_id": "4869f1cebe81453e8e7617db1507d0f1", "version_major": 2, "version_minor": 0 }, @@ -970,7 +766,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "561b52d296a744198be189e03f9b121b", + "model_id": "4fcc41b59c544b3b892fbf8490e389f2", "version_major": 2, "version_minor": 0 }, @@ -984,7 +780,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b28eaaaba6e5468f85eb7cb7af4fee2a", + "model_id": "3d0d5ed294a84dcda7fbea5d0924f761", "version_major": 2, "version_minor": 0 }, @@ -998,7 +794,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "be01f92e949f40188c6d96ab67337762", + "model_id": "7c821a5963bf4fd18acd4918b83bcbce", "version_major": 2, "version_minor": 0 }, @@ -1012,7 +808,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "636101841bf64094ad905c3e33d74897", + "model_id": "616318b1991342bf8c49292828b48c7c", "version_major": 2, "version_minor": 0 }, @@ -1023,49 +819,10 @@ "metadata": {}, "output_type": "display_data" }, - { - "data": { - "text/html": [ - "<html>\n", - "<head><meta charset=\"utf-8\" /></head>\n", - "<body>\n", - " <div> <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script> <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n", - " <script src=\"https://cdn.plot.ly/plotly-2.16.1.min.js\"></script> <div id=\"654a2982-4d89-40da-a928-0b7a8a5b725f\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div> <script type=\"text/javascript\"> window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById(\"654a2982-4d89-40da-a928-0b7a8a5b725f\")) { Plotly.newPlot( \"654a2982-4d89-40da-a928-0b7a8a5b725f\", [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-13.867763,-14.070641,-14.292063,-13.88009,-13.918364,-14.162065,-13.887416,-14.273762],[-14.130864,-13.992345,-14.256775,-14.155781,-13.941737,-14.197601,-3.5250063,-12.322446]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"},{\"coloraxis\":\"coloraxis\",\"name\":\"1\",\"z\":[[-14.145252,-14.111187,-14.104684,-14.144794,-14.097358,-14.18644,-14.14221,-13.334492],[-14.156257,-14.152325,-14.145813,-14.099783,-14.102302,-14.143165,-14.180906,-14.127906]],\"type\":\"heatmap\",\"xaxis\":\"x2\",\"yaxis\":\"y2\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"},{\"coloraxis\":\"coloraxis\",\"name\":\"2\",\"z\":[[-14.173992,-14.066993,-14.224934,-14.190774,-14.171983,-14.200967,-14.232817,-13.33146],[-14.075601,-14.0040655,-14.148338,-14.322131,-13.960637,-14.202975,-14.39004,-14.127561]],\"type\":\"heatmap\",\"xaxis\":\"x3\",\"yaxis\":\"y3\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"},{\"coloraxis\":\"coloraxis\",\"name\":\"3\",\"z\":[[-13.92904,-14.090907,-14.142209,-13.940542,-13.977854,-14.179171,-13.993744,-14.195768],[-14.183683,-14.09655,-14.20812,-14.206975,-14.004225,-14.117093,-14.115167,-13.735591]],\"type\":\"heatmap\",\"xaxis\":\"x4\",\"yaxis\":\"y4\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"},{\"coloraxis\":\"coloraxis\",\"name\":\"4\",\"z\":[[-14.171475,-14.059196,-14.290342,-14.145466,-14.130345,-14.177188,-14.215987,-14.202147],[-14.0839205,-14.024414,-14.181681,-14.247328,-13.972576,-14.191546,-14.334505,-14.078006]],\"type\":\"heatmap\",\"xaxis\":\"x5\",\"yaxis\":\"y5\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"}], {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,0.18400000000000002],\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"title\":{\"text\":\"Layer\"}},\"xaxis2\":{\"anchor\":\"y2\",\"domain\":[0.20400000000000001,0.388],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis2\":{\"anchor\":\"x2\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis3\":{\"anchor\":\"y3\",\"domain\":[0.40800000000000003,0.5920000000000001],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis3\":{\"anchor\":\"x3\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis4\":{\"anchor\":\"y4\",\"domain\":[0.6120000000000001,0.7960000000000002],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis4\":{\"anchor\":\"x4\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis5\":{\"anchor\":\"y5\",\"domain\":[0.8160000000000001,1.0],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis5\":{\"anchor\":\"x5\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"annotations\":[{\"font\":{},\"showarrow\":false,\"text\":\"Output\",\"x\":0.09200000000000001,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Query\",\"x\":0.29600000000000004,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Key\",\"x\":0.5,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Value\",\"x\":0.7040000000000002,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Pattern\",\"x\":0.908,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"}],\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-14.39004,\"cmax\":-2.423922538757324},\"title\":{\"text\":\"Activation Patching Per Head on BA (All Pos)\"}}, {\"responsive\": true} ).then(function(){\n", - " \n", - "var gd = document.getElementById('654a2982-4d89-40da-a928-0b7a8a5b725f');\n", - "var x = new MutationObserver(function (mutations, observer) {{\n", - " var display = window.getComputedStyle(gd).display;\n", - " if (!display || display === 'none') {{\n", - " console.log([gd, 'removed!']);\n", - " Plotly.purge(gd);\n", - " observer.disconnect();\n", - " }}\n", - "}});\n", - "\n", - "// Listen for the removal of the full notebook cells\n", - "var notebookContainer = gd.closest('#notebook-container');\n", - "if (notebookContainer) {{\n", - " x.observe(notebookContainer, {childList: true});\n", - "}}\n", - "\n", - "// Listen for the clearing of the current output cell\n", - "var outputEl = gd.closest('.output');\n", - "if (outputEl) {{\n", - " x.observe(outputEl, {childList: true});\n", - "}}\n", - "\n", - " }) }; </script> </div>\n", - "</body>\n", - "</html>" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e7553938fe4a4ef38f47745d4d9d929f", + "model_id": "b105e0baccea4c9ba18ce50c81f4841f", "version_major": 2, "version_minor": 0 }, @@ -1079,7 +836,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1dfef314785a4693a404b0d57d6141a2", + "model_id": "5f3dcad087314bcc8601b1e670dc3eec", "version_major": 2, "version_minor": 0 }, @@ -1093,7 +850,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f70a64b8f32e4ffb88455075dd23e56c", + "model_id": "9bfc4459c53149eab261e87b91c44d38", "version_major": 2, "version_minor": 0 }, @@ -1107,7 +864,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "17e461d70ab84db0936e273a9f3f6c14", + "model_id": "dab353f6cd264fa8af7a65142bdcbdb7", "version_major": 2, "version_minor": 0 }, @@ -1121,7 +878,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "165236f474284abfa9172e4f4074a7d0", + "model_id": "d8e68420704141d5b9f52597634fa457", "version_major": 2, "version_minor": 0 }, @@ -1132,49 +889,10 @@ "metadata": {}, "output_type": "display_data" }, - { - "data": { - "text/html": [ - "<html>\n", - "<head><meta charset=\"utf-8\" /></head>\n", - "<body>\n", - " <div> <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script> <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n", - " <script src=\"https://cdn.plot.ly/plotly-2.16.1.min.js\"></script> <div id=\"0952c2a4-788e-4fcb-8126-c71960e40e7d\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div> <script type=\"text/javascript\"> window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById(\"0952c2a4-788e-4fcb-8126-c71960e40e7d\")) { Plotly.newPlot( \"0952c2a4-788e-4fcb-8126-c71960e40e7d\", [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-14.09771,-14.076301,-14.324807,-14.055652,-14.2750635,-14.43573,-14.208144,-14.39448],[-14.496584,-14.453276,-14.416288,-14.303434,-14.365909,-14.456058,-3.973495,-12.939518]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"},{\"coloraxis\":\"coloraxis\",\"name\":\"1\",\"z\":[[-14.46471,-14.297178,-14.419741,-14.417672,-14.432617,-14.414541,-14.416406,-13.488615],[-14.424882,-14.388939,-14.369636,-14.391188,-14.375077,-14.482475,-14.476969,-14.316024]],\"type\":\"heatmap\",\"xaxis\":\"x2\",\"yaxis\":\"y2\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"},{\"coloraxis\":\"coloraxis\",\"name\":\"2\",\"z\":[[-14.415234,-14.298391,-14.433281,-14.460214,-14.445007,-14.449017,-14.412723,-13.529374],[-14.444474,-14.397686,-14.466362,-14.609969,-14.383108,-14.516665,-14.440855,-14.234399]],\"type\":\"heatmap\",\"xaxis\":\"x3\",\"yaxis\":\"y3\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"},{\"coloraxis\":\"coloraxis\",\"name\":\"3\",\"z\":[[-14.143188,-14.189437,-14.264787,-14.212107,-14.322887,-14.436652,-14.242862,-14.403327],[-14.506277,-14.485699,-14.346952,-14.442441,-14.356551,-14.429957,-4.410964,-12.826897]],\"type\":\"heatmap\",\"xaxis\":\"x4\",\"yaxis\":\"y4\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"},{\"coloraxis\":\"coloraxis\",\"name\":\"4\",\"z\":[[-14.402386,-14.191995,-14.566069,-14.393083,-14.388383,-14.455374,-14.440852,-14.378222],[-14.4480915,-14.420664,-14.465406,-14.520906,-14.461384,-14.503322,-14.643165,-14.520193]],\"type\":\"heatmap\",\"xaxis\":\"x5\",\"yaxis\":\"y5\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"}], {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,0.18400000000000002],\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"title\":{\"text\":\"Layer\"}},\"xaxis2\":{\"anchor\":\"y2\",\"domain\":[0.20400000000000001,0.388],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis2\":{\"anchor\":\"x2\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis3\":{\"anchor\":\"y3\",\"domain\":[0.40800000000000003,0.5920000000000001],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis3\":{\"anchor\":\"x3\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis4\":{\"anchor\":\"y4\",\"domain\":[0.6120000000000001,0.7960000000000002],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis4\":{\"anchor\":\"x4\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis5\":{\"anchor\":\"y5\",\"domain\":[0.8160000000000001,1.0],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis5\":{\"anchor\":\"x5\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"annotations\":[{\"font\":{},\"showarrow\":false,\"text\":\"Output\",\"x\":0.09200000000000001,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Query\",\"x\":0.29600000000000004,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Key\",\"x\":0.5,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Value\",\"x\":0.7040000000000002,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Pattern\",\"x\":0.908,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"}],\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-14.643165,\"cmax\":-2.423922538757324},\"title\":{\"text\":\"Activation Patching Per Head on BB (All Pos)\"}}, {\"responsive\": true} ).then(function(){\n", - " \n", - "var gd = document.getElementById('0952c2a4-788e-4fcb-8126-c71960e40e7d');\n", - "var x = new MutationObserver(function (mutations, observer) {{\n", - " var display = window.getComputedStyle(gd).display;\n", - " if (!display || display === 'none') {{\n", - " console.log([gd, 'removed!']);\n", - " Plotly.purge(gd);\n", - " observer.disconnect();\n", - " }}\n", - "}});\n", - "\n", - "// Listen for the removal of the full notebook cells\n", - "var notebookContainer = gd.closest('#notebook-container');\n", - "if (notebookContainer) {{\n", - " x.observe(notebookContainer, {childList: true});\n", - "}}\n", - "\n", - "// Listen for the clearing of the current output cell\n", - "var outputEl = gd.closest('.output');\n", - "if (outputEl) {{\n", - " x.observe(outputEl, {childList: true});\n", - "}}\n", - "\n", - " }) }; </script> </div>\n", - "</body>\n", - "</html>" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1fba5dbbcedc4bc0b043600453e83365", + "model_id": "cebaba9991d94d0a9b73d37807905e79", "version_major": 2, "version_minor": 0 }, @@ -1188,7 +906,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4d5677f76854460daa9f66767efd34c4", + "model_id": "6d2a69d6abc647878aa1a6f973081eaf", "version_major": 2, "version_minor": 0 }, @@ -1202,7 +920,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3dea495e15c049b9b3a199c11690fea2", + "model_id": "00766f3a21a3438c86cb58fb7c2cae2c", "version_major": 2, "version_minor": 0 }, @@ -1216,7 +934,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c8c90497d57949db9dda2c139875165f", + "model_id": "1424aa1274d44583917c356f741393c1", "version_major": 2, "version_minor": 0 }, @@ -1230,7 +948,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5766c84a6cdb42538c358d800529d374", + "model_id": "9fe9690c8f9142948d0374ee7a153c2f", "version_major": 2, "version_minor": 0 }, @@ -1240,45 +958,6 @@ }, "metadata": {}, "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "<html>\n", - "<head><meta charset=\"utf-8\" /></head>\n", - "<body>\n", - " <div> <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script> <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n", - " <script src=\"https://cdn.plot.ly/plotly-2.16.1.min.js\"></script> <div id=\"d16e9f50-5942-4952-b67e-4175fd0374b0\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div> <script type=\"text/javascript\"> window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById(\"d16e9f50-5942-4952-b67e-4175fd0374b0\")) { Plotly.newPlot( \"d16e9f50-5942-4952-b67e-4175fd0374b0\", [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-13.19263,-13.347814,-13.800685,-13.349333,-13.313317,-13.398775,-13.256893,-13.844354],[-13.515493,-13.312331,-13.429906,-13.25336,-13.257504,-13.468011,-3.568708,-11.623617]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"},{\"coloraxis\":\"coloraxis\",\"name\":\"1\",\"z\":[[-13.367426,-13.311,-13.537926,-13.283593,-13.539844,-13.424672,-13.343373,-12.869039],[-13.465478,-13.322073,-13.319036,-13.321832,-13.2353325,-13.387451,-13.607008,-13.318724]],\"type\":\"heatmap\",\"xaxis\":\"x2\",\"yaxis\":\"y2\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"},{\"coloraxis\":\"coloraxis\",\"name\":\"2\",\"z\":[[-13.467334,-13.374045,-13.299824,-13.393491,-13.411175,-13.448505,-13.434408,-12.896077],[-13.411268,-13.151008,-13.409945,-13.60983,-13.258074,-13.524189,-13.5923815,-13.428212]],\"type\":\"heatmap\",\"xaxis\":\"x3\",\"yaxis\":\"y3\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"},{\"coloraxis\":\"coloraxis\",\"name\":\"3\",\"z\":[[-13.274189,-13.530174,-13.404912,-13.456364,-13.177133,-13.377117,-13.33579,-13.579447],[-13.591475,-13.398362,-13.351272,-13.371054,-13.374702,-13.503289,-13.359222,-12.856175]],\"type\":\"heatmap\",\"xaxis\":\"x4\",\"yaxis\":\"y4\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"},{\"coloraxis\":\"coloraxis\",\"name\":\"4\",\"z\":[[-13.443732,-13.369116,-13.786501,-13.2416115,-13.579827,-13.389201,-13.463222,-13.468414],[-13.461549,-13.296544,-13.414983,-13.481478,-13.375094,-13.410861,-14.115074,-13.372341]],\"type\":\"heatmap\",\"xaxis\":\"x5\",\"yaxis\":\"y5\",\"hovertemplate\":\"x: %{x}<br>y: %{y}<br>color: %{z}<extra></extra>\"}], {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,0.18400000000000002],\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"title\":{\"text\":\"Layer\"}},\"xaxis2\":{\"anchor\":\"y2\",\"domain\":[0.20400000000000001,0.388],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis2\":{\"anchor\":\"x2\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis3\":{\"anchor\":\"y3\",\"domain\":[0.40800000000000003,0.5920000000000001],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis3\":{\"anchor\":\"x3\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis4\":{\"anchor\":\"y4\",\"domain\":[0.6120000000000001,0.7960000000000002],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis4\":{\"anchor\":\"x4\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis5\":{\"anchor\":\"y5\",\"domain\":[0.8160000000000001,1.0],\"matches\":\"x\",\"title\":{\"text\":\"Head\"}},\"yaxis5\":{\"anchor\":\"x5\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"annotations\":[{\"font\":{},\"showarrow\":false,\"text\":\"Output\",\"x\":0.09200000000000001,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Query\",\"x\":0.29600000000000004,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Key\",\"x\":0.5,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Value\",\"x\":0.7040000000000002,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{},\"showarrow\":false,\"text\":\"Pattern\",\"x\":0.908,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"}],\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-14.115074,\"cmax\":-2.423922538757324},\"title\":{\"text\":\"Activation Patching Per Head on BC (All Pos)\"}}, {\"responsive\": true} ).then(function(){\n", - " \n", - "var gd = document.getElementById('d16e9f50-5942-4952-b67e-4175fd0374b0');\n", - "var x = new MutationObserver(function (mutations, observer) {{\n", - " var display = window.getComputedStyle(gd).display;\n", - " if (!display || display === 'none') {{\n", - " console.log([gd, 'removed!']);\n", - " Plotly.purge(gd);\n", - " observer.disconnect();\n", - " }}\n", - "}});\n", - "\n", - "// Listen for the removal of the full notebook cells\n", - "var notebookContainer = gd.closest('#notebook-container');\n", - "if (notebookContainer) {{\n", - " x.observe(notebookContainer, {childList: true});\n", - "}}\n", - "\n", - "// Listen for the clearing of the current output cell\n", - "var outputEl = gd.closest('.output');\n", - "if (outputEl) {{\n", - " x.observe(outputEl, {childList: true});\n", - "}}\n", - "\n", - " }) }; </script> </div>\n", - "</body>\n", - "</html>" - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ diff --git a/demos/BERT.ipynb b/demos/BERT.ipynb index e8670d454b..649b80f775 100644 --- a/demos/BERT.ipynb +++ b/demos/BERT.ipynb @@ -492,20 +492,17 @@ " \"google-bert/bert-base-cased\",\n", " model_class=BertForNextSentencePrediction,\n", ")\n", - "nsp.enable_compatibility_mode()\n", + "# No compatibility mode: NSP needs no weight processing, and without it the\n", + "# bridge matches an independently loaded HF model's NSP logits to ~2.4e-6\n", + "# (compatibility mode's unembed centering shifts them by ~1e-2).\n", "\n", "sentence_a = \"A man walked into a grocery store.\"\n", "sentence_b = \"He bought an apple.\"\n", "\n", - "inputs = tokenizer(sentence_a, sentence_b, return_tensors=\"pt\")\n", - "device = next(nsp.parameters()).device\n", - "# token_type_ids mark where sentence A ends and B begins — without them the NSP\n", - "# head sees one undifferentiated span and can return the wrong verdict.\n", - "predictions = nsp(\n", - " inputs[\"input_ids\"].to(device),\n", - " token_type_ids=inputs[\"token_type_ids\"].to(device),\n", - " return_type=\"predictions\",\n", - ")\n", + "# predict_next_sentence owns the pair tokenization — token_type_ids mark where\n", + "# sentence A ends and B begins; without them the NSP head sees one\n", + "# undifferentiated span and can return the wrong verdict.\n", + "predictions = nsp.predict_next_sentence(sentence_a, sentence_b)\n", "\n", "print(f\"Sentence A: {sentence_a}\")\n", "print(f\"Sentence B: {sentence_b}\")\n", diff --git a/demos/Colab_Compatibility.ipynb b/demos/Colab_Compatibility.ipynb index be122f7847..9b3381cfa7 100644 --- a/demos/Colab_Compatibility.ipynb +++ b/demos/Colab_Compatibility.ipynb @@ -1,5 +1,31 @@ { "cells": [ + { + "cell_type": "markdown", + "id": "ecf1ba08", + "metadata": {}, + "source": [ + "# Legacy Compatibility Ledger\n", + "\n", + "**This notebook now tracks legacy compatibility only.** It covers the fixed set\n", + "of models that were available in `HookedTransformer` (and the other `Hooked*`\n", + "classes) and verifies that each of them **continues to function on\n", + "`TransformerBridge`**, which replaces those classes in 4.0.\n", + "\n", + "It is *not* an inventory of what the bridge supports — the bridge loads far\n", + "more architectures than this list (see the [model registry](https://transformerlensorg.github.io/TransformerLens/generated/transformer_bridge_models.html) /\n", + "`TransformerBridge.list_supported_models()`). The set below is closed. The contract at the bottom is\n", + "unchanged, the untested list must remain empty.\n", + "\n", + "Runner notes:\n", + "- Plain `boot_transformers(...)` is the unprocessed load, equivalent to the old\n", + " `from_pretrained_no_processing`.\n", + "- Legacy TransformerLens-format repos (`NeelNanda/*`, `ArthurConmy/*`,\n", + " `Baidicoot/*`) load through `TransformerBridge.boot_tl_legacy`.\n", + "- Families the legacy loader force-set `trust_remote_code` for (santacoder,\n", + " Qwen-1, Phi) get it passed explicitly." + ] + }, { "cell_type": "code", "execution_count": null, @@ -46,19 +72,29 @@ "outputs": [], "source": [ "import torch\n", - "from transformer_lens import HookedTransformer, HookedEncoderDecoder, HookedEncoder, BertNextSentencePrediction, loading\n", - "from transformers import AutoTokenizer, LlamaForCausalLM, LlamaTokenizer\n", + "from transformer_lens import loading\n", + "from transformer_lens.model_bridge import TransformerBridge\n", + "from transformers import AutoTokenizer, BertForNextSentencePrediction, LlamaForCausalLM, LlamaTokenizer\n", "from typing import List\n", "import gc\n", "\n", + "# The legacy ledger: every model HookedTransformer shipped support for. This\n", + "# list is closed — at 4.0 it freezes into this notebook when the legacy\n", + "# `loading` module is deleted.\n", "untested_models = []\n", "untested_models.extend(loading.OFFICIAL_MODEL_NAMES)\n", "\n", - "print(\"TransformerLens currently supports \" + str(len(untested_models)) + \" models out of the box.\")\n", + "print(\"Tracking legacy compatibility for \" + str(len(untested_models)) + \" HookedTransformer-era models.\")\n", "\n", "GENERATE = True\n", "# Fill this in if you have llama weights uploaded, and you with to test those models\n", - "LLAMA_MODEL_PATH = \"\"" + "LLAMA_MODEL_PATH = \"\"\n", + "\n", + "# Families the legacy loader force-set trust_remote_code for.\n", + "def _needs_remote_code(model: str) -> bool:\n", + " return \"santacoder\" in model or model.startswith(\"Qwen/Qwen-\") or model.startswith(\"microsoft/phi-\")\n", + "\n", + "TL_LEGACY_PREFIXES = (\"NeelNanda/\", \"ArthurConmy/\", \"Baidicoot/\")" ] }, { @@ -73,10 +109,21 @@ " untested_models.remove(model)\n", "\n", "\n", + "def _boot(model: str, device=\"cuda\"):\n", + " \"\"\"Unprocessed bridge load — the modern equivalent of the legacy\n", + " from_pretrained_no_processing. Legacy TL-format repos go through\n", + " boot_tl_legacy.\"\"\"\n", + " if model.startswith(TL_LEGACY_PREFIXES):\n", + " return TransformerBridge.boot_tl_legacy(model, device=device)\n", + " return TransformerBridge.boot_transformers(\n", + " model, device=device, trust_remote_code=_needs_remote_code(model)\n", + " )\n", + "\n", + "\n", "def run_set(model_set: List[str], device=\"cuda\") -> None:\n", " for model in model_set:\n", " print(\"Testing \" + model)\n", - " tl_model = HookedTransformer.from_pretrained_no_processing(model, device=device)\n", + " tl_model = _boot(model, device=device)\n", " if GENERATE:\n", " print(tl_model.generate(\"Hello my name is\"))\n", " del tl_model\n", @@ -91,14 +138,11 @@ " # sub directories sharing the same name as the model in the list of models\n", " tokenizer = LlamaTokenizer.from_pretrained(weight_root + model)\n", " hf_model = LlamaForCausalLM.from_pretrained(weight_root + model, low_cpu_mem_usage=True)\n", - " tl_model = HookedTransformer.from_pretrained_no_processing(\n", + " tl_model = TransformerBridge.boot_transformers(\n", " model,\n", " hf_model=hf_model,\n", - " device=device,\n", - " fold_ln=False,\n", - " center_writing_weights=False,\n", - " center_unembed=False,\n", " tokenizer=tokenizer,\n", + " device=device,\n", " )\n", " if GENERATE:\n", " print(tl_model.generate(\"Hello my name is\"))\n", @@ -111,30 +155,11 @@ "def run_encoder_decoder_set(model_set: List[str], device=\"cuda\") -> None:\n", " for model in model_set:\n", " print(\"Testing \" + model)\n", - " tokenizer = AutoTokenizer.from_pretrained(model)\n", - " tl_model = HookedEncoderDecoder.from_pretrained(model, device=device)\n", + " tl_model = TransformerBridge.boot_transformers(model, device=device)\n", " if GENERATE:\n", - " # Originally from the t5 demo\n", - " prompt = \"Hello, how are you? \"\n", - " inputs = tokenizer(prompt, return_tensors=\"pt\")\n", - " input_ids = inputs[\"input_ids\"]\n", - " attention_mask = inputs[\"attention_mask\"]\n", - " decoder_input_ids = torch.tensor([[tl_model.cfg.decoder_start_token_id]]).to(input_ids.device)\n", - "\n", - "\n", - " while True:\n", - " logits = tl_model.forward(input=input_ids, one_zero_attention_mask=attention_mask, decoder_input=decoder_input_ids)\n", - " # logits.shape == (batch_size (1), predicted_pos, vocab_size)\n", - "\n", - " token_idx = torch.argmax(logits[0, -1, :]).item()\n", - " print(\"generated token: \\\"\", tokenizer.decode(token_idx), \"\\\", token id: \", token_idx, sep=\"\")\n", - "\n", - " # append token to decoder_input_ids\n", - " decoder_input_ids = torch.cat([decoder_input_ids, torch.tensor([[token_idx]]).to(input_ids.device)], dim=-1)\n", - "\n", - " # break if End-Of-Sequence token generated\n", - " if token_idx == tokenizer.eos_token_id:\n", - " break\n", + " # Encoder-decoder generation is supported natively on the bridge\n", + " # (the tokenizer's own pair recipe, generation_config honored).\n", + " print(tl_model.generate(\"Hello, how are you? \", do_sample=False, max_new_tokens=20))\n", " del tl_model\n", " gc.collect()\n", " if IN_COLAB:\n", @@ -143,8 +168,10 @@ "def run_encoder_only_set(model_set: List[str], device=\"cuda\") -> None:\n", " for model in model_set:\n", " print(\"Testing \" + model)\n", - " tl_model = HookedEncoder.from_pretrained(model, device=device)\n", - " tl_model_nsp = NextSentencePrediction.from_pretrained(model, device=device)\n", + " tl_model = TransformerBridge.boot_transformers(model, device=device)\n", + " tl_model_nsp = TransformerBridge.boot_transformers(\n", + " model, device=device, model_class=BertForNextSentencePrediction\n", + " )\n", "\n", " if GENERATE:\n", " print(\"Testing Masked Language Modelling:\")\n", @@ -160,13 +187,14 @@ " sentence_a = \"She went to the grocery store.\"\n", " sentence_b = \"She bought some milk.\"\n", "\n", - " prediction = tl_model_nsp([sentence_a, sentence_b], return_type=\"predictions\")\n", + " prediction = tl_model_nsp.predict_next_sentence(sentence_a, sentence_b)\n", "\n", " print(f\"Sentence A: {sentence_a}\")\n", " print(f\"Sentence B: {sentence_b}\")\n", " print(f\"Prediction: {prediction}\")\n", "\n", " del tl_model\n", + " del tl_model_nsp\n", " gc.collect()\n", " if IN_COLAB:\n", " %rm -rf /root/.cache/huggingface/hub/models*" @@ -510,10 +538,13 @@ { "cell_type": "code", "execution_count": null, + "id": "4ce9f7b4", "metadata": {}, "outputs": [], "source": [ "broken_models = [\n", + " # minGPT-format legacy repo. boot_tl_legacy now has the converter for it;\n", + " # promote to free_compatible once verified on a Colab runtime.\n", " \"Baidicoot/Othello-GPT-Transformer-Lens\",\n", "]" ] diff --git a/demos/Exploratory_Analysis_Demo.ipynb b/demos/Exploratory_Analysis_Demo.ipynb index b0aa0c9453..d3fd3a947f 100644 --- a/demos/Exploratory_Analysis_Demo.ipynb +++ b/demos/Exploratory_Analysis_Demo.ipynb @@ -531,7 +531,8 @@ "metadata": {}, "outputs": [], "source": [ - "# TransformerBridge doesn't have tokens_to_residual_directions yet,\n", + "# tokens_to_residual_directions works on the bridge too; kept manual\n", + "# to show the underlying W_U slice.\n", "# so we implement it inline using model.unembed.W_U\n", "W_U = model.unembed.W_U # [d_model, d_vocab]\n", "answer_residual_directions = W_U[:, answer_tokens]\n", @@ -1475,7 +1476,7 @@ " ),\n", " )\n", " # It would have been great to set start_layer=B and the inputs to the cached corrupted residual stream at that layer;\n", - " # unfortunately, this is not supported by TransformerBridge. Legacy HookedTransformer supports it but is slower, so\n", + " # start_at_layer is supported by TransformerBridge as well.\n", " # it doesn't pay off to switch.\n", " patched_logits = model.run_with_hooks(\n", " corrupted_tokens,\n", diff --git a/demos/GPT_OSS_Demo.ipynb b/demos/GPT_OSS_Demo.ipynb index 8da0e017ea..882c5498ff 100644 --- a/demos/GPT_OSS_Demo.ipynb +++ b/demos/GPT_OSS_Demo.ipynb @@ -6,16 +6,14 @@ "source": [ "# GPT-OSS-20B Demo\n", "\n", - "This notebook loads OpenAI's [GPT-OSS-20B](https://huggingface.co/openai/gpt-oss-20b) into TransformerLens for mechanistic interpretability.\n", - "\n", - "GPT-OSS-20B is a Mixture of Experts (MoE) model with:\n", - "- 24 layers, d_model=2880, 32 experts, 4 experts per token\n", - "- MXFP4 quantized weights on HuggingFace (dequantized to BF16 during loading)\n", - "- Custom GLU activation and post-top-k softmax routing\n", - "\n", - "**Memory:** The full model is ~40GB in BF16. This notebook loads directly from safetensors, bypassing the HuggingFace model pipeline to keep peak memory manageable. Use `N_LAYERS` to load fewer layers if needed.\n", - "\n", - "**Requirements:** `transformers`, `safetensors`, `einops`, `psutil`" + "This notebook loads OpenAI's [GPT-OSS-20B](https://huggingface.co/openai/gpt-oss-20b)\n", + "through `TransformerBridge` and walks through basic mech-interp workflows:\n", + "caching, expert-routing inspection, logit lens, attention patterns, and\n", + "activation patching.\n", + "\n", + "The bridge loads the checkpoint in one call — transformers dequantizes the\n", + "MXFP4-packed experts to bf16 automatically. Peak RAM is ~56 GB; a 64 GB\n", + "machine is marginal, 96 GB+ is comfortable." ] }, { @@ -24,18 +22,9 @@ "metadata": {}, "outputs": [], "source": [ - "import gc\n", - "import json\n", - "from pathlib import Path\n", - "\n", - "import einops\n", "import torch\n", - "from safetensors import safe_open\n", - "from transformers import AutoTokenizer\n", - "from transformers.integrations.mxfp4 import convert_moe_packed_tensors\n", "\n", - "from transformer_lens import HookedTransformer\n", - "from transformer_lens import HookedTransformerConfig" + "from transformer_lens.model_bridge import TransformerBridge" ] }, { @@ -44,23 +33,9 @@ "source": [ "## Configuration\n", "\n", - "Set `N_LAYERS` to control how many layers to load. Each layer is ~1.6GB. Use fewer layers to save memory." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "N_LAYERS = 24 # Full model. Set to 3-6 for quick testing." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Model Loading Utilities" + "`N_LAYERS` controls how many layers the *analysis* cells sweep (routing tables,\n", + "patching). The full 24-layer model always loads; lower this to keep the\n", + "analysis output short." ] }, { @@ -69,163 +44,7 @@ "metadata": {}, "outputs": [], "source": [ - "def get_model_path():\n", - " \"\"\"Get the cached model path, downloading if necessary.\"\"\"\n", - " cache_path = Path.home() / \".cache/huggingface/hub/models--openai--gpt-oss-20b\"\n", - " snapshots = cache_path / \"snapshots\"\n", - "\n", - " if snapshots.exists():\n", - " snapshot_dirs = list(snapshots.iterdir())\n", - " if snapshot_dirs:\n", - " return snapshot_dirs[0]\n", - "\n", - " print(\"Model not found in cache. Downloading...\")\n", - " from huggingface_hub import snapshot_download\n", - "\n", - " return Path(snapshot_download(\"openai/gpt-oss-20b\"))\n", - "\n", - "\n", - "def create_config(n_layers=24):\n", - " \"\"\"Create TransformerLens config for GPT-OSS-20B.\"\"\"\n", - " return HookedTransformerConfig(\n", - " n_layers=n_layers,\n", - " d_model=2880,\n", - " d_head=64,\n", - " n_heads=64,\n", - " d_mlp=2880,\n", - " n_ctx=4096,\n", - " d_vocab=201088,\n", - " act_fn=\"silu\",\n", - " normalization_type=\"RMS\",\n", - " positional_embedding_type=\"rotary\",\n", - " rotary_base=150000,\n", - " eps=1e-5,\n", - " n_key_value_heads=8,\n", - " gated_mlp=True,\n", - " use_local_attn=False,\n", - " rotary_dim=64,\n", - " num_experts=32,\n", - " experts_per_token=4,\n", - " dtype=torch.bfloat16,\n", - " device=\"cpu\",\n", - " original_architecture=\"GptOssForCausalLM\",\n", - " model_name=\"openai/gpt-oss-20b\",\n", - " )\n", - "\n", - "\n", - "_open_files = {}\n", - "\n", - "\n", - "def _get_tensor(hf_name, wmap, model_path):\n", - " \"\"\"Load a single tensor from the correct safetensors shard.\"\"\"\n", - " st_file = wmap[hf_name]\n", - " filepath = str(model_path / st_file)\n", - " if filepath not in _open_files:\n", - " _open_files[filepath] = safe_open(filepath, framework=\"pt\", device=\"cpu\")\n", - " return _open_files[filepath].get_tensor(hf_name)\n", - "\n", - "\n", - "def load_layer_weights(l, cfg, index, model_path):\n", - " \"\"\"Load and convert weights for one transformer layer from safetensors.\"\"\"\n", - " state_dict = {}\n", - " wmap = index[\"weight_map\"]\n", - " prefix = f\"model.layers.{l}\"\n", - "\n", - " def gt(name):\n", - " return _get_tensor(name, wmap, model_path)\n", - "\n", - " # LayerNorms\n", - " state_dict[f\"blocks.{l}.ln1.w\"] = gt(f\"{prefix}.input_layernorm.weight\")\n", - " state_dict[f\"blocks.{l}.ln2.w\"] = gt(f\"{prefix}.post_attention_layernorm.weight\")\n", - "\n", - " # Attention weights\n", - " q_w = gt(f\"{prefix}.self_attn.q_proj.weight\")\n", - " k_w = gt(f\"{prefix}.self_attn.k_proj.weight\")\n", - " v_w = gt(f\"{prefix}.self_attn.v_proj.weight\")\n", - " o_w = gt(f\"{prefix}.self_attn.o_proj.weight\")\n", - "\n", - " state_dict[f\"blocks.{l}.attn.W_Q\"] = einops.rearrange(q_w, \"(n h) m -> n m h\", n=cfg.n_heads)\n", - " state_dict[f\"blocks.{l}.attn._W_K\"] = einops.rearrange(\n", - " k_w, \"(n h) m -> n m h\", n=cfg.n_key_value_heads\n", - " )\n", - " state_dict[f\"blocks.{l}.attn._W_V\"] = einops.rearrange(\n", - " v_w, \"(n h) m -> n m h\", n=cfg.n_key_value_heads\n", - " )\n", - " state_dict[f\"blocks.{l}.attn.W_O\"] = einops.rearrange(\n", - " o_w, \"m (n h) -> n h m\", n=cfg.n_heads\n", - " )\n", - " del q_w, k_w, v_w, o_w\n", - "\n", - " # Attention biases\n", - " q_bias_key = f\"{prefix}.self_attn.q_proj.bias\"\n", - " if q_bias_key in wmap:\n", - " state_dict[f\"blocks.{l}.attn.b_Q\"] = einops.rearrange(\n", - " gt(q_bias_key), \"(n h) -> n h\", n=cfg.n_heads\n", - " )\n", - " state_dict[f\"blocks.{l}.attn._b_K\"] = einops.rearrange(\n", - " gt(f\"{prefix}.self_attn.k_proj.bias\"), \"(n h) -> n h\", n=cfg.n_key_value_heads\n", - " )\n", - " state_dict[f\"blocks.{l}.attn._b_V\"] = einops.rearrange(\n", - " gt(f\"{prefix}.self_attn.v_proj.bias\"), \"(n h) -> n h\", n=cfg.n_key_value_heads\n", - " )\n", - " else:\n", - " state_dict[f\"blocks.{l}.attn.b_Q\"] = torch.zeros(cfg.n_heads, cfg.d_head, dtype=cfg.dtype)\n", - " state_dict[f\"blocks.{l}.attn._b_K\"] = torch.zeros(\n", - " cfg.n_key_value_heads, cfg.d_head, dtype=cfg.dtype\n", - " )\n", - " state_dict[f\"blocks.{l}.attn._b_V\"] = torch.zeros(\n", - " cfg.n_key_value_heads, cfg.d_head, dtype=cfg.dtype\n", - " )\n", - "\n", - " o_bias_key = f\"{prefix}.self_attn.o_proj.bias\"\n", - " if o_bias_key in wmap:\n", - " state_dict[f\"blocks.{l}.attn.b_O\"] = gt(o_bias_key)\n", - " else:\n", - " state_dict[f\"blocks.{l}.attn.b_O\"] = torch.zeros(cfg.d_model, dtype=cfg.dtype)\n", - "\n", - " # Router\n", - " state_dict[f\"blocks.{l}.mlp.W_gate.weight\"] = gt(f\"{prefix}.mlp.router.weight\")\n", - " state_dict[f\"blocks.{l}.mlp.W_gate.bias\"] = gt(f\"{prefix}.mlp.router.bias\")\n", - "\n", - " # Expert weights - dequantize MXFP4 to BF16\n", - " gate_up_blocks = gt(f\"{prefix}.mlp.experts.gate_up_proj_blocks\")\n", - " gate_up_scales = gt(f\"{prefix}.mlp.experts.gate_up_proj_scales\")\n", - " gate_up_bias = gt(f\"{prefix}.mlp.experts.gate_up_proj_bias\")\n", - "\n", - " print(f\" Dequantizing layer {l} gate_up_proj...\", end=\"\", flush=True)\n", - " gate_up_proj = convert_moe_packed_tensors(gate_up_blocks, gate_up_scales)\n", - " del gate_up_blocks, gate_up_scales\n", - " print(\" done\")\n", - "\n", - " down_blocks = gt(f\"{prefix}.mlp.experts.down_proj_blocks\")\n", - " down_scales = gt(f\"{prefix}.mlp.experts.down_proj_scales\")\n", - " down_bias = gt(f\"{prefix}.mlp.experts.down_proj_bias\")\n", - "\n", - " print(f\" Dequantizing layer {l} down_proj...\", end=\"\", flush=True)\n", - " down_proj = convert_moe_packed_tensors(down_blocks, down_scales)\n", - " del down_blocks, down_scales\n", - " print(\" done\")\n", - "\n", - " # Split merged expert tensors into per-expert weights\n", - " # Even columns -> gate, Odd columns -> up\n", - " for e in range(cfg.num_experts):\n", - " state_dict[f\"blocks.{l}.mlp.experts.{e}.W_gate.weight\"] = gate_up_proj[\n", - " e, :, ::2\n", - " ].T.contiguous()\n", - " state_dict[f\"blocks.{l}.mlp.experts.{e}.W_gate.bias\"] = gate_up_bias[\n", - " e, ::2\n", - " ].contiguous()\n", - " state_dict[f\"blocks.{l}.mlp.experts.{e}.W_in.weight\"] = gate_up_proj[\n", - " e, :, 1::2\n", - " ].T.contiguous()\n", - " state_dict[f\"blocks.{l}.mlp.experts.{e}.W_in.bias\"] = gate_up_bias[\n", - " e, 1::2\n", - " ].contiguous()\n", - " state_dict[f\"blocks.{l}.mlp.experts.{e}.W_out.weight\"] = down_proj[e].T.contiguous()\n", - " state_dict[f\"blocks.{l}.mlp.experts.{e}.W_out.bias\"] = down_bias[e].contiguous()\n", - "\n", - " del gate_up_proj, gate_up_bias, down_proj, down_bias\n", - " return state_dict" + "N_LAYERS = 24 # layers swept by the analysis cells below" ] }, { @@ -241,50 +60,12 @@ "metadata": {}, "outputs": [], "source": [ - "model_path = get_model_path()\n", - "print(f\"Model path: {model_path}\")\n", - "\n", - "with open(model_path / \"model.safetensors.index.json\") as f:\n", - " index = json.load(f)\n", - "\n", - "cfg = create_config(n_layers=N_LAYERS)\n", - "tokenizer = AutoTokenizer.from_pretrained(str(model_path))\n", - "model = HookedTransformer(cfg, tokenizer, move_to_device=False)\n", - "\n", - "# Load embeddings\n", - "wmap = index[\"weight_map\"]\n", - "model.load_state_dict(\n", - " {\"embed.W_E\": _get_tensor(\"model.embed_tokens.weight\", wmap, model_path)},\n", - " strict=False,\n", - ")\n", - "gc.collect()\n", - "\n", - "# Load layers one at a time\n", - "for l in range(N_LAYERS):\n", - " print(f\"Loading layer {l}/{N_LAYERS-1}...\")\n", - " layer_dict = load_layer_weights(l, cfg, index, model_path)\n", - " for key in list(layer_dict.keys()):\n", - " model.load_state_dict({key: layer_dict[key]}, strict=False)\n", - " del layer_dict[key]\n", - " del layer_dict\n", - " gc.collect()\n", - "\n", - "# Load final LayerNorm and unembed\n", - "model.load_state_dict(\n", - " {\"ln_final.w\": _get_tensor(\"model.norm.weight\", wmap, model_path)},\n", - " strict=False,\n", + "# NBVAL_IGNORE_OUTPUT\n", + "model = TransformerBridge.boot_transformers(\n", + " \"openai/gpt-oss-20b\", device=\"cpu\", dtype=torch.bfloat16\n", ")\n", - "model.load_state_dict(\n", - " {\"unembed.W_U\": _get_tensor(\"lm_head.weight\", wmap, model_path).T},\n", - " strict=False,\n", - ")\n", - "model.load_state_dict(\n", - " {\"unembed.b_U\": torch.zeros(cfg.d_vocab, dtype=cfg.dtype)},\n", - " strict=False,\n", - ")\n", - "gc.collect()\n", - "\n", - "print(f\"\\nModel loaded! Layers: {cfg.n_layers}, Experts: {cfg.num_experts}, d_model: {cfg.d_model}\")" + "print(f\"{model.cfg.n_layers} layers, d_model {model.cfg.d_model}, \"\n", + " f\"{model.cfg.n_heads} heads\")" ] }, { @@ -454,16 +235,23 @@ "clean_tokens = model.to_tokens(clean_prompt)\n", "corrupt_tokens = model.to_tokens(corrupt_prompt)\n", "\n", - "# Get clean activations\n", + "# Get clean activations. Keyed by the REQUESTED name: inside a hook fn,\n", + "# hook.name reports the point's canonical bridge name (blocks.N.hook_out),\n", + "# not the alias the hook was registered under.\n", "captured_clean = {}\n", "\n", - "def save_clean(tensor, hook):\n", - " captured_clean[hook.name] = tensor.detach().clone()\n", + "def save_clean_as(name):\n", + " def save_clean(tensor, hook, _key=name):\n", + " captured_clean[_key] = tensor.detach().clone()\n", + " return save_clean\n", "\n", "with torch.no_grad():\n", " clean_logits = model.run_with_hooks(\n", " clean_tokens,\n", - " fwd_hooks=[(f\"blocks.{l}.hook_resid_post\", save_clean) for l in range(N_LAYERS)],\n", + " fwd_hooks=[\n", + " (f\"blocks.{l}.hook_resid_post\", save_clean_as(f\"blocks.{l}.hook_resid_post\"))\n", + " for l in range(N_LAYERS)\n", + " ],\n", " )\n", "\n", "with torch.no_grad():\n", diff --git a/demos/LIT_Integration_Demo.ipynb b/demos/LIT_Integration_Demo.ipynb index 8defa89afd..40fe3df6a8 100644 --- a/demos/LIT_Integration_Demo.ipynb +++ b/demos/LIT_Integration_Demo.ipynb @@ -118,7 +118,7 @@ "# Core imports\n", "import torch\n", "import numpy as np\n", - "from transformer_lens import HookedTransformer\n", + "from transformer_lens.model_bridge import TransformerBridge\n", "\n", "# LIT integration imports\n", "from transformer_lens.lit import (\n", diff --git a/demos/Main_Demo.ipynb b/demos/Main_Demo.ipynb index 1c16438f76..3a9eeb9d1d 100644 --- a/demos/Main_Demo.ipynb +++ b/demos/Main_Demo.ipynb @@ -135,7 +135,7 @@ "from transformer_lens.hook_points import (\n", " HookPoint,\n", ") # Hooking utilities\n", - "from transformer_lens import FactoredMatrix, HookedTransformer\n", + "from transformer_lens import FactoredMatrix\n", "from transformer_lens.model_bridge import TransformerBridge" ] }, @@ -220,7 +220,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1712,7 +1712,8 @@ "metadata": {}, "outputs": [], "source": [ - "from transformer_lens.hook_points import HookPoint\nfrom transformer_lens.HookedRootModule import HookedRootModule\n", + "from transformer_lens.hook_points import HookPoint\n", + "from transformer_lens.HookedRootModule import HookedRootModule\n", "\n", "\n", "class SquareThenAdd(nn.Module):\n", @@ -1934,7 +1935,7 @@ "if not IN_GITHUB:\n", " for index in checkpoint_indices:\n", " # Load the model from the relevant checkpoint by index\n", - " model_for_this_checkpoint = HookedTransformer.from_pretrained(\n", + " model_for_this_checkpoint = TransformerBridge.boot_tl_legacy(\n", " model_name, checkpoint_index=index, device=device\n", " )\n", " checkpointed_models.append(model_for_this_checkpoint)\n", diff --git a/demos/SVD_Interpreter_Demo.ipynb b/demos/SVD_Interpreter_Demo.ipynb index 4a2f93a693..790fd43f30 100644 --- a/demos/SVD_Interpreter_Demo.ipynb +++ b/demos/SVD_Interpreter_Demo.ipynb @@ -120,7 +120,8 @@ "import numpy as np\n", "import transformer_lens\n", "import transformer_lens.utilities as utils\n", - "from transformer_lens import HookedTransformer, SVDInterpreter" + "from transformer_lens import SVDInterpreter\n", + "from transformer_lens.model_bridge import TransformerBridge" ] }, { @@ -170,7 +171,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -178,24 +179,9 @@ "id": "EkTlBQB6sbvS", "outputId": "a7b4df48-e052-4c29-e8b6-6a351c802798" }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using pad_token, but it is not set yet.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loaded pretrained model gpt2-medium into HookedTransformer\n" - ] - } - ], + "outputs": [], "source": [ - "model = HookedTransformer.from_pretrained(\"gpt2-medium\", fold_ln=False, center_writing_weights=False, center_unembed=False)" + "model = TransformerBridge.boot_transformers(\"gpt2-medium\") # unprocessed, like the legacy no-processing load" ] }, { diff --git a/demos/direct_path_patching_ioi.ipynb b/demos/direct_path_patching_ioi.ipynb index 9ab323a98e..451ff7f618 100644 --- a/demos/direct_path_patching_ioi.ipynb +++ b/demos/direct_path_patching_ioi.ipynb @@ -15,7 +15,7 @@ "source": [ "# Direct Path Patching Demo\n", "\n", - "This notebook demonstrates **direct path patching** \u2014 a technique for isolating the direct information flow between specific attention heads in a transformer.\n", + "This notebook demonstrates **direct path patching** — a technique for isolating the direct information flow between specific attention heads in a transformer.\n", "\n", "## Background\n", "\n", @@ -24,8 +24,8 @@ "**Direct path patching** isolates a single causal edge: it patches only the contribution of source head A into the query/key/value input of destination head B, leaving every other component's view of A's output unchanged.\n", "\n", "We validate on the **Indirect Object Identification (IOI)** task from Wang et al. 2022:\n", - "- Clean: *\"When Mary and John went to the store, John gave a drink to\"* \u2192 **Mary**\n", - "- Corrupted: *\"When Mary and John went to the store, Mary gave a drink to\"* \u2192 **John**\n", + "- Clean: *\"When Mary and John went to the store, John gave a drink to\"* → **Mary**\n", + "- Corrupted: *\"When Mary and John went to the store, Mary gave a drink to\"* → **John**\n", "\n", "Metric: normalised logit diff (0 = corrupted baseline, 1 = clean baseline)." ] @@ -62,8 +62,8 @@ "outputs": [], "source": [ "import torch\n", - "from transformer_lens import HookedTransformer\n", - "from transformer_lens.direct_path_patching import get_act_patch_direct_path" + "from transformer_lens.model_bridge import TransformerBridge\n", + "from transformer_lens.tools.analysis import get_act_patch_direct_path" ] }, { @@ -79,12 +79,8 @@ "metadata": {}, "outputs": [], "source": [ - "model = HookedTransformer.from_pretrained(\n", - " \"gpt2\",\n", - " center_unembed=True,\n", - " center_writing_weights=True,\n", - " fold_ln=True,\n", - ")\n", + "model = TransformerBridge.boot_transformers(\"gpt2\")\n", + "model.enable_compatibility_mode() # fold_ln + center_writing_weights + center_unembed\n", "model.eval()\n", "print(f\"Loaded GPT-2 small: {model.cfg.n_layers} layers, {model.cfg.n_heads} heads\")" ] @@ -130,7 +126,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Direct Path Patching: S-Inhibition \u2192 Name-Mover Heads" + "## Direct Path Patching: S-Inhibition → Name-Mover Heads" ] }, { @@ -138,8 +134,8 @@ "metadata": {}, "source": [ "The IOI circuit (Wang et al. 2022) identifies:\n", - "- **S-inhibition heads**: (7,3), (7,9), (8,6), (8,10) \u2014 suppress the subject name token\n", - "- **Name-mover heads**: (9,9), (9,6), (10,0) \u2014 copy the indirect object to the output\n", + "- **S-inhibition heads**: (7,3), (7,9), (8,6), (8,10) — suppress the subject name token\n", + "- **Name-mover heads**: (9,9), (9,6), (10,0) — copy the indirect object to the output\n", "\n", "Direct path patching lets us measure whether each S-inhibition head communicates *directly* with each name-mover head via the query pathway." ] @@ -190,11 +186,11 @@ "source": [ "The results confirm the IOI circuit structure at the **edge level**:\n", "\n", - "1. **(8,6) \u2192 (9,9)** is the strongest single direct path (+0.083). Head 8.6 is the most influential S-inhibition head.\n", + "1. **(8,6) → (9,9)** is the strongest single direct path (+0.083). Head 8.6 is the most influential S-inhibition head.\n", "2. All S-inhibition heads show their strongest direct paths running into the known name-mover heads (9.9, 9.6, 10.0).\n", - "3. Standard activation patching would show that layer 9 matters \u2014 but cannot distinguish *which* upstream head is responsible for each name-mover head's query input.\n", + "3. Standard activation patching would show that layer 9 matters — but cannot distinguish *which* upstream head is responsible for each name-mover head's query input.\n", "\n", - "Direct path patching adds that resolution, isolating the A \u2192 B causal edge without affecting any other component's view of A's output." + "Direct path patching adds that resolution, isolating the A → B causal edge without affecting any other component's view of A's output." ] } ], @@ -211,4 +207,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/docs/source/content/migrating_to_v3.md b/docs/source/content/migrating_to_v3.md index 938c035083..f520b67690 100644 --- a/docs/source/content/migrating_to_v3.md +++ b/docs/source/content/migrating_to_v3.md @@ -129,6 +129,8 @@ Post-norm architectures (OLMo 2, BERT-style encoders) and MLA blocks (DeepSeek V Additionally, **HookedRootModule** has been moved to its own module. Prefer `from transformer_lens import HookedRootModule`. The legacy `from transformer_lens.hook_points import HookedRootModule` still works in 3.x, but emits a `DeprecationWarning`. This import path will be removed in 4.0. +`HookedRootModule` itself (together with `HookPoint`) is **kept** — it is permanent infrastructure, not part of the 4.0 removal of the legacy model classes, and remains the supported way to add TransformerLens-style hooks to your own `nn.Module`s. + ## APIs that are unchanged These work identically on `TransformerBridge` and need no migration: @@ -150,31 +152,39 @@ If your code only touches these APIs, the migration is genuinely just the loadin ### BERT Next Sentence Prediction -NSP runs on the bridge today — load the NSP head via `model_class` and pass the -sentence-pair tokenization through: +NSP runs on the bridge today — load the NSP head via `model_class` and use the +sentence-pair helpers, which own the `token_type_ids` plumbing: ```python -from transformers import AutoTokenizer, BertForNextSentencePrediction +from transformers import BertForNextSentencePrediction from transformer_lens.model_bridge import TransformerBridge -tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased") nsp = TransformerBridge.boot_transformers( "google-bert/bert-base-cased", model_class=BertForNextSentencePrediction, ) -nsp.enable_compatibility_mode() -inputs = tokenizer("A man walked into a grocery store.", "He bought an apple.", return_tensors="pt") -nsp(inputs["input_ids"], token_type_ids=inputs["token_type_ids"], return_type="predictions") +nsp.predict_next_sentence("A man walked into a grocery store.", "He bought an apple.") # 'The sentences are sequential' + +# Lower level: pair tokenization alone (input_ids + token_type_ids + mask) +tokens = nsp.to_sentence_pair_tokens("A man walked into a grocery store.", "He bought an apple.") +nsp(tokens["input_ids"], token_type_ids=tokens["token_type_ids"], return_type="logits") ``` -**Pass `token_type_ids`.** They are what tells BERT where the first sentence ends -and the second begins; without them the NSP head scores a single undifferentiated -span and can return the wrong verdict (on the pair above, dropping them collapses -the logits from ±4.37 to ±0.58, and a genuinely non-sequential pair flips to -"sequential"). With them, the bridge reproduces the raw HuggingFace NSP logits -exactly. +**Pass `token_type_ids`** when tokenizing by hand. They are what tells BERT where +the first sentence ends and the second begins; without them the NSP head scores a +single undifferentiated span and can return the wrong verdict (on the pair above, +dropping them collapses the logits from [4.36, -4.39] to [1.10, -0.06], and a +genuinely non-sequential pair flips to "sequential"). + +**Skip `enable_compatibility_mode()` for NSP.** NSP needs no weight processing: +without it the bridge reproduces an independently loaded HuggingFace model's NSP +logits exactly (bit-for-bit against an eager-attention load; ~2e-6 against HF's +default sdpa kernel). Compatibility mode applies unembed centering, which +subtracts the per-input mean of the two logits — ~1.15e-2 on this pair, varying +by input. Both logits shift together, so verdicts are unchanged; exact logit +values are not. The legacy `BertNextSentencePrediction` wrapper is deprecated and cannot wrap a `TransformerBridge` — it reaches for `HookedEncoder`-only internals diff --git a/pyproject.toml b/pyproject.toml index 6776916d0f..cf3924c0b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -146,9 +146,11 @@ "ignore:HookedTransformer.from_pretrained is deprecated:DeprecationWarning", "ignore:HookedEncoder is deprecated:DeprecationWarning", "ignore:HookedEncoder.from_pretrained is deprecated:DeprecationWarning", + "ignore:HookedEncoderDecoder is deprecated:DeprecationWarning", "ignore:HookedEncoderDecoder.from_pretrained is deprecated:DeprecationWarning", + "ignore:HookedAudioEncoder is deprecated:DeprecationWarning", "ignore:HookedAudioEncoder.from_pretrained is deprecated:DeprecationWarning", - "ignore:HookedRootModule is deprecated:DeprecationWarning", + "ignore:HookedTransformer.from_pretrained_no_processing is deprecated:DeprecationWarning", "ignore:BertNextSentencePrediction is deprecated:DeprecationWarning", ] markers=[ diff --git a/scripts/capture_ht_goldens.py b/scripts/capture_ht_goldens.py index 25605d9660..fdc0e8a3eb 100644 --- a/scripts/capture_ht_goldens.py +++ b/scripts/capture_ht_goldens.py @@ -252,6 +252,21 @@ def capture_one(model_name: str, config_name: str, out_root: Path, skip_existing } (out_dir / "hook_manifest.json").write_text(json.dumps(manifest, indent=1)) + # --- short-prompt loss gradients (incl_bwd): the HT-free anchor for + # run_with_cache(incl_bwd=True) value parity after HT is deleted. + # enable_grad: the surrounding capture runs under no_grad. + with torch.enable_grad(): + _, grad_cache = model.run_with_cache(SHORT_PROMPT, return_type="loss", incl_bwd=True) + model.zero_grad(set_to_none=True) + _save_safetensors( + out_dir / "gradients.safetensors", + { + k: v + for k, v in grad_cache.items() + if k.endswith("_grad") and isinstance(v, torch.Tensor) + }, + ) + # --- long-text loss + final-position logits; Main-Demo ablation anchors long_tokens = model.to_tokens(MAIN_DEMO_TEXT) long_logits = model(long_tokens, return_type="logits") diff --git a/scripts/capture_tl_checkpoint_fixtures.py b/scripts/capture_tl_checkpoint_fixtures.py new file mode 100644 index 0000000000..99fe1bd1f6 --- /dev/null +++ b/scripts/capture_tl_checkpoint_fixtures.py @@ -0,0 +1,66 @@ +"""Freeze legacy HookedTransformer checkpoints for the converter tests. + +The legacy TL property-format is frozen by definition — historical checkpoints +(OthelloGPT, grokking, ARENA) never change — so the converter's test inputs are +captured once from a live HookedTransformer and committed, letting the tests +survive HookedTransformer's 4.0 deletion. Rerun only to ADD variants. + + uv run python scripts/capture_tl_checkpoint_fixtures.py +""" +from __future__ import annotations + +import json +from pathlib import Path + +import torch + +from transformer_lens import HookedTransformer +from transformer_lens.config import HookedTransformerConfig + +OUT = Path(__file__).parents[1] / "tests" / "fixtures" / "tl_checkpoints" + +BASE = dict( + d_model=32, + d_head=16, + n_heads=2, + n_layers=2, + n_ctx=8, + d_vocab=16, + d_mlp=64, + act_fn="gelu", + normalization_type="LN", + seed=0, +) + +VARIANTS: dict[str, dict] = { + "default": {}, + "gqa": dict(n_heads=4, d_head=8, n_key_value_heads=2), + "lnpre": dict(normalization_type="LNPre"), + "attn_only": dict(attn_only=True), + "gated_rms": dict(gated_mlp=True, normalization_type="RMS", act_fn="silu"), +} + + +def main() -> None: + torch.manual_seed(0) + for name, overrides in VARIANTS.items(): + kwargs = {**BASE, **overrides} + ht = HookedTransformer(HookedTransformerConfig(**kwargs)) + tokens = torch.randint(0, kwargs["d_vocab"], (1, 4)) + with torch.no_grad(): + logits = ht(tokens) + out_dir = OUT / name + out_dir.mkdir(parents=True, exist_ok=True) + torch.save(ht.state_dict(), out_dir / "checkpoint.pt") + reference = {"tokens": tokens, "logits": logits} + if name == "default": + # stacked per-head views for the head-slot placement test + for attr in ("W_Q", "W_K", "W_V", "W_O", "b_Q", "b_K", "b_V", "b_O"): + reference[attr] = getattr(ht, attr) + torch.save(reference, out_dir / "reference.pt") + (out_dir / "meta.json").write_text(json.dumps(kwargs, indent=1) + "\n") + print(f"[done] {name}: {sum(v.numel() for v in ht.state_dict().values())} params") + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/tl_checkpoints/attn_only/checkpoint.pt b/tests/fixtures/tl_checkpoints/attn_only/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..4609d8375764c00170369642cc7fd32394ff2bdc GIT binary patch literal 51189 zcmd?QXH-;8v-b@M5>=v#B2md8InAy{Q2{|k5EU>WNks{6f*B<z6BrOsk)VhmiV20; z)#iZd!UP6_U=}kXW}gAPc=X)YbDwq2dDl80-f`_EYx>{6uIlO9T~oVfvTGkvAt4C~ zp?`TP2n`Slm>(3da8Y=0Sk%ZszbHTBMGHe59i)YP{_iVRBp@I>BqS&xDmXkWQpznN zFeoA@aBOfuloTtZ=fMi=dPxe<L81OZfyUE)$FU*-QQ;8*^NoF@qk=;srF<iT{G)?I z0)3-`!Xm>Xd>5NZv3+!Py~G8bA_Ypnc|ojbPZuf2kZ`}KDLrpWv0`4r0*wjV*Q<}< z|A0l&toTGBktkNe-Brj{)>S>3m2_2i6;5WQ(&l$rA|1ubc(DDv1_;!P!XthEHc9X| z+kd`b%=x18dzhx|fcXNJa1<-s#ggb^$@Q@0dsqrltfB|2<kerm@(&3QSQu$+VjL1? zYP^IU_y@|29W-CuRXvIw+(lAPVwGKmlh`3Wl%Y|qiU&LFUn%~q>K_zyR;`Dk{)?i~ zMbYe`XhpHZJ=hUma=+W+7ZnvIm~MAg`wx%>tJCF~XcVj4MG_G>C7Z(P3Fb{OW5Ow{ zeh<<hiZ%3Ljs6ko#v1>Dv}8yALYnj-1yi5Gnsy-tv!BA6^&rioSPKu<@*j}?0xPZl zKw7cZJ?`54A||u8UG^ojc0Dj&6w7$9@Q*Mzmiz&;X6<`mqkhfZ=&reQ=mCw1V#j*0 z<Ng6O-Pe_M`~zgeI{k9g`M0?n-#d3MJxJFmc7g{x@gI@X*hzmNZCSTp$jQGU-FuN9 zJ;*6htfvP%^&gP_0xPHWB1Hr%z|@4D-s7)V@8o&+OrB2<ZAKJ3(}SJ$kF;s*>_2Iy ztZxr(PA|=`hvwfy3y5L^J=h>G`F~k`p?;AI*||NqUjgh89v;#gzw^Ah?KNZP3q}x% zVuQOz5d2GK7q|*{eb@_oMhJ;wLp|6qFU5ZuVZ75MH;-{__@5@5vx@{kLGZEypq^e2 zBqEB9^kAcY4>i+Y0UP~icMEoLkH<^8>=8ul(ysNgtOv6^ie2HsuKZVwKO6HW#*$st zgIWCx6WfJ}>%qiFu?Ze*;_n}Usc~<}CjCjWVv~O@l9Vpgub55kT_kBe%=9RBjR(8- zADM3Kx<8rL?D}8K4L!{6pv~wF+RPs2#wd1^2fO(nm|a1;<xi#!o7Lm-)?dhEHoIr3 z<n+*Tqu6a8?Dl`8xv@L`q}j50J+z&_<}bf%{t9|Xg;8vg2fOPZNL^vO`wx<t30wTj z*FC?@UrF!$?d@TfMzLic?7n|wPGigeWSX-3e=!gI#;oXN9_(Qriee9Yut)xZ*%i1) z|74o6l|3FG>z%*jJ@Z%9L#vKr`5r9ykF;ql{z)@uX%Fp0FYROx?Nkr#bQF8WgFX9u z;Qm^J-GO`V7p^Bt%`DjS0-PX9FLdF$qx51=l-Bg%E=94IJ=iP1N9ixzuPD9xr+t>} zH33i%rGE*4e@AI;6nou+z2PM#2=$OK-?_nIej$P__2!>lt=L;VuHEjkMG&ENT}z|B z$BsKu>|GD`-k%76_Wqv;Yqp^W(fA9|)P-p7K|F|JTRhlSFKK}l(P4k52~*jJe==;? zM_p?`ka0ZjhPcWmvrl@a_-PO5Srq%+gMIM_$lrG=+x92OmVGHOMmCCl)y0ubVqbR| z^QMRKHi~`c!M=}<VcWYCDc{hbD8H^;jQ!A^)pY233Ct8?KgO_~-AU8ppomC8D#m_t z6^`^06Et-@z<!Qlzs&D`uWNGWhlelhNr2d|G3>XVhORuVH!)(r$FM(o-tS6sx;y;e z#-A|}LW1n8r@gPBy(bBZ5Ei@{6Cu*u)Rk8LogYQ?>24J5ZIt<~vFlzOA=WME+bbCK z9|XqTRzwE)g#<;2cWWhjwJQIp4GWJ5^$Q7J9uycM+1){^w?n_*J9OQ9Bc!_(GQEm_ z-Eg|cUl`G^TiE|U6S5Hlx`nd;nQ(E0T(?mEKNBvGQ0Nva_6k-1qZd7M)a8$FWI%XC zP-KKscb9>^U55N~mqmg;K@njQgSzE|d*#aiEEkZYL!u&-yVXN_)k?psyQiz$l;E(4 zq1{TAf1_l5gTod@M@0<lR;vCR<-%V|wQi;Qzfms!rPSzFYW6Ds_-woF3=LT%&_-x= z3y1d#2mZ4|;Q~8HbW62+rT^-C7qd&L)2-C)RsONCyGIvHXK<kJqKKdfy>6|3uXgai z81PFQ9vKy3&@DFX75^4*rV&QnjmEu=zr|a3?h`SxTVT>F_|2@I<fz-Uo_I6u)|&Nd ze~Y)@w7<ohd3Ohk-VVRTThGQ2VcD&)>Q(&qnf%&1y2oD_VcjjX`Ok!Ggl)Ib?mrVQ zj^K3*ng2|<JOa9fq*wS`y!~$D@9}2e-DOm7m*3*;_bz|O+vsk&L$Ca|c>7)cE8fO* ztH<`Le_L~=zlFYFA&=`;I{q8w-|^<ut#tl3%D>}ne7Dl&-zfi%H`i|EgkI$zpY8AQ zHnCecsaN=0y#4M_R}{H*ODFeA|JC=u<ITNW>CvnFV`2Y?c$?C#_3YLD7H_`{_$}V1 zc8jO=io4?N??feHdUs=2RibBKRQRp&_pQsjTj$fO>l*QINRRHfFq+XVp4ltzs*C(e zN+M=;x6STt>#A)0ov?I!DOd)+-I_VQ8rlD->6%%;Zjt}rqUcC}<H@e#5>{HGEA9x& z3SHOa|7V?Oh>-g4bt2=rU3+en@2@<>(Lv?+a?$@S9R2#sv}>SaZyFK=gLkDA|7M_J zLKc4v^s7ANE7;)Yg@p!%ML9Z{{M)hqWpKe8Ru)|b*e_w9cYPb}y8hpuDhe^b8{oez zDk!|G-sR|E^IsnKZ(BlwqoP8Bx@<^-hss^4RM++Ywn0va_qz>UnczqhM+fWwt^q$J zwWv{xHr=2thu)qmP@%R10zxlBV?S$5)XkwQ_uGM6yA}22yFhG=8QiH$XXf(EiD<P3 zk%a-cy>$$2$d0Bts=Gk>=3^MkmP47jA1&v`p}1i(F>$&JKQ`^f$kvP8*o<4uF7xr^ zN>vbfEGbDEd_Lf!%7Zw2-bUV)gX{6Y;{ovT&?*?gi)S8gt%Bq=t0AO+C}))T7*sCn zpxTsn-s$kIu>Mdn`EbmWzBwO6x4l1&DObLM*zLi@U-BAL5h8|Vul%veX9svZ2}BWy z!)r11@Z4c0ne+Z5;o&TL`M4QXJCi{CaVMHK@4^SRVnnP_os7I}M}%dkQOk!_<d}FG z8u&WEj`oFA!geQSd`?HRstkObFoZse)*uC+3ORA#=Wx@-i#z=NJWRd%9xvI8+U~Dw zhc#96usUTRd9v6IaCLw5I9^PS+NaXZi=Oa_=OC(@C`GRXY~#_rSE0@;jM#i0OFwOO zg2v1&SYwM+s63jQTZ!TgeNFf*F#z(k%^2qL7`VSJ0fu=cLO{_MT<OvZyRG_Co#gM_ z1osL${=F_YZuJhnRe&@}c5s2=tQh56h;Wm9j^N8l$@oZn4S8%7j1FryFtl8kh#VIq zO;4Q3%zHm^*xm@7FyuM<UoV1d7b;+iZ3?K9bwuccGHuc~=T6^EW==@%!zL{YdT?C` zw`!jfU88J?mwfv}{M><9JUET1o7@jR7V8pgJstS=))zY7JmVyGrgDFkHZfm{nlU9O zjb;UIWgNa;#)*@x;KlSxrry>U&h(cd#>oR{;89~pEm_K?ZIvZU*$nJl)Br^BFl>Ec z2`9BXnOBNyxDQt*!=7iBSYf3?rB_LVu$c?Za!}^(Zg#`@TK+UNY$AHES0Pd!*^Ga4 z0EwB>3b!p(C|s$<=|4P)MX4IxRy_shgk-4%7*ah`BPutr5ZA`%!tR3$p*S-Tln$R~ zu2g@ARTH#PtX+Z3ony$IFi@q!0|Ow^bRU$|o`8=P#`MkW17No-4->ZPkx5VenD_D0 z#KONF*tM3FKI>1S$8Ny(tG2Y#Q;M8xu_Yz<PNL@iB6zq*8~T*y^0h7~(^DCVTtSN? zYODz$YrZ!!-<4*<vLk!Agnh^1>RU}T${Iy)C+*^rj^v_M)>b@__!$f%glNE<_aM@t zO-@RzrXoVq=w|y>RCa|4Q){L|BeqN?4(hSoHibRV=f@dtjJF~;$!{Qu^5{d%tJ3hI z={MA=*u%(QOe5yQ{h>ChlD}5t83rbAzzM<bn6hLq%$l!5qX!jp8@IP`!$&)E$KQ`9 z%Ubq<^xZY6E$4}=j7HHzvC-t=`Yq&Ek~pqxxB{Pk?tv={hJvHg7HU_P3tDp9@a^y~ zD1OqLFIlcfPQ057ORS%8FN1WbOSm;g7)X%QgSF||I|bn5{uol)Z*zOz-Nc*CcHD+n z{V1y^!(8354paBH;rmsu;IscR)Qsg3wfZT<@6cg%*Bn4Jy_Vza*dw?)`3pBZZ5>*@ zdI+5^%a|hlp|nUVg?lkhfzDiML`65I!~K3{RC=Bxosr^8=C|hX*Q|=bef<@2+p#U2 z+s|)&{qzu6Jxq&yR9Q>PUdfUAm2zaT+6qj0Z%%KB*%5ooeh{@i2g3LD<*T(8(3**! z<X)NyJ(pDteVzN;_tDPaQWOoqv}6|eS?|O>?E`4ea|cq~vX;zfw4*hurPTWB7JBkf zAM(k#2JKtck@t7iDB*91@W$Ds&T$wiUu{7w+>;p7*ahU$gk-osiy@jXzQL_01DRPL znxQgAh8C_Bq8kHjaLCcosJtlwG!rsli|Y?i+3E)wm!}ZhmUOU=$;Aho^7;9`6fQVf zpv?I~td&@c)iyTtRBjD7J|-UyM&95i-4Msfh5GRF)f4DYuqI=ciIbAAJHWnZHof=N zLU6CCgM)oMNJxyL{joP1WX@B6`h>lN7Bh<>#KoM9OsYay(1>yI((w4&N#=A;0exsR zo>azUVgrdJXPp&k^N1*7Io*VATi*z)W*U(X>^1!Ub`WP+vXg9gh$D2x{p!x9eq@>7 z5PErHEU7u8fzor;aq06r;4!Njy>CfU1Lt9o8Gjl+w<nRmmKYMN`ciB>EKP1~@}Wvo zi^$#grsPEKT6CD50GE!AA&Y0Mfa!ZSBQLR@D=Jrm4Bb`C9P`QGGwB{aZ4rgb0SdU} zuRtQRt{=J-8_^}QVf=)+c;XPHOa}&jBvX&<LjPlF*c`r+Z2jvc7c$-&_xSKgZmk2c zuXw<C<^~Y+BeO7D+=UEEcZ5!#`E<aLQdE6Z4%2gEU`dQG_uS1MPLR8t-QG!b?9MXq zsnVh5$9MCVjQ+yp?G>dzANr9ZpGOoD3ej@(GIZ-Z8g1VBQiHh?eCGwGWb?(jL}hFU zom10?3z}UC?-Yi?+WqJ8)=*XY7$S+;u%B4>V-HAt^MFs56PS7jpt@OCaHBotL=MPM z##xCrEaij5?FMdf#$VuYD}zp5zMnp|_`yVpEvBD#0;$=&h<0Tjs=WTmLS3n>;QDu& z$LN2_JWL%O;J=%B^khKyb@Q+}u@V>89>cg1{Ycu-cE%(johP5Xgq$+Jf$mZdFn|9( zSgvSE-JXmfk2LEbVTC1`dm<Ls%De*R<xAXkY%(a`2&RUE4%4Ewdw5MeM_i*k3nyIa z!!4fn0i-UTgW3xn#Hy5~EghRcbngf>RdRs#N<DIC8;=|o5hhX()?x8=J?i!SJl+_v zmu`RT&3qX4g3D@?<i_sHB&LJLVCPK+Zi#Gz-7V{h^!&G6;DL4giPtwWv+^g}EtY)( z%-w~M(cc}<QXmGWq^V!OpP;cQlpCvb7TsGy(IapH9X50m=$n_2d}T2RoR@<a-+aUL zufy!7=~+>0nPIfwz<VIOoCnDh)v!n&$QAbvW^>vwD6;8GUp+iWKg=z{gU3{H;oAZx z+jSEJC#XZAa)0{sm=O7H6+rry0QW4=1(gzCFhw#txZqkDldO4wTyz~pRPGE$3AHJ3 z-{Ls=&Yh-OjTL0#q?2~plVs`P!~IC=(bX7eH3)-MpTY}IbsB!MgcNRU#EOhG9K>!w z!)uSZd%pQ(?u<OHZ-fH9bKn{?@Z@HEu&fB#AENa9;JbMH;|xwaM2*Zn4<x)$56wIu z;itf}ob2))RAXB$sCbWnVP&E)wa%8jt#+dO^R`3%Xi4g1@`AZ$=!%E$?tphDzNFvX za@cZZ3_hB-08UI3CI*{4XlbVb*&S<yVdIKP$5MOnT)Kn22(zF@=Z_Ez?g~!I)h6fc zcA|*+OK7~8K_;wTh1-pn<C@Q@SX#iq?H@8!ckol#D_x9RD_=9y^Ew%2D=9L?{T38+ z9`u05X1Z|1Rl#3JV8iApdgJm-xLWRq8$@l%6uB+Xr*%ABnJ|zV-<G0dJ4K1s#&z&+ zWq*<|T9yRf(51!cdok>2JvZ0foccx>5WPjM+=UIJ=`5SU#QB&a{hF$c+_fZfBe<Nt z)e8Z&>t{J#v%N4|bujs8U`HATFQBT&n(!f-(AFQjVRHN>a1WJ(C$1XsXxt3W#5$Pl zZePpId^DbZ?AwQW-EziOgDdcK%{pFta3_B1za3gvW@FfsQ8eYWFUdR{0=v#_rpfNv zuytoSbnrrOw`Bs{d+(3azT^_6N3Xe!p8`0WZEL~m%@vrOd4X5Cb``eXID?a}uOLCu z5_FEy5PD;P7|wccNyR)1;6zR(<HRYF`=VyV*w3;mv~L29$hrsvlZ?o;azfP3jmOoE z#$;OoAEQ@iLsW$%9W!GaO{$m$HQu49@$xnn9StYsSQb2>eX;BQqg6IWO-OJGq3imu z-G9vf$M+v0p;-I>&8y4*^I1dGV+HPaUH>)z&Hq2}UohzZ;;YO59bZaL#|mudy8dha zTl|mxw~@<$MQet_r{<~TP2xldcV0;Ho6F!_<1)HW=Lqum&#PWkBureZ*H`;`Ugur8 zya&7tgQy-UMU#GsT$Ju*INp|z2L$_#gv3>T^~x;RGQ0uK^;4oB6)b7|u>|fx-9bFu zzY}~99A>g~)Jec9Rbpze8x$KJz^0h7v|xWDvU=~i$=j82nM(`yyW~n#x7E|<l6$z( z`X8|-$q;i5bIHtYnb5pgh&JbEkl6NUs#$m%{dY9r&ir(GJVKKW{cAtI)jmL{T@|JB zj+epy`#o$G@_||}Ez<GUg-0iQk;A9+s7~cd?yhA3J-)jDp1Ka9d(DEO5*Oh2juPB= zQVEj-4&VZ%d)&BADYC2mEI40k!ne)3+)Yt;8hF`?e(@=VgAcx=!Am8Y_jM#qOgzi2 z+Fb%Zw^GTb+bYnx^d@Bc+0x6)-SE-de%$gGqlu@NGM(~z5zKtwf?Iw3`A;rL(0#5Z z442eGuH1Nms{PdIyO`^E&GiwS&mBi}STDHuK$Z%RooCw~oXr^ug)=&q)38+e8dlC; z!^KT8C&51+Gj-#hV&+pd7#APF2vr>>lO7n+<98Lv0^K<{t+o$0BS;n77x&|YRdPZ0 zl`#FhnxUcB9BEo}6p?C5!aeufnA%JwDqX2XT#R;OjaoJpZSn(2KNlv_{S&GxsG;fX zSh)L0ii~_XisXE524&Ai(x`caJ}8K$vL-M2zNO=d+RS3I<!Cmj`yF87$7aE)bEWw8 zxh$!@NT}luH@ol4c0o<jM0(|+9bSpAVoudhqjAe#LximYiJxvqnoees=k_U#Ms*q2 zIYpSvFK$51`q^}jbsU`)UIU-*9HdP*?Rjz|Gr4u9QG|DW7umF4ik^IP4YD?W07)rL zrd@uX-NSTOIxird8QEx$Ct}<2?z$wfns*3}o&L@(oiB|tGDor2?<H8z-Hn+6>0Hga zCAj`x8*Gf3Kt~$S<CYc}F;$;S!MkZLu!hIc`||{B(N?2oCpS~sCw83k#k}MB4@S_H zqch3uVOP2DUNdNB*-+AA-^LTV5Kf);CSg-#1y7CI(}xH1q0QquPA|zJq3KU~OnW>W zd)WpWNd+A5izn7DPQ>)3Jld>u8MK|#>BVh>$@<bkG<&WHY<zMS^o(-A&152UYL>uj z;WFOFh$G-)(+<{=Kk?B?X(F@hD@bJv5s#N*w2<QgF?)s1Z``4HuMj*h(165DD^!{^ zAEjP=X4VY1BU|PTq$_rY(3(BlAnJk&xo~(1ef?d9&Kt3XyX|le7as)LhdBWXg1J~# zmq^a2C&CODHE3#j!kFJ!$z<w&z<kF8F#I|pLzRb;Suag!vQ{PNoR%lE(x2kntW56V z*)x!T?K7whi6@t*`_MfZx=hTE-83?D8&rm$1G|G>RNkpSJs?*LtA4J6LRyG#CY%I` z3EA|j)era@D2aloCeBAkQ1?40sqpO_QuErG9FT5<cPBVTKKnFo(2k~Yr9<gfwi8xP zEW;hm(NL9TPGwGNaW;$OIi097SSS;RoyjINvS1F?yPgWg-sh>xgK*wVuZNsee5>HT zu#Il@7!7^%HEI8QhoS1$cib8oOX<-U_#LOxrzWSM_Q)xyu`}lGc~R!}gH^mYvr0fg zRRS)V4kaTNZN>0swRko3HMfksz=F-mAbHLoTD2Ugk7xrNRJ;i?1w%N0;R0BA$CWDS z$1>%zMZA>>Md(u~N_<R=Ku`ZF^U%f#4<8kUlh0Pds5liE{P8KbV}$~llfYt3{(An; zE&XXy^b6+LsYM{=ZcM+2=5kv*^zi2HWEk_L7_1+NaaZ@~k-JZ3&@H1Hczm-H)lLm= z#QT;URAkgaw742QHRUj4vDt;57>K;P*EQ(C?8`9U;}mK+WHQgg7Jy3yBD?k-&*<w| zXc+jBE1K$09?$MWYp?m!=+6&$9nAws`ivY>y6+~J_wxZJ=f{%zGnrUBunz+?4_m!1 zV&YfC?c-{|%`pW{H=pL_mEVUt8AI|grxK^G9mGAfISi^D8AMCW05qTGfL-8LY*{sz zaS2Z00zXcN=S>Q<#zTdC8K_Ns&RpaU?o?nVKeeXf&ktkTGfC7sG>Hifh$r?wTj=XN zI~dbGoL;|WPENj@L3QpKky9^YvDxYg&wf}97c#sF*8MyL%_5h`t~JZyq(d%wU38qx z_cDT?vU|}koq^mrxs2id492U-i!T49%kwmKhVtzhJdfA&;C}vMSft$!Ll>HpDYz9D z>Mo+2Yx7{|Y8Se3=mvUog&FDSlZSP;)wwdoTVSQXpR2fb1#izy0l!cUm=HRe?3|=S z-m@aytVPS9YJfPM`oo+42nc6Xy=RiI$ELuf*OSO}=_xR(E|NU;ktc^$kAQsGZXz!q zNc@`2aa*nzIWkd>EHIc%bB|r%wND76fkQlL*bNu(2#DrpF0I3wVja>c=1vOkK4#=( z)M<6>Cb~;2m+WYI$*tU<0(BFO$&WlssNNhzq;r3OliCIH@=zAOY}!qSUy-Kn8fjE( zjx!1s90sw~7s2>!J=E^&OV9b7$J<LQ;lTB^Way_k5UVUi<h0ZU^&=y?yss;XlFWdb zmuF#TsygYP^&I+~ImkKpRl}(PBcNlx9LbS45bR|Ur0{+m#S$KMJn<awMdXmdo7}+R zwG$|b)pKUUj&mB}CPdRrlbStF#8See-UcdUz>Q=^t8OUxk4dFL{qiuYN(gdob+F&v zQ*a~1izv?V;~h25KzislDBQFmJFE_KPwIxF>#RyN{yG4Qr$2}NnQHWPMjdW1e+|hy zC296Y3bW3g$H@1(m>QHq^5oU&Nr^q&p^eUHWAYkQc@jjkbtiSf2)HgRi|Kzk(9}vJ zoYR^|MMO<7E#N)pS$+-%b#CFx?u;bM9FCCC#(FGXxs>)7awMMvb`mAMBF01NI-FkO zNjfq-X!GU}+V^4|cGxKsd9itP_`RcGH{F*upZ>u-jQ9c7hTr(1m*OzZM~AFGNQic@ z3f(h>kK^C!kwOE1=nz$>9dAB>a!CrxPpCrkwrR|~VdliT^&aV4x)vSFWgu4LC`a?7 zD2^OWI{Y$;;v;d~y=5rv9P$i9?g^1c;abRJQ*lDhS|WN>u!s5-W5y_HI=D?3b&Mxa z#(oGXagPAS>}&89r$DySdt5nS9ZmX?NFL6JK&7-yPJEm@wb$QZd*C#X+vBn^#3YYz z`O}m5S{9Rynj^_jn@>Du$21(7q73hSkx|-z7t8ENz^8;raLh6!jW)L2PH$u45tBwp zz#O7{<2~bb?*^}LBFCIa66b!#x8SNHisbV7BQX7%F1+?A06FhF`0}(QV=zFI$UVPf zw|{<L-r8B$(eU{*PFUtEy4|uM_mbbEzrF~B&HaFqslp`PP!!gFKFdUSw!&-ATx^u= zOKZlraWB@afos{xVB*t>g~|SuHLT<`lqATD3;Vc9-o@P8hQ9W<+LjZ$Z*};6(;nvY z6>IPht_1JVsc1X=GsE)2(PeWYoqm2hKS0(K_QrbiXBEYx=!Bir;*2LT@QI`|pK6mG z9q~k>&7ZzWbs_I&tHFL|Gzw2Vi+A{TM09xue00b`Q|Tql)e|z*W~B(Mj5~^$u!uz4 zKV=>{$&d?^bI?yRk_4rHM5cH%oqT!>6&6qCONBmzvAn^M@ZbWp4VVi~_Jv@%IEqW> ztHailgGgIIGTk@g7S4(I2BJ6b<LFNd$yn#Tbepp}u2`)Ex`O$f<`6{IKU{{f(lfzj z^#?rfSi(Qx9l-oZlO}J)n^F3e9(kZSkFHgH0V~F`P}(elp+0uFOgoM`<)i?!T8M6| zu;+%~Yy;YMhNNoeqKu6<Nt+%?t%e_=HA_uszhXt~nC?Ozv`uEX<BrTVr8r#b?FN@3 zQfS%1P8?h{ggn?Zn{hTik3kCa;NA=+D9(7ou!+YwHJLh4)RV)iBagV`s~hN&kV-6h zTmkA|inysa&f(i)XOdtB+;yW<I3z9!((lhl3H>4{G!my}VX1g@#~`|H*<#pOF^7I- zyvfTYa`cDM>*^L(P<v^1Cv|iC!Z~^W>esGQ`9In;i1Q(FNU;l{kxmR5nkfY{-IM5q zh-f<Np*%e$ABU+M?%~=|BS=xkHQcRu32rVMNh^*NaQ8*ubLqQ=lRi!l@mX0bjMwf* zUu7GUX_B{@SM$~3Xwo7SzA6Tj<ZdzVzp%LNRVh6sY)T`VhR_EU^Uz2^joSL^P?0Ra zJ`O%G&q<cZY*!~E?hhx&je|gE&RTdhXA(&a$fv_b4@Aew)p);UA<bH31Z#$-!^##h z=766pUH(>%iCC0HCN9*W#?|lfo12c^!w7rMJ7ffXr@a|m-l+n|v!d~_ci~IpcTRR- zCA5s3O&cH8;Om7S7?;_3U{$U}cRWs~YHkXUZy82sx30jsqqFI@@O4DGEES(W@gX-I z;<?$p?_k$l%ad(dgZ>4AHGWbaE8bd?bB)>Xg6Bm_Ml~^y_CDkaX6L{QnO#V&8n_iL zam>w&yWn&)OWqG}WajkQNtLec$FNTdcE0UH=%iD7!G1+5R>|+@I=uFx-j5B$tK5rU zKI|IYd8JGP*C}$zyYxxwdkz&BSrKNr51IVLnv<FK3S^k~u<?pJZX2XS+fxfMK;kOc z-wWs7jMSw$iVk$uwiG)5f;r7|+(guxb}%C}GjXx<3sCsUBaT~x`Q8qeq=YX->#khK zW%<kC&W{1~M_@7+@1jTMw^`WjIBZR>`E4atqn6UwO{#)=;}E)W^C~zrct7sU-AOg> zO(bUrOH!*zwoKffLxOy?3Y3-g;X}wsx@}Y%Ml49-X;vRY*_agkvE2uDs9nbG@}lI` zuu^9ItOLMXtPDEI{&4A`Frf{ZwBK`gJXKr98}jxmGb3sdwqDkwrF{-_6WTsw*sTL( zsL^R`ZvwKX&nYl6e+B+zHR%k$i~4u2!_j37T_sn-yv*+dIfaUJXJ<0m@%lU&8!rRp zoB>rC#0T@+mr(SS5IHtngS5&kll68E(6ow2wObW2Yejl@{V6<qYq3z*!3e?iU#mY@ z{*UVq5$$QzZ)G$U734tP6@Q_FApetSFyRd6#e$7+IgWlLO&f$~@HI2_$%E`FRGn!> z^0Pvz@%2gMS#5vP_~8(Ejag138;4<$K0^$5tJ5}%RC>#gp%&$h*fBF5huWA?%-(<% zCzQ$lkrw1h>Q0P5orR}teaWYqV&3QTKk(A)essBE6xpdO2|7FdNq=8qG#z@GnWgT; zq%4~Q-k0_1$@PI`v%xzqGD)!iZob4%_e$o<20Vq@6*73FB@K<6W?&n=SzS>Pj}Jd7 zlgL6@$f^0pcZg_%$z@XX%jJ>e&R@H6<zLp!raVomDLRpg{Sd_^36)I75d&J#ZbyII z5VHI3(Tv66r^tclYw4)j`6Mhhg3O;Tg^TW`Q{9JW(e>wXdgzV=ogBE1loh;((8viS zzhNF+PB{n1tc2+FmKl)cvmQ?_T?abGjyPyXBDkkGlI5|tc$GpTG?V85i(fs)3D;}j zprjZM8(WGc25H#1B#hTqk&Jf?Yj9-88UE~pA~c%Y3!8$HAuQK~-f%lcKRfejoyj-6 z|0a*Q8oGv8b5Vgz8|=p1o3sv>P992Bv&1pO%z(b!?u^OSDsVXYA+8Ci2Sv>RG(I$r z|GFuU|0cqlY7H1j?Ft9tMnSEHt-OT8Rn>@6<yvB>G?LOj8~BgM><5>I$y7onm>XRr z#ff}9%`{iNfMi1+Zt!1AkFA!YA;yonO|x8S<i{LTtI)(ZtP?!tl40(T40xS;92bp$ zjB$C_z*B!3by1hYHIX*l_MkFuZK(>eXb!fU@;n~T9oj`I28hz#cQ!#wXAYP>k)k$V zlDMrix4=rDMDBJ?3A!I#%M?qT1F=mmsKLY%uK;l}DY*uG-lvgf*8qN?RV8?}sA1mc zQ+A&vbxDoSPUi4~>u4W~cw=KaItpoF)uMIG3S)VqRiQ+DZPvjHgTC~{@?!iHb`$Qe z>`SLVt6}uw5|}YhoT=ZdS~^)@f_s#h1IgZVXz7CupfoCiY)G_1`wIpreRL8=4jaX6 zX9T(YDhE#20m!((T6puNjZ;{sLlz6Z`(DgA1~1o4qf086lc-`9Ji1ereD{u^gSZLg zNW&F8ZT}Nzo+!nG7Vk0pNg|ylk`C(c*K?Ixir8?;8qP(w<5=&<VA**Y6C7JPg>CBi zvc3dHJ}>~IMZ59n{Z7amu!PfiScY2#wg1BQB9Q2_klZd);?`LlLGdYHK`CVf{t!C= zN9GwY617k9$9z97^W`jRbbBW1?q%uzdBe!EYbU`<RfvpsNddVB)#w<w8YNonFsEez zy_Ww9%g5Q`u2OTfy$}Z08SZq%Eq%Jc?j0WSOMsQ9U*Uim@+5!HXtLS&8k73nl<!YB z;aj^bp24fZwCw(CPU55t*&d42^VtHfZsTWIvC;y)oSWh6$#z^gAf9+NH8J-eU*$w} z?Lbs9l}s*{BiYv9VB5nB7*uOWFV((<X7^&?f|TKPks^uon1<%#?+eak*utPbv&oGg zFJae6OJX}G5%0h-s=oUI?@Gu9&{`o*Ev&0C=e7;)(|8iCOCEEptn!G7WGz3=Mwc#+ zDMQsS^>FEt0lhK!HR_Bx0G(k2D0@L3H%4^ueA~i_nNU0|{VE45ChVe_4>d?TbTUKk zo57{$*KxaGZ?fzdOiNlLh=Y_Nl<76YhDqBvdZB@N+vZBNG7DkS^jC1cW&}0(xQD6N zS|<2zWJ6cKaEA1&4cyKpa$q%dEO~gv-F9S>0=?)XNiR>o4D!cRIQQi$<Y9e2#@S7v z`=>YHno}7#OIm{Ng+q{#`hz?8M2Yv=e*<~pHH$>=N~4`|fgq|?0v~D#Z^ht9m{KpO z5rkM!w{!B$xO5Jt+*hHu9rEBt$tZHr>nuNaY#~mwtN{-LO&am>1Ao-62B=@qfWx)g zagg@|ytV8RCdK99CPP&m(kGsb*S~@hRiiL)^dT%ZW@ur~b-1}Nl9b<Z!uP|inJdyE zFn;ksk`Wn$%4-G?pYfsKHEb|B)c1nmoP-n|x%CDF#x>!{SbM5wxCU}&EkhHBy}W>a zn;5;z``}!L7&ebw1-{iYK{ahD37IWdJ>$t{>XSB?zUkK=Jp)I=i&X|BF5omwo+?M5 zo=Af++uy_0<oA#$@dVQ^x#RoA+sTlrRxsG{0TC{LU}s+R45k#;V4(I8I$uEw`flm4 zlM0K*^2BM})#{J<bmdombNDk{b5)(T8hmD)ttR90b$bBiTVZY$j|2+1j;lw|X4gl& z*!`hU`Ewob)lNC?(SY69f3-1fc2R%}2Nbx#x%Zi$5^q33>oV3FJOXB~8ue7%#GDwC zNZd2_aK1uI>8^BlSkaM$l~1<Oprmc^D!K$ZTkPpte?r}UZpCvA4BZl_O6MF{OpRZh z##i1O!JyTPeEYlum7GRX$IvwR;IbCh6|8^_;kVGqJONB47|@&!v^lK`-_DH3{Q72? zm2XR)B(A~p-ljw;Dh73yPlw`7wm2zRo=z$^q6dd{GUCGV#7rR+q^FD`BBKV<SSb-u zoBNHa9?+Lc9&v(uKJIjwAa+j0EkUuIOnPE`2d`GKAG%j~)BZ<Gpo(**yMr}})1=X4 zwc$08KG}l7AKa+^@i}yMgeVlf&Scg}4<VcOno*7TA+*N8n!DMNOjA}ap>xECkVDt3 zL7`wOT$NwPyZ6SQ`WtJ|R<YAWx}q;_la#^h!-dE@<5x_ZtTG)X$XmY*3TB4h(4^)Q z79m%<3sRQ#rwX&hh<nUx&T~f!)$lZ>>|8ZW+d6=Bu6hLf^9Pg8Xm_fllYz0#tLcR~ z1Bt4#2$QhkDt=4Z3Crnh(kVI?MZb8GSUo#5;P0e+PRfz*3S+@SyOJ?_w1>2ZKHz3B zbFs<q9d3LsLc^Q_Al&UCr@yTg-%gRI-^cW$b%MON@UtUPN}oaY42pzBmxj@nxGPNU z#rvQwugF;zJ3-%&&CoKj8a{T4QT<0xU_5u&&c<U5w|w|a`mjoc3`#8lY1s*cd9O|@ z?<|8$hgS*C-N?~`x>&roQ3{>2yx@cQ5mGp19^AIy1O~n~+z*{R{#|h$^6};+oNHge z<X0+kPp_I$W$|es-xLT#zw6VHmEriHLz*+Ux1eu3>QUvEFll&H$PD@zO#DZO;^5&8 zd~bI-uJBYdUduU)F=Clmcc6yOzSy5^`VmWnM1{$Qr)4OvK7e}8>qE^{7t#eH&%k7F z2HAIcIq7VP;8Hf90+Fydm{gt!>n@ze`Tk=`^j{`))aNGrI{i8BFKGkw6I}I#uPM-c z&kg#BzvG<5W$B^nNw|CID9G|%Pru%-2Vvb5qRp40%ZFA%&E*XKtq;dwM!6Pmht~oU z);5_Ne>)T&%#<T8f$FrmhDZ1Y`#8PAjilw(Mo=tD#WAWK%<lGqv{X!pJRNZdZz?76 zV=HtZu+W}NOOT_f&Xw@}$Y{n$OcqQnU*ov-9>^N6=1Wfb2A0EuVXmArhHe!mNwx{N zsm6s|wQU3%)5wfnwF4JsC{xixkz{;UDT!=iP;$&o3}_oi-bKgJwVz)yyALm-#~T*Y z5l;^y+2RBvF3y8*v+B7G1;G%%_BP0A<$?8cQ*tNAmX2H`1MkMahxuPd)4L~yc-T*k zE?klh7JE)q?|%3OTW~g=sX7*{tA>+?CDX~im@#BTZW=c_S(l`D45f)vjxZ%w`4Ic~ z5NLYj!SvI~MDkV!yhtJ7-0F%uTusPU1$DBzwiTAPY0{cDU7Ec04L*973SS0y@Ry%Z z!c2?tG<cf}&-r*UjP=Un&$#PCE@#v-cRuZBoD_6u#WfMZchXBtd+;^LR#}Z^e_7Ds zclKgSLOy&r;zcDR{Fs<hb*fijNuNiq#iHYBoaAn4yl%yT`)WaL_|jO2&c4AP5M)cX zC+y=tdTvK=3F$-X30JyeMIra&*mK<bLW`-n6GLaJ_oq*`3(f<m#zSnnHgDzAHh!I3 zJQ)zwNHv0&QyD5nYp*_qU5h*T)26-xn@4U;f>k-R&+bpe9A;p$wgy$#8&2My`h-&Q zi^1)k2L$B|BN~Dy1{X$Y(q_dIc6OsIa7;adrcZxn(Ox;!e0344gbp)v8oywx=5Rb) zzXoSU%MjZR3H%U}LfbC`Tw}MQ;mN*&+OaToSewsLJHZ*?O(*z+nrcAycmcO*&kvBW ze~#01w{f{=KSRd33DDQ|0}3gXaO-y4GIe?rNzCRPBBXf~uUC7~wh5wiN$^>mv`U}4 zX$_?Eb~>2xW*^*+)}bR-ePZq%RisHzs=#GQ9G%t>OFwO2f)WX$w45DBMPCDzF{$8R z_uqx(v8Ol)iY1r61vCC)9&~l%INB^K0;$`^kc|@`a5qwha%19b$eo4p#8B`|w4y+j z)S7LfPu~}TewHh7u3rrSZ}(u<?t13>QGcvox|~_Es*|xXaU^$I=fW1NlPI=D7S^uJ zs8)I5N`(#&re7*WNlD}>Trxz2^m%@T`bp@}F@|?hv}lp-E(3eI*Znp~<fec`XDSY9 zQXpHmZzPjOZs#>ki6VZX1Cc#=j~O63jahI@f#!7pgnD1Kn>agzpo|E)w(Bm)l*hoG z`4uo;cn-M~zXwZA^2w$c9U`<xaOPuZ8iX%g0bYXV2W47?^gY`Si(6`#qpGHuK2w%E zHu^aT$vQ)zL=zXxNuj~*HT1@=p|p~>jyrba1@HUmI$TsGimh_)IQG+Eh}|}uXd28X zlW&{T3{gKU&Z$72*GwjPUlu;QrAdAqP~jf9?IZ<>byzAtnLd@PW(rI7i7zWH_&#d@ zZ-->zqV*XhZuoedwnB-h6`jYta+ZtrQzWe!^Wg(F;G)!v+=f&2aN{jc@NBLCrPe>f zEiPiTbb|_AJ8L$7jOq+ppwh{lI`<v5?e@V%-Ia9n<vwKZAQ2+|bqD`iaXbmPi-q`? zlyT<<ksyBu+%j(iXzD2NjPxGD1I>8Uu-*lMP71Vs9z#pgO^EvIN4U2>7%Q#-wGR1& zKf~N`g{lK7AL>OLzqX_K(Tm(xTQB7MxpJaAkKj#CmyCQT$bTkgz=JS1=sN$PaqEfj zf3N;v^*^pZ2&x|ctFQmr`48*=pZjm~KlXqA_z|G;U6nI5N#k29bfCENP1GJQWczLR zGwfXMMML%<;@(c)4r3-5lMKVd;Oz2*+3FZgCTHCPk<mLCVUhb-bs(2KSQbyVan^MG zEe`jMn#orv)DWC$-T_zoO{CKdCz8AIqC~h@iGOUjDCykoN<`!PL+%hUI(*C&6jZ%H zY0Wk6%6EA(e^f4h-?5kXAb$nMD2}9)zo}4RlWNEw`wGuY>rYLkV>qe#W^~wH!8w1G zZ`e9%BC)@C7T;)1qg$4}VZLO3!MhVnv9$Fj_hrx)C`=y+2cj3^Y;R?<q#}-4d2WR5 z<dm`Ss)dh%=`7S->x8iqHl!$T1U;{<1b!)&)KcaQ*bWw=(weR$e#=~(k+lXQ1BX!~ zQ4>08-ziA5R6Oyqeg<c3BS=+NDbbqN6iiqiN(Z^MgPPbb3Y+Uu`4TJ8dl3DL`|<Xy zNI0u(K{qGqlYHNPxHQFpsFpck-~Jlp<d$$||HBGy<MZjn(ApXE!iJ#ix)Ef~BuTEp zU!Is>kEbJZCBR@$4p!bwBEq`q%-v@@NW@=RFiYNy8~A!TQE8aZMMMNb#wBlbdN7Q9 zKctUMwGZ&@Sy6mZuR<FY%&EdE16uKXBbVc_f>b!j)2Ctj7;z#PPdMLZs!tsS3+Zf< zIjaeRMz-UOTN7|X;8Ps>W*aQ~BuS-;Q((6^A8p=S(<S{&sy;Xw6OHBj5N=76kmRk9 zvd|STid}$}9U;8w^JkFK6Y+HYwtjSn#RRVXcpTN4UXA^db!h4K3{JV_D6GGI7@zHn zBWP<u(q671{rDv~ON7vmVs#L6$Q#dI4IvB5<bX68<ANpq!1dry9CfLcXS_U;3?7m~ zk9)4esZV8Tj7TxF^Mg3eE8mZXF;BS<=ZvAOIu>3j)Pr}}b2Qxbg7Y0NOiYK}XXa%t zAkzzH;~|w)dUmol45EYJQlcBKuDwcH1FJAky%bNBXpthzOpFq(z!^h!aMHIcabokS zYU7!8a6E1)4f}bldR$!%n#nAo?bfDvr+OQySNz6&bzOr6OAo;hYkgdKKm}f}c?@IL zmEaT`!S{!BCz@E32PczulKA>YJR>%i_!VZrn|FRR?4A~h&wb8MI-dtk4Q5p6MjBqJ z|BeSM6hXai38{MY9d!qO0j<MN;miKX+{xp2ap)m2Cc;jf8mNm?|F7DDHLFg_(ggd} ziCU=Jf13HY<Sn{H#-f%%q~O`lHYPx~KY4pr@cWP**?4Z1JxOfHpc`wBqiUfZ9f<Ad z<{t+C+dIy~n3e9Z>}LG%mHi#ay@8s7Je$Gw+lQjdF^sYHblt~^(bf+8h6UwQ#k zMru>Byz|ibHHE+7`BEykSRX_7&nDq}Vn}n|eViWZjc->PQlY*@FuF34o(=yA4gO;& z^E4ki1humf-(GMc=2J1AL0n@gObt?mQT%=k%zrzMp3x7czMdR-e9%VU?_0^JD=)aT zoxy^$vx&q+@O)0DQihb~-Qos%9)hRx*U9Qc!5NrIjWB50YHXdDLX%qez&S^2n*HW8 zWaCD@UacnH)p3AJzRGA4bXV|fSc(~YR*c>;&g3pmF2w1mO#);kA(J<Oj4f*cx8`Od zR9Qs#9shxEcG{DEvBIDh+>g{|uICg@Uo%plR>PczEO6Cyq}3V~yk;tj(Pd)z_0@Xx zoEk(qhXC$rffDA)58>pNiE{R{cQZd*bV%-oyC_+wN$Qi_XtT63PPVcD!EtXKP_Tf` zf3Jequc?rYMK*L(|7CowWw&79+hE+Z-v%XGi*R9H1xgmlGcyIhw~;bAkE>_-k!+i} z_-M>05PvtB7)wea9rX@un-lO$<}ECSiFAtJQYOHp4UPJ)qB_@P=+c08EXOk-d+|61 zOczDlXGa(=`80NfuV;27w}b6{L*Tu#hbM3MVp7T&&e2H;brW*&veI|-(pZNZFY`HF z{Ulg0y)U_tJBYmec8c>~#E`FN)4?ZlIQ6%EjF5U2zPEni#pnsn<tf@xm;ObNu*Ht# zsf?pkP{WASJjm^xwuXMqUQV^u@|kn|bjlsr&m34#%hfsuL%QJn_M+*rB=M0vQ7p`a zF$YZ;v+bY3)pHK5v_6IUE0k#G)a%S3Z!0p=x)WW5`_R@a@ubT3CxlKFqt{1G=T_`l z%RQAj3zrYb5X)nEAV1?I^V%SRe3($l9JV=w5AY*%-_nVU+TloQBXg<P3>VU79tojU z8^FJM9;aKXO`T_1a+~_<(eZ+M@=^6YVBA{57$rx+AmiCkZ*dS46yC#M{j-@5*_U|n zrV$`ye-y@VrDVMLI<(JWP`Pw6iIjcBDBl@M$BsWtX1rLBP630lUTiLR>7Ei^GO@sp z4#cO0qp+@je==d@8QZfO8))Xeeq=_aI*Dq_#is#Rz<Pc$S1-CBO+Q+}P0a(?SJxQN z&h*Aht9)RKLlNiqMTZQwE+hO4ws16lE@*r?i?w}>>1$R6B-Bg!=UQdx!q~^KsZsEo zuB;*aOWRN4w$(4d^-Uzza(c{*o<5lA{G<WSJ}NXM%@{15<jMV(G;Z#T2<|b862-8s zxMuBIP`GVEr}f`VW(%FhsmC`nr<d6=A}!M)VnjW^)$A4y85@Li*ynZ$YtM0i9l3&% zw?E^pmHO1aoCi|<Wa&M9D;nsrl*Vz^FwRqpj=>7(Ts?|DnSpp|r2?YpQ6}pC4k-UY zaA@CM5GDBiqT`wKXz)Re|KXG$NVtU{f6f@N{do_jZPO?3-+E*Ajx_FNTO?LRETNmL z#zNDn!E~el7e+)rl}ubECaA~vry~yxB2GeoQSWDuaQLv{43jyYzR5MF7R{T;n5>tO zsjH0<=tP!nizIu6708RhK$Y$8Fn!9EX_`?68O;@O*8&WfyHSedY(grp^Xpme%u5e? z*(L=p=^?6o2?L8e8z3reKXGXkrgd?P@s{`rp4o~Wm=-dC#Ep`muSBfLv$AB6Uo?SE zyEueclwTn?jRSD|QXMk8&jkA7iX>58Aw;7cZ`)PP{0J7F;-p#J4~{YCiD^{^iT<3A zn&t19BX0`mUxT!%%s?43bl4dH>)A}5|7(~jnnjM!_6HN`MtHI-3|BmIrU@7GP^Q8K zU;PzN6!*zf8TVFDe8po-%&o|P06CgzU`HPdY;#!o7jv8)0y|$_;H2Mcfaq0cd^d3~ z{w1hqnmgU)J{z8d0pnl7H<L$Dl~4<hXSFk*5>08;t!%hZ`G&V@@^kp2w3eIkyb$%6 zHi)0EOiEYf!MBIj#A;tT`95O}-uZMJYMWjnlT*YDB*oa)?qjDHxElIKJ%#1^67aR& zkQW+wmuI+3gO-hZ!M7Rl20w%!!pDwd;8;yIb0(EXtrDb2uHIY7zNAUL1huV;caDJg z@VyWxr;i6mxx=Fob*MEYf$od=2~h<KR9E^Y6a7ktelU~=`^I^sDTzYR4I>=Y_8H#% z*o!}>DB<<xH5j8*O((~mfF(6eTw>WRF!el!N-YQYiEp&2Rf-_5HR(_PQl3ua2QCLc zv87;NTn{$8-RX{@h5QxEEa9GOD>G898loG`an6SsxN-D)6zW(`kB^xTOXt2u+gUcy z$T(8N*B6+hGY6wiNg^IC$)Ou}e8eI5w{siSEvPT3kzJb%d24c{$dIi!kiSh#@I#|K zoHEpsX?-h&7EB+oJh~QU?vuul6_nzx0u=l_5F?t8^OlZ#1Ui=&!cF56i0V6l1~v2{ zeN`?HPsNESdncN+*qOoSeXe7SwDoaCj0U}4Fc3yJ>r>74g+Nz(L#~MkF4LKZ3c7Fc z+2TX|@j~AK3We#8mP$~LbAy2)OR!97Cr|FGDqSVNneo(M$PbqpxKnW)d9Kli2t_<# zCi#8Es}9*X_3TABT4PQfZZ~6n**@;T<xo0w#%|t+NG&jT*M~mA8%VC!9IDY@hiov~ zLGH|zW-NY8<`M{x+}O~9q7kc5wM>yHdU(=eO<9^eD;z7M$_2k&jwi!%S<>Npk{o+w z#mHKp<Z?xe@Y{s|=AeBN-Sl0FYJMCKPTXClb4odcp4tuO@!8z`GBxhHZX3*Mtb(5< z(qtlc5vv`|aeccGF?2nJ@#hY}!fE}<g*_bYaR0zmdWA7Z3culq7&ZDKYz(wy9f$pU zQ?b=FpW7#IL-)Bia#Kd_BX1py1+|E?;IzsEZw|VJf2A684jToWlb%#-ixoFkDxV(( zVMM#6FSU`+;hF4Dr9y48q+Na}G0`o=sGZuhwB8Hr?w^FL^*VG4dmW?3$WxWsr{Q3~ zSjrFSL#reliNf3{lB3^*%i`pazL$WotuCPWsUNM*y$@?!@4%eFQ!zC~m?UfF<LR4v zjJkRsNIRoT6b2MPbnqF>TX7DI<{rjlGz&&Z?B(WppXb=r(ayYcMaB#^k{!PU=L zMD{pdWmXDpsQTg)4{Zk?!%;!*nZDZy!!zzPZ;}U~NYK0Llo1Y8(#wswMNfwh9mD?* zd2b$#W&Hj97O9X7nHngCkW_}keeIngDnz15b4sJqJj)c7DP$;9D3zjU5cjor^MD4a zG!Hb=ph<H&*YEl7Jm)#XdcJF&v(7nxSeAcW_jO;x=d<7Y{eEp!${z@$R=4My1N=GP zD}avva!1u};hdxIC7<GOm(s>9qgAga(xy`xvYoFq*=~6<ueUPBF87|(+BR2;Z86|Z zk-%AHmuP3-TESJRnVM7+DW<v`Cs!OMH|1IEvu-AbZao7FmOQ6DuSVc!|Jmf<eFW=f zz7)%58FSj`nZgtOrx160H0E260S)Q>QIfv~JHI;shr_a9rGp87t=LWdQ5O%soQ$iq zb+Op;E6n^60S|_KAfsOAMgKY%2+Teu$VV9Sgz6OdxZMDjZ51%?iZ;5vu*6y3gTeN^ zHhx-gPcWHQ3q^XiRL6a}|3Et`$-0OicaP%lANNqG%1{{7Sp^ql{elG>azMYoXXVaU z|KQ;~4^&tg&5GI2p+fg7*n5t~rUlzrNm&;T{QS!M0v)Mc=P8&zr7L${EY-;SpOc5G zG0a(10;N~0g|v@eSYPA<Ww}{mW7iR~%h#i2O|Ei!Z+{F|zcga?M{=lG5e2_%XA7E7 zt*~xUnXvM7Fi(G_0S#A2aQV{;cogzTtodi_U_8tOb7&jwyI&)wsXE|=Mf&((`pYs8 zDE*()pB(;U{b5?I3)S_!AquNB*rhp=7lq!Wf_oYmxk>>h{M*Mmd*qcv3?1l|;%j;2 zwJyAHw;em*Y0qBe9r@${H7=?Rr*AHgVcz63q;9m2799=7s1Auz4kj4<%YMps^qk1C zff>|ibp(HG0qz<-gm;%JVdwKF#X||MyvSlCCzOVAne7u<-2)w#FRH-d2aJW=oxTcl z()Liv1pyb-wBs#K8)<EtGd?X^2Z`MZNWs{Ew-%0p^==7}<Jh(GamXR^46+s#V{GtJ zc{|>idK@%<KY>WseEe0J#>FnLq5i!UPH&5&)$`)1!SWtmwD~9c{dE`CcNt5OH;?lI z%RDfC_el7mc|}@tKf#5+Yhdz6eYR2?im$gHr`_K#iSfyb*!5U%O#W~WR&Q|SzRgm` zQ^5>0YF6Qi_1!?edntT$$RNAH$f+t1!63v(W_$KJy{h?5OO=no;cL6OeeOXTIN4e_ zuvHJsHh7`OlTm#A+DW0>yC-!%ZZE4@^_lMGI|;kST&8!&Z75=z4`P4^q|UMxXE>w^ zN2c~h`+1!hGLF*Lo=M=>98FidEM{HXz2fMHR<uOR2tE3yQ0>SlIIU#`y&KQ*@ki<` zi<a^=7PIkd-`@N^_ot}*$OV2k6@j_IC;F7_&JpYN@LaPc{pqm}UbN-lWB1LD*FB!l z`M2Zn-@eQ8-yKE?WEu~bV}?*o<SmFle}xuh7*oN)B@lH%2R4rlg@Cs|gz4=vX;r-q zANsNbdJQ(C?t%?(+H^r&IqS5z$Y?IvoRYX39ur_;;zRLyup?HkR>8RX1U5b)?PE?R zLcLlKx^9|Cp7XsS03?sW1G)TR@g8!!I|2quT6pZ2wOn-mov<iChbXC<mc}&EyPRMw z`nN~?F!~X__P+-I%Ds5`nn!fcc|C_#n((#bLGbiytJtCYD`C<veZ1;XBCc<$hl#m$ zu+~eDejIq|Xb@IOdlytu=yg}VOnc$Y(gHg3*iz8wS|X~b<>SQf#iFa4A+No*3}wlR zY?Gx1X{mPdJ%;%-WpyHqTc}Jb?epk+i6u=5j0LB>*TUJ$U!h^PDoieS!)4o!P-iDw zjOwPtZW=Lsds98_?4^Y9#|j88*Gm1)6EgkePMrQ?1gX4OPsb#`$CTy>YQJR>8n&eh zH$3miG+*wc*ydGqWxOI(Jy7BJco(pFTL6<^rBdO(JWTLaLc_X~6tyXulUI6UmTWL& z9a2CW$uo9tMkClOr*Yo(hip1+J-Y{#fLF(O9Q8pH7ffFuj*BW2Rx8-y^1%hdQrSo> zbgYzVObeyaDNAU6hYz&4!%B7-J4v?u$4hu~i-cDzO5o~;x3b0|UC}PY07sOB;x(fu zG}~qrSG7s+wnhaWq_u){7KHMLN5k=4Sq~aer3uE*r*StkMcJw^NaDG>aMve=t_g?f z)HO5K_<DiVJ0GPE^)sk#{V$NDK4Hb+Ih2;ZpK8*w#MTiay~`lsv{xM4zw3`R_nYXe zVF=&de~|JsBc#kqHMzMNP)gz^NIsp*BOe_T(k1QfTv=aQ5E8@<nPqtYb&NR3<Pw=L zErz6NI`m{+5BB-~kkU<az(O>{ka4zx^6V^nak-c*GIk4|`}NQ?@-XBIU*PFhM?SIG ziGRD#pxzf#$-vGW!?$LW{kK@5GHE90zTXG&UDd!wrUjcOeI%8rG?@829;ej0bFq^h zX6UUU-z$DNz45%9PgwDlU(Il?DS{6q9F(28P);js_KHOVKhvtjU1U+HLY^0WM0FVL zn0C|+lDj^Zd+$2|xl$&_S;~NQ-8lfjLkm@2dvnjSQrPlru3+JxFN`)Vg6DZx;CR+f z*|OJFFmGc|RyyM)et*-It3C8^@Z<{dJLqESUn|Vs=#KVV-^tB<&f@RhNfhF65^~Iv z#3rXhwCL+gQ1g6GvW(r_zmDL%T>&1fb%8lOM)DYc4Ny&Kk4l0f>o_~(<T1l&R!||l zk@{};*7)L)@ELSvjs-`I4Hn-9meXgO!JII!5^t8xgV0;;V2Stz-fl}KoeUG|Em!6I zyVFrVYdJ3t@u5+DSJQ+r7hc-5l6wDP-2EbzI*wZ?XccxBLkEtgMWdT5Z`a>~TIWEX zTvHFzcP!wXONNlM;S%{D*ePGdUFpdHAHlEb8GYD%2++iu<ID75x^{$kv1%!)$2}EZ zcqemYzk~F#IRd6j_5b%B6R72vB5V0|XUAUM(aXG%u6TKH%Z)BV;)agA?o}ig^F%uB zdXcX8x&zL+YG}OuiEul=JAU=QPRB}0#DF$K(tRAwTT>&&HQn#P#W_py-p%%C&{;?L zy<sSu4Q-<zFKWrTi$ARl)4}G0DqParf+H_mU}qOEaErYJ*?&7>a<{XreJLD!`n(XV zoD`{7##r3r=){+uztHgw_GnNY58Z=B4%j<}!h1V|aiki05BVW<EE@+a9=r06Z)%*E zJcdZ}!szu6gaEC@nD;=FZ}d4TOx7-x9Xc#S^F5(_^1T}cjXX*EK}oQ$B_CTJ_hQ|X z`=R!>BHQoOr^fWnw7)C_HY~Nq{sS!7e(*zT+n-1l-Cx43$OPd+Ryd{p>_y2j1hc)m z$U2-)l)k@wdhgp=eDpm{(76@@K`-0kjHsiOKQvdY?~n+&x+Sdk)j{GywCA3;EWlw| zG^ypsiL*`?fa=e?^q?eMsLE;&UnI)L$`OBs2(wZ6Zutw*OI4Zrt_y?Rw+(Q|G#4S* zNCEq(3)rQ<#KcLEa(%~?VaRD~mbL80yyy~KIVl0Ij^0DNhW-=dbW^~;c?~bDZ-k#` z24bekK}@_{2QHiCB!Ao<wyjIYSIJ}G(|k|3>aiD29A8V$`4`}q<fl3(^^ixdm<^w+ z+u&br5e=2tD>nwtr+;@Z(#VdAa+S<wq&2*ksGK*OQv;28Z(|2~Q}u&9Ho0M}cL<k9 zFXV5JlzFeiE6OgrDf}JUo^Q-;rWlXA!q;<3{Gs$7ZGF-tz8de1YI;GC-K&(0RpZ3H zzn78kif~>&Y7o6Ct)X4>_rrJ<fy@0ZxMulNYCAid_l3n_<vk_tJbXW^`r6~{yVAbb zU;;l_^PA14*zwHNo>Jzyl7a`V#&%9Kxoxt6&}=)MXN=n^1e*;bSI7A@SFr<?6h(k? zxa8lR`+~IXZ-Z^GKA3P)l@-mSB~O?+mrgt>tIpg*gAQ212oqiQPwc_R-n-D^p4X`+ zbqgFByA5MHhGCxhVp`MELoA&9i>7qbz)vTq<Eaoy_uiNX)yexr&lpv1_kKDLYpkO{ z$7nXFGa<ju5_7}KhfcjZic2QC!luLykg)L)1ny1|YS(IWRlDE9OqH(Cp(D|r%+CDT zyq=z3S;F4m*W=A?FQ|(W64x<#=KhxNFK>3tcCiryvMOQ9tD)Q{S_!+Zz9Aay3P$_I zLxgF;&a#t_EQOi<R%3C<axwDfZMy1zT-=u$MxhGnv|?@%gbdKf;zyPgJ~xMFEGvUW ziF3j0eGsZxK9$eeSp>(%v_kGe88-a5D)t(#iMnHEQ_RaKoUb>TgD+H5>2F`*;_mB} z*H*+rho#Q^BD4_dwQs^B?KJFPa7bFuCQ)|v4>&Yz0vDScr!`weS*ra_aq6!dyxF2X zihsgz{ilqI(1H5|&yQP3Fw(_{?)@P5up4e(u#^6zguxH@yEOc$jks%aaiwXj8I73u z8;-QPiR&lL<G;(Q;PkuCpqaiC!ZXeYf4&ydybhn?eCa5(TGS$&m3owd=MN^Wz(cTZ zmxbuDYXSxj=)+yAwJ|5BBSrX4!K#b7^t8fH7Uq5c{I|s7@l(+lwcrLRDr8EYJ&{U% zzf}r_w;}19E-(C}jP>f9;BMc6v_{DXljp|Kt<9fd4dwB;etx`nz$^M|Xvc-|Pr%l@ z94<}yCi)#q;HPu$LUzU+s`}gwAM_2tQ~lo1<&grPxFRRBR5f&{|3y9bhl4+D<i88o z(2w~s{P9M#_@coOJI_BZJNR`koE^R&ZiF|AXP&6DYPA7<JlBD{b&RDoI(|$?gT%7O zSIImn0fwDx5VNPLab1E2#xB&RrGs4XZ?XxbXE}kQwH8O587~ZcH<{i_nT?voL-D7y zubP<Cn|G&ZqDF%PcW~M#ym8wAMpAA$TUirV1XhC9&N$FXc`xnXtax*g7OozeEhGlq zgfsD#pegB&k0)xPtp62pT5JIfYEgsNUrg}eyE$mQ>kNf;)Pw_9w}_KNm(kv1m&7YS z?$b6s6+ZE4mpIri8+@Z3@vwWa(CBK!Lq6`M&0o?`A)+q@znu)r)H~pKnI6x0l_5Cy zXPH&@c>apEbU42qoc}eGY&Kj*%a8N$W0VHdHa{rUDHOG~my?9J#PAPtVU_PTnd*UQ z4yLYSKy>;b23h<8hr^#J<dY(ew{I0f&PwZq%_`D=G8=qrrJAD0ZRqfBAV=rDh11sm zV6pZhDA;uzx+sNVOz=BW^H|GQ?*D*^;hpit%Aa7CI7OWLW)(g>{11A5YK8l=CQ5Zr z3Hi<`A)npt@x`&Xa7SUW(A&M3S6;6sBdK2<__PydFA%VDj61n|c$1=5KVA&p@Sx8! zDrh@EV{7~IJ-;1fGyaU&FFSx&h77`O(@l7JzkxWuuvON5`+N@WID#ka7|e@LjKWc_ zL-=V*3LG=PD40a9hBggDSa|;mZQO7YhRuBldQzYE_hAn{v27+MwZDmaHbt;!(0g#t z3!uW5rLcByv{<si96csC$mi~t`W-d4_%z-X$K*T#_mVwgr~W_T%c{O~VaY36{kMwF zZ?uOa2`6QfOY3NN)h6&Lk4LMlQ5<v2hHAo}!@2WI+2O%r-aM{eTqbFDb8r`aJlYG5 z4%JY>l3oaR=Rjz57S#rI8h)tRM_ANtE3GV_j>o=F;|7;BVdIf_Auhfd&acRmCp#>I z(z;af@ed8oz4#1!Ln9rv&Ja8@`mjRKL0ovhhLruJrT?9jGmCAb6G8%bOgTt{CdI)4 z<^Fi_&S;FnEj-=OOZ>g7;J^7_p3fTT`hTuJXti_t-}0v<^ri400jT~THNQv)|G(~k z`5&IY1mODr;`#qK8rgq$Y5s@jF9Cr5v-AJo&wFV}x8#2}|7g$<3Aoece|5h})S7?) zbqyaJxHwG07q$DZpZ}x(`M)B_{x^E-5D8-QKhKYqj{pA>e$oH%`bh!0|LpqxcQ?uZ z?SKA<&#z?Q`_In*eE-*iw@!sHDcDnp%+}^lZx;(wI=+ToJ<ozynGLWT_Rlc~Q+ z3>~vEmsmqdT(Z&%BQu>5*Ld*tLWvz0D}&B^Q(@@zA#fsg3%XithnekroEkWBCXCwt z8|tJ!^Rm2?!itNdp+Tc3*7b|z%#XKWi<TXy6%E9YUsmv7Z#HZ^swF%)X~SDHzfhNh zq2S<agEb{>usY`|n4N2c$Sa?zXRIc-y7mxkKa9crbNggNgS*q2BRk-p>1x^BPS?Z_ z$2?`JO0lAu<2W(?;bb({{z}1`k&bh&8i+OOJH#J(WfV3mMtIT7kAJT13iGAR>mmbH zo@qFWKZU8%9Qhw1N4X4cEcV4Z^@s3Y+Ap5W--aa?51{3Osw}6I67GD|24j!?qo-jZ zv@Ai7W1Z$>_nc~STr@>6mwX-PytQRXb$&Sf$47avZ!APauHf#?@u=cd4tHmtg|^lM zz;n-vPHP6@l=AP?YojY`ca^+GeRsgYi?LXExD}L>$5Ppu^EB&uFbsVDhxW~gq>HOJ zLUOgl8+O?xwA`1IM{^5AE3`n?1j5e8bH$cD+o8AJLhe?)A1|G+tjwC&mCf8Y%l-Aw z3z5pZ#b2FF&`$C@1bN<oOt<+|;Q0Y&%nQMcjk{pY%&|1{cRSWT+yyjOf1nw!jl^yd zYv6p+IjS72i<*WSxNKbzJTN>hL@fhZq^}hk=%~VzGiJD<dJBKBkE;A5I}3Xsr4z57 zO}Yb<xEJPA&pIP`eRc}BczdvmPX_)Qr^ga;0w#!Wpf)518t-%$M!t@b@7rA_)c`5t zL$i&nRQUk*TsTuX`d2J$o~wb=b2pRK6g%9Vsf<NujJVw#161g~3GUoHAkNLO13ir< zvB5S0l^!?1g60i;HQEDCq)$RoZ7bM)It|@Nw1M&2W+A}Zm{kJ@;u3FnS<?MmwBcPR zo;h?DuYdT0hMen28+VGl;7T~Vyimr}L%S&?tP%1?-4pHi)}dcODgQj8Ox=@%xurB7 zpN#;#{jQO=#~GsM!66*GB2o10lK^_A8eBfY0E@Dg;gjS!a5d-z*=S3CB`Y_&6vi;x zvWU)Za~Hn#c@6Q8KEs3vFRqVIq9tiMTsA2cihm}u;-&&Ty!;L6I!xqqD(xgoj4`fV zdXqe-sPMv1df2Z|GERRuf)D(hhRq9}kxpY2clzog&TBgktIN;Q=aWvntLtZ(@}hL= zl4M!YTDe2m{94MfjT{6?bvxv3rwaMjpcsw}xh{Uo^#b$w3{iPrFxy&|vgy(`IBndI zCO?RUR6BdxeBeCzzdRy#QAmdI)w>|(j0HJGT@fBsjNr`uTfu6TI*s%A4Gw0*;2Vuc z`t<>ZmaV1DC-V49Ibvrq>A(0eXg#=?!xw79oY6yR_Wf!6v?zw0cJ~DHbEbUXcndGQ zm&Ycix2V_m@fhW(fSwcbNo&I=IDH_OUsc=}jvXJ&Pt4tTe8vgkfrdLQxo`{=9t-S| zy^t^KSczV*hrx?2w}lY{T=~h_2V%kB4yY>MLZ$<U(b?uVf@R_#&|c7!pN~EQQ@&@j zS<7QOm{TN<_~C%zCn_L!q&^Rwp@qIuhVXvNQQp@*8Sd2d<Nm)a_>)ouT$?_M2d_H| zBfBhM@qr_2Uh6NoEi99`ycg)`#s(gxe3zFc{h`xKFG{TC@i6Y=A?nw&NxWHbhIUki z;WB%P?ZJbvuSXItu=MAz4~T=)df}~;ZV);u28tHei#bz_>?=1I(X6MlIdFOuzeqSJ zcUd<?xUqUJHYWTLH|U$-rv@G9^?V-ss#vm--#SRKzef}Mnt{=-9{f=9hj@OHgID4V zm|pG+Cljb*?H6wzkhc}~Ug}M2CT--dm&_?4J_XyWFU5*NMXc*FQ_Nl=<Hc8(2tlW> z)BQj8RPy*CEH5#}q!uU8UXdy|Wi0^DwM$_8vT_ROQBSU;Rq^vpsjj@RhMcEdg;lCD zWGS<v)%c1!whn{Ccis4M$~Ee*IUIgRl|XdR8*yA_85ua#z+EAQ@^-K1OWBjSP)U;- zPOI_ijoECdya0|G$*KALaIX7s1IB7@!kVbgc(gc!a&Cv<-%s;s+?p)mji$EPp!kNq z%_|l2KJOH_r^JYhbWVYddI#aOZhP5{cmttoWE(wRsn3FKDim5z5?4;PpgDRPygXeK zvjR4w_ApmgOBn@qJ9^>2fK(yW!4%qtHp{lCxKrSBWzKneo3DMafQ8?kIDB=I@b~>N zUL>(HI=nnYiU!I!B%)H#U7d?o+9vqBlMdeeqlZ`ROXTeqJMt%O8N_c-p%t5e|0Q)7 z8g~?nL;4knYb-knuEU;zj->^h8Zcj6>DEe{O4dP`*9yqDO@P3#_H6jR8@)bSBc_f# zC_f-r@us0?@#A9;Y`vtzKL>Uv2@;KIn~t5Dyt05yB*w(slbUR~up6eXd@Mij62=ix zD$u`VEoRPL!=8nM5L<I)?n^oeYdV(DkLoqjGpEQkv(7<9_6r#EIhPD&>#5#6p1)-@ z!VK>kVdJ(mA?(c=vMnA6Mmi(JuMfu2yUHH?StEg8?^EGLv4wQ9kGuHF!jBzHzQWC- zzBp%6H?DaohnO{|#SO=vgHf^<zHZXs7lzv0!RIEt4pM=wW7@;1ttI$%Ock9@G=lb3 zd+B?}d|Li<wve3M0ZS%Hp6M-?7@nw&s%b~WJ?l(J-p`T$-B4xsBy~KJpo5h^f~4QS zD6d@x>)Bc0ck53Oo!FC|g60V(U6oOxekd%w(hqNx=<~gHid?>MxKR3~v)H51gPzT* z2e;Gn`RSxmy3}GvC&h(m{&_BrAt$JkSHi`j{q&SddC$dW_|sAjxoSybH@77?Mah+q zc)bGAvKaPQ?&I@sHVM&teX!}BDy5znOqo5l;H)1R;;N9l!qQ3EvTHjn`EGL*70y=T z!~yxxdhj=m^fu$#Lqlcl|GX6L?nuJNhI!aOW{A+Y;1pH<y+$GBF@na&B0f1NoUbb1 z1fTK2xUirvE*ulb)2CMp&XN7$UT+O-xIYNR1@;*Hp;eeNNC$o2YvYYhW5v^(q<pWp z8vh$ZkS)BY>MONyxuPemm+N9D-=6qtSU#0L(#N{ZN91?yUy3!XgEjB($=xJ&;kDx* zp&-MVbXL0Xkkze1X^tMYTh&0WeX5|pLI>>nGL=UhErwFF&RDK@jGFc5qwS0bj#slv zV8l2Dn9LFQdRGU&J8BL4Z#XTys=4E+De-&0b~ylZBz{n#rU~`w)($6^PvLj_wh3pS zb>hOe&BDbpFFyP_fbFk%!QFp1pqJifXq_aj%a6nPsfq}vu1MLkf7`J4qa$?1(~$!* zGjOh{<Z<=AFHF4fTUg{Hea@$K64Pt|maMu3lPyeyGmoy(r*%D1wP>|GH`JK_KJUvq zx>l@{c?;Il7vSg!vFNANKTL|?P1Qv-CiJk}c3fBJdVCZ*nVsM;i=Nn`{~jV=)xztL zdvxg8J8}IteLgxS9DSTF%PQX4@%yp9+^o@?zJ$B;+MRi15;lN4h5|pYo{d+hI$!`7 z!`{YN3b_>l3swc<NT0!w9597#RYg)LJxf295mfIjq07U5((H@TFrZANdAG;GJgs)1 zboMRm+gk$-4XMJ<UQfjHsb;iER|EBncGIV{9q^<oO}JH+1aWgezzP>{^g12Dk|zr* zZ|acWjWi*xP!}4{+T)r9VK7PM5S0|GU`XHD5VCM9>F{ctH-0}&EnG%F<|%`s+>6JY zoy9i~M~U&jlKJg*YyRk)M}c<FXmI#B80xBt>(WBxRkwnn>47m^NX&t(`Gy#9!w6%> z<vLD%d|2F`Y{pAf<Eg$rN*GofBlxKa+<ILJcMpCg<tlaY&aDCLa$W}y-JeXRH{8Wh zL9@_5C<^3C0v1XmB!ZHisIx~B#r^mPIXhZ~yh%eaIr$ZANUpE^-p&JGJ_-`XDD4O9 z4Q8Uw%5h?K%S#y9y+t^sd=3`<v_MF<#ygUpe&b&W4O?b|>!SRydDTExINbt&4X=nA z3&&u2;WBvNZ<ox{GM6j*ZkOv{dLy(*4#I6MuAK93COAs0%z$#cicyv_T=rLko6gmP zOV}q0Rk|c3`sCBa7n}KgP$UkRu7b9UK2W%6G%DJM3uf1+fO^v=deU73qg1-^`|yQW z;5;3SyB;IUx8Wdv`BnHZWfX)MOk*_vK?g?|3dIu>;Ym+9sH!dm&6GVn)vuY@puq8F z;{Xnx=zw(^=c(~az3h5-b4r)L7OPZ0iPw8ZvU+4Z_*56u@E*gtR`Qpg!`C!_>M<(2 zWr-`>8{wj?Ui`MRG0u6s0FRB5W*YoifQC!w^JvGH;^Up695s56VD`Q%S5`}GpVM=M z@4Gu;&WU0e>^YR`v{Z3|>RPer!3yd596(LuRE3K{l84LfB*g7MDpp>Um_8wLp+nke zns)63eI2xbe{|lCal<NTTTwVpd6x|j>kG-YgE<YqbBfkpoes5rHRPY=CK~Go;mz5d zIpXnt2pL}|s>mEz^|v{Gp3@#5S{1>WbHy}Xqa(-Z&!gJQZ|PP;BdJUN&t1}5)phYS z*^lybs6TuzFAH-ZpTA*P<Zy*DJ9NT9BTUfBvlkcgZlTI4Nhp{WPi?O>aop67v^!Rp zJM0?+e|J2m2<*Z}A70C@eX9kFnJ%0RCDeSUg`O|@O|h84T{<m=Ub0M(d4CcwR4wLR z3wPoh$@lYSuC%vkR^uyk1K`^>iP>aT=eV-^m~h8u3(Tvjp~RW_m8}i!`9ii1Ynmss zwxnPd&Uh~C=-Y)hTukQ=OQbwRks5w6{=_qnBI_uda^$x@tW|fNHYPgaR-H>!_Croh z-Qv;RGl55M&jP0`X-<QdANJSmN%`HqQT<sUyS}M~KIcc^(QCJ8hS@(c;;{yf5+m@M z*%t^K8bOg(r)h9NI{G#{vr1HF9@*`p=n|>VyCzlB+n1f8MqVU#dmhGz;=@5z@dDhD zy%mlog@b4=?a$BNhF9NC(4iy4h5ynYhE{#M{6DFmN&&k6On>;&Y6bS(AgnNG0O4pM z>7E<K+p>v^i+%{lB;Ufpry1hPGq)gTUj>v7si0Knx3Xtp>jeLMNu+x-1x92=!}-%2 z(P~g4<&D?H30Jbf#!31Px^9PXjdO6IAQt-F{Q{>I`@+Q+%21mz4Yk^x79nA|Xf$%G z{9{!Y8hNih>jzrnnv1EzM1vGDX;ohwW6G2>U=sIf-NXD+0kvISxmEJa+-N9;%X_SO zsHDGVzNir6#~Aa7EqUD9Hh|TQyW+7~iD=-}39K&8MVj<g+~_kFoX+i+`Xukfc`J>v z{FMov`Z5sPyti@3=M~_s+DgN&o~5e4rPS)3EkBmlg~#@r0ztP`c<17K6-_NN)~x9Y z9jkW3`~zP6P;(ic*3-jzH3LvK<2v;^ZU8<l5_@e)43A&1gd%^3;o@(jan`!~<R&{w z=Nm3#fM+;&dAA$BFHVMV`(T{Vt(vmpd-J-S-ZY_KG+Zs-C*)n71W7h2v@}(Z>@KO{ z$}It8=#fs<%gwn#L-NPF<<i3G+W2G6A#Ajm1b^2%iPMIygnpeaVDqMzQcZV~dZ^CA zQCZS{31-8_Eggmrw1~o9VG*(zvpDgGzNY9jY@OI&*%!2LYjc+IC!ui5EFsA%2##-h zAU4(N<F1+%>h2dzzFHdO(6twiy?Q~Yn^;b^)5eQ-^|NV~#8(R){+${O!sy&%HA<Sa z7#^F>7xJbpMD^H?Z18a;-b8yYS5kmm?-aP#{x!Ixk4XNN0&aX{$$c9#FvdEE3_~Yz z$B}dJ{MiTOH_?a|NE$&q-9B{jM1ye6T;l!=?t=D9Epeh*0eOxq#&w76IcE4)nsC8_ zTh8ymq&`a6q?F7XUiIc@4w3w1(mXgE;6~rle$W}s1PVS9LY>i@Z>aCU+Yi6f+ZEdc zlXW+s%jd@w(;*ME+l8}I<6MksvSQO2H^_5i68D_>m0mVxQdYw^icddINfnLIEwMK@ zJMQ6C&*fazb1<s?)S%<z!ohE806n+##K?WxqUXp0sA#vC4>h@f_#hEuo}YoWc02jo zlU$hHe2xRpB(i7Sde95rL%}}XxOql4oT-*^52LT-wn-7CCM0iY*oOKE(KKpHBW-9n zAvXI&uuASlc%;ar;_JZHIqfh?vsvcRX*$+kQl;_pMu6q#H^PRAm9WiAg@;`lh1PRQ zX~KXUxOluT&B@#k84Yj1Ud37VAX5SJ7ieRkfE4dOk!IS=<?U+D;BRR_N$sK?FMEH7 z-_JflAIX#4K6e`5-w{qza{EfzjubIP-4ZXDcnU|yPD8rj%l@@*VA`}l^!S0qXP(_h zR8^iv-fsnd9OZ=V&4<%_qa3=n=L<NE+bOPc@t~A-=HlQSbMk9&5tiJ|rxWw;Rb1LP z3&qz~aNutbbi5h?{Tp^c!Rk6_us%i#%LW7QHYEGuSv1dj4W<9Kl$dXCVNp|GyxPqg z{&>4%_{!lhIbOt1x~UM8rh~7}XW)Rc{v31THjG{qilr5$<n>XD3_5u7nZ)Zb_(~dk z_5qY9#KEKZ&K&2SP62<ssZUlY_<h<97jr!@a^)4OwMiylMGJftxLWSuRSrA$=2Gvq zO61`i39BcoV9+Uv&5$6lJVc9{mhKh%K7UJ(qLzqripIl|1xSu*j?_F?ndUxe6C-U_ zfO$#{%v%|RwL_Kh_@;+6K}%6sWLhgu3ZIRC4sNFb>XFdS#ubFI&&7FF^TFtNSDbV$ zkjLAqF!^fmTc^ELZ`K{VhNq)K?@-E!B!22-K~I-h@t$S#Al+p!s{3`sZtJFzYDtT@ z{*^a8+C3N3g<kBT^+g=IX}L7FBbTqPd;;(1DpY)%`jrAl9H!vE=Qua5MKqVC$g2FS z#n`C!d?`bT?zJB&KV4Y>j_nSJY2Lk%tL{R{tT3MDZO<z=x`<B}To-?cksKDNBtNkk zP&IQ4&j^vz)sBNuGtwEJda5FKK1QK$dSG|k7=ALnKUZfY<KaV2yl|7`<5Y<kmv7oA zcDqtcgZ%_Pr+66*3v_w@&?vE6k}0YfMv{X5U`#D;f}dS8aLA2*81>#2JzY}i+8?AN z^(tK4<%y%WMJ(L49}k|_j0E*XFQBQb0h*@A;=uSla4PSotTC*DDrUuF={f~mm7>6{ z(W~j_yBJ)5!WeSv1M!jNHmc0MOCy^W^XDzDyzP!QPqP*&wEqkKaZ=H7q3QvceJ%_i zZM#KgJDOp_Z3Q7ds2e<NaAUgb1J|w$m)AB#R=O1>pw|BWJf~k2?3Vh&?`j9~<3=^! zwyQUvE>+?VO<l2F=t6ql<2SwNvz3xp?4@5L`oL=`FRLl-Wxtdri5DZ4pn9YM9lmx# zoD<zl(==@9UhlD@1e6x13k7`gQy6?nAA;Z392H6fP1)|vLmEC=4K>bJ(<Ni6W@w4# z&@7QAgqVsaG9OaUTV3#o0>eg=1iZCa1tSM^1mAUbu;$P)2x;}f%JU@<e;}43CNxlm z#Myjfnn&)v$Kb*`Ww>S8i6iulvCwR+805EwKl>`6y5#!|zSV;cc{f3uQINc<{xbAL zH_q<l%!ZFdEI9uWmfEUwc<^(od%d1g6!btj%!?LpOXpg>Q?T1HfPb|75c_=bfz}5% zNO_1h>FH<C_-pl)Id%>0mwZs$eEVYm0~vI3Kn(5ItEGp(Z_&rQhL|m@gjnP6aKzyy z*ng?RZ94s^7f0jTo7!}-(UkTrQiGk}mgAj1uGBg{f#;r?53>pu2?woT3w4W<QGf3T z0EK;25huNes^ajXlQYY<Wy0Gv*5v4PyfW3Km=~QGA?f`3>}*_4lhcREn&kJXSDl^U zcV`QyYq*IHSraiu?;fl#9EZnFTqea?%5cy(Nf=l!r(qt?Vf4;WN%tQ@CuX&0?V~Tj zy|@UB4^PIllor@|DH;r?s^C(WeAs<+Antgn%BhQsX>*XYUS6L|A<Y%yVs%rz{NlH4 z|G-U<Z(b%>^#4bTvvlFacUN9f{Ta**BnEcolFGjkZ^d@Yq?v<yH>JM32R<4+j4nLZ z$GGuvqMX(WHkU@?BsU#Q|D{Y6I7?hyp@g|}E{kyi+QQy7j!-l!l4r~)#qyiCK{m!7 zno|Zynr#ft4>ly9RW)?wnG?)#=+1|R&%*GNcgZN*Q|!JtQhvk7o_E}@Czs`tX0F%^ zZ#yc;tJ>0d#$09SmoDvR4v*wcA=2E-=*6&cbR8(adnb<_Q6_fIiNfo1PlMI~0%1>g z{L2J)pM}W{G$x`++Y$`@m&{$m66tuvbwT@zE_xsLMV<0XVz9<inwm71-r1<q@kiHS z?90cXT`tw|r?kZ@Lj;~8=EJ(o-=f~;1dxpvaN4MQ!V1af*h{+4JO7R($A(V4<JV@$ z4xSBFD=qL>;9{X_$!^&5>A2wK-U;3vOvd*eqRG{_nL>`w7oWA<mR-GkL-sIXD=TNX z;oz1m+09)o(malPbTEAuTmA7QWA_G%8|nr}lS|>ypEBz9AsTlSlYDyZ9^98{38rsv zLVqcDzZ#E1m()3!x5|w_4erOX;_-Or$~W1t8K*$c-V4npcf^Beh}U}Er;RgYV5c4j z5jWcNigF$NylodN9x#{c`8XPB?7?=?<0(2^4Rr>rzz-uPv-!jiq}^-_%X2k(d`NG; zegC-Z*^Yapb^jx+?pi2h{VnDW@}DsJh&uH4OB3+GJ?Pse@ZlfI*t6my7@vy<o6f7q zMyf%NjV-|uDoV6$Wf$IYb}PM4V!X0KLFxyl;`tw0^j*V2(o!_pdA2T_r=F$YWmjRV zrxwk=lSA&e6!`uiTj*kSmS#(HRBS5RvDaK<Y8YV)GZrdJzAQ%;b;jVZjZH8!P6sy) z@`0(hTV=g{9dYQxT)MyMs_1Ie2glD3<1t^hiji%HxO0pS?pZNN-n45kCTi}N<`eY7 z$_1n3>dz+%w!!xWoqH!lHN9X;b&$c>1>rRJZ9P0&Webm2N3xrf3tnx#QE49{z0;&D z`K*95a3oZbH@6>(2Hsmh{=keKBE9frjya~L^kmOTM({Ce9Ud6!4x|3~qQbFsdXv_b zKk6v6w%sD!KEMP|?jRi5#}H}pXPEd$7dK3^7fc73u>S~qJnD6iR<FK6fA?>I*M%3b zeR&f-Tkwyp-c(oi_;8m3-hU+f$@hW0=ApF#(S(w>vf5e$zWD1en9e>VD9Lw0XU{8i z>vbY;>o!kVtRrBr&N~I)gm<$19VzG$xr;YBAv@>mQ~j?NuswAf*8IIdT6f-4P+UL0 z@iT$SlO1XA^68cLV+OIPc%2L^GGSWmM+#Pu?)_b5tU2-=Tyis}ocLbYFnItc_n3j} zpBBQgDt93#q7G)Pbbx=)Wt=XtXNT!o;mnFZqP{e{WY<tdiJ2(vIZoXdg&{q8Mq5Ym z?2=bv*qpIEz<(kcIVDo&mesKEK_i79kD;a?*XhXAQTRyjAbaoYgkB{Me8%h^ooZ@= z7d{)XV;@y`uN5ZncpKa+>Ck~&hT|vmtz77Nh;&-};k-{BagBKhuXueBuIl^(y$eYa zi_jcj?i2wkZSlL+0nuUCaPIR+noF_r7unkxQeZ|P{QeO_r&sTX3HPS+uY2n7Z@(&Z z>V6Zf*Gt-1)HNaLXD6I`D2^id3LQH6o;SDj!chI5&^9>&PdWgaIv#~L3A)(FCI&k+ zS_rX;W4Q0|l`MCS2JKiEasS{7F+zPfg}9ku<*${ns-;Dk{pTs18y*CPqjO-c+iU8p zvkYEFY=+yCr>@@jr1)vuK`I?!fp2bW(ZQEC&`{A8cko|W@1H}_%8Hz4V#3ECNATg+ zV7zcyMyBT3?7vX*r%CfL7cbtA<=Hym6V(sm#&~ha-brY;!UR_v9iV`3UunN>ow#sQ z6eMQu7Iu#|#apfEqDilO@H0;36KmT@w&gy=woayfSG;(KcQ$=aYb2LTrZ_NhEwV)v zCCqSxv44IEN189eh9Gb3Y5R>nOZ=t!!KLEM9OwV$-#Rm7tJ(jg{vZKt{xkjI|8~R2 ze|NzD(ElL?=>GqF{?`Aoe=oLM4rIJk<L^t#E7r|ukzF_<>9FTEi+>dkQJ0nGJmgw9 zUCbX#g;g=o<~V_7zaNIT<2vI;zYuUU>x<)tg!7y`nmo_h04}YF6o(`X<;>s(QVwqn z?b(}1l^6G5Ys!1kGUmLXIZBalyL^REpXrjvAeNSe7=Y88-?C2CapGEqE0tHQe^G<( zPib$}L+)a&!489U@P~hI{BAyugZI|cv>vNqt(rStc6TC6-|M34qjq?IY<Fy${{SV2 z2o7ED4}nvLV%(lfG<CyhigJ}Q5+6LV;%p-&OB{&&_eVj!X8{k(iGlk`6Cg_ZJSR)1 za>P(mT2LRy!~1VXa6B$GlH71)Vt=+x>%iiia6xrsDj5|Y<ip#7=uE2x)tqe4>o1+A zkaqsu_d*X0&eWCI_$go=n=9$#jx_kZEl-TAlDt3yz8qI447q9tsR=!WiG4TXcNaNj zT{OYcAC~;c&zXHDzJ^$#j+z=`xpL|y9>3oK@5Pimnl6*(r?^YJy=lL}v+@A#I<{A= zeUu_@(H}0iIv7GFYsc~~>kaaC(GMwFH%8vjBMqGN&aukrsW3LCMh1l!DD7P^YHvPA z?e9$Gq_MuT){SRm-+2g!b?Z*<TVlj(eLD&7HZSLazfJM8RIhbvRur?|2BEV4Vki&X z!sfEG6sowCol3PZ-qMa04I?n$PU6bCXyW;@g)r$^I6hP#AijQb0Tzz$2UZGqC~(DN z@r?Ib+0otCXkywrEcCw)JEk3mLmju^>1Ph8{kRuS_K2jSs5LUh>q97g?OEA(v%B=_ zOjnE^8Nq(44RCI<IiJ$pPXT3l)Z3^Zr56_p33`37etVm+N5xuNSGw`|Z_7dc%o~jQ zo`#yZjokeA6m^$;^&?NOgSLw~l(8iYM$I@#7H5-$=;qVH*aMnakhfjdI?WUp6}qy? z&VJ}Qw=<r)g~F3%5p<xXJ&wNhgr>i{43l45p~*WHerk9XI%dA04M&gAFzZJ8AZb6* zwHrxq*GxEm$d=!jI78;SzSMrFgSc(=4Nx+cW{O>`<aqy=q?T4K_{ndH$M(%ct8L$* z|GlelbisAGdVL)274t|r;w-#YU(4P}kAzpM5p;R$RM1Ti<ei#5`CCaho^p35YAu}1 z7gnc;zdpT(@PlgHLE#g47OBIkea&Qay(86V50QW0s4V43>p^{Un~;BE7ze)}E~-va zp@&U&_^)qwR=Tzewuh}Cht0(_CSowp|D%B_yMBRrmmKmnc|+}b_og4q3+PEw7hI%Q zPEU_5!b=@ZIse#a*w}6?UU$tE`kYl@?ImhBLC+p<t(uDpn<kKY*?HL5-<j2Cy2!nb zo6+19Gwz+%0CT_Qix&eWulO5BOfVS83HO@F`BySb8#jmS!$yjeqX=>%wsXMXs}#E{ zAC5d83x<cU!tbi*Z~{DWm5Uzhygwm6JEDn~aWK2NWn*o%9Cj(4f+ed>#Z$|@MBk3e zY_%*?aFDzeW(ggrsdy-y)}9PY`hKT=Q@_JWi*$0HI)R#+S_P%kE9u&V?l?E}Fwb8e zflJEXh-#&Y(AK*nn96UHPQX_Fe4!&JeV)xbtgg#%-1`SM6C=t0lSJ?=PUAH~GFT1> z<=uCNaM@ucIx~DC7aco5X}W9h$8jC1PqgQj&~rlW2B%6jn@mXhG6`2r3geWCiz#D7 z3{RUoN^JLQHZ}el&v7<3(w?O!UK_X!r|MhtyN)(k-KL4=XG75>y8tiSnPAosFK(Qm zf&*r0vU-a>Esa|sGZ&}B@=*#n^`yk;b+-q1*O7Ql;<<SRx>gP?%z|$Zdx+IiJ<=3z zimN{UqK-4B!R()ke9a)0JM;~NRHK)|z|Cf8R$xs_V$X@M{SK2~9}9S2&>Q_-^PouU zoUrPTI`8?hk6(B1$wncO_^?MDtY|#WkC%jCpB-OlL)!#yHzkwq8yoXIX>f${xZYym z_MKq;@edgJX`ruEx2-?WABr#aXa9lGQ1RIeo8r5PxT*m6l%9ce?X$$(<_MhUY%i2f z-2iL<WWg8RnX(?%(mK%U4!%|Y=<WLtw8wT49gE&i?Rsq%?mSt9LFEJR(Uf@3>y#ya zG7!=6KmfFrjl_UdE1I%)3pb8C!%m>a4@~Y$xtT<=y48nejiXRwfhJ1>xTqxN1Z|D= z6Q;k+2KRCMV2WR%U{#VNe7_z;aig|Tc8}h?AVLRqs{Vqzy9=AV^d(uUA$Ic_$>UBo zgT`YUN4rXONRE964<{{$N9`^HteYmDmU#Gzfzj|F^{k`$shQHe+-N>4_QXRQJL2X< z7unBtd-0)7Z~QxJA3oU-4;xnuWNv&UocONBDX((H`R_G3+jkE6WK;+q`(owza6R7f z?!z1IC(1wT6~oRv0oNW-;cxeu67MVHr5n5G)6WSQF!=#Lag^@AQ!>YW*QH#h{UX@p zISOZK_rf<pKWReF9+HMe&=JFQ9NkkFP+S4Km$l=#lgYe){a}by{3713>__9IT%EV2 z7C-xK$nAbfbAJu(vHj&VFrWFC8c*t?*=~F6_o4?DxIcmpE%vZNCe1SMF-3I#J04Zv zTi}gZBgGBmiG7_VR?-$gj|Ymlxo0Ted3#qJx>^k~D{{r{Axgqu&q$8UOGc%@0=Rlg zi3&VYF+_im>`U!+k{@<N)1dxfySa`2>?r3AdhU3^<qo^*eh~L6I)aPtEQ$aBg_dSV zim$Z?(r)X^q~m*&azFoohyDp@3%|t?YYx#J&E@Rxaa6E-X-N;39KhKtZql0z+lAYE z<9MUV1^8B*D0kR6gNj_!XhE4JJeVFuR`K^M4@P){agq^s3y$QrN7dAGi9YCs7vj^c zOZnI&Up}&~4_B2g#Q+^?4q*>h96s4w)C%;GZ9Cw^|K>~=e7z4=-v07kunycoK{M^3 zI#kN)%ufa-8>yGC8VQYRh4jQ*pBEpTgcCjf3H|?$z;sI|s+iLZ-}-B@`)5<s9NeG( zESg2D%(jzHuawFlR>sY*gL#ak&piHfmU>ptr<poa_~fa()O+y(=$>uD-7ejR$&q7m zw_+#k+@{Rw({6&|?iQgfXFJXHkeCdnv9fQ<w<xFe5ICv)rJ6IB#j_SWfa{iF<%_kv z#;(6$JTD3NZWs<TyQ^}GNf-XN(wRpzz7}0iSO`ZiY!km^oD_QaJcs=0Um?1DCjVTz zjjo$%vHq6xsPQu$cX-x-e`p~3Jbgz?*LC4l8?|M7r@L@bSpqLRz8(u|B2j4YU_R|A zn>A0GLt5yAci!|9kM7(~%g-6%cWrZ4NH@n3U!x#QjFKfbq{5{$2k3r42&~+60iybO zgYGGPjOw3^y=xbvMb{*Hqc$9^H&sEh^BC~|H-_(++H<buU&?k!q@jMcJoeKde*9!8 zYV1mdi8XtL52go1x%(>|?N%fvez&4&DF@+^qu_XGgB^GKqypEcWs3UgGaydtH&=}b z<CUJ-VqC-+J{FiqI?LwEGG~}lamyq))_V}^gEh|_Q3ubvMq)^IJ@s@yD>&+Fa<|#- z>3(CYG*dm3Cis03ZtfW*z8-rE8a9MM^$A<S&U-uLPo2f`^93*>!-tIbxnRu*UEDaX zKMW2zgm1buiosf5sBevsX0Hb`cP7BK#qUW?SyOsur5sk@E71G?bFl4{O0NcL;^X3* z)E5oF`^zlQGj<{C@iNpqKTCLN`b5}XGD2MKd_`QVHVSip&BoT7^SSiTa9G#TlCnZg z#CytJ$gxKfJf2+&TPL;`b73dVU!~8t-<Q&BL526|>4{UNS%uEU)9LefCkz_7jYjwS z3)HO3UTwi_(;6v`eg0M4^dp?6?#V?pdvD=eQVD69Y-V|#H^uckK|VQ$Vbiipbm8WC zDznQ6t1U7vJZ6oS`*le^wh+X5#=_6E{zQ)^iQlE!qi?%OL*ZYjbCmjVTJ+=y70>pR zInC{WX2T^`%;BwKfQp`wIk79ANxlJU&Qh<Vx8yJOaU$#Ek|(;g8q2T@{4P23RVxRc z_cVx#ZkthS4-h>r=%Bx!RR3G<hLh9)QJai0#wbv1+ISaI${Ju(fiGnouYy++w>oq| zKApBMf%LR~+$7sCKI<7ole<dZ0!4w=k6Y-lL0238NSR7`K~G+}(}nXU%V0_eTluv0 z?K#Y?JK9G*f*y$r$m_i#)i)dvZn{2{Pq^L%o#OscTe-xvK6#Gr_S+>I{!YN-T?f%t zjp?Gp)z|PyD;etAPo!z7H8j>q51PKzfQr`(aBsOu2D5s=m>a=#p`(>BF3(<Ee*`$c z$c9gJaHb%|6#BT{7+#M`mbiMZaH8u&;hmA>Rq;3Fh50_B<}oW?+HnPb?Xe9Ks%DB% zW5~ab48Z}OdGsrI0Y9F!o$vpBA?|l>hc^NyLBk|M6^~Nk<*9P|;BStzLY8q`(|cOC zSO<rXwPpTejg_Uxp>=}?!Sw))aP<{dIK-p%=R<PU%O7NR&UxURxk1)(a3L1=IYMTd zejwb<kh|sP!1$>hdB6K;{{F!Q^=K*Ix!R4#wOfmM0}j!RaVILhV*i59^c%8;sKnda zVufS15n%K?jMP5G(<`2Yb4M<u(P`=YZYRrL+5Hv9>+TlzH#EQ{m(zISP8_9Y{-QDS z3&ih~3ZGmR`CiKlnEQPTBwg4{s!b0Bg`Sa6y1563Znoy@FLv;yy8E=+Xfb|r3!qze zo5fQ}%|dtI7&Olhqbb82c=5TVZ1J{8aIK#s3!JTpUSG=KRml!=t#~F=@8yPb_Kc)X zbqD#AzX7(-IuD*>x?p|Bah$&VJ{UeQr33{f?6J2RwoFpyIToHUVcka2A?1pEmv=XQ z*|h_AK}FPd{7F|HMx%QA0qF3>0WYQ-Qf=)G&>PoCNf^!keV;g-sMX>FV20IOI%7^a z^89LLn4n@I+$ixywYRErtE6`@V2}~mcQj(>v{?TBzm)eSU^%T{-xW=oXr5DM6%8~! zd)*b128lvaNSc(QP{yK?DN2;0LMk(nA?<ZHh)#$MNg8M{ONJs&cz5Ug&;KfQyu<mf z_q(p|+g*3ha6P~KUi;Z=KYKmvS?l+kR)nbqY4nDqH&Gp|0LBkB$wnCp-t7Xc^u}*^ zXV6WyO)MG7-s^BoSdn~xs)0{=$+Ro~GCwzG+5as6Wblvs=h7bP(!*}kiGF?zGtqN0 z)7dnf%a0{eD@$V<xzhte4f;bzYz95KzYU*USwt`7mB6u>d+edTqe(%vBbj+qkY0Sb zkNDLmu_a+A!MlYadz7-U(JP;R{GiD?D=N?%9C1-hKm;vsxQ97vqv`Xo0mQiDJ}UW+ zVf}SDuA|@!RAfs%xNq2qN!kv`@vTv*>zR-!yAF<9R>4v(wlvlar<-^_l+@Rt_(?A^ zHgr3h92&%W94%*;UsNP{@)M|*$#B?yG8&?v4(96F)X8*10kZR|CETe?C*CJnCia;# z87VoFtLL;PC$hHS5Wgv)<?Br^6KP^BEDSYe)5${nP{!u68#a7S1FiXDC|~YDFCT2c zWv``}^o)g^KG9iNQLP5$W7X*c+K-&%zmF?zCFl%Me(FB{C~ol2#b+g6Fm9GOo}MI% z3S7@>ZtiBr&)kSCd@ez)TTXHPxATY_++dDw--&$1!bDT|9q;DEAmVjl936HqpV68A zjM<()h`4^$BKEhmNv@v;8L~_W^bfURqd_hEeu)k%v9=I8@+7FAzBgNTT-NN!_Go(3 z(utmmlA-EvsxUHDoXVe5r|WKf;52mXV0hFMHmN9-c6!929?8T@s%<E@aSYYtJ`>i@ z;<(s^eThudLp)Hx@yxoVpvv$GFe}@YwR^V_TGVXW2YT+L^a}UcA~*~#kBOuSclP1x z+3mbh0fsnh|1Ln?K$xl9jy~&Zp=zrEv7c)MA^|qwxGjk-yS14y3{Swp1rM1+Mi<zJ zXQZjU-UM=MtS&kao`(KSl5qO)SSnqk4c;++z_-!@mYykv>8<+cdiM;QV!MUWSn5Px zY`KQBHrPY$*(A&>x641i?GiIeQHi+n_#t)I8T!4EAErvFkmRu*RIo^k<q2=cgaNPF zAVm|iH9{Xj0aHk`X)P19;Rn_TSVP-3YYbX2ob}IW0nsEwc6aqIUZPeFXjgn^9>iy{ zJKaR-c=8p#?=YZKZkUnB7j@YKqn8t#ij5>?-Bw&SNu6q!J%%O1Z^35i1kyg%7|)E) zf~V89=@oYydiwGrs(iSPm3a|GHRMOIg3o-3_>!rx<NJM#TsEFD<a-V$>}EmkAr11$ zOP9X0)<*5bn=ofoJBpr>C7+vxsoYovlt?gziH;GBdDsS05Iu`Z=g1KMqNT)fbUF+k zq(pV+N0GD3{pp^AA9%k@pTZe&bxa-CkH3DLH=P!j2yVP&tmkw%tO^X!@<|5ViGGMz zij}FbWEC5-SBWcTc#5M%B}w9>>uk<p0di!*YqnsN4V|}SJ(X+KBqJx<&{vk9VUXNe z9B*@*In|yGCMTcs!sX=2oiS_a9PYcIZ{rWJ9b`h^jB11zQ8$@`$BSWlyErW$;mmxj zYDRe>9lAVBjH*o1A^i9F%p5)Y<1)kI%jzN~aPEo#z2#s^4yc@jy!Fw<;7m;Z{Z0j1 z{9K*>5Q||Jd>V;s)Y{?YVo74_<N~(}<>*r1DmXdvBpte89|}~x;OgU-^On5$0=IS6 zz@!2@Y`rv@D*J}B8)HTgCsjjI{nZ~No2wYVD{U~#G6-%bH{z3zYsrJr!mPk9L1MA{ zILurA4&ED{fcmAvuzIiq@lqZ~SAJXtUJq=j0JDeoljNt*4}8OOx=+AKaskS_88WGG zmid@?7i*FOFzL*0v@J3uV$lMmGfJ26>(;XoJH+6j(F%IMa0`g=pFr2od(p9GExD4C z1{O0X!fKmJXqvViJvUp>^PPnlk}XUkhYw=HN-u!!-NWoU|M&R%%_I08UPufVXtCNQ zE>!r?JUZl24!K^Q%-XpRqx#SE=x+u(@TL9>JiC9GsXV>~pII#>$px)gA<&c`%l96e znue10AsaBi$&ar3aFAxGd18Op8L(|g2)@HMd}y7E%4&IVt|pdYgM#Tr$&7s66h4?c zHlDhA6~p)6_RzrlHLOYOB=&e<4O?X5Liudo@#GUs*_cPq*$-)r=&xptS#vW`YPLGb zU(EehD9>Y;I2n>SYbRPOJDUcG&ZKW<MADM{Y`D((%81DkjN;?R;MB>*bu|abjiO4l zEVLlsYZ_7URuR0c^uq~Z*RkV*8m2WbhRH{>X>pMhagmvXqj#B730+6JcGP<qeXx|M zR1by5_#!5uV-h_*HW_?3-9wl1biy)uB=cnf)C3tbl8>}GZqXQAG59O0H)k`ic7&7d zuHs~Fs1li-6b7C`!|36hGI-1BylmDTOXC*$VpM7bk>xZyGB_>D7q!=zQKHjm)z>$0 z)If~L3b@f`IajRMEKU}hT0@?_3O#md6();J0$%$7a<ab~iB5AysRhA|s8JmFs;Gcm z8COrc@B?N=DdC9~5hQ+p2zQR%2Sz#)HH2&+vP%b9U=udW9>Ds0arqPFl1O6hI_eh~ z#AtkyfiquMfR~pB4YYfQW_l->dxg_scIhmlKkzv6f-kZXFVB<8Mr+c>={jb+F2Xgw z;+UMhA4WV_3Gbhof}B-92)}X*cdsaCUK)%eZ<hp9E8X41>+0{U%B@)HGT|b~SUh9P zBERLA#MYum`EH(fn>9V6ButWr$&;-W;?&o+feCGr#mV#?^ZTdkDD9TRY8E?Cx7Xn~ z|N0J86d%eAopl_y9}8ez`D3B!+$qpgEuqhIv*6mBk+e3<h2oeoRH##fG_Ouz>{e|; z4;^XZllK`!=WnA?l8Z^I^Hg@dFQ*focNnDf-htaXWtu)vgkxIGA~k8!WUhN73O3K9 z9pV}E3IA)Z=Q9y~54h9&o42tWMhXy*)<75#wvDb5sD$wIj`(Kbd{#NP60hs#z#{$r zth7WNlcD<&hN%bRk)0Xr`F0tOC$|Zr*cRAQIGuW`ZXs_XRf!IND!4UeLUqApYSkP~ zZ%5iwsm`(a^Ipg0oxN>NlorG@Mst3cjR|#u#1CS0MUFoVKOF-(dNCxe?h+&7w2+D$ zX3>?^Tu!}qF137mgf6kqU>s}gc;9dM(Zu8`QpxAdJRF~hbw_HT^7~w>RcVZViX0zb z{Yxmly@^bFAV@!E#*>DT4d_;L4Re~5NOng9TjCkTMkp4+x39To9(uz_Xw*Kg2UrJ3 z#5b9hcM9U6W&tMbD}%a~KGfxWCw~0qf|vZplkYOq5bE^kWE*FkEFwa7F5gJx?p=V_ z_A^+8u@caBBOVl-b6}e5Q#0f67zm&Chz^w7%Nu*k6prcH(t@3*h_Kj5qV!Ri7!CEH z-fJb;(MvDFcBTSvYvzJo0Yl5x^{Coc8B+3U4_pmzL1t$SmK7IZ(HkFnP;e*7eO|-F z?K%#hQaq`X=6!(M3RJLq4zuj$I1)SrNWwe?WEDdobD$U|2OeY%It%lkae1gjJ5$!% zy#Vj=Z=lDc1?ZO(XQ{yyCF)g?OhYf$qTj%sXkq3<T)q($klfG67lq&@cP$ikw?fPP zR!~w|kH=PAWw))%hHD4g@tfU1+V9>rC@D0h3;8_Qd4t|EA8g-XnX(Ai$29_7wn~ts zgAw$4OE|r4p-wE_BxzNt2p*E#%nB@cjGePjVOCoy)6n)CZ&UwixGo}}T>zZki127q zCmKVp%yz~_M^rei&^<IYB#H*5t)a%l`V+_RF>uHJGjH;lMB@7XCe)u;1<}`(P$={s zo(MaQR|MK|(BzNc&YKK-%J%TWR=!}FT}EUxUpW?dSkmz9Nc4{1N<7|5gXb(ckXyN! zEOC&f?Xh#nj*DCAu+}@QhmI_#g*=Wbw;Y70_dKcUo)$PckD+C+0wCGKlM1<VJeK@} z=!PxQ#Kez~Vz$X_<5M*<a@r5}lf52<w#<jDR(`6p(SeDz{)jDcLTtvx`}zFbb=XYN z3)Z|+rq{Diz|-<rxbU6R;d;^l_Je+)VBH`(?eGy+WM~>=dHXUsF>@<^Aqg<x?K6xR zu$T6e9|`(-M;ZI?EjXd53=iAg;w2ZvlCbctP&U($)+Fwx=NDEm7ley3|GPNbFXJ<; z(VRpJcbB94Z*mY=RRP!ZBQQ^57?D)3W^!iTW=hLsfZZ?%3vSJ!<reFq!}KGl)s(~8 z3u|cL%Ph>cwne=!G34}I9r~TTXEQUC;m}Zfx_yWRIk~O{*y#`9Xuq|2GSb}d4?JZE ze8Tl~d^Ja@h$yVS!B5OvpV8{-ci=tV9)IWhtH!>GC#^$Qz?bwWCRNIi?Y~}t9{Csr z%5%m+OJf-vIm;twZx*6x+IrmZxP;gm>(i%~4&X(R#c)?Y3;1epG3Ndou)S$FtvPxG zRv9Z%Wv<6%+Li<w>^hm8I_^vaohxuz-8QrxBZ+f`x3bUGSJ7>kw}3^X0=*kiz|-Qi z(3@-{nNqF~bXV3VklGvyW|ccoP}PCN#S)T|o<g2#USn;SH^Y_aI8@jn&H9;-W%mhH zFiCA5<j2VukdyWb10@A$t>tqp9#@8SJDOoAZ#22OH4jW9GGSqoJmrPP6StYc^zZ;n z$P?vsFET5Eug(f<(uL_T>(5My!C12Ou`kr6FCh4vJCh5(FmKRBa0risXYEp`rcwd} z4@nUwISy5?@1$qsPvbec(Rkr{8xvT-N8DYOVq~s0J@;0K6l^YK%Wj84yPz12II;>a zuk!$bSXpv!;eKkrAd|i1H<$*VT0vh&+0b3XBVdO{H9OdXQ*O;$ajAak5F*_%l_V+O zW~z0K$$$sd*eEAUv@n_2m06KGE+$c3t3+R`>?9IWTwT$gr=YgxD|`^2NZKcb!*chf ztbeH@ak6>>HSWbQ_YRle?Y9Pa4cVX&U6;ReNf9~?li-Q8#6m=>JCP6=j)JXUQFF?9 zF#F^{L&BD_inDn1#U@jb5=^Gk_l{)bHD=RYVU{$dVh~ALWKO&+XOr_DreJ;XPJY+{ zKWecugmo9V2l{4WoQAIwYR^oBL$-P#68xFO`l{pFWJ~%vl8@G_>yR|FOs2Mm!2=JQ zm;{9gp1naTbALA%Czl7|mXrb%cy$jI*H0z=-YO9Xr3-ACdn4Q#!jH+xckzCvB6V+_ zM=~#PTzbEWlaFp(|BtNSAN@C}lO~4z5AzTIn7`(ya+Lq|hySep#en0}`G2wg<-f{b zWWaI%{ImW471vO2{vrd8Z|DCse~|&lVe|hhf04mI?w|W#KVg4b@E@MP|G0i{*~-0G z^EePY#bco9%~5=QXdua*aEPqQjv$v$%p@DMM$z9LIUS;4uI8dBkr~}EkStyM1+OHo zC3Er<*gfY)lBs%xtmO1lhpZn<!f$UTAsI*6QBFy?=kD*|R_Y4ou_A=u*AgF;NK=J} zX7DBF5gRf(n985{#<V+kV*0Rqc(;nnW1muH)h+kaEjNZ^#-ezhS?goQxIl>>4G_hR zom#NwR0jkk4#yK;BJx)$uOKd54lw6cF2r@_;ff(nWN!-}@&-tfq=&Xd#ZDPil0@jI z%xHXBZHrSf^YEIMG8K8ON+XAkqMi?4;xT7cdU`n@&FaUakAK&v16)n1wnZ_TOd3v~ zlyZ7IHWFm7w>^3N+L*>KiD!mX+2A^}eNcWNlXr0F2y(?b1$|^%^WTbCW4LE3HmnK* zOGR^fX(b<N7^q4+Qz&zy?HE*?a>m?69T0PJBsqT8l5S-+X}jQhOsy+mcih-TJoXQy zbBJ0&tfC3EbQ(eO?%U9I3v+f^z+n{5)TBE*3b^kd@9>~R4J2d>68ZL{nAE6%GZ)UF zhu>|d=~0L9k=1b$a3m2ciZ1g`K26Hkn0pkOJBs0;$$YZyv=df-e9GLI-Ug>_g{g($ zBM_3yWYyfcT$Yh6J56*ptyyyg3k~{_1*aBKzcx3HgVU0TISY`DTBTsTY!sF4NT;6) z2}#|U2O1`UnDuraCmVGEZi!@&xDrtsBmWpy{vhz`lp-TF;VP^T(;x?gmO;>+C%EKX zDyTIKr7z0wF+$-}&`R16W~7!gpS4ZtO4(#cak&V-%w{y3-k*Mxoq9RnX)?YopMh73 z`jcESQ+P0rflHsZgWgPOC{+uhYH|VefYc1!ncs%H=C{M0_r{=Ib(1)MQJ_8<;k1Ll z8hF`)G-KQpoReLQV?HR7{Z|7a)eLbQ$2VRPmcX_)`jXn9^<0g`P7Lr@MxRGtSpG{q znl`r@G`u+K4`Y4ydV~jAYOY6YSIohQPhHUMk_FW)96=^NF@)kw1yV5mG3btr;#oZT z9d^7<WM4$LfY0<Q{Cb`rZl|Q8ScpGa7C#I1rq;8XQK~e;K!#pXJ)IxA=_sI0Gm7oF zfp47nXx_9uChw~yY+IiMr(P{UVS(pxuuYj7CvtT--c^@lrH7Ht8HxExBAo8?zAvnc z%m&8&bOQ3vEJ1?@sc_kJ6nOT(51x@7__%Nc-3I%~`RHY+w{I#95S$D?^Tdg?+7GCn zQH!a+mEwjuXW7=G_h_gP0^=mgkvbdE$0wR#zWfE;<|sv?p67x~qdJ*C+m5JziiSDz zBiX0`3*Lk&BgnpeLcDn0&zCzy(r`~pE~Gu%3~z)g*=d5|#PByD6Q-)5_Q;bs>isq< z+5Z{jSsiCGbnMZ<d{4f>nb{crV+@<H>pB?jqL|L<wdOWBk=a}yi%xbbxMa_v+Zzjb z8>i}G-Ao;dPu4-d_#9|*;50g3`O@ecgQ+j;i9vI=VS=+OU8@;Lt6oNNT+E|MMzt6{ z@NyN-E^r{L(>YFjuPyAb(YuI_OFd4g*JhXT9cGwHuC~VKFm5hyf_XYa=zMWyda!sC zQ@Bx(j+?!kp7v3pK?&A4*O%j%*p<v~QxT!(-j(8Rt3YU#wIUOAhhR*p1M{wU4&>t- zW}oXkJXSpfziBJuLjiFbyF3eo&wj+{h#zn_U4drLi6tXz)o8PkEKS?5!<)nPuSd&Q zu*3Lnf<xtScB@P@+pu>5+w60V-21#2nlmi$$?75$o0^S2tpUvW#fvaKM3L6lXd~YM z7jV%^B7BD@(;KE)AU?2#chKQF6gYXa3g<7e?pCHq2l$iq#FxCbo6TvM?BsHy6}Uvm znW}1*^L94A1R=jW7}DPyV$|<riq$1@_PqwVeQp{t8Z830&y;9os~UcDh$m8l(IB^1 zn$~fguV?1XrZPvW*~S+dY>d!ntWS5nEbrIJl(-LIHS7hkpq-zDSRG*V3*6{{>SE%e z5KM`>AnClJN`K$D8k@4LKyX0<wrz+e{k~MQC8{~-FB?VQmhj=H1UJ~|-GNW6Rq@1( zt&Hu2Y*gKHi7EO#E1!l3kll;8e#b=(c_QUnu=N`E`(EQ)2u`yA&r>4#Rr`!^L1;R> z<Bg`fRa#)m2^-imM49L=uEXf#(o|U+akH`yd2sb09?V%lQ@$;roeL=TbC1SJPCxQP zGe3aWjL&pp!&Y)v%oElH)idF_l63R@U99tV%3SRbhKki%jLoz{+#;}t&YY<QVx$_R z`sL%o{m0N%bTIOpb%5#9TKus-55GI;kj<&zS&`RKIM@9wYibrwp6Aw~u&M*dZ1JY% z<SL92y9JF;zp)b=G>HP&m$Asl35ehTN-Z*Q#oi=VqF$1|o)QUK<1=yCBy;AY*Kq1E z$cPM@`WgLPR7m*NELy+ZgOLzYXZ=0C;y0V?=%DWjVGj;7`~0VZ%EYbkNbVJLOL+v{ z<LU-=uf*xgh2@}r|1*{zGp7TT&9T5D0#=7!We0DuW%HWF|InYg)MnNAN5m}hYwpb6 z{I3T8*X!rI;wK}|*cBFBT-7yx6_59@f2CJ&h+k0PKaKwl2Xt9*C(UQ30JnPA_<IX9 z`Nno<G#&E?{b-0+u%AbO-)fKLoHg_P7EPX@-GkuN_1Bx8o~e<szM-DJ{&<t|26`rX zh9;)^M*8FR49AZ%)Hg90XJ}+>YBJuq%gP@Pk<lYL3%iD+9e<axd?MU2@7j!p^k4IE z5FcOHkooheZvTJcVI8y7Kj7yHF1FT2@^g@`q1+A1Eho*_W4FD7f|qzKpZ~7|@bf(8 zq03(W^;W9&m?!?Hc|w*41$+2-&F9`dpTH$vfy;lMitB7t65^)l8nQj6lIC`?+deH^ zwcINxc#&7|&oij=BscwaW^ivzw=J>f41XL-y`SMjj!)G1gOQeFrR@nHsM^DTKZ$66 zhF?om9vs1~*)@8?;|BMJ8~hAU{d%Oh3)(e)f@}7M8~zLr+MAU1S61aGxK?lYxS!$O zMb&%42O9Q<kN+9|3*#sDh8z7i;FEg8xwE6^+0b2)tEcgCR=wdSKf}8dYWIW>oYfm{ z`ZK)yzH>g`ZHx2n4cG7GcE7;;^nus^YyG~nCf%0ji}URR-^=m3b20a{pU<xkd@s-I z?)N?M&VAu~x!&LJzvJRQ*6-ze-T8cauJ7pI2fml{b??XPiLY4G2fmm0{lfn3ec*e! z-!Jg7ec*ffU-x>hp7uNL=#4k%?SI|!u|4q>X?@`T?SFq=i+*AM=|1qi{O=d|GkxIy z?SKEd|2yXNf$!yifA9Z=ec^lg-{1Sc?Abon@8y4gzyAZz^?~o@f8FE3p2u%sZXftw z{?|Po>xqwJ`@r||zwXggPrM_iW6|xp{Wl>*_c({U(skSJ_QdxcfjQa<3F-U`q~1^W w-|(UTy7$+uis+wDgwa25Ih=6Uy@b*~A9qDUj&_1w(Ffm9?yu|hf4=rV01{EaMgRZ+ literal 0 HcmV?d00001 diff --git a/tests/fixtures/tl_checkpoints/attn_only/meta.json b/tests/fixtures/tl_checkpoints/attn_only/meta.json new file mode 100644 index 0000000000..49280d3105 --- /dev/null +++ b/tests/fixtures/tl_checkpoints/attn_only/meta.json @@ -0,0 +1,13 @@ +{ + "d_model": 32, + "d_head": 16, + "n_heads": 2, + "n_layers": 2, + "n_ctx": 8, + "d_vocab": 16, + "d_mlp": 64, + "act_fn": "gelu", + "normalization_type": "LN", + "seed": 0, + "attn_only": true +} diff --git a/tests/fixtures/tl_checkpoints/attn_only/reference.pt b/tests/fixtures/tl_checkpoints/attn_only/reference.pt new file mode 100644 index 0000000000000000000000000000000000000000..bdd30eb1df096a3a4a9efd2717806cf8b29080c8 GIT binary patch literal 2165 zcmWIWW@cev;NW1u0747`3`MDFsYR)I$*KA&i6x181=%@nPCN`zNT7kKwva(1f(@v> zBtJVfuegvgxg@_RIYTeLv?Md9m@B?0HK{Z+CndfFD4AarUuMKr$fThW!4A|>3{;tz zo?6HZ(Zc1EpO+pC5#uUkiC_dOF(_n>U<T631*L^--Yg}B?1A14-i+QXZG{}(EZ&T5 zg`Azq$@w`ssmUdo`FX`${zWN3C#Se%CYNv(a%lz?a+ehH1QqgvotTrKo>@{{$d>_h zK?clLx19XM61cVeU~3Hv1v1#Y9ZCuXq1Jf|v=<6_J9rCtGqx8BcV>W07AYwd4Js5X z)eG=u=a|@2q`(SvF$f3X3{*)56Qn@ZOUp0HO)QBoOD!q}I^NC603~dZf*B{%5M=lE z_jj5>fd#_2?UrFMMYbCn`0<H3ndy1Csd*)CPKG2~A7y3&a)5n#;Y%i<<sgjP0m2Nn zNDfG<EJ@7=MV6bBCF%CU0wgE1q$DR5<c3z;$C99A2*SACAi!XQ<OWcx);Dl-vLM?5 zptK1BAfJN(H-ut<iGy6`ytnGE0#F=;@wrS1-DQSuPI3fX#uu?+-!up5eOI&Z>{sPk zV0WhXo!xEACHo3qrrPDG|FlzhYqW3qasB;|jwkQ$mbTf?FyX0P`I6TC8LK_*SEu*v zfACq!jyw9YU4fwJft$Z|?0wW6?RRBe+$UFHz3)`pT>IOyI{QnXI_wXS4%k17Q+R(n zcf8%Ei_-i3{#>;aJHBOq-ZI_&=j9gIom<Ip!2OAfy*XE~{l-Hz`w#4r+AHvIk$u7x zuf3PQB<%mS_r|_evXkvBKCj+;O8UmW*N!Lc<fJ<GZ@BPb|6)ct`^JYc_AHs}>}LNJ zwP(z^WOqZk$=+h~BzuON(`^ktZ`pU(t99QZJ?VWH)}`1Mn0~en(%WU<Q+ve@6uu^& zr6J(7k2mWHGB_fIFFecH5(r>eLc>8DbAeoNab9{T;q<S-V1VQTJ#gYHPA$p==D?~% zQ0Wz)nd0W8Lz06G3=GUIO^ggp3{8!U%#AEej4TZdOiV3JOpFc9ER2jSj7&_-3?R8w z`(VNr4PY7s;Q((&aFT_U{K&<Z5KtZ!VAccZ`jLwuNfgbYz*s@(M=SEsjX^F*WKfK` zh-wT*>4k0*a&aJxVp0&$;|K@Aiy?FakaMj7iUHS941km&=(>^9B&zNLV9|iE9;_Q! zh@-{^a(WU(vHJqBwnpfNrebvM$Pul8qCJQeSv&S57~svurUTU_$E*w24{Ia9XlZZ) jVqo|IG!q1X`aqRBgD_M9Fv5Uhtf2CkfddFZ>LF?YQdAuL literal 0 HcmV?d00001 diff --git a/tests/fixtures/tl_checkpoints/default/checkpoint.pt b/tests/fixtures/tl_checkpoints/default/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..9536b177297f92eec7139a1aa8972b989a13d75a GIT binary patch literal 89525 zcmd?Q2UHZxw*L<Z5>=v#B2fVWk({P?p{R&p0ux|Bl8O@SVK4_I3MMcEiVBK|qL@&a z-enFLF@XsX%sC)p_SZ9b&V_r<yX(Gp-&$||*B@5b)O3C8Q@g6VcUM){jPz+IFC(L* zB=eU)EtyU-q0_=cXUvR=h!*t?3l;@i&zuqI>7gnU_>aGMxzNy<$jI<eQAA91tjY*s zShz4eY+yvFNQIT@?Z?WR`z!P4@TidRFzazagIKvxQH(Hjnsrc|C?YaeB}f<^5*HB} z79<Lfj*Srp&9YNr+nJmDEAp9Qd1dg_a8|x0i;8DtOt5HF%Sjbh!C#iw*s$&W+wq^F zGviprVKQ<eR>{{##z)=9Fojk2G4zp5VO7$mNi0znv1)#72memIdS*;)&>tr8akk?$ zzRYRz(^{CeY^P~FOIF0HOIS)0mPQLpvxTK4VzvEP9siCzD<m=|bVjVTjdf(St@UiS za~qT$+hv-fkD-X|Dj^vrv${UA$!xb4N_P>f=g0Q=D<y>0Z$q(X4O%FMzbQr%ig63Y zM8x*=V|)EG#g4bdv<=09HEW@m|EBboP%K&~mLk^5kL~kUN(gJ+hT_QfZK2rwrr1g- zb}bZp5$oW`I{It0ZkJ$@D4OqjU)HG&$cc59w6eU2b&-(d_y$%_WnK9`<a<vxm33=D z3Pdd9$HLztN3f&~(wTMtjqKNg<S&C%w!Z|)UkRzKM+<U*h#lz14*DBp2ydll8>9<6 zxTU&YzlkaA5Q%*$tal5{N5l^GV~71MYy>;J4a}7t(E=O!yX|}>ZRghl8YN;!`>|vG z1~e|nhaLMTNRDq?TN`%V@0$9zYP^6y8!xbhIbOt0@M9<bEpse8sV&o%4f@TT+=?0e zCo`ml87gAK{MhinVTSN_PHD@uW2d&%c-o)M7tzvu(_3gWL~NuV8}+xev21i(nmrrS zLYw&~jcuU`TWGN&R^-RV`D^~=`ilyVox#p(!Tok89x*YIe|nSI{?c}JVCV29$cWgv zk`nm;6n37Etfaf=x0J9z#4hw>7x`=drGz1ahmY_Z#4c`YvLhSM19?}nga`KS-2zDv zv59_cQrk*LMb6~!l89(Fxh=toO_6kql88-}*ktI_FNIy|(@$bkS_>{+#4ht=m;V(P zBEhX_i*sgI@&@%2v8yDse#vZx<QmLuA*~j%Yy8-?ZQHNKu$VXzyRI$Dg<b#KvJFy} z#7}N)@sn9CylfGh<Hv6LD=#ES!rR=I=gQ``bng~vMU&aB64SP|pz=g)z8|~2wS%(# z?Q(at!Pwcb1udAJzcGap%&rzpk%-;x$L{$nCWI|+i?L<*wqW-C#*|1f`&%#vMC?I7 z_Rv3LBu@8mTZ|oB+JZUq8*@~GDQm%$i&(KA%l#D-!eU#DJxg0K$9`juOE4!|FegRq zDL?jf>kGiv`cJoerY*^VJ^TAMKPN%`_Pgi*yv-|Gm={FsML+h^-!ezAm)kNO*(<-9 zS6i4;$6NWQ<Gt3xye?vI_^~(thADBpx7sqD*s7KqSN}$)u(w-o_B$=KyCU|UAAA3A zX(QO0wlrt9wuM&ryZs(W+ONKa)F5IX`mvAxh9q&lkK2-5*eAd1`m|O1J^Qo$p0_Yx zh}f5Y?5n?Jj%8oBWxBF&ely>;V!r#6`M!nuLBuxtu^<12DRI3`ZJ72p?5CC*H~-mw zpIh4ROAGC*i2dfre*atASoTL-nl1aYh4$+Ynox!xe@THtSw1FZ3g!5iNZ5`K{e<#- z*!s@@eMw3@u~30e`&~eb=e4&JwwIifIA5V6ACnX#@xMYP{)^NF3zhj;OED^ZOe9q0 zLqDM!AGUVKzl&*c$HESLYTKIF3p+~ANu08<6CdLX`t6s6>ik!cP=gQsgqnQVwrT!w z&_XRfsV&n%s4Y1sb<;u}J}R+Z;;Dt5`7csuE$qU_TCDHN$3#M1KJ*iI<HNreBXQiq z?tD_)VjP8fl5-OGE$qR^B&GCA7V7g+{%RH)@SiOu81gZZ(1;KHgvNZ>wj2I*=t2`d zt!)ub!k&_IEj8)IN2O)(o?U3le~~(Op&1`*DaV|TiG;oR&`)T=hkq?c>ga`*d|KOb zoP}1BbCR*Kun!-THdwOInvY7V(wF~iDZ+-2iG;R%=qI$}Lw^<i$r~9RG$kTBIMRBy z(4Ifn7VRQ*kerjE9r@UAv=bkdpq=^87PJc=6A4}U&`;>bhi%azLIHoSE!tJcNX|*o zz{h^02_Kc9-TBWJbU!{O687gqKcNR7`m6F*$3_1!=NKa#z@KcxcCZl+l$_&RUQQ$& z#7CuYAN3TWC;z48MjOn>S_<&uV<O=YKJ*iM^I@9;LW0H!efX1Y3$PUqm7G)bQ5Ok^ z@i7TsJy|%Mk4lUm!GE@}NAfX|(3cPWgnoP&w@^5W|Bz1Pf}+Aj!IHV4a5R5PGO-el z;Unh$eEnpEWBKSp;W&v{GJ%^FE{x@;g+j?p%||xYUx7a-Z31C{R2ayI)Bi}6^y{>k zm>De-GvRotVnS<$WFq<J%uP5^Dwxz-AeoLy5&uvHNmY|utJ?FbmPwm1SehErI#n_g z{$olf43(ybwN6)Sl`a{72*ahCDXleK{z+pkH7YhVI5J!~RVts>TCVrE^5__0RB&X( zyznq#gf!3e)_MNP3_IZrsV1_u=AX>4Yi&l9R36=0{!eE7mbWq^Mw(}4>pUG=n=Tn~ z2wAB}*jn`0v9h#1W(Z@YTG79uWrcB4?W})8J4-lQs-5$1Xy*y%O11M^YxV!RQY{T9 zsa{ZQXpAsCRybdpYeDN=-Tpq;Og>w<Fj}}!nqyJx9J+s>gXhOZiiC@$8RA=KXw&Ju z7Z)y(Y7$y&bXr3tp1$RV5QYmArQ)R4;?93Bj)@fslcnmE*6KEBDOTDf{G%#WDqi}p z#Vr3&n;9n(rb)%=|5`lbw|JRUy!>B_XZ;qhkcwBf7Pq<fq&D+yB6+&5l4>(rYumKO zKkCQNRD_vQ_3GB@zqW>iED^7fir2Olx49LiXug*t!h&Y<{jyFfU;ppOqawnD8>I4$ zt>s-?cVlb1q$9JW`s~*FRzA*Fm?KT!)H=P@btau-2{%hMxve#=YSc0hla|}!<F-iU zTU*OpIhR)QR?cOcG*4dZJpW{doiJaj+1^_7PiD07aXX~)g4XhXGUKm4Zl^R)Ve34t zd|b=eQn*VhDrzlib)EelfJ)nAhH$r3yXW7~vch7icJIHToh95S)t3Al+IhnLQtg4( z+EzZUwY{x<+(BusL#=bQ@^P(m{o&&dOLLUA&e6)pwa)R|#~qPoINCZxn@*SbxH74x zytSs)t!Hc9#>a`JVy?Bg)m`0MEV(sMs-~^gZP3y#{KLl`lZuc3Yw;gG?u1l)@?VSp z@NuW4;?w_H{D+S_BNd-*EpBt|Np1e^<IYL7=UZ#rw8lT`CvjU9QuT$_>c6(eA3pA) zRD7wmxXrC7MgPOcU6#tP{5x`qkGm?BSGJb7@^Q8;&$QokNk?9j>aVxfOMKiP(*)rS zY5L99=~}JQTMvwGNu^b-rIMomK($C)`K@ZH{&s7<Wa#^QtS`JHO}^VYS@HtnkEwvP znfRyCJ*n(|Ynl2#%OrhNBh}Tm(#6GwSda8kRMKk5n2^lhZIWZ;fB90Un~Z(ympax{ zB!f&*(C>k|r$^t`?{@z2mgm2Y*-FYj^7g)Da3Lc({_C=P$T+kq`}gaeAbyZLH99Ih zTIA_r^WQB#(8)n!fctFrxn%uTa{Si@Xv;9I4G5Vl3XhSzuk!S8`EQr|M~g&8h(wX$ z5*w1?p{`_VAvykQ8#H7Dt!<FZr25);dN}`24fv^SLi^O4(p9D!7%+N1YE?EtXw-SA z>EMh>=2>*XUN;!g=tP6W-Vndg4sKVaGgAciM84dJs6!{*)G&bF-xx=;^tXfVjmI#M zEr#9p!L*nggo;)v#Afgv__<~m#x|Vi24>u3w%ZROm&(G)V`XJ>Kky^YEZvWjrmhxT z+P@O_J?;db4lIIRf&}K_hB8Q5wiqHiMsa<T9)sRRGc*|0C^#9j0ahL;ARmv6rf<)N z)0__{G4;}SP`K5VgeYHON+K0-_v;X>4crWVPr^_Rmf)3z)$rV70-5~b6A|D<dhw_o zH8_<>LU0S()@{cJt_nn<#*p;A=tgAK$5O|KW#our8d?T<z~;soRLOM<W_(FUyRr;? zoY;*%iZdeHKId_YLC@iaw?B9A$5|M2`2${Xmv`M;)d<VVreb+&XYyp$2*AZ1(eG#h zIqbfauATWrOh$L1`bjGEQfQ8V?z#+B{?Wwc%Ru^h!(gb%To21!k;)XuQF|wOyl!C( zUz9pQuBjcvTpR%RauT73e-eb|f5ipf4Y0$hJvB@D!435-p+i2HbAuLd7CVKik`xbb z=*cQjv4b2pJn#^{8lHlWOqY?zeIn3f*(!z>n-jUC3Z(Xl7nyMP7xvgC#G&1uW5~69 zxN@!pM!BYfAz49WKI+n13w!S5ofPJn@*b=;aiIHGL~@Jv=+I@lj(8!cBP2}ej0IiO zn5vN-;8TG)aqevf-`@p6)7xj9(w3#%)uLMFYknQ3W~I^fVH+5a?-y~{a3^>%u9T^E z4T4i0)rfUUCmMFx8kQE$;nFs!lR0b#HqX2dMEf9Yc;N`gO`Dn5+RM0)mqx<QXO39n zq(@a3se-JXH(l?c%iUQ!0;idT(5UEP7_d^0sQ7JULh3@v!ch%y%R!IArAi$4b2M=% zGJsq9C*X{XI#mKI+S|4d)##ju%M-F;$Nm{mkQoL#2hTE>%74J3p{A(Ns70nsw&IRi z>QmXyp%81k2MQ~X!KV^y`u5E}aGRTpi5q&8;ZK8^4+*NoA*2!5<&Km->qz1TuENI4 zuC#Qt3OQ5nN(%2DN8`Qu@NlOov@6OMo1D|7Co+<_ZS|gLv@Db?`%%OE(3t>p5AEa< z_Z)@G?~Jj}`hN6Q@^&uyP&PWP-+=p)zJQfbhK9cV0CG*H<haseDkn3Ru618T)#uwV zm3DemxNaozFpTGNw01(fpQpG10ovT~;Lb$k*N)hirQvzo?`T%ClhHh%M(lfrK&7Zu zyxiy+hNY~+p%K2AI(rvPoMuMjx)gA$H`Q}J`+IUnKMWyr>-T`_on>gMF&Y>3=|>O5 z$B~CC*O8main!qZCHVYnCtRA|9Xxf`QMcXMV4{(O?|Obk#p3~D<>KDt*!w9k+xZFi zD%^~E$2g<VQi+`GYD!Pv-Uflbk0G`37Ps^L4ZKn3#;tnYfwH~Tn9G}2;L^P>@x!9m z@FnC38pjKWLG>sSeBdDZ8h0Yb{`2rn{2^SN@|EkEwgR1AKZIuQxlF!AcbadK%Dot* zMJFuiL*-Ye!@Ul6RCTH+9iJLRrZr@Vmn{<Fo{rj>b7UPi;@5YvMS3JG?qNbc=`AO_ zUu%%+1sbHQ!F){pU{9|rxDj{94j|f;1u=WtiwzpK(TZWC$=x&?dS-n&wD;=h-p(|G zOVzdn+ro(u?7RhcHg=*}&pk*%{c<wC#*J3!7g6WS>*(<V?Z{{A3UseuK|b6uphUb0 zVrnLlD$gFIc(DU<@J(iH<EN7gLsQ`1M1~l@_zpLpbY>=gtb@{pYBX=T3|$@Sg53`H zN8L4vV4Ro%>wJEK-iBbvxHyWq)~AE>!fbr7W~+E>5QTGt9Z>CT9#$$X$8r}JdLp}m z8?taK?2o<94Zp64u`?{-)$1qFq~%No%vB_X-!_AL{v>+$n*;A_s$hRRKN7i6+x^H} zBQp7E2z|m{K!*th5b14C`X-klOs~Nu399h;%5mmo);9XE&k$0&Fca^SSaRA+o7VLb z5yx>hG-qWEESk`Vd}OcSk9S=-tHLd0lgAQ5=ie)DuI)hP26v+uhsBeMQ%0yd#SrH_ zzYTs9%Q4`lGPU&T0htLW;Y(vOxjNg5I8_&6%|TUieN7<M8Iw=$e6S_QvX`UBq(r!I zcmSC-em;!bxfTUU)m(nD0c4mjVkX;<guvl<@oBw0TnyF1*;m7e+KLY7UC@WlR*x1Z zE=eFBB3;@!>=PMtXgh`+NyECB1!Tk3S6t)}XWSVmAla23#J%JJGdeqz*dLmR8x_4t zmvm2P4xC0ibt^*sN5wEMdm+qT7{omv;SR^h9nNjna5`|yZU`(hqxMI42xj;H%H-~n zr@tNsll;I(6cY2$vHx5g(Y`;rybq$5Q<TJB({0Jx^HYf4z(_i|q8%4LsTAI8^?>Dj z&*IJQ`t&ix61yJ1u<GYdQ2Oo%pB;xX)gD02*I&Zb?v#_;r$!ks9eRI`7?f_^=VoPG z1&^B<bj-ZH^r^#7Mx-!{e%=D4V(m;S8F}anI;O}-#=HFSuSOpI|1$Eh_4I)M>BysH z1a!|l7weKraaQFKT+*upN$cLo*d(S4G*f1i6ZY59SLFe2-Ma_oX*<#pPkNC@##N9w z-;qo?7LUu-UIX*$6>dK=611;JP^&HnY5ww^f?9zmF4LWeLoc-BW{v#_D(BBY<+&!} zRK(KyrZphHs~6hpctB%mZ*n_FKn}{u5|sxlu;5y6>i^>`UhlMvZh9QReC+XpTmMp- z8@MNv*mfC!%{LghDVGB~>Q@rgY45nOeJjMnuB~P!ZXM<}OZ^EjcV<9FM_)WmfmohU zrNJG3fziw;ZlKO-^sSFVzp&}FNB1>gVP8nL>MB6k)GR#z_B*D3>)|%Gw-a?%>p?qo zz6<K}1duY!0P{70T=H#V)~5A<e3$n0^}{pt<CJ{de?%W=yxYcX^jQNDiH4A;+mZe{ zB13*Sg_8D#z|MxCS=Oum{KPQS97Nip6>&inKudbhixlEEmr=Wvw#;7(GLni4W> z_;I(5!`11*gB{4y!;3M@sS8HvKZO^g4Qb5rLXx+-21_#1unW5it*$)g?gnioQ^x0V z?S)$O_P#4j=i_Vf!Q6agf6CLdUGLznPvbe$NCPtEERdMI-e@=a5q=Ik&8g4ZOpS6X zK`)>e^w=#AW2#)qyYj(wZ|)|j?ypP-+q__|Soz?=JDcIXO%UmDrx?~<8i0?cPKRS- zWr^h)KU&mmNp{5d!RSE+q-l;jjGnWZyoh$7ea;>t4%{Uio^49bxNSi>`&UqNH-ija zya+d0&%<S3mSWL125$XSqvl<o!Y<VU+)(<48JFA4=sKy8QNA~!fb*mKjMmZ_y)N^y zL$GSCh+e<=3N9B1<0^SqGD>3|v}+gwmxgwx*0)sXz-D=3vU&x)U(k^x_E#ri*Uf1` z`Ywz<T+L0fx2HiuOVWF019xs!e>%~nEAcv_O}{NQMea&6xgJqW-}R0JgKMWbbGuzI zNxv)kWa&okcb!i4kJRErw4n_@cfiPm3*Z~20Z)94;L)J*oQ-n?+0nS1oA77|{nWl4 z^}p$b4VIVS>9Q4q#)xM8+;I~$EZB(APx{f+lR+f&U?glmvzDg#ZiEe6ilIpmi8~w< z;qHeJ9Q!q!=sbGEt^OR!x#TPdr?;12Wac?R>GDO`aQzexzc!zQ$0^ateY(->ofL55 z2S=(fdK(<eDrE+9+T@<R9kC8}EQ@NNNQLXqL+9i^WNa}Z24{xg;u>p`vrUX~i#LL( zM41j4pF@*NCPGC(6dJv{h53hL2syGI9?<qE@joWJ2N=lky9g!6f6xEe{m1@CMke0< z|MHO3|Lpp1hVgv$CC7iS{`UV1_2&!yKm3!_fAiPElkvO_lH<Que~15t`u~5q{~iAq z>d!a&|KZ<%*8fibas6F1GGOMi?(n&840)S03}U=ykgavQ;Y`h3y2tDgiuX<}pP4U9 ze9Bjr2aUcaxO8zR_*;e3-lPa^IwWx-^NVow<yPFsk2jQ*E{n?-tcP_y@57l6I`os4 zBTYDx$UUgqj|V$8L(sm1%z86J61qsA*jnxY?fVa4&BB3n+uj;vdw<|YZqmiM-u2kw zf)CNpsix1BcXItLK4C?&6=qvylL<MQP&Z44)@{uo@r`lRIPWBeY`%|Mwx-jgLSx$f z>Rx<jx{r>%EKfB(FM|7zyVxKT2$lXOr0JcvfR6Mh2T$fwv(g3J9mi05bjLP$>eG$x zvWtLHoQ^-53USYI9ZU(`htqZLa)X*x$o9t5;B}!E-_@COH{^Y3*hMG$HLwWwKlp)` zuXJecx4twf=`^=!M<E2>TuRp5(u3wXH(+D1E4?^x1U`D#ft&ZDKN;<>OGmw#2@^il z<A%Tx@so2(bdQe>!zI^~OV?kZeg{MPe&IE|;`0d3W)C7}tUug+piX56PIYaJ*vMJQ z#4u)#W3fp03YJb<#w{6TPa=LkW~v4~#muJ$Feo9EktsV!hCk>-kKWND)6FO2*vfX? z_;7t}oYjGo)yoF;*Ru4>VunUt@uX>WBBD~8j63hWWGXXtsA{PR@$Rz&D-1SL`PyJm z4)$hZeLthVmI2yMiibOoR7l^4{YciAI?x?mLu!l<(Ffb&sJhK7aZu3^VlbhAtUJ6B z41@PE2?N)|i8Dp`?YTOsJWr_S<`Hf`=5B|I<YDyELpQvXP{y379!r<ZeFH*QEs`+K zjnp1rPoBG{GDhXQx#m%_WLm*}G_IaRCp#~p6Jsji^X>h#_J+Gaqi-g+!d65C*S3>2 zD^=+6w^v~O+K-^DV$3vZPIY^j?n9@BrZatO-0@g^Bi>n&3{F!Iz>$+bxH;2QQBCbI zRtCQU=P5fdGc=v6STP${-hBzH7Y?O;t*3Hxw)J7kz7#<~?G#|Gj$*)<p;&KfK<$pN zrRq=IIIr`$N4GxcMHlqXB$Ikv=6?8(r<uFElX~};0-19$bnvcZtc@)Z7*KcmaQ{|# z>30ps6=spB^rr%*F#(RedI?6!+c?43(O5Yv3Df5YXr0bQF!f5O=X1J}l|^0X#wl{J z`pIeN-6so1*bIYa<3e~NyIZhYcnG{*8o@dC7d|?!O4PQ01C@<3#P5{?&Eo_>>|UeS zTVE*HB?Hg589`E}6Y30~hAJ<<Fw1(nk#$o$)A?H>X~oVQ5S`N_=MK)MZ+_^}slC>5 zw>-|^to=aSF~>lQZ;M4$N#vAa5{&mYfZEz8jQ#ZmOs4rq-0HaxdR`->yKZ+f@s$lt zF)0PJlbU2=`cr(jK9hTR`V?%v@&)v|C6J5b0_n~Sb7tYs9W*vG2TEhkfZKk5syVnL z-KS9ri+(MFJer4ZhaLx|p&RLCr=RdGOd0vRns^=VMSX7{r?R)QNW~j3vQPCTyg$Y< znj25zD$_W+q^LXHz&68zVY_j2T^y9Hx2I~yO*ogCnw*(vH_T95g3T#5G<Mr$+WXp4 zC<r)9^&Z3sCip+(R1z9^zmP*W`1ObOTa9VQy9c4{<`3Ku8&B!s7x)9m(5E&hpz_cO zsBp99?)p>a)`LZYw-XCNOJ4~t*mftqW^TZkXO(z4>J2xSyufX1Q$YDl2sD^@&_Mb7 zuwVNIsBP=Ug~)D$8Ml3?jzv6Ete!7ekeH8wdGaLCrVsSCxXe6s8H@)H%fs<!3!vW; zJ?Q%BDYtpP7MYyL;=-*f#lO~dq{(qFm?I}<f{L#-{SlSTZD{I^H+H1JfF}ju{6K-b zyt6mC^JF|-*YCbSY<H}D@cj`a;HCzYWK=<%q5(ZI>LBB=)|(#djDkDYjA-YL7h#&; z2{iG@WS&P)2k#O@cKLfjpKk-<e&<hI{+JN*cv3rBc_oC#eR&{gs_R73$7hkEJvX@A zUk@;4Ydonwm5G&|+c7|MvBCd5CVfNPG^hebc&4K5+LPSW;(Jh~W<?%mmExG?UATuX z2SL9ngP15-g7MQVa0}aj^^2x3-Vv!>*r##uyjF`=`00_aolQyLsq@_aEn3XTr_NOI z`9Vy3ri>;BhBHy23B)~c9etDQ1_K&<(rY*E$?;d?soC8=<ix9ZtaExIaPLvUMfR+P z6~7KZo!kYoec3!X?vYL2<R2x|{QJN!^<C(e&Or9$Y{qJD2IHUaPv?C$7mT*`g5pgX z0>3v?;ojC+Fw?XVy3epDqi_SvFrP`+R_4N%#olyv_f_=9d^^(AE*GnA8FIU|Z-SG> zUasWICA>8y6@sITU}#i-vSqj#`M}C?6KBqavQCP0%+CP&Gc<<L512r{9T^3~-wY?? zR7XL-s#x+gP?H?gKLnc5JBX%c7zwVm$DC|`a%h+anQl3fW*<2xXdD_%!@7;8(bv7f zFEoyuFsBMD3d~5ef-l*2=P{$9W=PB9*U;@I*<^F=D{jHwRHzzeO@8J&LiyToqMH2^ z1{<6suMVuoSG7B6&r7P**C>seO!h*VZ3jVN@p-U5T@96c+S4<EXYtmYQrLHGIqCj+ zGQ{i35DgPUe*Lfyo!8!nh?Fy+;?-%`veb}tT>l)}o!ZZNwKu>qp}nAKng+?zwB(;< zLXvlH3B^JI^*r_*?+UX>*EJ)+<IP~uQK;tZdK~49Vr+=9oiVk0oP<S0Km#oGNT=&5 zj7e2@2pO=HhIh!t^<^@U?P`V{?wo+@k^V$`a<JgAbq3M{w?ONL3)$>+kb6?q6MZI@ zqV=~<STOE6?9DWwZ!)TIQ}G)}*`iE0exfk(%vp^6V2(?}Q%SC-Aw90NlRL253tep9 zfWAP97&mO8-YA4?vg(+A)q^fA?SqpWa;coW4W@;D;6@jpfiBJKxZSt=lDQs-NK{QV z7A%-UJIZ*HFQHqAPVaoiPvsh%oIRQ}W%$v$wUM;_`6_I3(<Pb;Q)$n;hrw-J5Uo4; zlX)oo3FTJb#ZecQU|OIVS-GDO(+EAfbCei|yz5Q!EJL74-jFuE{Rp~+si--$4DDZz zWv2G9C(aFbN&BMZ=vk}=@kWO^x>ZE6Z-3GhoJq7FDdLWG-Dz{TXBc@`hCGTfK>@oI zhh{A&@`w3nXkY<m^i!o>U&^AH^-#*VcO!+qLeSoL1-{`Z*r@XX7j#-dlYb_WhvS8) zla|RT4)Ue$7OPzModj}g&_;~3$rU^P8cl*63&?8YzNEX$X92T$EcQ*+g%3f<=<L0N zyWM-i=fqg>TyI5cTwJ*=0oKHCVHzQ!lZo#24~+la>w@-49CIvLk^7ZUkBbgzlZ$5$ z!MH2t@WyW&XawBGS0|Mj%TC5b<N0m3z0=wYmQTEfR?nYtvTEOO#7zfsH{}C{Sja*2 zl#i&qRF<S$$;0w5ry1es26!_%8*7x?(~2Q4xfjcp!Ih0EU=!Gkc_|^3wJPO|bd<=8 zb9=bq0R`N<`|aItzMMzgzE|OoH9MIvmz*Iaq7(x9FGbg$Ul>*pgWhYC=(w|+#G&e= zVOM;Bcw&A6$`9Q_9ZroVmVvQ!!c$YSxha7ty$qpmmwJ=;lMG-l(;sDroyOZ@HzGeT z13r0Vp{?p{=JGK$>asu%7A!f8m^hQfxj$td4OS!PMrL8Kax4i?|Ab7zS~~LN04l4P zB36ld1_K3MA@RXEc-d(R40g{0$5|pSU2F&&j&~t1LsRIUUN>>F@H@!gxQG2e&maT6 zcF`O!L!7@@2h92Q9P1HIRz94I@v0NRd+|p+>sct?7ZA$)Oj9NA6zfp+b#L;(cq(14 z{{rR@WTB`|4x<9yaIWbRIyfs8n8h+Qr^KD>dE+I}m#4^5(`;082_R|XVyRQlL$qR! z4ed~%jZNdc$%B_88Sbbjb46zf&IuR+7lf&F_x@(=TGov`STl+7vObI9T2tZfcpWIn zc*3wrM>qquD$wq&fn|ptaVeKq(b<uuSopXE48P`cW3HdUcLiP~(GIw4eNJGvCCQL} zZyG9D<U?K`MY=nBDIVV3g|3)83s#p*rr($V@@lpQ{n_VDc|FUoz18`WswwT^jAlsr z8=o=apD&Gw*8xRHb&I62gBj92Qw1jYCexw9I6Cp6COx6K1edP5i_81<BKa9taEJB< zxG}dcEjhG}yC?sFOW)p;v>W^opY3jdA*LPZ>y6fAtnw}9^)y2`oIDd{FDt-sjhoDe zuPo-gE}|!7ZK<%f8+}kR75iuzP}d+cDz_f6okt){9js2&HW`v$_j;0}*5P0_c{x0q zJe(wjZlyi?cSg^V<#?}t23<e14=n4R4h!lPn0>+Obly7+MmTdl88*X=T9<#o?<36I z9tz#LfXH6-z3E!;ey<Olz=<Zr-+`|+KRETyrBL5@60Lbufp2DfWV|Qkf>W^$-TXM6 z8jR3_t&Y)jQp0?l(tjh(iCIBZcQ3`~PXfsej|6U#;0L(XRSMK=mtn{@{u)27i6!qG z$(fps@Iv5E3j5VEk9IxewoS@{7i!y)I^E~y*DqmioZk*7>sa!kXALvC-4?2IWiLj5 z)^ZDK>_&&5*ahzMmtvXbUarZ17xw<Siuf1%i;H_)f!nWjY1j&FE@is~S^9xP?U_!5 znHNY#K5^#M#=ZtM<^!z0<cm37%xL4%JPcL34DNShxVL@HX_mGJU6hkbr=7E>xt?o? zLG5OymvJV}(tQD1zXZf{LxecM!;ut<WoXr<YdCl7T)6$S6a5*M!X<e3CevOzxNSb@ zOs)iPAZ7jL&^NXE{CaaYx_a#*IM8)3Zpq$4jqVO3r@Jasr{S*5lAQ<m@n{+7>RQ0Z z$i6hEUm6OhrwWYAkD&U(RQ$Oq5H=fJ#7&y=<aLiCX63|vAef~KW+@?X;h`*{_cLjS z=e~HNa<`z{yKl^R(M)W(*qau$JID=v`30kI?jzm%oW#ajAUoTgfIjxGA%rX@%`tb- z;`TK-JeQ%1GzytlTiZcao;KamoI*CgIZFm6s6jDjN%gvj!T#0-ls_Ruj`TDl4Vt=S zrJD!TE)r1F25nqFKV7>17_)IhfsACk41fIh)}NgJ<Mjt&V;T)!5J%<tF;GCsRrKJ; ze@gdlIIF4g;38X${U52)`?BN3#+er6!NxMwpWsBcu8*SD*M^g4l^sdV#{=L$U>=RF z>4Etc46)i_NMAZEr8k2a>QG#RO%u|wyNeygjjOQam@e7d*MU4)x&;$XuE!IuLF98q zf#A#8pLpR-2RhG6M7EeKgV~l4(lJODZM$D&CK?WAQs+*FfQuIN_{uP{*77|Uo6J9d z*Ip2(`=@ZbJ3WQU`D%EmJ`JsF$Ky+Sqr9Xf0Uv(WC9!$xkX7+r>>+#!BX_INuNV80 z+gEqsf~(HVnp|UQEI*7Y{FKMpiKR@&AxpZg(T)DRF5~vYuMP`hPLO@im(zZewvy<0 zA(=K#1!vw(r{)h&qtCCS^uTQoIx=hp*}d%pM8yszTklVWi>YVeh?5K*S3e%s2d>29 zb5?+vwI_DjoCLn9o@8G9O+l%Q9L*GXz^vDgap<)Q*srXBJq8wGp=BD@%#IekEJ?xJ zRu$N{>6CcVemNS)?SeJoDG;4)L$8lGLce&4X_d`)y!SSjxg51jP;p+1jO{vtxjTFX z&KcRA=&x7A3_DBuYLgeHIP1Z|l!v%1v>LRHJJE!wCE_==x#G9N0BX{yGj+@BjH~&z z8n*NT_S82ZI;G2rl}=wu+pQ8m9<Ud@?~kNPY7t!jd=*aa+exOb>;<G)32;@&a(ZO3 z2930S%&nQ|Lt{T>p+Si;zGVl)Q!WLj{LFwi*++5akjJ<r_X>=*7)!kkHE>z13%4nJ zH@Cb<k2ustxQ%+AfM*VDCncTa>5kiLpuRZ^?4GDlm#@j(h6(FnL0}ShtD+En_b+D( zl+J*{8gDdW;)#E#A{m}i0f8UVNS#lpILxUO{Ob)c_sa>lFUsbmB5(_H@WC~7k4L<| zIvqV_Ot5U`3TD2wCNU||Awe!H;Du#-dTd?+evZBY_ZGCL<DOM8y_Y011D<%%;MbLO zq=gdqC@Bk40w&X<2dhA*pOCCdazgiWmZ*AoIL7wq$82KwvHT(r&fEjYps-4K`}HNK zwZe?d;^)5SGmgNkWn<~=(s@KwpofRI=#w7-LfVBJN)Fw>geTp9;e=yFxZmLeZhVqN z$I7LH;fIx6sfjk;zu*jKVjFQ_z+-T1zKDsQ4V+evA-<|EguV|fq0h`6c=%p3<aV0P z89m&M8~ArWd5!s?)NTg3wOfZ<;cy5QM|}gG)L!^gVILftYRM>7KE<EYg1O9B6KS7Y z6VQAYOZQIgLFQgL4o><qq`!A6XgnxK&#=X)RPTma^_}RIt*^0okSlI4vPai*(cqln zOMBh4pwr#n<DuY0Sa9++b{elqw(jgt)&^Z+mj196htM_n&TYNG@^x3b``#N)>9`u% z6oqv3v*}#b>Mt;Vfdl$`)xo#pjX0xI0`ae{W$rz`%*mO%fxPxoGO|E}Y;^t(IS<ca zc%>D+Q27q(d<%dJ*M&Fv+9cL*EZPsb$M4B;g)Z$Tk?TKS!S=q6#C38K-i993aK|~p zrN~uaGGCE8IG1DAEf?CZ<~TYRKIRrV<q{j^O7RjGb2@L~Zq)x;4Hq6+((7H{pxJ<Z z&>Y=~vgb5$wXjJL^fHFn$t1v>ZyGRv=ysa<(1<ibGt=#!9b9;R4L9-6Cda0(w6H-) zJXEY;cken_H9Uu-=k7D_UiuJ|%sd!A?lqjP=tV6*?PRJ==JInR7rOX`7o=al&uy8l z0Z!cql82XkUHc|$(er`I^y0XSpm{`(^PQ(h9#(I~C2m9M-f{PF*@+CCsH#MF!2w8I z`jgxLL`U!?WEFYgKas?3PovFC!a&}n5I$BC!TheVFshngBZzdMBhF|tgVH$|bx)7p z^2mkjh5g8W|I^~^fq6LAu>$-ojj8a{M{&RH_n~_Fee7w{h+P66;LW*@FnLKXuCda` zZtW7t5Q|GFEbE71{SRP)HAC~VuEC8xv84F+VEoY2nYpAI2}5ReCK<5{QFmD<5;!CZ z{CjjI2il+G_sOWxz8kJX*pgc88}CjHtd>F6#JOnWu}cuzVGYy!;vP7Yp@4OL7eP?@ z1kg{LLn0?>l#hS1mIkIxp>I2M#L;1W;l(0LvLy5*j2xpupB_tt0h>O+<&+PQr1S*S zFZkkzS(`|=F;39c^8t}9e&A-G{|rXuRbZHDH#$vA1=_D`a#M+p!{Vf|+~x95_;kTH zab3(aTz1)zHduaPyqrejycIhEH5*_`nSg}xT+hY5Xr0d^LHyn*DE+lU@Op~|_o&kj z?6}yP)_H5exqVt(*pz$BFQvDjWpWWKEgu21%Ycs7Uc(&gmPC9rc5*>7bLjSTUzp#N zjHOR<Xn1lCypAh`=6ZL!JcQ5@zc%2R`wU$drcWpDn?<c(oW$1wtHH9tpM3wa8FdEt zr=C%1@X>oYtk^amR>j=J!S;z@tHgluIG}ZDW%zDF0&cCYgNa*R$&;jIcs9V6=!h1g z*}QR3u*MaKM`+UF#eL}h9?guRYyz><iUQS9{fJz@&NN;{4h*JzXUaRZr^<&0!`(n% z+JnD$PAr*?3R#)-*pMbcrFIAOEeW6<4;Mli=S6o!7?Z)n`;*01S3vc6Jw|*SK`o9> zrjvy7kpCu=S)tmEtl4EpjS{-i3QK41MpFt+T`-$YR_sO&TyX}iZDZiF<_f{xw;?pd z+K4tNoFu9x?deNpHN4hShP=0a&Ae3CrTzGE>(?$3O!w=?)PCqp<chXK>fDZ0Ymx%- zUAUMVy*ZT{jkcxi6a!4#(1|oJdIWp7b|uYmzEsC71LNx!({qzM6MbDdCUMne{GPf6 z=Fv%{S$-hOe;rNYd%K~fcnjTmT!Z}38VC-irA(hkJ4s{I18zJs1#5%f<Lc*fG<t9- z#Ef{zS>!a}yHT3-$AAvBiXZppeeonZ>Ep@HF0nB4LJwNM<PuYP{vMcWYIBYSgQ0!o zTBsjZ4xgG8sKuivFoZkk=HfSio7Zy!eORVPx-2aORrR5S`Cv#(Z_kAb2N&`Ct~BVj zs(8G+S_Qq<`@=`YLnLq1RJi571}uYIxSwXZ;ya3F<kO7{IK_P%v$a&4dwSW9>MD)} z&Dt>N{=<UyEsenkO{$!|y90gKRE>H!Wy$?Vc}$m25hSF46n5=-UmW18!R4K(!z)>* zaiKycR_&{xlg@V}YktNP8F^W<>gjG&H0(r2Pi;r-^k>lNa?ikKR|eU0aUN-|7jmg< zPk>zX5*S{b1S`&+#AzV|N!(Q%+V4v(ejE23_ZGec`(s@B&~K?wcXtG|Q+&@2R#c}4 z%7^2QIsIUL&`SF4RyD|)rxH`K3Z2)z6e=!eh;M#80^^HK1e^V*ljxTtxgocr;K2k9 z;vHs4>na39Y`KT)owu6Qzg`X6`AczteiO5!u`?}FkReZd-NqX_$>R7DGYHFbCu0*e z=u)pz_;IK|(?>xaY#ra=pp|~eS}ztWkNOUdJtAO=h8IR{kR{2kiMXc1n_PCS0lKh; z8MtUO&dAWE@&{tcko83*ww6KV0XH!8<skAtZV6rf<rTBz;7oe-{w&(->3$^Z21Bp& zQ{nr>YHroG2uN6d3p7k}!TGr@xt--o`_5E@_d`Cww6Fc?o#QeB>|j7=%uWY~oyW>| zJba7wIEhZs9|+E6J<0vq<H(+c14yszG_HS&IZ1EoPLoC*VhWwMLj2<cVC<I*<4&d! z<(nDsB9(wwgAZ=@u^}6@49Vik2AK2Gm{z<rrzsoW;-hCv;cM3>@w{U?nCUQtM&x)4 zyp9&YK>u9v_&eU@Vn!u%`}1CAu$CDuxgy8UNiQ&s5m#WN-eR=7>Ogzm-i7svTjAp& zf2u4DW)>D1(%#z~>GRm-n13{lQ{JJ9*PJ-;UCgfyUl<5+8?TEyg}ahXiF?G4p1aYT zG8VA(m=B#lKacx)<T>toVZv10UPvbxcBD@>@%usa6Cgg_RIuRbOL5hR1kx$Ih8ji8 zqiR%zR$hJz+h;Y2$BuaoE{{eqiB84PIH@C1@EDINrbg7TcTe)}#Aj5|oCPD^`$2eC z4`Rf>F*w)HnAT|@b93wGfCH)#i~~C|Gk0mA@$2(gCUcOPT=NwhjC<ng>SZ_~PK~%W zDdES+RN8nE;0n6|t&X?n*N$bW$MUTlb>sJduQ?{}Qd<G)N4IfncK!q<_vbj)Jcr9Z z{RJ}442AYSA5li9kXy0Cm8t4Ij4WK6MP!T*<F#^s`f{i|ogHx+hcB|ABTPC|O*b>l zc)JH~#hKAwi#{`V4{OupCuQJ0dkGzTKc0TxG#ix?<!LcHh|0eKs%BFnz810_i{npl z5FSr1e2-v46#VGonnAQqUJjP#3?QqAJ>agVcIO5xaUr*7BoHh9ooLB6eNt(+jz0a6 z4;Jfvh*$Mu2z|E`*YBuit{o1+>N)e6*^8PP7aLD<yI~5fb2^R+>(pWSf{b##7d}+x zU|0IJRGt*Zp1|4N<Vd^cmuRq(869AC2j%l;x^B00r@MS_fl_uVC^av|Znauu!=}|_ zc;8Ke+EF4B9Mu`w{dbv8@?)9lH??SP6F^kJWw&9IG6<^4kt^HpfLifFxIL`|hR9AP z7ZP@2k<C`JW}z98*~#zu>7E8LGv<Ro|NdaNNgn;cHo~m>3g)oBEv8RU=Z^G$4l?Rq z5T;biMQ|!;d21QHzP&pw6|CTnTz?_>ak2_$mdRs-hA$5M+!f+;CJ|%HX=LOrdzv91 zj0IUGDDa=ar0iLb&u$u%pZoN<2P3wSZAn#Fq&bp4)hK83iY!PFtIE%3E#Y0a^*D27 z23gW`2#%evLk#lIVs0_Z#RqGXhKy<O5%1&7rRTX-C#vE4I|2W0ZX2qse1z+~6==~a zJ-U42B=G?K@pPMBGjrn14>Wb#1Lw^b(6txakzHNnh~l@+;wuFSB*rZs5?)cpm+L~p zLp*TZ)Ky??rX}dp`yo6qPCz5)?GQFti&jr%XkoeyF?{m~cU4DV$t9r9k)QEb^az}< z??H;Y`_r0ljc9-PJh#EsAH~5wocxwUc!M)1ec$urpJ5sBAbJEy_CFZid?NclTYqr* zkJlgm54V0ui~i5{f4KhtsQ#KV{r<ZDmjCGw^B--%oi?Nw=>5>=tZdT6&RR{V=yd~4 zhsd~o-|-Ba=lRpfy$85=BR9c-q1GhB>L7S|KVdd_#*vZh?}A+a&5W$vJuKUoO&-im zAUT{fU3rtkJ^dz#wepPkJ?)#}Qiox5tkp1bCqbUb7U+nN?2sqTJA8<ILPyB%ra*fR z7=`?*7w9az!d?2INv8G7#vhw^2_9^nj|;W?(vjcwsH{ynY#jI+PmS$JZB-X?D%0#} zk30PDJoLU}!|-9m{rqWsYciIuoBNjen)wy)3@gH-h8x`1F6$sKy)*2Kn}L%8bjj?J zCCq{|y<A784useBVhl@Xq2fw23{-L<`MJI5SyLSdPIaV?YNx=ps|-~&_8|%Drr`MX z%OEza2kj$oLx=A<0cnof$39h$=d4}$p~@m1TG5b-iSwdpml2I%ps=07+G^Cj!18)O zVsU;i-kKN-r*$3Z+GGo|HK+s5Nwp;UyFIXdM<a54T@17LVF|bT`8Z<b?1j0}-B5i+ zFEV+!GIu{jlh|KNpnbEIz;b66mflDvvgYZ`ooAbg@alS)sA<P_elw5g-Jiw@g<+6! zApi$I=s|uQu)x~N2YC9lJie&bqcvLgRBMqXEqT70%kr2{N<1{_(`XA69*e+ZUbmR? z6NkY;btB1~SPS8O8*%*2p*S?`DRzIG19LwsQ<Z{L*r6y!mk-W#cE`f9kAtm=(Y!qf zH&sbw$_7ZC;e+QD&cTAFNWr*i<4Mu61iCV(1KsQ}lxsY?gqn>j#||lGv}jWXr(1s* zR^B>@&-N@K=;}byUM(XX#DzFfj?hmERj}|t0G_@aNoMTU08(p>(`R=8pZ&kE--QN& z^}JZpwOcAZI(h|;d8$qq$`vqMJ}T1O;=O3K@G1B4j5X{okB8S<)esQ<9Idv$;DUO} z65Ag4n5mi5$+)~pctCF{Jw4JCy3j6gA!!6IuDnbd!pbn$un3P8nvi_QOccqN;P`Hv zIn`UGIIQkOx%Gr9IJ#sGjsA6{d{9*d+NsT^jn1}syF7<fYkz0H`7Fb2a}K~yXA4}g zPY>QKdkh0s6yhise*Ta?m?l-^!tvxSB%!(nPbmx}!Fd_*_I)sozH34fvY(5S&*nny zeLE_1Jq<5a|G@nv+F)2Uo0L8Jf##jRg2}<B@OAG<?)cF=*!_S4BXm=wmWGNn<eMpf z%^H&3Y5eo*SS3{LJ;{8U{SLij<I%)2mVft?!-Sf5B=1i1zYp5H5zj1gCrS4+=<152 zsGrxHcE(2Z56q&UmpO1#JRWoHzHJ2p;q<G(pE=U=3f>!?fV(bEpniAPQl*kUxZq4V z@{5%4e9k#c?Q2REa?e7|w^Z?}=X0pWEDMa>JBh^XTuAD2@8P(p0DQN=ipsRlhyJBW z^mNQGxF0frGEcWc6Tf!W>-!5%&VCFgFo??>WvOMVEGphx2-DsTqNgk(XwYa5{63mu z(2otI-=!B^+Lj1@?{pHe;or}xm8y}V+?!nI(FfqE<~6c7iQfY=yau|=U5pLGQfYF- zPB`P~OgFy$0vmC)xOb&7-ZArl3qiVQ6Mlz(H>|=8Jgq=)TW50TN9N%;G$o<x%8)4- zN(Sz(hY@vkM5Z*K?m7As-)?ay9pYudAff}Q%v{N7+rD8`J}-vJ_t%4uu_rAzDiPFC zd5qhwfZtxP#L;8IDd!Q&J>8~*xtiTLjk)ri`=lMruX;0*z3L7sR~eJ)<Po$^RToD( zIRL-iA3JTEPN#j)!)sUc$m)C-x~Ah?vB}(<(D_{iuG#B?N)7oqBew*V^EH_X{O_$) zY|i51iNR!}%M^Sx;4>(`A4#m0Rgm_353Y5I_%-t;7Qiq%DtHbPYV#8Nv|mKcuBg#D zp^aFKr$GJuQ4Ad?kFL)SF<i<?Y>HXQY))we*LzkVc<l~P-tEHV)B&95U>!71%*Km4 zKhWQ31+Km*=FBaUVfwiC<Xm<a^6L8uE@UP{zMW2oz}TKN#Pu=4(#!Cp;j3U_Z+>5% zwj1^Cm=B5T+(@q8AWHc)j9BCS+^(_9=(mmYsHwqL=8QO<a{Kl&`=(cNm0l5$&hOu! zIWC?gJ<=rFd6_U^zYSxz=?nObo=i)fPoTwo9ojtR8q+1fiS%`DMsL}6wBb?$DRcb= zQDYS7wSMEc`8$_$Pt{Jt#eHhT@klOcjz7-4u}mZ%hn6x2T~6Tx{KVXI98CIc_9T_D z*;HY?H+g9v3sGgOAf$XMXI^AVy(T(xYufjwL-_UN!-hM-x}lKilOlpH){~&xVLv8n zeSoVSH!>eLUJxYM3PHyGFbvs1$q>aA=$^%(ZqY~*tNw`5z1^J-9CDD1f3Xq=hjztk zg(=*HyE=Hm#sQm}5TEAt!>W!Q$<V&1Tu&RVqM3I)knyF4MD#KnpN3un=V=98wftVR z{p18UjQ3%Cb89?3ApkEd3WRkY`CRZ<Gt$+0HxZw6g~JI`!078~tZZjZ->`b1WLP9V z)1XFY#6N~LHT-Y-)^`(M*mN9o7QX<Wx3Sb@@MA&TxUNj|XCv?m)T5DU*5Ei;liaIM z<EFe2a*t7-Xh(0rWy_a?)-4-4w&PkdN#-PuIl7iPIoFMms~-o#Ue)3TyPMc;U^q@@ zpSvY4Kf_%;bP1JjeZiXxEU0_208~1t)4LW<G|X=fUBWrTpwT9D0G2@W;(qkWc*F|} zv=HSFGopK&q4+1k?(MgOi2wbn=c%)3`O!f9@kB5vjfh0?<N@IN>n@DVu^=Db1>nZb zY22%qu~;UYP1lwUgxV8b>FSWLjGX3DGHj6ozaHO__TATo43@b{1D-v?o;`XpOy&^! zHrtvy)U6={*1v*Gb5j)JU@|u+mh6(%A}_iE)pfhgv@6!7X?-$Ce=eW95^BlZ5owdt ziAx2|-%fL<Uis0BE~#*#H=^FxXmGf_3PfppiFb`Gty(e*Zz>)W*v;RJX_1}Cl733` zwVX3~wmSthXAY%f&vzpZ#h1tp>rmV@$BayBH<Z4(q)hbZ%g{K_TW)0&K7qq%MN+32 z3`dx=#I`Jh#C_R{#>MZMLvOdyt6fZ~T4y!Vy~im4=SfUe$Qzg-zn&bO6aqG?HSlD6 zG|qqIMHA2GqFRYJzP_43wD)LIHQxr%el1{Z?43xbPz{=C=|&&(ws|bL${c08!IszO zIMokEAb;5l-w)e`SNZi!`@wg(FILB)(~wv2-R2RLC04@YiH*$XBwH%Fxe?Blz7;GQ z`5eCLEa%2Q&%@r#OGuceONtid!uN;H#A#14`7wSO-u`?GDr;XMla<eOCI$GiG0?4d z*kWiedJ6L_l;B&nl^`nYj=*ZW5#2rLh1jLnTl^Sv03Uk}fFl(fnNv#z)G1MgWcPjt z8!s4Be|~N2{Ov=a*mD;w(Xhb%{e0n3uPQX@mPq#qe}QP57ByGB!Nk2*qaUp_!M$cG zsZFL3e!UO&d-(<4{@jJXM(N<Sc@?-&r<{(AKL)caYPqD{H^FxF5!9*QCr)~6N}W>q zaji{9dR2EE(d;}Af)(a~dqFk0?C_<VyXT4L&vk^mJ`GG?gK~(gvB$|D$K&e$D^aFt z9z8l>8qAsU23;q*Kn>$bt=^nt4o~QcW`#+3xG;;Z-uwx>-P^>iHguptU_iF7u@WrH zQX$<oTt{(^0{=s^TpZQik!g6Rf(}eOa6G&mChSqg$oZ7w_HD@jJQRg>M+I{RJp!|f zGvJ1GA&A;{qT%=3k@kA$$Y||hsD3++bJ&t07JR8<`j}ea{DnsJ+P2Qnzs`ahH_iaM zH~_M3<Z!OpRMawmhtFmm5D$_04v;5HH`kYf?vfGEIdV4c*4ZM^xU5eXX|82P8!`Wn zyf+W0GLHIwizXyPrjkk_Bn?#9_gaal5QzqrCZ$nPnoFjrOd*mCp;U@O5&K@N(kP88 zO&VyVL6he6-p}*j`(Dp^&voACuXB#;+L!;fy|;VczqP*W`~56zbD2TeT29zxsEF-W zJ`>#M{-$dVX*Bu#WjI-F#SV9$(!C=`#bY%AtUu#`?CYxjVC88Jii;C4z5g6G>^1-s z24&)Xe-**HZK4=~GOSH#pbjfTsP_>q)SBeY`Tf*6%6A2o1s6)$^>8#u55iX0Gg$t9 zgrGk1jF{dbkN#Zp6N*P|<>bHG-0$0X7%M&yzIzoyz}W*}6`m$8I?_kHW%?F;A6LM? zLn=5yyiAqORuuoG9a^}arSJ>KVCj@@c<G?Xt)5?nvT4hO6S;rLcyk~Ay4(>OQcpqA z;ce77bf0)s(}s_FJ{G;~kK#uMOG%434`V|n(QVy3bYq*P=#V5m&lzv-pE5!mqr6WZ z49n5vP$#z0+#?%Ow2j-nRmU%y`Z&Zimx8lRxZvJ2s(W+>QsW1(SI{jAcGTovv#Ow2 zHI(Jc6uF{{Gj{h6#y#dwXnmLl@#ikEe76f|{ZQq~^hXfecpv8IO{Q(p?J=s~KB~HH zCiLyA2;0w@Vt2J1SiSfhWo^6wgZxXWoKwNL%VBY$&qXn|UQIfS577C;TcOMSN#ea3 zfq2mQny{%|LdCD?;qdm@OE@Xjo;wZ<q7^$I2`y1-q_E&qWwfyaD^2r2kJVG*YpW4< z%pC;d*0kqa1HC!dZ8{zQ<%}xb!#GRNO+Lf+9;HqSqSdda(&kfXvR$t<c=*aB-e75j zUGG1qb#0Cm(_+A#BY-nXF4C@kHwDLz&D5llK+#vab5g}oa#EVh9_#0D$hI@Ec=>bM z_i7A&_MS)HJ;t!k-j`y@Tq8~$KSy|?_Y`99jK|zz6F^<Me-!1e#V+p-!O_qR2(~rm zuNCPu0Cn*2OCMaVrGteQUt!LVaCkWK0~z)@CwkX9fM4b*K|aQar(8*fk2?$?Xq$ks zm$lI8g$2%a9|G3rwD8m7`+~9WO(@W{rdsaD0|pJJqKpgpF?}3=|G1Arlt;jXF3Pw# z;}<O6m<4(RTq}3I`Uj5|x}ZXE6f0&vhYFpqVB<O-n-*{9j!HUk=;v2H;Acney3D}c zGrDn?Wm1oP@Hx4t7{UCdMNoY8ijexz4eJUVpd>p(Z0t5hcIkSQtjSSM?`=wP%}Ya8 ztCvH?sz~^KbDp5_)Dml#mI%S819;Xeb!fOchRdH;Kz(4nSo6=?)@Y<LX3=&!@SsLa zRk6kMOZD(S^p_wPDE`0KpZ-hzK?(r>_w)zfn+{Z~e?t^jtFuFM1TPJ_MS1ttF=Dj> zO!;?!wfD&@hxWFmSBkIY5!br%lJwy``fhu6EAPZ*168@;W*B{Qcnk}D&XAhn0a|)I z03$mjNI94Q@Gki&+o?a5WBk&n@0xJ_*aF-wYAB}{cf>B|%ETk_j=a=-EXNmzaEbL3 zS?xn@mM^WqQHP9#JDtA@^HcXx(Rl$E*R<oU_M2#3>S%mgupSb+=aGVu0dLEn02`d* zA<M2?<>SC3<mx|6RE)O5i{<TjQ_2ZY|NR6a9CPtkWhxgsyoS2>mN=^|mewqcqXvun zbiwMM==IlG*wA$%Mcg{Ui!E}%=v}?=L*ue^=6-_nf7gP~M?JPw9f7ZRoS^jY7sa?F zMeJ7E2a`VBhcz1=xnHxC@l`Md^_taqazl5Jr>}sIwrMnc2y%+@BQOZ`kXfI-POoZy z(+Z_hIC?Fe+h-r9K|aHTL)&z*WTP9pJQ>H=uayZ`-1Vu;2^(3>>d$m9*Iw8&;S#+& zVMXD-9*EOjAZ4z#INLTwI5x8n+AQqMkanE5=_i6$a}-_ex{P(K_lx5nS<-S%Lv-nv zOgG0y!f8!Q=+jumC+gK$7A56t%;(|Petr0R_D@l%-T{6$6@Zz+C;F7>%;6h!v8vgE z{`5QmFWR#3vGW$Y>n=~|+}la`@4zMb?+)VxGKqsr(L<>w;x@#cyG%>dj41E$a)>;y z4O=FL!1T92gjwzO(&{=ZKJsNJ^crGHJp?P>y!pHsJomJ?)Nlb=oszgKE>mDh!Xxo{ zfE`w@QO4N1cs4pI)iGrWP^a3HuA3x~>mqlU4wA><p<Mo`a349{8v}zREj;GSIxaZ( zPFOl!n<()Lt%z=-cUb{g@Nb{^VSGKk_Pz%H%H4S7+IqS_dIN`48uPUi{_yl_tJtB( zD`DC%J-q5tByMP`gQ?lIu+B}FejIvfXAoLR`xjSJ$aP1)MEl{*iaa{=*g{b6RwOE` z=Hk@vg`%TsZ(esT2xUo%Y?Yx3sVT$d`+Dcnj5P@`X^9dkx6h&PMHV!}F9z&$UJGY0 zeT9a3D&SM@ghAVnQ5Sn_jO?z>PU_KoXLB9x>eUhBO7jRV-IV5?CuMp`oq6|*F{J!r z1C>gCj~UJ3)PCzy?A?|k+;F`s(|CD+VwzXe<;jXr{ZN_X;vB&0Z65f%N}>D%IT-KR z5qsB`QRL<*P6~F%4A~IKIHG`7l4q=Hb|cs*rE<>oM{MG|ft{xpfm^3I9QQ#37tdNO zPKqoM)+h|el|%A`6|%9IZ&xW(_YI-($;)X`hYz%@LonM;oF-fO<0ZVgO~R{HMR4`Q zTUq1KZa6&90LK)C;5EZ1G|y@rSGP&`wnhaWthtJ`7l-hN`cYU_(vt>OYk<*nU+!+I zC|msnNvygD_dJs6nsAg(T{C6%ujfgv%W>*ZH=Eiv`~pep6IKnGPpP{PQcY@x*g8g} zcWES?c8g`3cLT8IK@)xL9mw|%9;V#A;ZkPh3OP9$P;$a%NIIR(W9v(W-IDfJRnm_Z z2l{iv-V%K9I$9iTe38sn6hfk}Ha%J2lRdsaqTME0U@rE?z)9AE(!30MajB5Z)6xak zgSu!EaTKzJFYt7m9iLog&%d2#Q=bbdWH8(e!?tCT&9@k#GI0**ygvYO-BiI!rU{#; zeI(_`RG9NS4rknS=0f}7n5MgyJTH6Utj2S4K55C9e>Fo@Q#c=rKP)?QzMNKB?H3CM zeWuk3d&oRrnOrY;h-xt2F7>z*Bz1c%cRz3vvZYMUXek5MZP!2m7fn=t?aulo#jy3+ z0>RunR~T<n0MBzS!-<StvY^-1uyB(;cRb@Jet*-Auej*q5T6S1JLq7_UrWr~<cv1k z-pNfp&f@Pri4<sC23e+wVw3$5TKe@RsJgx<Sz0;|s3kZzJP!}wbb$Fi$MOVkbx=ue zj~xX?)*d|?eI|^gx&Ha^Mw+|bU+am-!e-Ou`Q{uxF+hClS5BX;hH(7CO1xFF5JGOZ zgXQ8Ec)LA`w9|~Kk6eXw@6AH_+?Bj6(1XVHTSHSq9e71kF!lMvnEoP#I!#(4Xy*42 zLk5kfrQ@3`@6_Ffo1^{Mr=|{O?Oe=R7kfk2#*5^AXqS97ccUi*Jp`|&XY^sq5kTW% z99N<Xv$Vp+3)L$~E%vGK!aa#2`X8o`&EYUh>i@s*oI)+X6j{@&2ix`Pfo^7vblJ^? zTW)j}5;k_?^{*nhkf+jV#|w15*IgK$t%^oFo(Okxd*D~^>r`4)Bu;PZO*)UGcw0(@ zxVFb#xG;YO-oMoz4Z3Ixzc-Fx(-Cd-<Hb!H-PN0dL$$H_ure3*G3SU&=Geu-4V+>w zLgwGjnAH6&Yh4UOeUBG{rM)8cN}GuL?Ckl{=r43)qYWBdiGv;iB2V8xfx`NX2BQd7 zbRYUd=u|QZRy}s)o!?YBCusta<b}~4;0Mz+mtoFB4ZhL$xZtCeFFSHnhGzRhxa_?X z`Hw9lJ^w^F(2|QSk9)CB*+IB@N0DuI=~3hEEp)IX5H_wDh64tgv(1o4)OIj|%zM0q z+Y#}?`HV11`Pqw-q6y}?b(M8Emmqz9x%A$%i&+0XRnWc`2>vhI;q1uclsh6@tm}{f z**Zn6`qftAg0yG-+vZ>!6h*4JvEtmaJW%<0j~*6<3Dp_x;fq9B2_ExT2sa&v?^eDL z-BgsQ-}+EUzhi(qeI0}VLj~-sCScbA5)&z2%Jr2h!O+vgSk{t`IZ;IzJS`rsj^9Un zM*I_Fb&|omc`YxgYlNR?2H{@g!<cZX796(7N&dJ!Y+t_{UnNa|Pm5gPs>^;jd14)n z&OHymBtKP^G(#S{Y94&P(gy#s3uuJI-nlVo5&gS&fyQ=Hlq>HIBF#~~M5Ua0oZ@H1 z`x`sZo9Z9rve^k^+yl8hY6*X<SK|G)uPC$Rmhg8(d%m%tnWA0p316!^@`vL4wCzcg z_-e8{s_ObfX0Kv0Qi&D!{|+M0Rbjkx++ccBTtj;n9fZls0+)N6bIr=9)OL0r9|(=b z%KII;%cz5_;%S5P?n!m8!4!VD_BWf(7|wH2^rg&mB?Sy#gYE3+aGQ^T&}==6XHVKC z1elH_N4rI|K(PZA6@-IQnB?DG@Pf2#?tpc#z8GJo!ir{5k|)fJi>H>!uI$}MgAZB4 z7-JpwPUy*{?;U8F{&lKJ*$T%dZpY|Op_pT~jMldF6!U$4(Twiu_^E6bo(hz7?@c*y zCFy|Z8m+?Z-p}HZjkV-w7sUp(#^lvSVy;+v(5Y9)arsn7*qqP-;y2ZUUwX1|bDb7f zxBD&3QSJsEIuY&L+l4=y)zQ<-%h~<=2E4WX1$FI+#I;PW*}vrn%bV>o9jwIZ8I>^O z)d=nz)e*a`xgi?t2|$}=Lj~V}(Xz673t>+GHCPz9QjGX{hpu{`5D%n;Qi#HCTD716 z0tf10VZ8-~Ey&{8K_##>VF9?k_eW)mr}Fu`3ZQgCD`YQ`VZ)EBVy{ses54<6MZb*1 zMY=v5aQ+Gv|MnCvq+hSRwkifXtQgHNLh_+b>lV~&rQ(3RBhq;`jWVzNfFmQPaG~)D zTDwh@rP$mOXa2guTg=;|_$L%Md`hbb8FWB!{kW9`Lmdq7(I2vpI^mYZyXa4HDEx4~ zN288giF<qsD@|fdY0T8$aIDoy+%Rn+{|%~!)9*fm#_nAZmUc$?^EICqcK8hEipQbl z(iYj=l;aezXb5Tg9f9?G%te<yQ!rp)U+#KE3$wC1QMlI(tiF&<Pb<7+q0WcEdut4y zI2DDFi*Jyk!d}U<CsMKJw@M-Z4kUil;U#~Ruug3=-0L@p)^_y3qy@2bd&_57OF2BL zzZdTx_=^7a9?to3Pr%x}94^lICVG{|^V9kFATw<~Re$b|5Bp8WQ~lr2rLh8^yeucv z6jijX`$hT(!@!$1@!utD>BpjI{&*uwe9_PwyDT~(JN$J&oE>!#ZiF?8XP&6B$`u3p zSk-~McZ#94+Fne@{l${USII0f9!6F*h?%~sTpO>BF-x>)#b5{gn`8{TGweZem?npx znJf%?=R<F$%tp<!5%^Q8tEOi4;q+t;RBuq=4)zCxH%=SDP|7W5Drw*<ze>>D6${$Q z@1^?9lD8CS;+heeLc;W0a3-!2G$h^e@l*|z4Y(}&#^k}^7FBrt#TXC2n~z3&&QNG4 z4LEdltLPIFMEgrGikE*ppzXTKeDc#CamesY@Qkv<qs{?BqoW}Y{kWgDd`U%x@O~8V z)(3*rI^blPF3*0ICX61yGRw@#{1vU~Xl^?=_iGMWZM=jQ9~a`sNOh*|UQn!^FKX^6 zCy9WHVISneYR~O5l|#O^CXN$8wErOboBsjZqn{}7lOj#FX%zy`O6P>tYSMc$4?J&5 zJw?wu(Ba)6j>>rpr-%K6Wm-!iZ_f$n+A$QP1KyFU%R0XN;0H_%>w+(We}ZYk3~}b0 z)%fV>KhXcw3J>N^mHM6{@|<5p9_j7zMd@3(tFTPy<6OwW*RPPFG_Ur1+8Hw!3s^b9 znVenRNl~*uF9UaY*f)ss+78jgoBjE|*G{sUd`9e_Ih}(82jlix#=NrsAe@}vD(kUh z5eIY{!&7z+;bkYs;W)>k{4_ZkO3f|^#u00vO}#fPd2pFFZ7hS43m$>4G^hQ2)P+xO zpM#0*Z=tSL0qh(69-MQgQ+~?|Shqh)EZS&>E<O$N1qY>hM~yW;jdR2aSx>;ZXrI`5 zz)$$Hx*wfi{)*Q8t)_FEY~WaYnaroSmeQ*?gG+fFT4s#n=-XCQ6ZRac&aGhEhs$`& zq&hK3((LBr9{hN`7aAU^p}gh25bn)~kf;p0>EC(Ok!BBJY4>duTs{j+zx#57L#nXp zSey_W*9_-Y<;at4gP^!JMST22owF}I!#>bR$A_f}E@^#P!T&HWc~C=2UeeM3PRg0Z zw9!c+9$aP|roq!<;gHe*yl{6sM&eeU)!R+{y(jNK{V&hwj&=M$)E^`O(0@;V_<xW8 z<-gQlBmmI=7vKMX(&+w|f6agS{v`m=fA9VOU+29vrAzXE*<U$$s07^U`ak(?Bx=#W z|IpAL9=b46!dJEXkH`P+IsYeu@PE>Khe{Be|8suuxdE>QckK#cT7at%k*UR>-Yyen zbb1YY^v{A@i4{AC4yAoBlW2f(G?iMJNvxSfE(*59h`pl`*Shfae2E<wBZDscQ((lb zp>Q&0D>|C*fI01Zo*FcD4vgFJ8)~IFV^B_+u<F8iXi(S3+Wrx|_v0PdsyUof3kG4} zFH3m1KNB_`*AyO>S@E{LU#RQh5U};M!kVHsSd(=XOsg6p;__$GkI~>($DV@qhY6Tl zbwD;Epa-2fwiE7~tdT9~d`<jN>MBd=7$ch6O%mfC`Jj>3R|?RGu$zC?K&(;QDgMYQ zq0qU}!i!#B{BvD5SR`d0ml~+>oZjR3Q>Y5fm;Vv6luF>nGEb~kdj#*LdZ8?LI~JKg zgqHIvvaHS>aaVmCOf3CJPeTJKC|;Li>=$8=tSe-<bcSFi&GoC?wPcC4UO4K<M|pr} z41`Cl;vUU$sBB*j_vW32w$?+y3(ku6YX{+s^6%7ZlOt<&le{MVcEaHcF_?d}6_k=D zQpuTfH1~M`40`^D4$O|A3u`t((iMp}?65~@c_1g3<`#%jXn~9=gk4T#i!J+hK%e1D zxO?G2ym+p%GGl5tHg(=2_trZnL@1?;zd9S^aB1G{?|K9FIxV6+*AFmzVIZb$+5>Cn zOr$x#+p*TsuAs5z1I>PID0UBD3+EE6sB(x7YV=mep!NRnu=i;pG6-Z5o|b5!tpZQZ znBvAOTls@cWaS^(S=e8{n|RGU(ixP<y)c*bYYpM`*%{p8?!pcpY4~rFE=$A#m?FM` zn}N~Lc(;cz_I0%UKzfN(115`)OgC}I%7?J;{F%z}zhYp^0(G2~y@f1i49E1nN?35l zklW2SK!qNg;qI+N;)1l{psU^_Hdx1F$Hxt@xOpRAjdFpLyQiV3x($YZIt@L>w1Lst zW?}j;BUbSngv;HXWr+`N)5dq5dCrKryy4Lc8d}wfHtiC5@#Qdfc%g(TN75-Uv=MT~ z-4|{4*P_?-V*YtdiFzaja7%F<J{tpg=UpT1i0zH8hlg^^ssz!sZ#?LlsB`%k11!i0 z!Y4`d;i~^hveJ_LDwa-kF_dAvMFE}N?ks%k`x@fvKf{!8H?E6Iq~)pFTrw>M3V$ZB z;^sU&y7CR_*iPjt<#v)K!U)%`xJ9lrlzGV~UF_dC31>YT!-szQV)No>q}>?FoxeJW z3)@b>n)0*sxy+vTbo(q*TDqIMCR$XqR_+wGyq0oIV+TWG?M`{yseHaYIGQ5@uZ!QZ z-M}m^O;lPKz}6PUY_g&aP8;<npNBD!GTer?96ATyFOP{`6_Q}`l|2xB#+>XUFAEPV z#_-;Q+rV<Q8clNf4YsBu;Tugx`t<=ul&qsICv*5qIbs(v@jv(=IBZBEhb_^9`Qt~> zya&Ggv>=-7)Ahlu%7h;nZRI8RbJ*DAHud^G86)i!&~-{KX>R-krw?WGtBMCg>4_oy z#LS5&r=1iYsyoB-^QEBhSYVgTC45QSQgnMg5?*Y*Ba9j7$WP8b6!ZReKo$8`G8r_I z&NjahEE4{J)?$5rKK>ZY_@2q8EsyDNR)IL?hb@MktbpvXdOTvbCVEO4vIi~4`9O~( zxLebo2mCVUPaVVI+N^OrWc^tf+jTLE5A9gv+5o|6Nr}YeJx|9sHSjp4dmNPbhfc4! zAhDJw!=#T#sK0)bcq{J=?W_*PARCG8z=N@$OCm0|@aC@%i33u5;q5Xf2pJa*1xxD0 ztQm$jl^YFd?$defH!G4~#2=PBtRE`eShD~d<Nt^o^^EaTgEsVfz7Rc?E!fa&JtW)Q zr>Xr+!EjGcekAz=TtCUdEnzmyD))r4c&b?U#hnM{Y=iw5`_S5Ho4DIWGm4K(#`bC} zup(a(YkSTSGgrxY+12HO|LN=W;ExRzJ$?i$i;OU_#U8X)r3m&Ji@|l>a@Y}6PSbnV zk>hw3{CrF5E6%T_(KD{XY84r>lv&Vhd_|pFN5av2PJAi(8V%4G1-~PUAj<!ZIB9PQ z8Q9jqJt3KL(l_wM%xRq8QG*&ztMZynncQ1xF&sCPQ}ek|T>Ie$Ow`znHIZHLcwri4 z-3i3MpBB=jwHd-24K1-j@eO@jSS;pz-X-oxjuw|{p8_kj4#H`j_Ocss2151NHhLVa z$AWbV<PV!B2K$)Pd|h>3xmyD>rf)&5k&djIJPvAi_QHSDQ-ly(6KEUJEZeH=On%Rm zIP2*hzV^W!mVCG8ur-Oo-}fVVsl-a?@bU;L8Ytn=@Jc~vO*UF;8RPHH+IZ`aE?%`M zlDAuC$Dg!h5Vs?lR&56Um)JvS+*v3N?Vl&EwdgE3j(i5%7Upnj;36^Dsg*Vtt%p#z zRgh~P4}PKTx%d0-^!j*>m@@XT{E%SDn@60*kB?oj^`bWa9Mpp(NHnHyE<NQFoJYnI zW8iI>23suYjw!*9<>wqiIXqGs1{AHsy$jZ|YyM!w)@+&c^3KBAPDS+N%3A3=r^q#P ztDqwD1x)yyO}%9ssLm{ozoj+8Z1);r)Am#$^vxNvE*u1g+GE784=2&P%AWjLJ)U15 zQ0ApE`Bc`|S$t*g#kR&@;Z{LEoIkBQ*F2I#^xD(n#?t3tnB<19o7DM5Z!PZNaSLAi zE5o)4?cvn6BK$g`nocJeLi_6d^u1Fqt^7GpNJ{E}MbjkD>{bg5OVC1<)MMhl^~NOc zZ^!>`sIYUQ8Xk++#>yZ5((hlC(=Ls5hnwT~VV@u>L7(mY7YfGRlu)5=1T4ASA8!=t z@%?s+T)t$KQ2eEf*fZaSp3SWTr_+o0>9k_H*fN~T#3g9<c>zu!d#ILI!i9o^^puKu z--TxQ(^3xEs)=HEr{y@Kqaz=4dj+CJA?&j_z~|m<7NYihVADGlN;x@%_V(P0bAP0X zs{`)|E2d@2uI;kmd(DxQKd&Pv49tbr!@p^)yD8s1GD6n=&r9Ln&P06NI|m0u4;A|5 zoubOW*C?<&T2TL3z-5EO_^Q$^@R%HcOY-{Rk_oXqYt|KEbi@F--$xxA9t=itu?+@% zXccA*)<)0wT6m-LMDg@yDc|X?%Ks)1WD4);%H^AIsX`w%$aS!@r#`+KnM=j>dRV*V znEam2OR<KvvF81Kxs$}MyLRFu<fV-!?O+EUx~5eq&eFwps~gC%Z#4{1=z!f`rtp~K zg-~qT1<Q3ysabCkTF-uHcQvC3#!ONG9}dUYdphvFackLo<7wel&0RYUiQn?I>misg z@niBejHz$;cIZ<+gWnz4E}VVVne*Q^3l~b<_~`5DY;)NS?)|#~y>vfA>on<HejLV6 zl|?vpS;`ju+m3zekI`jUJD$Eb4HuY59!Jjy!qoG>g{2<S>wH=(F|7t-(dygaV{R;* zslP^_*6X86!5VpXh!OvN-jB6)ELnT+ZP-9xfTP02f}hg-C^4KjUn!soAxGuble$5- z6XVd{^dyIx>tl=Fdx&^-6J7`2rz6kai5tG@@$m^^=wW|JR`G5)zn|#I&FX#VOPDjS z+m%Dcp#!;72=MbO^YH3STb#~?u)i^e0&j=I;?;gQ)?)}HO`pNmDk3QqpQRr`1XuPK z(WQ|;Y2Jk>7+50G!aI{-p=LYic=jzE*k1z;4JpFUUQfhxDW<epM;-MF(&<y`PIywC zD%`G4gxCciV3mVAx}BcRk|zf%Z)ua)jZ`5uUk4h`+Thy7p)gJP2o)77V_?5|5V&L; zY4aLfIQbyW%nza;3za}o?#2_&&gEN2BgMF1N&NQuF#hP7Lw>`b(U7nz7~!aa>r(^e z)wctn>7fyvPsoCdMZIzQ4MU8clx;Wj@lkO{k}0oHiKDu@NMYp7Xu(TW;MVIMF@4A@ zDOab1cW)15hjZF^<be;F+;A4h`Oigf|45K`6fj>}A>p57PhDIRDfY)d$lBQ|<V+ii zNlC9@V^Uq^_jWG$vff{q(D5J)+h{6k2Tu~Ow7i6|Jz9iQN>#A*r#V8>FuW`2sW<)= z(a0bxTp#I$&8r8o!s!<H+xxPpzGMQH=Lf<2{(EE=7TH|UZ--p(;v1nwa*%CrapbIb zbHGkwB~C9NUNO!>hCzSTxv8oS96~=)NXLspf=4c0c(H}w`$yowS;}a=^aF*NM4_Th zm|%K+2B<Y{rYAkrF;clJzYkl2d8226QMXdEcpC=tmtTbsGsZ!nfiI)k4>~-iw@^4W z0iNi~K}BT=Xe96BnO@Dr26=Y38V7Q~R9mc7KSzyU>SWh@n9*+eYq478lXzV}g4H78 zz~f3Gjp{jyZ%Y2cDtt|gW|mUPZ3_%;Z-`4Xdhy#XMmYcRVk{je?KJqa7<;c+#N+K= zijQ}NaOC)Xg6aEiTzN%eyPTdcd{6I;Stko&i0cTd)l|VLD(l38hpVLT=Rj(jq#|7K zmpnX9We|JtxLA2XV!8y%g$}9X$@khx`Z{<q|LC#<V@Fod_JS~+@h%e{)#a0Q2QwOV z_Y|$WIty-k)sT0FlW3&lkGJM^;qb=?A#if7s4TN(mEUIgd479*WLW@bstRecdMA$6 zTSzxAy`|d?jie^||Mo~{RkvlnvLEGDs5fc>2Zh>_$KOyau)R!sJ9NguV~o+#wHN1e zx=?MODCGIZQQIpGoHVl&rN`)ShXWJf@6P8Gj$PUC!)w{KZ#Thwjsqt_5jEd!q36qg zQw*kY*Ul@TmuxS{+&_uutC#VfCA;vA<okHDK&ma8Rr&IQ>F{m4#O$%GwF|ydD%|zh z3JYs$C}B=+WotuwKA)-08fHnXB`H|>v!Ba4d3L3Z7k2Z9<&q!1Ko!3jed0OCk+qdf zIO1Dh)~vlwn-c7BoAyO2`5~vK?s4es8qec*WPp8!w5LJS3kPWEQ*IA;RD0&fj&E*4 z-*aQ|__f<K+w`9p{#YHyiQ#z7^b3TJ2&V|k(==rIZuD#(&B~Eocx?9zqC<op@0oUm z-oEStHSz+n`}0sf5*G$4is#|3?5%J-F$_dAsXjk@2VQ+UNk@*2627!rf(<tas|*@I zIG#^BRpWSjCUIfG51~|=8y|j}CI+9m4Os^&pm=BnrHp<ndltH0@V=i&I=7NxOhy!( zJG}`l2PaU@WF4GxIRmWhrPt7H2ZX6t!J)hu=zs4EoL1}y7hWjA%`{)sY<F6O_?4pJ z*lqHU)m>@q{r0TqHw@QaND-zQB#Viw`{4u=rmTU}xNqw|=9dbn<><(*l4s#YLor<1 zH;hL}`sdyk6=K{3BObFghg;hQvYJshES;Nx25z0f^1=e7X<x-n9uvX7>Y$`kzY`Y* z8)5k?V><O^5VpB*=T6Tnz+I)4MqWKj)qjhr)jd;Qn%b2o_MZX%ca(Y8vilWHEi%@q z=?0yu(_zseH-4lMgr{|Nabe9sl+C_Qy-pZ_M~lS1S{}`l7cZxX-=Vnd+jyM2{sB43 z%II9fC7kXW#$Df~!}n!L5M~pAQ@UTFjJQ6$KC2H+=^q7G3l9i6SEoUuRWhwe(WT)R zRWW$$bn5N0o35-h;|6ufpX`)POJ-@|kNHQi(R>>G-C!^JjtqwWozG+Q=9f}WRYpBk z=Hj>vsa}D3uxV?DQG?7Qu~%rgEZQ_y{Gq2I+K*f>4p8z0tvgzrVf0DJ-#S-FwDgA) zn;(izH}!B&O)~ZH3LsBSb+YZ&3nyMZFVs#gCu`ry;_$k8G*{wVg^c=64F;i9^;ne> zr!9lWCX0j|-zBIP(}@i}j>TJO!{r?n;PyKO?saf2?(8d)ccp-v>MgimLmEa8%c9;P z)40>v`FQT^L-LwxNQ<Q&y`4^9x^S{VxMn7CSB7*&n-vx~)ijS>Cl%uQBQ_j8Y8y>C zZ_X{}c4A`Rj@Z;Oi8sFL!_RCZ_{p?|aCEv8eM|j8XEfp|;8-AaL3h5PwiE9>`c7|G zZ5NE!-+->4A5(OP9MEbP#vL0MV04ouo6NpJuA35Bf6iBW*|?W78op87?h}+)(FolW z`f#(|K3@G?&ei%uQ1z!eotP8`UMr^4a|>6DIG`oEj?IILcFXuklLLqk6EOPu8CW-b z7k_(_4fC3-*zZgNyVh<1-GF@*;L)9%XJ^8hD>CkB_?4VCE21<6;*Aa4Q7=A<#%*n+ zjSVNoW{+@I&b|QkicHF$wmdAW9Y$(2%Un9o!kZUWX!61_VDb5luyJZ7Y<E-Ukr&6| zu=&L_WndOuIMI*h?>z`<4R63kd9>`|UIolutc898Qk?Ttnq#$qcc_jAZwmuTY!_vB z$^AS0e)b9aN}ka6*}nW>XBf@M?k9Qclf`5;3p{V^Djc8ai*(+Ty>Gq&U*A9U_@Ttd zo!3`XQSv4Cw*o(ow8!>lqv*Y17G2x-1?(s75?4F8Q1W^+aY&XKc{Ml)%kSmV$%XeT zE*_YR;%iGd^tUJ4T@8l;4SOJOO)WGGD<y@XA;9Uq$!1grEgZI%cK^1Jm|JgQX;VME z+I<-Oad*bB;8EZcC*mia6o^jM##iUkaA3&*j=pgR#;*;*;)-H&`>06<9bEZL!gUyO zIh9@e0?OlKp+2q)$2#w(>3`g*Z$=1seM*N5*)A9ne3@=qC6TA1Ill5+Be!)chn@Sg zsn5EO<l-3tYkZW^|CE%Cj~7@Ts7XyL_KW?Vzoq)f<>LH;$*_Del3l7DH7`)21y9<< z2&+|KmRti1gZ=U52qiqR`4LUgR1}t)+!UvU&BH&3chEq!2xw>J2*SkY;=<}hV0fY% zPP^vEldYARJk|NF{eG%5?Sb9GcB4X{5K4<6erj({PnTQrzMzG$+hGW*d3D0>>wQV3 zs72iH${p&{7vOH87rSVF5l3uZDecY3=BvR^;Qa!Hif=Q&lHZu46!5o-vr}6{Gg-2% z+WU$a6WN|GrgfzI?Z?VbSLT6TyF+5CdoSebdr&ktlzrW8ICztT_+;^Q@rM|}p?)3Z zC$|8q?A^+<1LbtJ(_qwy7!6NdRgk-sQplU0*uy%SpUfJ-SJIO3=n;EfvRU%cDaVN` zH*XTVUoNB}UIJGsUV`3vI=pB^q}V;t1eJS7kb>S2Oet)FpWV`M=#Bmu`Q8y-9a8Ap zAEaY-%3Rp>iJiN74BWGs46fG<1+}FwpsAYyn(U6jL2>)wRL)OXV`v3c%#Fk1^$NH; zS%F)l*3i#)(YWEH5oFi-VZFt6s?5GeW1E)o=dF&s{jL`K4ihP4zzhCSrf9cB<q*uP z3dQ>Ex5;#8GfcUoAjJ81her)gO!qwC+T~I5n+*|_P6hF(d2j&F?;i>2(tPgS%|ZOQ zQI)sv>BFasJ93AnZrCnl2|e%mn_l$YMoFvo)2}go;kA@k)R1b)FU5)Cg@}%DWvl@m zy>?QZAJt60>eh6>&qPrI`iirJJTChb3SV{)#cykm3&nmWJp9cg8s(#k>gTS|MI)(Z zZ;9fN43VY;nusU&J|g|wUGbO#!zSZ+yuC~rBL;Q?&-KG$?U7OlZ1up(b43t$D2BqP zG*E@aIecT1L(Y9B;F4M;xNXsy!}W|X-*lqr@3oabdn%xs<hu*F-II>EH$j`Bzr4Ec z66m87XLcUVz3WBHJNFS*SgUbZz;mj7y@8SybU`W9jh1cS%{O&VLAu>^{?YP7?EA$7 zS|8pZrJ-7+tCvQTuhr4siEHVg<U`r+*$)RCN~5xY(R5JvCO!Inn?By_jhV7ah%x#O z$82AM&6ir-uHBz{aTKn*r9~GSP3XW<RoL}yCEo4pNUf9OdBK@QFgI_haCq2jp>}B! z>h1pkpm2aHVx{{~bu3=6AI-Aud*SWcVPt22qB6y}ke8kuBkAmVJld$7e0GnNHOU`P zuiD{)*WInWTir>t&6tYGy7ys2{v<3td5IL~D#2mTL}5^!oJP7lhw-~YB;9=|ot)dA zwT{08=fVOoI_iU|$t|$!VifeAsf;Teav}ZJAl&&<g;SOl(iVT|yu7}E0-GzuWojmP z>BVo^!9klL*Q`XY=>3nDW$3`k?~c6c%4aY&kQl^!mskD`e=D{Nl6L;--je2`E?7Tg zB%Ob(hq04mMLDe#tS*klX-?X>`<D`t-&}E7MMun@e@Tp;t|jbWYX=2$BY5`gVl2OP z2V@g$pgDP<q?tz3qJZAyvATvXKeLC~wmtaBsJR$cc8?4*UBw>DBIGwbY<TB`I&xSk zY1WFp@Q$5=yt*xwXD?8K{=21m=IB`N94PIbj9Lbp#@B+<yLa-KF(qP`tVq1R;529s zBoOxXz`smz?^&qaKz%A2w=Ktze@WaeG=WZpUl+74>!AAyPt-2IC<dsnpqYsa=$(}c zov6PC6JI_Ct#YY{KBXmI9xCt*F&EbF{VnQli3iza0e#2a7gk9=vtH75-sN`$*)??L zoxip~X23kC4mQVMe#?aF<>|2R(+R=Nxih>woP_T?M3JLsGX<VlBtC1oBfEO(hU`)N zHdab=!XYggvRivvq<skY>G1BkZ28BPjGP-Nc7zigPb!8Ze@dwPhbY`xNb*@X_u+xP z7GU!B77UPb&uj2FbWNF$Ijf!c(~$lwE1ZmXFMpGboP7#(ZQRh*rxPB=-gvFo1KKoO z2E)~2A^b*rURAD*pSSN}#Y1LNKOIYBja+zm)MSbZQ$_88tMJ1ZA2yr%fwY>fVP&=k zPY&$EcOIOOJ==MoG#`AVHQn-sjK784LH-lQA5()qUa0~ex)1%@1U~vh3H2*3fKgQx zSan%VR#FdGI<W}HD0ifw;I6#$>^6Fz$ar~`f;5jy!E-+{=)1bDq-ALE=y^J9mU5N? zg08|gS52CCH;bHaEAWHC*3i}REX|YlSXfoGW48rH)G)>xW-n2cd@*(`YEQtCo0?!w ztTt{Q>;W_Hw95K;+Tn;t+4NxZRngI~FHT+*$`ih96C>J=aF=Lp+_!45ylKyVOwc$e z@!@-6<>GO2wdYd>>wpJ>_WhHhs%`+K*veqy;xJn9who@HwuZ-RBG}2^0k5{+sI&=` z?rBn%ckc8va4bZTx3nLD2JTxy{?L?dBiyho%M4SJ_1SfrA$*KnkB3G$!?-`5s8G6_ z-lTTpkJ?JCHGC=V7-)=TI|;}3?TxhTGfe%XgByKq1e1Zr>^;T?kGtKcHEV9r--8?B zb^dv5U*1H|7XKs5H&-fqez-@|-+v?<p9es03vrkM(UhXMvYR&z_`<KhU^4HB&{4hz zy0~7Z+piONd-sLHGHn5Sb=f6&#=n#0?o39Ph&{a79(i=G9@YJ70qawDVC~=Yq<Qx} z`N#I>8$aWzJjsssubfr+AbK#1ir2}&d@uOMe53#c>Du2@!Wv_%;G&ZeWySTv2A_eP z)N?j&c$yET)y_gzcrDBhwuOJsWxQKre~#3(#5omzL_KMD#GVm~60=UKIZizgg`xU9 zyRDOWcKItYbpAvh=slGT?GtG4)-|x?VIzf|h^D3=*Xh{IaagZ=nB5O_Mz<ncK4W^H zPBk^b3y+Q1sjmvW*9;YSvK8)^bYQ=&qwtg2HqLiFLfWnUap9*<xYjI?SG_(ASG9kE z?)gNC1!smYcZmR%*7)7>kZ8MS6!)!{_5uX|BAem8$uG?he*Xxh(`ydGl>4*z*L^kk zcTfd7_qYYaHb~l3<TWAjXJ?#wB$mSYG94*<&s$o0VThhSwE2W%nJu7+-EnvmuY-N9 zqOn7xxe${uf%}aLX1QY&XvH{)2ZvOM;cBBO(8(Ape+9$pmKI^&pQlhY${%`<&w>R` zuc@DQ5WEcE0(T@&Q=Mm-_-XrLDjsN#Z|-Q);g?p>P|*!{@?Y5CokdYfikxF?%qJd) z^U>A-Jby_>CT5xJy+rZ{N&6_5Ejx(incCnH*&kvjxN+eAX*hh9F|IK@MAN^0rGwVB z;*!mgkgzvhNFQ&4w_A6M#=UaE%P56Uu4^ON)&~&N>O%)EyYWu<O!}PKNDdcGa8SZJ zWb;UhpX~$_|NIh;HD82{{_d!6{f$0L{F}NV#p27X(f{e6QOMshr(IvkcpyFgciw~l zf*&FQZ2x=y3F)u?|N7@YydTrHrmFtu?rLd&_5Z`qUjq32_dfra?pcubQkB0iFRxfX zyG3^Xn508iZ4v(}9HFkkW<2y-7+uJnNcq*#&}KJ<=Di<@cVfHXCa*wnGVO<xhKBL{ zyBfT3v;kaP6(J6dAHjPA7E3v#3AAs20##nvhpox)MT_Wjg2p&SzT@x}LOf<kGxit? z3N!%wH@{__uf&S$6fRd@9`=hGbbd;;O;5SQFm<*aqK!Yi``~x8NgS}hj(mHrhIOjW zc*)tGEIhA^D)sH~!NeZewCEv94hbBw(i{9{jKJ7^7is3k@f7JOdBi`sV#V1;N|JK$ zx$lpIntmRS%!-BwiBlj_dOc;uGdX;O2`#RR<xvB6AlRLd1}RQBHemo;r*>d*ewd&# zHiZle5A)IO{&c3*oNCJ2^M;G3DX^V4_dDMc1NQ1jZ17|l7LzUM!*(>}oHb94t(Lqn z0=}G7D-69l98%)-g{l2E;dci)Wn3`E;vW`V?=_k|roM(4p_ZB&Vz_eVMV@@n7Vk%w z+nEGO`y-qs-k9%iaIHK<drJ3<H|vwdt$L&6mWKnWXx&8KGi;-Lebgh0(utNg^h^bN z-6~c-JrgEI*T^9MJf*%1K&>rR)c)>FPMqi|Yu$84_ML}vX!joEyfs?9)~~bhZp%s@ z^xFhKOZ`&kW<@dMtv@R1ErW8ut!yScOCgFY*uGd3<1B`=V()Ow9WHS-9W?M<$r70M zEDRs14HRELIS)%F_XkUbyX3d(v3SP)tn7IDHJX~b9`n7g!%p9$aHP{#JpIfTwI26E zAD0L!h+HdEygrn6uRAOIZhDVio#}>AW5d}?r2(pZ%=nbXL7HBYLwyYU)9%82AzrsH z*6nB$_9+jO?(5xo^0$>Bf94K`{Z2zo>?Ushdy0BUzT&Z`*F)QdEK1uN3gc!UCiAn2 zLR9l<Vd5bT%*)v!YxOn3rTLC*ysJODF6e@%ZlmxdD4Y(pw8!zcpU|v#m%!(>B^tj| z=BK@{LZ`hiXyfr?G;&xYeUP-5sGFNech4L+am1S67>|a%RsE>_E?aT?nj6s3NZP4% zp_1dgUy^F-6~Rk>TP!^=2Q9aMhXMDm!tuq|<!W`Yv|r32rSP-xT5TP>C)NwERKn@f zwwa)_+mCl?=<~Ot?mXk(F4SD&!{^r|i@!d-hp@w{+(F?JxE83v>I2PWc)b(VXbqKr z-=rkv=;}aiOPi2;awG@5A0?_xQ>I5v!|`9g9^CQT9@r7Oifp$O(uD9Kyy%ZQD)0FP zW?i$$)A$Xw>(hsRtjwb)iCuB2ZaF<IU5XbwnQ(6DXV}zkB3^gQ7W$r5V6Ek<I7Qb6 zZ?9f}3Y({pTFE)sG+;EV&2f;soiL>Z$)?;VwE-4<%@r^BNnY7Eb{KCkh~w`!(db`E z;5%tP*@TW2eIg06!*}rXAy+A8Pc9sLJP~>yy$ZjppTkLT#nleFto{C^`0SVlUcw>l z<dlgwugGCf$5XI;jfr?_rJLy4Nr^3k_6oL=x4<;M12q+nfYVw&u)N=Q>Ob>4l$r0Q z(KDw|Q&X$Z@pLd<o6-XpgdF8XE5mVl$s19%I04%FbOICk9nzk@jX$67#EGBh@lMO@ z@*DU6fz{Lq^8O?d3=30vt&jv314B6d?oci{+L6wTn#u*GhbUENE&e#6O?3%2+!9hH zWN);uRJGa*iC?DS>S>{zJarkRjfrO81>?kazvfZnugM&1WhK=N`gm<n5YE&a#_u{= z;gvQGG&>uD#+i9|X}B?F40YqiDatr-t_G{M*wBjD#WFK-7OWhnfHTV^Myaz6I6IET zYZA}O&CjuNM1BT*d(>0BBJ~4JVJ5ix<1gwo+ZX2jROD+0Dcqr-AEX$*6b5ZEMbo@t zv^=IteC>6Vy!x8M`@BBr?U(}vnpMK;KWe=1#{qubL!S);Bk)nrSXk9~f*&sr#J)Sf z(8jhY+-}BRdSGP4_a$(?(xg73-;P}{?BgFW^ioGpsc+hFXaE#m9Kha#qM+ilDK^D* z7jbnS?khe6RqZpx?B;M>INC-ip1Bd${mFnYI&)+_he_u^t2203|D(6>KhQqwrBoVq zklOXyBHVql6#dHwV*QLb&gq;Xelifz?$C5-D;bN^Q!Hu5x~<$e=?vS0DnB%SAms)U z$nthymNkw;^~D-2Eg+(z=##W9#!Hy>G83F99e^2L`GRFpqVWBCG{uhFPMJOX@ZxZ7 z)UN&u&dv^O{L+(TDZR0~$5@_J)(q;8t?Y(ZszFlBJ9sp0CDgaO1hC#$JT39S75$>% zVai!Mvr}`VeX&t|R@BEMn>yi^1P9sAcKh*>RUiC2_W(ZG7zdkH4PtJr7fyax<>Xh{ z;-dE&oas5AJklxzmjf|!e6#`Yy7%Rc4-({`bqirvj)3b9Df71nObHK^@Zya<^y%jm zobL0GpV&#)-zk~hf$LH(&1NZVb{&UvwR+(j|DQA^YadC_d^*;9H;&iW0Tfq3dQdx# zElc8q8-_rP;urBjWq+C^<%--bH2K-@-rVk&wD-2R4Yt3O3TAWuQe&A8nx@-e{}(+m z&$%8twAjEZnY7Eh=L~W5-^r-*-W+et9V>1mSL`=hV#RC)ba|+VTl7Ql?%R9fh&8IP zw<25I5!g}q>l(okIZ4>jFAuJs>PUGmDHy0XSoY=Sb&?;oLlgf2V7;Y{{_HI0jk?Zw z-r+7g>U<FQE82mB&RmIq{)JX#Mu@Mq22uL3OQh|2i?ToefJff(Xbr!`F>8;|U5%CO z?Q&cg{?dXTE<c3xR^6gE=XVHq_Q&!j<MZ(CW`f*y(`+hmOr^yo7VvOZBw5Bis5~6* z4n~QF*gYVE+v=~7{&GFg4a>);+g5PtG*3RZzAsl7ufXZr(jK;+jyTH4UDWjRkZnI? z&;RDn5<J}xSKj&ZT^Q!Kll<olhbtjcR$);RbhMIY*(wpxsG3ht-1T_b;b}P4<)1L% z?-<-|VNVtFo8j94O?Lilf*L~x@Smk~X|?GN^5~UZ>CZ~I<#hm0ko1wqf6kKrl|?j1 zdj^-Cx<`GM9fBU2#@zkl9q@^mi0O)*u}hm0@Aka~is>yvTh<O*;38$xO=4u<lx|a2 z>k+V5{!2AyE{SK&cLLW2VdaZ;ymt5i!DwM3?%y~H=JZhE7UQn`FL*SMX?!g@o-`MZ zpWiNiNh=e2dOU~RSzjTld=CFyv7N4)YO>zebEy7vH|}(;0q+n$^mzJ?R;=&Jt2b%M z_Rn(Qf|7U+I<W!sY9df*aA7`eC!4!a+QXRdfp_2Z7mx4SK`W~a@w=88E9^GIF<&Dg zRE(4*G^D`AGl%HG^gsySd>$hEyMxXtJ&YWXgne!<L-THl^hR|Q4%=J}Nuwu#_rD2z z*TjaiE&fubZ32z(vgV1O2J_=5BT#)$3QVopFMKdLB+8v%;drM4G2y!<`6eHRdON}H z$j0H^{gX0W_uVV%?Vb&>(!8~LLMR8jW{R=l6S&kbhqQwh$@b1Rp~9AFP}*lO>%lOd zGo}`vcZ<Nl%sSF{J}cPiX>j*>?dd^dtF%*mFHQ0KBHY?HSbRP4HZ*Jum3GNn3&Y)a zK<>=BEI*eAW70gx=zs&(jM2eOlLo+$z$5skYoi#T>4tj45K?V)Va~32xVG#)sVZqm z-&rY#((f|#dGH*pJEzdAK^pkD@D}w$191N`7j%srXxL;K>Yke`yfk?t>?j%|t{Ht< zT&Fq?vwzLQ)?16X_|GU<-^qe9LX5@xN?pmWXCgeFR}9;xwimNu7cE+?$9LWr(`!MQ z_vz}2Go@W^qYG!z=kNCDKXyBf@AVg`S%=-)0@$iGLY(;gtGM|`7|q<5jjA^8!nedC z(lp+}@>+L_?SGOyvW~*$po?_=);TH}o(q;+Wt?9+3@r}okbGi3hzpH`pQ!_g>Zgg{ zrQM5fyGu*aU#M}U+6h|v<QNssbCuaI=zyl9Bv!`JZQ^ugU19IkZg?i?2B?mfW(|EL zf31f-4Lc!uf?Ka(36_A@#nF7#(v}xK^{0Y6rqtRKM3?j0=<Ox--xldmMhy_T*$ATz z{lunC_aM2X0XFA(Qrd}XcqMUrLl)=K>0w2%JGDPI$qtIo^!>@Fn>1Tj6llYwCAJ%N zwBV1FDU;{vbMP()&he4Kj1Jav-wo|K)TsyBMAk#kgvI3cUXkh=4hgp$AIYa&?~3-Z z|ER58V)~X<(Y^kA#NNN-@kF=5v`u}MXnXZF)N3X|ZTqR@n^Hp)?RBB)OARQyy#VKy zTVyb|Crr2zK<7JI3X^hd#0|%Qa|^8aWQWn@ub509HyFX|aY+(a%n?p@dnCLwl)M_= zCcGrqL)0j><Q1J(;n$wq{|9$(8ctRC_J12QYcOOcQ!<nciM`GXm4uR_NE%d%=6No2 zicATqC<&<~rNmz6Nuy{|N=X`!=0Pc%pLO4F|IeG}{yoR@KmL#7*l)bB_qO)h>ssga z{eC`>Qa_i68wT{VY$UVt$;Y3|BB(7Wky`$A^2P2F>`rhHya+<3=v~geyikcB1BS8C zm8<CQn-92YsRncPb0GR{$EwRK;M*2&gxkR^X6zJhjdL=y`*K38RQFNj=$;Sm2eyd% z*%q-<qcXG@=nve(ePWM8c`z-c9~F=HqTY{V*kD{i4{oZ_REdo&-|7V3nR>2zV$vVb zn0-gIn90(vze(KL#uym#D+*P*ld+3}SeWNx^vcSn_j^gy<@kr2rn#Rhe(?gXjj3Vh z9_+^K13z(6cmdywnb18}njU@Xgs|RtNWT(^N^c%>Qo6BF9;rjyBkkyRXBu6**MjSZ zEM?su!Pw*&$zMo+%V|$ZV8f0?;Y=rIT6$>(S-v~Yjcr~anm<pPP5gQWx=y9x*sABE z{(2s4!C_C7yH`Tp0s2hx;AQZcq|BQ8O{MJBEnx8298;uZna<G$h!0Yr1(rTAebZLn zIpexG$5)MBs>+ZulV<8JKk$0%4%R>W7|48eW>>Qfu(9zD44&GK>1+oD7(aDB*Eood zfdy-bmuGp=jKUifV7j6ucjwd;rt(fnJUsn9SdAD0qx%gZ_pBu9oqQ2<Yx3xZ>O!Vx zG#H#74Ph~A1Pi}O@q=Ri;N5{Y`5vVlWcwQ6xSSUI`_d3!igGCLkqsRUnZibj6S%SS z$8*0s%!T<-7J0ZkQEX}`Y_OAozT^TbJ=lZKFE60;RafA6LKA;vpCzkl@nh3&%2MT< z11#ccHh*PfDJ=ZJu}tkEY!4ID*Dpi(0Ik7vLjb2FERCXDFPgAi-;!Q!RAf$l_fb21 zIKRYNV6Ch^Px8B-!mQ{R%(n7Hf%ioH8ww#yeH)x`e+VmuwVjhqB<&C_BKEWul}p3e zhz+}O^pSylXvr;pZKW2g(i}s^qs?J=X*|TgG!pVu25gGG6idD84tLuQv4y2Pm)sS= zEL5inc|uQivS=5YL`(qV@P$;(1~Df&IcTk)!sbofz<E`N;EUcoFrK4?nzur!I{yW( zde@&jR4`BQqnv?tE&6b4gaI{^G%J<7kL$fvXsUuF&2lWk=p_}{btMcOrYmFFI0YOm zbml55c5)G}qu9JxD&+a$l+gd1%|hS?SF$@5B`(RaAvT{yH^;7EVdovltVzsSPwC=z ziwCmcpT=zBZ7Wt0VaQBY^@EW|da&K@3E#cUnpfFc3w>286ftrkUw=Z~<=F0cy6Nsu zr{dJe;Qd34%~dANvj(*7#uvfi;REJz&-v_&8|Zgv8rrf#yr9>E8ZpDkRyZfOPZwBF za^Xy^;}IUN5jdA2`%%|?3`{Q$=6ycLzz2PAzS(vbtGg_m+p=a*Jv^2&?jFF+f!{>K zmfGX=gL?pNmculgZ@6gN6L^?t$0p7k1@cS1z%MDAufM&MvyaR`qnbzDkx}RQN2dqT zMB6d!_6QpcFq(`@I#i+T=m;9rY6S}uB0yrj8>~EC2U9+c#Nd0U`TgFzIK!3x?Dei| zI6ZnIJUNq%RkwV^Cz393!?d(nut*Ye_nfA`?UFF5zb?xe5lXTbjd_vWZp=`8$FI;D z?XsocR~U@@S*P<8ZbkGzY?bnao+M9PG1r`5Qt$y3vhDf3Eqg>+#;su0_?K%=FXB@} z6v&bNguf|vG~tE|dsb<~AGTb}yc%QJ{%wi4YMcRC)jxw}av#BK<rwyDgcF{2EP|I) ztmyJAFDk2EKsra;c(vDYWT-imm+cB?%F8A}%HR7KyULNXmv{vyeWpXj5kvMp%!WRD zTA@|eO_(w48!DVuXT6<rq%mSJs$@9BSidOFbz?NEiJwk`%GKDCiz}I*<smQ{s7*F= z;@FwBODHq{i|E&&Q#duPjcXJ7z}p=b(&V%(2odGrQ^5n^QDcYh&kNvg{3E=4Nr&WA zAM$JVX$zbAFVRv#m1T`<;LDFnv14Q2@ioJ|X!f%0r15D8vl#0|Z{2%gpvD<=^t!{H z`c@31OJ9j1H8k1X;ah2ja4#0#{tvtdj;8m++TnHFO)me$C7AL}nQjdY;J!ZWM9qHI zw05Hs>5j8zl1&mWe)DB;mHnk^1NqT#_OcY+_H|~5bxWaYdpxr{oglvddoW#kWkCOw z61chFEpUteH+ZvHm3jLI!R=ZNS{eQjN-avsH2MHaJ$x<Xv)782z5W4rthd0p8Xx?0 zVLa)CNAfWVLz%yxJ!|>71XMd8auJt%V7mJXxRcY4&%bVE&6aY!)E-&pw)X_gUi%rk z?N7qfm2$Az$d`rbIMDj9n;@*&i=?<rl2(<ZSBL-LS)1qJuR0esL+rU+IKzF-x`(Yf zOELTOUi7|b&y?b&*zY(SCTa7Ok4jO3N2Au!{n}k1FL@G!d-tJV>sEGoe;&9^8w;Dg z9ze(B-8g@z8=d=Ii))JISgiR#Ze!hfu(@}X-?pS1-@Shfe<N#|{aj<->PisF9hprg zkIPxZtsLHGmKlxgvZd>G*6`!$59qpolzVVu7j}89WH~jTuu-Z*oGj6e9UZ3Zn@Kc^ zJ0fV)mwYPFpN}%ZQz6M@4SvQReB@byI{H;`wl$gKSFEN=)dI22ehHX4BAtT6F2UdH znY8?VD?d7U9DibYD}Qlx5J`A_7HMWU^9he%@n7=Vaf!Ys7R@Zc{(%Neyja*P(-iT` z{Owtqr$0SW52U3E)9C%wSh^xEh6drYGIV?tr}gX^`2QBxRjr5Fjf)S^z1EHWZEZ)b z+ZW-@g9se6u>t$e>tkN$Vi;diOqVY9XF+P?&~lG6so40@)?wXXnP10rTTG!n{UVpq zH;&3i<Ush2CJefDi1Az%D|}M}tt*^3)yGx>+b{vw8T~|q&SLIuN+jDItjuO^&}M<z z8)1GwGdfyc4<7~3#7>(Llr}FM<8q^zy5I~b5L|e#pIqaHDNLq^Ki@-%of1=*3ZYJo zU~Jr}%;q_JLe)fFI(}gj=E#o&(Kkg_Dx=Th^8&E{+|``Is5A)I)dh`uA<sGQ3l_y` z<H>bVEdAgb;T*dU+@M%A?B@ls|K(wo+JT+whw*7sns}^6Hp_aljUtw>;0(X3!Rept zAS}#~miv507u%CuQ|%N8teeh84mg3L)s?);n{(_zyC>@rymH0C3vf%gGUgN?grUvr zp}Wf&G(4ms^73umyY3eE#?FC#T(+7#Z1%FStG{^N+sPC(rV`ZLy7>CoKjJINPcZb> zUeUrHPdcV8$8yXxSz@Cyg?qo?Hgu@tc>2u!`rd$pLdy9emwYMYT_nzFNI@-SQ_ghy z3EX{rDIYAE3>{}rfvw&ZdR0*b*WO#ulZ`=y!-rG9--fJna|Y+LX$OW{4`Pd|dO=}M z62+-5X1M{A`0e3>r?%=S^tb&CA=`B5&;WUX^jO4N^9He*v)WO%b2jxU7tnLbcS7ed z3&Ri3qWe3O_-G3$7W!#9C~i!mO;Qga@|+*OpErlssd#`5Hs!EjqzpeuC5<bv`3hzR ztMOQB0e|kBn!xGofH?jG?5dqY^YwPI_py4+S~3?xItrnsW;}Ux#?zhHiPZo12=VN9 zX;o+LxH9dz>D;Ip|6GP|2!gCHO0=$g37D59K)G!K%WJ#9$@|YEW&0vp-y+PkpUfop zm&a(?!~)K*)kpOAMg(Q$JY)|f7IKdqv#{-0D?Io+lZ+oYVT6{z1AF=g>hA1dlbdDf zYhgNjVetY(TCZVwhc+wjd%<6sAIC>&U4%bBD_lZt&De&x140MY8i%HLxZL_Ji$^-8 zxQ#zKw0W?Ig3kTMuYZE@LWCpxt2P;-&6dV{1>ksjd6v31hG{gNhj$aF@`FdHK+lbI z7#vUzlY?KnI7KEv<mAURKx3b1#BFCdZYy}uQ%^BDB@3qgRgR4^4W)%!Rd~ylm9U#@ z#5+SOz^8_zTL!kI|5J@!d7BAWBR?RQ+KTm;YVhLwMU*d_$|_#9a%p=`!1w+0$$!Xw zfIEXpwq*vl>ZSu*Z2~M~_F&|-)<EF^CCpi#&)fa36?X}9pe!F}e&MVdY?6$o6Y)~? z<K!8#o1jf$jXAWT@(D%^NJTf7MJ(tKLn+mRoaV(fcwv??D$Mdg_k$jA<-t=tzV0fY zw5=Gf<$uFJJ_AU)DG9FBI@3IfP=5BnkK7mU_gJqZFZ76pVo;(A%g&FYh7XZ+$IXDb zhp5uSI(a;zv6GjY`wV{vp2DJ@I_^c!b<qx)$+#^_%+CeEcOYlU+7uGl<-h=3a7<Tl z6=qWInmAgKw}qU{WSHOI1h_k~S2X@~77Oma2~SUMg7|CN*l$A<p4?c5m!-bp!0}&U zmS{X=)@O<~u7AyQdq%OH61T7>)SV)WV{u`6A`AUE2<A`M0FCvF*)m^s`j$L{rBo)8 z*{8dFsI|J_Vs#*$5Bc!2X+G&?et^>19M!*F3OR1`sb8?b(UTlV(Ypq*(GiSY;yYYo zUg|T8$^ZE86K!F`hdEI6Ns_E%e7R)LulOOYA74;;Uo0tHhh4P7V9Q$_YA8MlFK;Eo z`M-h(==lqnIPf3Jwhg4oN00IHrg@zEooaS+S|a{n8KC&F3!@bGk+h}-jI1i*CjR|^ zV=mU?QJ>qQoSI~|F)|VAr`c0$)?PX{uaP@1cL~LRm3ircUf42Z9M$f<g|n_}!19NU zaBXB1R;idVRf86;eEJ=(u3ioJ=z&;sdj{Qd+YWusUqQe17MwZ1g_gf5!eUQvwEdC5 z%4S;AU)IeR7UsYa(}}d(#Eq41`vCluM^GZYwMuP}F#i|n!1Cupr{Je6_K%9gmK&1H z^-~wMw0wqzjuY{h(C-=XKAn9sT?ao7#c{d)?RlB)QgrNV9O%q&fDi5UaO{kTow-?y z3VGWx`q>rc?KF~JUO0@E@{8f#$Rdz<a+`Bq5{=(F_EKxfG1%m!O*%ryV)CvGS{*!| zojMV~WCI#;Ra+8z4_C#Ra*6yagH4oFy$jsh2h+W%8j-Q!V(##c<?4hU;+~@K(0}I! zaCwk|vU<KOEt#?XhxW6VL$2}OYdhg`d>Rf;8N^4pj^GdUYvi(fLfOC4*HE7K7MH6^ z(G&Mqc*&t2+fq8gRAkAnCRTxSR3XgE)+Et}bQUsgH62xShbjeuyI=SKB-%W%^^hEy zdG>Nw>_)J}XW`IxXfDI+v$zTfhuH%w!8bAuy1w;CecdZC;7EVQ<)oorLn@utEW@)J zmUzCQhg)7F!Da=m#MlZ?I{UF7tJzt{*WcLy-(;02>ewc%-WCc{$?B|W-a(o;w~)UO zVMNPMt)q8wUbM$N3Q`PPcq2E#99gyQ!qb%|Y*61MmaTJ#Yq4=+ip?$9uA$D1F^Boo zd$2ZPO`^9|o8IZBGL`;9t|s#(=x_N6UzEqPZ{s3i?W~pjk~%Hs@9`X3XI+AscZK<? z^cE1kD2BoDZQ|5r7t!BLMI`?r8KQD$F%>Cul>PJ*hfEOM^xu7H&Bm3y)^riQ-r)@W zWpilCJ_}CMFp&0abf^7|1KIust}M(wkev&42G7d7;*EzR$Zh=^ewI`djC4^F9HQE2 zH7yH{c-w;f>Ry%{Zh%{J+^IKKf}R>!vpknV?nx_$haYut8H1xl6YX-j`+J3T@~suP zYkv(&y=_9R?UR`FM{VY-eV(tM)ed(}BrzxF9^Nn1qFJA2v%>QN%j>!_`x+wjPt+s0 z|K_(2yR_whnEwz2I{#Vz;D2}c<G<d2?$2YFqJ$UnUyuK1`3t-M*q<XjtLy*H$A7*5 zYgsx*|HJqHAM<;KfBFB`pa1aq|G&Tw`ak;k|NoR9BrpHJeklL%_(696F+XV4`hD2? zY&rf`PJoX0CHU&d09G;P2-{K|#i~zEW6{RL=$D`1QCTfyCN5@imM;ddm0N${<*co2 zhB$-IJZr%w*)q0X@IRVtAHgE;>||>SO88;^*_e6n7lhOWgKM%plMHvq<|~6}@FN%a zQT~`;W4W3%PyXS)1^mWCW=(kSp)iL&rNbMzAEaG3%&}lWy2$0zGtQ|-n@W}{V9al0 z*m9~5mS&mb$sbYTO*-pXkT63me_H`*zpHSai9g%-K>|gJsx14FH`Dde0o`nQ`d%21 zZ(6)@LSYqNGu9#bXL=NCI*jHwzro`HdQ`Snf{LU?^z7G2QVez`E4NEHdYm~uuM>PB zUMg(g!intNJ10tCmd=?x^ulc}2jJG>LQ%fyP<GjKKQ2=HB>pJxiIMYj@x`W%;I8FL z7uHL#7X$R@_kQ9|_8f=CQvq19pbrvCE!c@O?v%(6p>ML=F}JOTPr0#&g&rJ0Gnjr& zvesyF_aDlt?t9TUH&=eu(xWI>ID}IBYJ~fj&zP^$3K_++O!Hd_X15Q<Y4fJi(a*c- zP}~uG>~VrEJ(h)y7pq03FSEslGfSYe?-JyXp2L#L{PDrpm)wmhJy7N?M{cr@p`S(} zuRlweg^g0@Co2R}>z2z{YbVX-o?1W=J%Zmm)14^=NU<2>I&fMwjMV!M(f3-$a$~B% zaP)F4`glOl-kgWq@&zpIiUK8QK7;lD7`#2D#q}R^6}E3QWQY5$f)#h4<Fd24p#Q>@ zUf*is`bADak3se@HTM?RYvoMq)pKBfP$h(OJJDr|4E<4`R4w)&j~{PM#mg6EScQ@^ zG&^u`;rni|oi+&S^jDC+#!@=me=4SmdvMR3Z*aHU39KI8WC1@0)1rb%>XU2%QL!u) zI84A9#g}mS7cF-1>T<|+L39v!$&DK`_)qQO?8%DlLPjDLmoCx4MUQ{*k{3jjH?su{ z!vu7N(@4G{DwM5swPoJxX5iSDK^StujfT_?Wn-V)!==K(tY*qHu(5~}xjp{{DetoQ z*YO`<(Ugby^PD8y*`JF_YnHH8>C@46(o=p&oE}BlsnKP<GVz8TC4gR?sFZR8-}_5Y z)#NI!>Zdy-ZO?{NZ|9<%)GNsE(IKZSAxE?DVRiB#Gq$rJOPnn)cxw;*;Dgknxmjfy zC^_v4+BN4wwev8TFLNK}$M)f~+M$#L2idv!RcL!)5-pV-4~u3ivqAd*pk?Y4%)MTR z(KF8QpDuQz{opm=pmIx)tB;~*Cp%z{=6Ou=>(Am|RX|X?0h<%(!wkO1!wgLeK5nU- zXv~D6?7)G3qI8?y>OT2A%=}ORd0ji<eZL3%WZ6h&e;wGENxEodQHsO5lSoyj3#vR$ za0S*A(atqfEOj~%BmWKOGxjuq{T{+Yf^V|og+B`vdLY)txe!!5gLb#qh+-z$VB0in z!spvSI=vh^d<DnA+i;4%VMO8ld|WXj2{QtMY3q>X^zcobz>>9O1uaT+_{}B^tnp== z4+%`~uwA^F<sN1o^c2TDwc=Mv9ObwNLN;d6QQUc}17=&B&>UqQ%D=ROtBsK*hrqp5 zwn&#&WO(AtaDkDqCx=hcm8Y|x>u|5fa`>d~!N%B_U_zZQ_xaKc5aWC9K=5ol-eQ7( ztaR{^lrkl+EdseSUok%FAKW`Mn2Kj4Gm9ts)HzC>@(x;yW(fW3c+E!MOyVZ^J}~DK z)#CXV`{weUi>|Sz-hI$n;D*mPUqq!z#klCxQtsU11$bzU7CmjXLJ7qn2r|xQ5=Y0= z4d)_I9`Hew@7n-1{tNlR=PvNGJe-jfm$2<DoW1i26dVkx!c4Uhm-P!Ey&<<msqJr| zU&LKpBjXAQ2KRBl#|3t#+mPKkJDH8Lln3uFZF=xYAOHBKv;MO2ps{ZdwF%71)3XCf z?N|%n{@RdF=+}!+4+U3iM*QZk%u?hHC(2^YH%Yd}<1jC-2_eOnODt&cYGMYm?Dq{l z`W3SoJBmC&c5VjtM8`AfA1(Y9y>eWl9!DRqNZ|L35QtgWhtECr@Z{7)&U;KT>g~F~ zUF@AMrpTpi?*gGOxZp*V{4HZhye8}qwts}xd2TTOl)U)ifl)Yj!y))AvZTGbA7H{s zFUT~}VK$4~F#g0K(y>C^sk4YRU(Lt-^0~DC&s_RFmr#0EJdX4KC*Dx_1;VEG(%2V? z?4Htm*tX&+7g?c7JLl}-19lU4wNDNjHyd+alWTF8R3=TEW(-QK1^P>iao)k>7_4A~ zk}iGV{PGF@+g^o#eXZHf+`qj1yEvRV>kRMg63Jdww4t1yFQ^F~dRKN8hAZ8M_LqP7 zu`h<O!9uTL!6JWPvWi4*1-NcsHm~wjmEKK=1!KoTG#e*$<HO9!ci<>Ca8fTu1nIKK z#3FjSHk4E8XTUEB{fU3P8qjy-eAw80lsm9w66lUigvT0hx!XEJDKj_(+TJSDn|Zgu z;C?UG9d{)~9apSzi-OG?uJT5^y!on5Ww!W{D~tU)lRbU^4)U(<gMnGX{@L@1Y<$lW z_`QD!e~RwF4AE<t@?C<?S>49d%GRvU|0L%)*Z}kw{>GC<RXF2|1W6}e6MyIqfxJ^v zwEY)BubwBXejspB)Auu{Sx3;}l?0WoQo*-%H?hq$oaUxTQPe7VZh2x1zTGjC>dim% zy~)w^QqzQDvc}P<J72jF#aMh$@DDyQMV1%mKp)5Dkl|Xv;XPj&hF*Ne6~%04=l-37 zd0Cc}_3$55{93?3!juIZEW-&2nylfP6fK(EiqkCCaaYEL;TN@=X!xZ=eD~KkUVy}q zN@cF-S(!7d?=6NpsVV5qoM1r5OHS)oD(-zc4cE1=WIHUIV5;nX^ll!88b-4rd1V;y z?%0dt@<X_)WuwVjYcC(|xsrdnpg#+%%z^tZqp9-kQ<!G37xAS%&9s?|Hx{Lm>)d2q zW-SouR(%BBTENfO2D4p8b7{WcS5%NX%RTMipAB-%f&|N2*s{Zc8(Z*~V?!!1cr3?q zqjf=Ut#E(se~yI~b%9U6RQ|!>&%F9ifn63VQ)9ww)5{P3^k^gDE4g*B@1H$B*0$%v zEwx$Q9c#99Sw23ho5a*FmU2^Wq;reD*s=02rFdF67ltZLro$1=tnG#}NhoBpS>D2R zinE}-G6(R+9aU&uzLV{rwiF&;wuJU@DTs4Eg0>67U7}v;QMc+T(EM$|ZraMRDP4}J zE76ZN`h@V4jW{+(wGUE9odhG@4cu70)7-mKAqyt-+^l^Jc;#CG>}*S%FrS}^izmBN zGAm{JrGwa}Uu$q;jR9p{zXY$0vhmU6Cj5TjBQ%-BGq!ROeQ~v8?nw(NDEAKkK}U`p zFDCOJ<`;6#!oKOR0ms<sCyCsEt$D0rtUXgbe;U;KOQ6%Yuk2#mJl<uFA=xbS5ppeO zNYnQSFYlrVMh!nPGN2P8x}wQi;6HiE3A{k3om7*p40Av45PW)vITtl$!GE!fZ+rg| zBtB_VgX1%9l+_y$+qUuhyZ57fwl|wSPmU%<CZN^q0L<zhMCvA8F8e1L(vF>`m@*`q zt>}uwX#-<XGNGEY3U~ptnvcTASrb{9VhaCce>eZ7tqPB6X~FzS_n^V?G@kso6f<kw zNnbmJ?QiIhFJ``l)}4c>>8JuT?oMKX6Ap{^|E`6LnqNe>kE*ex30~YLD+hE%GdB83 zA1@`L%%-^7)8#!;)N^k<DK|%vkz*d;Xle?t3q9df`FM=K2>8k%gvr?LhPrK6(VyAS z#WF+obBZPPq+8RXuqkxDVla7UKf<*bM4|#O>QRy4CLHSq!{3Wp?3a<$R_Mx}4LHR+ zvQG4A=@L3rQt)D09sZV2L#-_<Q52RdMomo|ej%HhOK$Md&HG$#{MrFqDx6s7F~KqF za-wQsKSm>c2GPHLY0M~JV0Roz!^RyPB$%wA3C)#U7=MKM6fOm$3=?Mh*pda>=Rr-; zV9I(X!uI$miqum^pPCpf&DsV=GKaY#Z*%GN5nT#7IuA6hlDV2|LOyAeGpimQ&dn0` zF&=pw;varF0!oeJ@JJOW8nk0I1}pAnry~RSJ5#(U#lBE<aZo!yP-zI&&f70Ocg=@6 zcbA~w1rHW2aIm&6_y_9_9^;RVIl-pqIq_@EE2+=AhU=WMiFI6ijP<dy+}3VYnD+G- zZ_~Metu&d#POVIamtoCdySxnV7p!&JwoI6XZ#sel-@JnYm!;I-JO;ZOcEcgVUW|Gp zbk`r3xjcP&38ikIfUQQcpqO}qw=f>Yf@X$de5Wqf3=fCn|IXraKg9B!&-_BEY3$t9 zXlN7fhuA%r;9|>mc&mfFnT0c%x2<C9Ht!ReL|=w%kIh_s_)2>6&yLRK+OZ{y`t0D; z_jqVbH{Kj+k0H?wP`CaAm`k*aA4-g2D$lyn+qeXCqZV@e^HcGwvkPTPFql|*24>u3 z{L<G6{H`up+Gg!Sx!Vib4rb1ZingQ5$x(1(O#y$hM3P%0k%;YOYuW3-45<06B=B8t z@Qb%xWj;@h(7j!q-zBBVwhc^WP54OQue#Er58L?H!n#jAX#joxa+5z<<PJv{hQo!J zl}zm7%(t(+3tpj0;AVOX6s6^v`M+MA_Dskv<xgUxjZSjI%t~44C@U%mXg~$Wa@=6J z6OTMv%xoOAXpVFOmHIa@pMg{Ho^dblzh8=mhX^?z7cY8Y`xYli|HL=KIwQkj7&QkE z!~l<-cs`?pOKG@HXC?+>OW-J)Fz+!qSIE*xqh2t*dIYChhcb(+ZrrZU@36-t7|is) zaZCO_hm*7RVU$T(wMTt6jdqH_1=XY3w!}1Oj`1VeRTYqXC4fBcWkKoZGCoF_fB&qw zg+Z^k!3aMiwxv>?{kkz2-<Ii6QPNPHnrO}x`mKjmBiCc`(|*kFfFnIQdVtEbZ_smn zf4X};7)GTJW!ei2SY^N#=-*Yww`E6?Z<+>cU4K$M>dr$>^^6ADjMRff&WO|(K1bIb zGGx)S;=kvwKm5+i|Ihf<b^=fMKbya*H)rxYDrC9plbML#WiaxQG<$xlr~0Jre4#H} zA$SJb!NcB|B>YzJQeIb3EIAkAKguv}snCyH<bXQwQ)yCs4XkKVz%SAzoPW6~MfHxO zOv}^U;!<DM(DWG8)2`vVe`Qd0`ytdFKZ#0>5*T;p0j~Diz~|T&!I!rpT6o%$jjrE8 zb;Ux@WywGqJAM$+jq8FF^^9=;d4!$)U`*P_8Eu%SM>pd}p*a2>df!oD{ETC`JopbD zPi_Re`TL<+Hi5Iv9KkL7`4xudDN$9|7x1k~gsXNr;9BX8dxSpd52vB@<*pl;3L*R3 z`(mAcJ*(kOorkjtcd}sq`03PkZz>rU_G4FlO__?PA)Dqr8MpZMa%pFmRnPR)hT-)_ zw132IoLhYbRFZFETDQO}Eq*An$r;P|tx5RzxxlGAydIa28poZ|m%w2oRq4mF*y@*W z1@@uM0Mb}j%eiFz#xomoaO=x2+&$AA@P8!9RmdCCkai39XJk0=6ZKi8(oQ@i9>~XB zD&T5bond8Trs!;-GD}gIg#ON^Ff8CP>{kCQ_%pL{g3D8o7`dGsj%M@S4~zI=zy5OS zVXrx#)Nvxu{lb2c=TTmIr3{<0#TORFHzJ0#VaO0&&OPu2D48Dw36fyCuQZ^m!GiSo z-C!{)2q!oB;#D?G+@X1hIX)NsEfejzw<2|DTz-nrH5d(_>n}k_^&6b;HWkB3myMXR z7u2Vm0)sL;e$L2r+FBWn3q$+FOJkP8g@FTU&XlX%WZy1MqVpIXcN&4(|4P8wC=Ip- z&SRNwMWCU*luQOMV;?d$3x~}dzPh`Tad0@9`DTmI@DSI>@d9J*BIt!`GtWu|JXq9= z6R&Qdta^90#9NZ-dVhfnY62s8-fqAx^_V%}DBJgHA0CzP$Jg`ju$9fOY*4d4rEW}P z3%ir>wb4xcWAp=+U5|6o^_93W%mt#i9~5}>rRXPk1mA2iq?1M()I9wlN@(8YJqKLC z4=KtZ_j3`g{`wYWw<L<UX0L&G<765ld{=K!o(5Zg+~iO5F}OvcmEU9ONN=Vkv-a6p zth=A^oJ%Ds_>%;4(mu$(<vqvBaeukobX7L+SF5OMOfC1Wb|mv{mgDtb*KxJU`4HD} z1GE)7A*3b~rq;cL-NH5aj^<)EPEQ_>4J_i=b4MyKN#K{N$+MR4AE0S~Jyp+a5>4$s zhD-C5InNLOAb99RocyDTuZ#-7OzH8g{&W;PTa%0vqc`!pYHe`xm(lblDTPmYs0`=l zrwP}!xBOFkYixI)ib3Ijxq<H|vu!hn(!uZEXgYrzHmbx>{y%lRymSE^Q#uK^=j_Bw zC-1<%@rBg6vI?&Yo%lzc+O)P$iF;HUiyfm%a9*+$%eCAN^~q&0eAi{PJ01g>2VX(Y zu1ZL0`T-RI^O<f#9lB*P__^~lWU1c8Z%m&|9i!k^Ks30qo7^!i8+J$Bhnwf`rgfVI zPWf(kYWVR0o{jQ`o-bmsy1S1XxKjr8a_@2*`bN{rR%LeA;2c~C|A{sZM$F>eSg5%F ziu*Kr7aF_VM9c5d5T_yZ>aSmgyK4?(b?_i|PUtMH?N~~|bu;PZ$4&gQ^HW%Ass?X& za}Q$&O=wVb7cO5J#C&gTsQw)LP;}q*5MEbW#uZ<4rFY3A@qWiq&<IGxPg`xMXGad0 zEZ>0P4)ZBp_&ZaN3!RF4XTa?IUC!R=BJNy&5yK@tNoB1vJNaW3i$6S<4}W9I>MK{# z%HLiBZ{Y!^X!VG9&(a`iU1iYo-$x6y4!}B>LT1;S52pm??i;Zsv-jJAXMc=<=E#9; zMyek4XcHvv3V}Dn^`W(R5$^nB&3%3SMCdCC-WZKs(z|dS^A^MjOfM<6J?ISIp_+kZ zdH$H6u0rE(^864zA6Riyo<z2NIAUWRJi5G!T|Ax44L%S@F82-T;e`bv&)jF+@xeU; z`!^9!w??q?q=%q1^*MZ4l>qB>JYhn-7(@dmL&Xzi_+h;qUnfPf&B8pFR`;jJYb!9! zFdL8F^JVV@zwVg1bNN>{vvF%{E%yC0Cf^`M%FMLKOCj%EBu}(K`10vuh3Is2w|fJs z3zKQM0pOSwW8mKoH++#T!ycw(;6e97n162ro*TsT(F^b5=&AYD8?JVOtin-No^cRw zH;!g^SAKz_b~BbUG#xf{Z)1V4D#<-|8o0i$1KGE!?51B7)IL!o*ZILTd-^I-{Gwy{ zC}$+Iyf}afzWM6Wh9`OPhGITC=?k7&EWvdAvT=2a70Z&8WlK$7LQjMZQ;|tyKS~C& z-D7K+gl{=6JminR!c5>uU>X&hyv9lM=kOkj7qVR0W-RL(#R6#_p1+(#Yi!5kKK*81 zVa!Hyo~8xX??=$K`@3M*jq%LkUI{;2ZxUGaI`cQhU7*?gRP;h}2z}RC&wi}+r|Scs zgQiM9;#zmYi_9WAEZGR}p#`@`mY^_F#YGkecxm}nkQ2U}_iNT8=Y|N9s>^2!%vO`* zkxrNF4U6%+K_)X_auP;=EXC&QJ1OgZE%)i+aQ53n@MJkifJ@XwIB_TfI{F85v%1xp z_?iOx8U7P;NPAe&SVg9AaXM}a7|X^w$Fn-UTwJX29$#NNiX%p8VxOulZ}!F-r@s^O zZsJ$)Jo6TuE0JLBa~0{;bZvoSH4B$-+eeeXWssrTIqGiP13^od(YZ6j_;ta3`1Ded z&~+(;*|+v%w!v{q*|Lc}`z6C-0|fW3oD_REVG;b!-vZY8`M6?91|E0)4s&8{q1IYP zyzRaU)4GrZn+KGlZbu3~uXi8>O5evaw}H_0=qG>9V=Yyz)`0DE?u$Kt#pAWBsx+uD zksE5b9XDR+hCiVq_TfnmTl`B6ZdsU6O89Dw5$(lNp9E?iREA+zK0=;YmFC$7Q0C!g zC{~e#xm8J&wDuK$_xeFNvVIQu_b8FaMsHdcWdo%)WoR<xH0@3C!?=zj(ezpCIfo}< zICfw)r)c}0*RGw$JU&f>>0iHrS-LK}dA*(c6ML1PvG)VaDba&>x}ETA(*S0zFhsmd zBa6v=75YyTD!D16CXr2366Nq)>8MUCNSm2Zd}<j~oGaz(q?724-bm)tPm`WX%wSr2 zvnj1IhW~EYf*os5^KZ6mlE!QeW+t+s*ipY-#`w*mR3UR%?czduu3Bv8P<1-`X$khE z8_)#LE0E$~%c2j*a7RaM;m(HJ0RL$^Zf$o!&u3*c&zs@zZW)@oNP%7c{Tjvl58yaW zHzqIZ$ZC(skj<@Ms1a6+S5{m{y8&at?Lq^7`C|({MsN5Z3X9nKw99bc_Yn4t-_Lj0 zXNgVR)!<i}C9ZoHOLGs*rjk%kx{_l9;rC62S=vglcyNgeJd?~j>FDCu>UykLXOAC@ zPSL;k<uGHit1#Ca#U4azvOMVukcd1-HJ(n88#svWol~Grr#F1Z%M>y%JIg6N`OAHB zv?A3h0w=%XIsA;Vpp90=IPHcH`Ml}^iLG;lO#L=ge~<`o?OKtK9K^hXM#9_?(d^}D z5jUo98)+Qu<%4WQtXH8B`DYtQ;)ov`H_Vsq@4GGvQFVlHVSgq-+JWQ^XVdqn1H9$4 z?QpsObT~R|4i>c4!?-Ivr_?tN4_1cpvr-L+Ok_#aUja(1OSmtEu@s;fM=uSpi28*E zK+o#k?Ba1p_I}_=*dtknt`;g742)e*8G{C49ys8&E1gN5Nmm8t_xoITPUiD&b~vG& z|31l&sXDHOf~?p4_*)5V_xbN!=vaicJr1zkrV6_UCgWiDSU%+KP2RNJg1rCa^65kG zW07ttmJ54iO9BE}T=*K^DKe1eB$dF<iRB=(Cj%xL>f*53LHxf-w~>D`mlM5t3VI4R zAWta)y8QK6NYV)WGb@=4J2jcx(l@BFU;v4hNQ1MSJhO~YV>4vN3o}RyR+G1niJWVI z-|q?1L3S+n>lN5A;sJPiRAZ`4AXEB0n6-H*qyHd#8k_S-G~w5N+U(X;oyT0*l%p=R zKlB9HG}Ul_Y?NsICKFm9Tqk8t9m2PY32a$~2|kjjhY2YTY}>;rNJW4x-!5_wHch0` znUYMiz6^9SjA>=+RS-M1@s3J=!0OmH@IK4)ifY4HnN9|N9dH`XJ=1}OLq*tFkS}n2 z*U&Q8Z;*U+0*iL6<XgY1V|@*AwHNDf)4v_~>$wxYve#y}WEO&%iW8X5@uJE^eL9}h zh<lvJv2`uy#J*L2=!`9(5pE=USNaWBeJsbqK_)D{FM#gBRX%Rx3^ICS26`+0aeKz* z!SbYIU@jBHRIOff&Yy)&LiRV%&I?1iGnGQ-JC2I`PZD~*rJU?850DJ-#(9UV=tsv~ zvBB{fu+`!Mgl`zngga6?v-A$M+%)Fam$dR<C4YjQS^|hSd9wSj=73p}9I1RN#n_JP z;v<<mSke3x>{_&pynM8AK#esYasC%7)CpOwxC$ZbcOACoX0eY~n>iWFLzMMz1dWb4 zjt_F~12+Fg%T75k$(w=a#+5OJx5GK<cT-r&nhI2CTnVQOrt-Jl{aAH}w7^blz_S6? zY(&LAGXB!W|1o}wqZjQ5m7g2f(8+yZ7-UZIz3ODKJQBBW>K1RmsRc(|WVxU*4Q%F> zk>pXJh6RT!F&Z~As|LY?sq!Ai${uGruj0uiQJ#5?v=HV2cGPbE2O4VUQTB^MHcnjv z*Be%I^BSDl$UjY(>UfrE-F{MKZ8iqkHh0)DK8l^$n#i1Fv*^#=1Eh36gYR<6<E)PV z;ajCXaVqWBtn`o>H_c-kV}b|w@jhj8z8J>NX%8paHd7kYeiN=2t6=zhM~rwP1rmKP zpu})0Q%M`n<_m0*1%mhL{;hpD)odn~9={A<93nBo*p-?d?LyP!ap>7OiB1<v!R>R? zsrc?*_I=`AY?^1t{ED(fRf}HWvakNMvMYrzd@!F6YAl9w<>~yyIa2tGg`<Wb9ZnGv z+IT35o|GH2b@%UJk9ZG-7Ml^jRENK(c?!*ItpD5Jy?CIH&;M}#Ah?JAv-!jSnzQ)- z_Ba1w{^$RA{?UK<{{LhC(BpytR5DhiK?A>Gd0agxMH#TEW*Hc}dM74Y#4=fnI8Jr$ zUT)GHZFcsWB`QBRWM^W|&~x4C6x=T!ou_T(GM9B>k>fo&x#JLcZ!{O!gI9SKqf{DW zznV=_5r~ze?{cM5lemeycXHp>T49)1C8lP-9KAwCf-Zd2VZ|Y{Y2<kWW)kxR?YHT{ zbD@vqpfZHHb*RwdFlY8`Ujn9{972J2<M58hR8WPHG|9I}T(M{zi<F208#8D2avy`F z>q6h`M-{gI2w=5i%E)W1Gsx=8({Q1CDlUklwf85Jik~&5+j-M5p#$f$ErO4UX@;dW zdDyOX0;Y`@GA*n3(vhNgN-wphm6~z_|L`GHS}zvXJsC8rYA%=`kKr0hCs)1IQeew} zcH@@IN7y}~U%6evjWK~IY$c_~3San8?Crx$!*B?5l-dn!mlMo=@*Z#5#xmaoeH`&! znF=k0oRIq@{(OEo<)n%*J-ZQk>%+Kr=4g1bzLl>&a}TqPhJyD7j>!!8hl{)K;g3(- zaq-Mr$m{lE&xifQb+-Gkwk;ioe|^kPn)wkoWmZA_c@y#bXNxGHWFm20k$kbDJvH3j z0q-~`@=AXyYLguddY?_%bkd`ru^JSAi?J1tGvMTvBN+Q3mphkKg*uOPC~{gdZGMu# zH%+b=nGP;MgXtNxK`x3M>&)@5Yavt>tzv5q3CGVN0RpAU@X?J{u4mB_G~RibUq56l zU%AAS<+9Ci|HfCmAh8#HLQLqcsux}SvZK0G(Uw-N`3A>EW#Y8b9xT~r#Rl|-b1I9A za6y&~_pN3-*W4Jz2d~?UPgTxCvhs1!(bFFwz&DWV%&di+l;!NzHDL|;u9)`On9;8n zx4C^E`f>3$HCV>8LcaHdAL{w5v#@S!`ebVh(`9$y-O#OYMZ6P6n00cN`D5s9Q4z?q zVl-(UN9KNBWPj_UFn_ATgJXnilyNyo-y1^rY~~X;Jd6&`wV}m|6X;221v+j#521c4 zO#bHzx??HEKdzB1vRj^2d5uSvk!GS%YQj3THBVTx4@cv&Bd8wUf?tXPX;p?Xl&k5| z;!C4NW1nY$Wuf4d9&?s!(z#N7v%-%GawTbKlRf!-$z=LRucB7hB=+io1NndYf|HCA z_!l+H`H4nObUoC8Ih%OmynF@j=F%j%b|{Sg>^}<=&7;t<JddsarU|tYswAOqOkX4h z^IiL%<C?f=>~>KmDd`ygMCo)IChV~es4j#xJ36qpX*fOl>cz@FsX?;472~U?Lt&vF z{`me6Ll?<0|M*}oJkSH$?{DJIt-1u?@7U1`t$cA%YJcYT$%eJrmQb>C4l}(|2CY8d z&|s#(pkD05*GTG<qe3~&DGq>;y~+G0pHZTjuPedbcQBrNcpsiutm7ZgekMv>8AGRz zwcx~+Z=u+3G5y|g7t57<pgS&&wnw>vLG%NxeDDM+cGsZ(-m6d*kpjzQS5ez!Z8~=K z1p1kOgR37W;kpAaMR8X)(Mjn|%tXN*u9ORoxgqiRUi&)VlB~<cPH<(%TqjXU*bpiT z5}Zu~N?Dpm92NwdlTwodo{yDc+7pja+f5VRq2w$cJ~l@zc_ABS`l!=z!+vahg%tDa zb7fOTDzbaRYx}!&KE|xP4oN*Otl!z0Eb&G>d#+rLf8Q^|;nJ$4VXDDO9xh}eOLJDR zLV+&3J>-AvIKx>?l0@FB6yJ#SS<xjGIDJ2s`MJfjh=5->a#;X;vK~NLLFITSK<I2A zMttqRq<XWZDM`q+q3>>2@zyohp=rE2*Vbgu4g{CcKfBqa*XPfqRT9wXu)qmECy)6v z7jT13MO?YTQ|Q}N#{GSh16xv(gnOb~Ht^M7m#-yqG*2s5;1ng&s>L6%Fix8(bV#!+ zum8(OvVwc0ok{y|xnqr_0es!_0Y8l~!t4h^enYQRG_>*$yle^NoF*@4=Wj*x^ONV1 z+R=AVq+*7*E3QLSnH&wDY|qxsHb%1_h(~@@g3PGX(Ed;nXAYEwWBG?+`T6UhcJd|c zET1gyxSxb}!%I+3rwzRxoB?yIo8X+RN>5YN$SiUaF5YnpwtaufMYOBYn8h0fhWr59 zpd+07y|XYiM~ioNNy5epeiWAS0vd*t;jq<{Nq5*BT3$Mbxexe+mT9uoP#2564+Qz! zrZ?bcdI@Dceq;4b1!%q}Fz2^<<L8iY*nCTo?fWQA@A`Iv$UmH_ly=bg!*?)gK|Ib0 zjbN_oBK)VMPaDU2aTaePxD|tIdFvOh1gDZQY^gOA345wiZc!+bGZow_Ysg~m39i~M zowbkyGqlPkmB0Pi1$TM2%Vi&5IjtHDb3{<8Z~?rdPjE#k<5>T)l`J$`$RDb$p@WUW zUYt-j>fMpR4<0@r7WNiW>Gs!E4focFQZ`=4p~ZDL?#&RUw1$U8PD63S7oqENv5uZj zl_q7g0i^qE0-K)rnwycU2D0O~;_qR{(AZo9mfaJ4hwtt1(H-7J`lsM@ZEXOx@y|hW z-zu2(Q<%+<v*v5xH^IHFJMou^BQx={#}Rq4FyYw<t|hM!zYc$gssg_(|3Exk3p62o z5{2nagcd)0@b7X1R-NO)s+UWU)@6Hib5vkzw@qQ5L@CuvpNA{sO}Sk-gER{o;nHFy z=n(w3sh`s*O?ii?=aD}(%pFF)-p=rCp};Q4*MvB^MfBmf0tOoTvyNT5C^t<3d>`8| zpUxeq95<aVrN!VW^YyH}`~ZEsQ3^9RpMu7brc^hgA31Fep#a+oE@kpT6l)B3$vnLt zzK2dC*RA6*sMwM<{OjfFEk5z}9xm*7UoQOG@(Xo_bwS^uKNwmvh=mFM3CGWYu>6k^ zdH+bjudc(v|8fE5H2`H~ALd?IEMP+;1W#O1HT>Hh#S{%!i!*HEs!w~QQ{E<38Ye$g z;4gZ!B=<b=->X?HNfr4=%kIIj>ARsgyMzUtn9kii7z<rrlXw*_hPr<D6WPgRGR?0^ zV38;0*VzkH0G|rVKm8rego8EWy*iuys}9!RXyyN{Q4lyJ9o+S)G1$IEQ}|9VgN~XQ z>i=OaZnCjt7P{*w>8u-kF0^Boyrj$K+EgeScbu(?vu7GBIW%p)fw5y#;jzbPR_6N& zZuV-5GH$3**1SQ~tfYbOO%!P34klJSBS*o(I(V!&j8}P`4o%0;;k{^CzQb3Mvf2jF z%_FhUKW8{>SA2+_Hnt#<H-sDvNWA;VC}vlGjCQ6q@hdMM1ReW6xS2i-4}a(aEop&) z|8p5r{xXfdube>UEqCzXazz%HdkU8}2jXt2Y$$o@#wP?^!8HL&7}dOu+1)&a!{trb zlc~Sp_URMQv-ld>ay!9Dp%=QHlv%6fYqYVS0!}rdxI@A~yn4r0Y+TTSC-Rj!Gt1*l zf96hjl;SI_5rVly)n)8ZxCYdYxJ<uWi}6YSPdL_eAg(qVN86IENy`5<IGNj`nnnb3 zb5>-bnGRI6%aV#DZ^4Xce|D-$2IU_0V*TP1+~s5UxTQrm;nR}S5UZt274GKTl6{h5 z62+p<_Gp%P_z-!;_How-7-0Rwc*^*gPG$}XO!4Ufs#WXHG}F`h_@UNpv-?>X9ySM# zs!OtihkcpmxPI{9J#c$14#3E5MYyU!V4)=rXPb^x;MN`O!rZo)`&kmft3NSf8;<vI z*ZL~~n>3hRc#|Xe;H$x#pMq<>#=#eZKvY=L&V8ymiwDvMkW7uV;HJ66M-@fD*U;It zC^8S6mW42t-&QnX<ay|xJ)V|&uVAxrAAhNB4%)1n&GP%Y_=A0)z+}fF(&+dH=WYL? zMxzD2-4w)ry|tstcLM+5-f8^poj~vVlHvICaoFT+&y2n_ihpj*Vp{k^e0o)+;1N<~ z>U!ImjA9mT9DRawaG8&luiRir&?vTD_X>`Usz&>HOIXAv6}agU3UBjencUV?yxk>3 zhLSPlG~hFM2^p_V;W;!j;WI8?+=9+*4S#%X49C1)z#gMC^6S)M=C9*nnyNQjbnq}t zjZ3CfsS9Y{B?fzyRdhc*g6)p)<jd6^Fxv1LY*H|1QKxlbpRG1#oRL9Gu>+*9k7Hw3 zKgKCrW-;l?YurD%A#_S4#c5>4;-hWVY-s%#v`C7B7y3i^mj0@2%f<#+-anNE{pC2v znrpCjg)w!znlg*FRKD_VKF*oF569Qla*^*0;AB4=df8;bwAvPP<Ggl5K!7|dYJ{@! zFLh8$sUD7J?qvF5&$;oQ2Vl3>RS5lljRn-d0#!{R16KClWzZu>%Km<c?T=7|SBcx0 zNc5Mh+A$O*il2#<l^42nWcskcG2v|Gf(O_rc<N?`yoBg=>6mtJB=+10!+FkA;Y{N# zc$8=id5(gcj(Z5ot90?&+h5R|pN5-%U*twlX#j7vyS(13T(t3=hLyq^MQfP^-Hr97 znT<SNJ|BW5XI-jI|14nzMNQSZBRavtVjYZJ<VjXH++h4-Q}El@%U%0EhaVHT6PRBM zTpgGM28t?lVP^vQADu^Tnofwjiy_100Ild&W-axrNm6$WJXzk$<xDPP2jev9)QL21 zN!wRQ-(<s^MEBzj9_XQKngv~(7{SXot!Ja}gwwQdQY7WQnSz%s!!tX2;8hx6g1$Z) zPe|udETlm@$et9p{)FwVN5SIMD7K}g4oI#ahGZABmHKY{)v!G@_i75>3K`D@1YZE_ zPsbsB*bfLZOMzt?75Hw0H^1g@0a^%i=kOZAllNc%>+c~#_gy1VzUd&Y-Q*67i%Uqx z%os=Adm{cBlYybD7O;iI16bkXe%L}|nbfZas6FpX0j0m-L!Aq2>hq<ms!4*!&I8B9 z)}h*je=xRR2{UbJ<W77Nql1GA4GfBBKNm)_j}4Y=?bv-x-j?8cq9HFnIg|B0o<OR7 zt*|ZI610BKr2mV&H;=|Lj{1IuGBn6oArX~|qJ-Sn-YF>!Dl}-2P%2THOB10&kui}_ zl1eI);lB1RC7Lu3q*5r&^QhFhp7*c!yzg4iIqP}OI)9vFt$Qv0xw!A!b^m_9y}$eO zx$}546m=PnFPnDa9rfj+&bo8r%ZE0oufK~uyhn=HR1A2-&g&3=HV+KsF&Lm+1$v+S z%ik6`if<cRgpsz&;(yd1dfRRA|1Z{`{@e41R(tAbZsZx7nOY7_cQZ+2T0hjA^+@*h z^CsC8hgdO4cM@jQo<SvbLp*kSH%C3#LG}~3@i#T3x)Z6mS*#H2-TJfT*m7#Sx*P|E zRR}SkBZY>jO%mT^AMJXy1D0v1)8bbQgC``@NogNotk?^SR@&0{8gE>`wFCB$Wy&?a z7{OwxcCk*3V&y_-ynJsmKb{i_<39V*`{HokQ_>U18g76o=lWo{!WP&P`;6*R4$!m1 ze$ex%C9D|YM;-i5gK0q%SdKp|$j}d$OwvYs>rIrYUgr>)7KZ*N{$gQSS6*q`0V`LX zh2u>-cx5kJaxc=rR-GY`_SXw*)-L0ZsZ;4_9~Er=a0B1BQ|0?bLHKY)4Vm`diLM<j z1sg#T(;-2~&Wz-zpO^C+d__&J?KnF!j!tMjp~kR9==U}Zi<kLGE+=PH^q<e4ddEO> z_%#}lvxD>_mGJ!Fi5zaV56zULq1N>*ovOJZ^#*tFkD1>o|9J@S@f-%9_W4kJq=xj` zpAuqP(<oNTVNc!@&TYa>{-O6&*elCGo%YHo)9TJF_LZ0cyC_#L9QTga<IDbvWGVR4 z+%XA)P3$gmY5gJS3|>ehL-vqu>?Jz%%ZrWvjp4(eLt+0!WsD9nq1a+gRPHc=J6w&& zAC(eopQ_1dSqHq=rHMj^SfTM6X_oxJ2e?yZj&kE<NP0TPp}U?8gKc+-qZQ|a_PAhF z-<k>COMVL4<4y__^dsmBf2G9WQsGg|7_nrB)R&4?!iEEytS~qMTKlZwR_A>B@OTl8 zZkj9B{#wX-XEManJT1(P$|9Ywrx9W|Qtul!B40*)pLRrI;8xP<zX7yNNy>1(i{hKh zlj&5@39@{=Re0Ou8fqSl=3iI3lF7cQaPr+&TCv?6I(U!eF&kfrpJRRb`(JO`c}MbJ zg?)!wKX=Zw%oo<5SO#J7dq96%Z$6^^P_$k+2Bwx4@wAGQthmaYpZgf1-&ISoQhgHs zD~S@+M#*X8=C=@8T?J-N2^5w48vgbUVOi;3sxfT<lZmhB{$&{|2Ac7gCVNiyEyrcq zTDWC|G6frU#Dra!NcH!2s7fsYeXou@q0Jt;D20ONXvr!2&=l`=-bPU`;~-;SgZ$Fl zuVS(5axATiBcIGCqT8|Qto%BVyY?+Z{ReuuTkM3@bAAaM9_eE3WnHZM(gT(EDe=e_ zXE<`MNDK^;Nw39ve9(0P9ho)^+_612O}9opt&V)C;{{rBx~Ih5QRAhl7lhwRDze;1 z${3aCfB`x6;B}k%g|8e66<Z-?NMBTRh@<2ne{n~rSjst~1oNeyulpr+RJqcTN9wrB zx_(qU8&T!Jr?+H+^FaZROAhLF2My`ylqte89ecWS;31ri+lV#kslxj+GTgpdE|#f{ z;;E8rL~Ghw9{6b}2?yHIx*U5hH~K67TG|y?`AhD~+&lzTDF-6S)Vkh13S%r2C`G?Y zyjvytP@LxQ-ea-sdSr|sTlJbI@5`dIdHNhwb(~U)3d9Mo68ULsB_CS6nw}_UlJZV1 z?yxZ$I%<p15NOStjAD3DkU93LJ}gfA>5k@QmGVGc7ykKZCcYVMC-~~O!{99{l=*QR zwbyzlewz~o$AS~#oNg{Y4?7B%wv5EW>5H)IrG+SWtENlm_F`Vwa#==^7P;MQ6-U1? zr(uyEyyRRsY8vc?IS*b^=!q;cxos%#x7r?qHP-OrvR3NRd_~$TY2!2NNFl7QSiJDz z1g2*7<(h{zMAmz_m-HSETV#o6y`EG4#je7p*by*hRs&R3SHtfLReE@JJLH@#CWG!D zgdHZwz^JWU@cGdQGk!k@<pDck)q+|F_m}k$?K~A~zxwj*+^Isz%0lteL*#7@GsrgN zE2So$1(T6J67yk<<YoT^q2uQB#G+hu4{*k9Pp*-(`##}Y={2|zwiCJy(ZZj-GI2tH zxftLwh~I1vCFQ>RvF+4i&f2g|FsVtWW&i5v`<6XC*;t25w~XfBEn(1G@h%*-H|KsI z9JryimVCQCq0X*T`B|kB4)eQ71Dldb=#>o)VN2-2jtGt!;7kis`r>)RWmITeA~Vf; zCB6xj@_a7pq+%H%FMfm=v8-{pvx?;YQt62SqcwO`hlw&L|0NQG<PZ$LlZL<V#6d*Z zZW?Rej#1AA#uQECi;o8I?=RNm<#3%AZ0d|<b`2DK*a@q?7K+c<7qTU0;MBSh*xAXN zho1{)iRel<748a3N6P59Vy4&?qN#uMUg6bdWsJF~iv`CoLe(x)a<&V=qk@9?SK|}> zdzZ~Go~?xDoJ!sq5-SwkTnq*G2l3*<6wDp}4UR>O;lCAfKG`h7KQj|Lls}aN&!zDF zAM4=I_u0bsi9YOMk;P}y55x2LMj=R=b6j3}k=9FfwdIp)X-ei^+BYMb*Ltdf`?o5% zIops+?V_po=!ZgiuUt_vuTkih6+yoh%F9|J%<$&*C9vFRIT#gbVQ6<{KK*H>tlzwG zyzKRI(opb_J!`)k20t!@S!0uUZZBoD4%5IVIv4P6c%)pXv?u<p>`v{?a%Djm?7*rb zjC_9&!!DcjFr>Y-Mr0gC<<Eu~ZWBsJrh5p@#|NR>rFb4ZFHxvi+z|(lS%iJ#cEjbB z5#r&z1H|(E&={kSW_mAZ%@ilr_D1Y|uruy&J5K(WU&v1?6qNgi=+XPr{kZGKopkZ= zKYD4DBHnMlMOAAFG!%@9I``ohlDBqs*%El`X-T8^c0@1sgvCpuc+iVi0*-hI(|-4c zVWZcH=?AZg$tkh;P`-waMJ!=cMR%xw-ATSNJBsT<HKBT$6CHmUh!w{Ma_Y8j>|<X^ z`S&aEWSl8#T<yhXcMZ{@QwfZk=Sv}~`$V~P-*^;%0f#}Sh0075EEC3HFW(2UKaGBZ zTmA`HpPvZ3OLii6h#>>jXi9M0O1Ink5&P_h_159S>)<2cZT<w(f+z8CmzyxGH5~ss zb;JdI7E#}0Jvb@juF!02%qoTf>};SexE<00Q)v$Q+Sqhj@I%_yPiW5%|4snKwF5C* z(;lWC_ZRjV1=6AGo%vDJPMX$Vg<D(Z@XVPXrS&s{;(u1c{+W}pq4@>eKHCA0RZ9-3 zo-1I6vNWq~C1s4yB=PF?Dd?Mih|{F=a=K#$jA&a28IyojH~ewvJWycmL=Ul7kp*`h zxSIyenu0RJ-@<owJt@192HRDm;GWz7?`lkhzHUokX;LH14_ir_*V%LX0h5p|_E7iA z3$$*p1-~uJf`ZCkY+7nBzj)@dP`k?ki|(ZgQL|e}GMCZikDsASydGD5Qpa<@orI1J ztMI;3sl)@8K9{{`z_C_|FV$rV9O`$FOY$<<@_Z-s|0fM(9G;8{+Y>q8E(8v}8p*cK z!}0B8X^rew3>N;CkTW<!`02~S$6X1e+%^J!jUK=QXRRfryBB%sUnkM!-VtcL;7(l+ zYjB=VHN8FT&xJS63U$X;z__KV{B@4IX!|4;C->bZ-n0G=Gxk_e?TZ0;y{7>_yxE;5 zo9?5WZk>c&*AR;8X(Jl-J_w&;BwvTLFV;@!%m?mnqw`k-x$kRtvdQTpK6O1wr@MD& z3*G}+*Q7jcR~HoKY~#B9d$=fj56pbg6A!12Wy|g@!Xf1oIBrZH_)VI}o7`%tZ}(f| zxJ@4{2hHQ_HPWnSz7}UDK9pG-@237)HDc7@Z9uu-XaPImZ;8_ty=IcoWYa-dqN&YQ zNB@G&)g18oY9dV6nNJT==Rv!sB5XJsQ+BZ5KB=4$0F48N(n!g9a#`Z2-5l8}k8Ek8 zQ`tMH^}UpLZ5qTbs}@j<SGjokWl!`ow?p4nRqpfYJe@LGL0A4=6z+7;Lg(5Huyvfs zS|#438aIH2d3rdl;wst9S<GIA4j6B>9&eV#5x+oClM_B!<;~X1K0@=kSV8o&W~Xlt zz;BPee4B9@jMgg_m%Xl*W|Qln(l3oBovH=b(Z+aS>VD|eP66AsN$&9IJfXQs#wYIf z#tGLCgF;_r+~Qd(TUvjfma4rHG`#a*OprP*47n^E(d+@eRbRoAui>K3fzBKh^@Y;r zXHxgep`u4eXSfvL&4*HliM3wE)X`s`XMUZH4~m|MM^-Fh@3D8t{m^heAKHoQ6Q|&9 zJ2i}XngO>1CHLQLW1cmA8TviEBl~vNfmeDw7jq>Z)zM3z#1`=$oLTBkoiEpuS8;-% z0ctqTvH=wD#qp9^wrqSsg-ULQi%VA<@RJM1_$6~E=)dj=FP|sTu+^z_w@`_)3gfxm zt{5JGid3@wG(PLlAU9Xn;~EcF(OXJEO}e?AOntY(gA>!}>zsFxHf<CZY#7L!{3Bt4 zbpz>YqpZ>D9(?V629A%3La(LM(EfV|Y+mYt8w#y>NVE=Jia#R@TyhMCNO@hmMLxno zr+qkT)otOi*;SH6GGs)yd_TgIpB8k(bm`t+VYvW1-+e_&9sUkq8`VN@ZdFo|fjj+< z564!o!4z+PimiK|5$BGql=YswoAftCz_F{lafo_nT&ODTA;jb2{7--AZ@L2hIB)=7 z@B0c18>+=K(+X+A-Xe;K)S&i1Ji*GugtN!br5>q~v})%}m|Q%B&vrQqDvJd^H!2!+ z&sBnjN0NAK+Z)lm<&V&*p`P*+?fC1HFc>_)h`NODgqtG{Kw{WD-qAW5>kPDnYi<|d zgjJ?sTXI_PcGiTt?Y>m<QHG60C+TB+Eu|j513NawaDbixjPH{OZgU?CR$mHbeMfhc z)(Q)nIzRzuNp<hq$0@^C1tydAb6b4B+ElLasv9oV+Rb8*8SIxc6-Jj{KvOdX)Gcd+ zuO4TpY}a4$s#O1-;dmY%gBcFmK7rDcw6I^d9W=o&Ty$R-PIk%PhY#tWD4rbmf>qWw z(Tg1i<%%n%x{!MYp6g_T?HUJ&%d0lQWQotIenn1B>uv#5uM%JGxj~2P3!wINPdK+f zo8Au_#=Rey03J-n!_PG_$v=i?XM4~?4cGsXe+jLhApft`ABO%v!2kJwH}CSl{<i;K z{$uF>NBKVr|2O$BL;u_SpZ|<E)m6c7(E`6E!OIpeT{U>V&nln6LnYo$Cxu!6)0f1K z`u892;lsR(cFNM5`yY@0?3%*=eocvCWB-4;=Kpm0|I2S{M<zSub;$><?Y;Twz@xD7 z_&b^<oyk7<9|tv?JeJRBfbFXd!GCTFb=#c-Z)$AtQsN$1Q2qcOC7dUva}KPo(GJrJ z56TUHdQtF#;lj&5{_Jw}3yio_LI>J9a{XlwDxUR3{Ok2iJfhnh<-KCyi}x;C_sosA zdY_=N=j7m=xR!UcQx)rce!$0fYFILQE#Y7-yrHBjSZ_<@Nuvjdqrz&W^LjgeZQ7UL zjGu+NjyBNPTtxxXt?B9Ob1=Y6Vn*IcqWSTY@Zfa=R2rTG*_M;3`^a!=j(SORJ{}Z? zmAMNW6>Q+)+c?3U74dbV8#hYp^T__Mz_+(8>NIy1!?x@Lt-0S|h-xt{k8Ot*Rdeyp zD_gExX3kswb;ATREi#`P#tSDrA_wsq6ph{nwh2w(^kWX6(@CeomVa<3;F+jmD4i{6 z>+sIzb6{*wJ-%ZZMIFlrgLg(Gdi9m+T$_$kwb@<hG|-R-J_^LGt}#$=IFnYmjo?{* zH%YbMNPhOygVW}orp{Tm*lp`h`qKU_9Cf$}yJ|G4=I(6lHm(@ua}U!3;~c?lO*`aB zT`{CBgANX1(Jx{MfA-PEfKTl?Wf{@5;cKX_iw-<WT+i_dOL*A5WKhqvhs3^-_-yo7 zTK=L2?7Mnli{~ilr9M~QF0mt4RS#yr)7?Z*-}d-Y>a!VooEHxpSg?5?MXqm2f);gc zIFacr++3Q7PIVp>?&$@p9dyX$!7!+)`zKyBt%HPdAED;hUT)}Whvhxyi++pSfxFl$ ze7YV7hGSn!bD^PdX7qkBBGweYnnuCcSS60Hm)w{O<Z#4lu5cn~HU!(XgKf=gX!)mb z@u!h0y_;GlaS7BYpdcShGWXEJvn3F>wIeyMX_5zj(qk3J7Y@$_Q&gWcmB-!Alg{PR ztbafk&O4PybGGbbZAfFs^IK_2hKv{gvlL#8nZ>^gi!gb5IyUun;lZi}FycuvySYm9 zA{UC`n%WJa*zczF91eowy1`h|y@+O8okXPzBjw^L75?(HH>S4l2$f;C!9Te#d)66Z zVnr`@-S>q84_y_McB%6(?e%p3K@SYMF8N~)DDhRBDe$TCGRWhSpxciVVE5-aC4PQG zBV(nS!<Hnm>})6O8gYWkK6b_OKTn}4Z2_z^HRi&vpW$MI3}^0k!?!<8d9gI#)NsoZ zudWvOY1kvW<h+toR>n~6=tuI^4gs|JM~3{s-@9UiU4e9ty)NwZh`~VL0NNUqh6dt& z$lMYRUrguX6w8m|+*K-g*<_Gdov8zn1%qhBVmb6V5(5^aq<*I2UY_^<7pw^E%bQ<? zQU1%$4k0!NXy3KP>{o37fz=V*f5>pInit7xe+|XyuV-W2Cp)~K`IFjpJ}LT6bLG-; z*Ezi;5oJTZ!`Xcr{7!wHxc#X;PM_dIhx)7Flvj?zgG>=3YDy@*W&oQ{k=!S1X2P2# zI_RvBDct&#DBizaDcl}o2J<{}B>#IlX&s5?cN(+#Zptd$d^3{kHe4hAY=Cp4r{I>l z9q^|%9_*~QQTkplq48rGeDF4bf)O@So+p6Mcdil-x^IHocITm+@+C6aIThVX+T*p{ zpJKMF8cehw0=-Rd!Pu;&XkK7~;mUg`bY3$#-Yyo;^x6%wjb|vO!bD8l_Ybz+J^)eu zuE6Tmw_y39=^T2uD<Ag#DP&)Mh`k(CaOLGb_^voZw)0V=cxd818a!zStv@CGzEd{b z((MtQlICqS{VmY_>}jE~)es$L)(8U~T+vgSX`ZQdmwX<YvVT#5%&I(5Hs5XsmLw9i zt#{`vwGYrEScMb9?J>b`3_H%<363TcF{-^4Q_@9_s(lBuzHG*4O(I==)*E}=Iskt> z?!o5;is);0nerXVA#%49%R?53^KGY+*S#KiqOJ&^2y?~JgQjA^&I?fXbTZd3n9rMS zjNwH~D#YIELHhr~d9c<|IjB#Od}<neYW#8RoaYPvI0O}x+q3Q0$>QQKf91*D&3U!1 zzL?^#&Z;5HxMQCyU^TE@NZ7ashCI*}cAQP6pqFOoVz&@oF1cc5%nX>FR3dswT-A^? zGj7uA!^<j;kxBGVQ0WC|cG4WzNPPySBfF@N*<%Nnzbatc|Et(oJ)3?lTgRuL7=ik~ z;aL5<E4vjKvs6@rAC>~oIJ#Hvlz)l(I&Ol8GZpD#Rw*c6E2SwZFX*I0JD%~c5SOP| z@X$YfxTILhcUv?HHtjX&zK}>4TDwE1aETEXx0nN*`#|o63jPw<kI#JEEvj{!2;Tks z;O(+mq&O}I?_T{Z6fR|GuVBRfb#Z*9d<?gb*#OPnx2bsJ2n=iMNh8;dg8aT0$+6}h z3@A~<7LOS?QjFocz*@@93!s%ox;$rl9;X!>No>|3^hdjh!e`sd-@cXc*FXVnMqbCY zKSuDP&Yh^>eT=v#*qpDn`%W7lWzwjOy%dyplfwQQ;p0P3AU|RP9`C7-uXgzG+t+=> ztV3zA^!+{B{lEwvk31C*d<ek>`;SRA`CC$5f2CAC+=;6-2eI-LT?~4lB8V*taJVfA zHtV*-gia;G;srg(y?Gb@j=2n)BT8Xou?0uQtQPLPAB;M7@v!sUdhzXmVm5nt5{{2A zz~Vj{Fn;AJa^BKO-mOP>$zwkapGo}Qh~=JO>1)I@5_Y3f#Bt7g<AO6+bb>hpA5%T- zMbAJL{yKiA7~dh5KJPjS6$(;MO{)DlZ#^iVsa-EuS=^yKyLmEnSmO&FKi9!>(H+aL z2jKPfe)RCfcx<;i4u_0Opz8U+JFb)oZ!_DYcXWT=E@eL>{Y}wvs2ZQw8^FI*dO)B> zEX4ZyLYG4<u|_V-exFh14TC3=%cD8aNw^|br&Q4dDF8FP<`^ZvFv2TFMKJ1aR~#0& zoj!H>D}J65EWF>ZESgFopNix6g|}NJ2Hk=|6l{A8Jfo|rXnJG$#4SImXwE&b-r6Rf zd=mkF%PPoqLmo|2JHd0do}mOaiP5EzD$TO!z=GyvZ1lZH%WEuP=xr@7lla++PVMk< zbra+~9Kz4Veo}8gkZn9_A$zbD>ZH_@;9fx$&rb?UH+oQ8hY|`NH<ibCj-XC{4Z>E> zv+#-<#FnD(u<Gy?@kNCajE($EVEsxwb5sMCNNxx778lg*KZxhZ;&E6)ELHVN!t+-u zLA|06jX9f0+rmenTJL!DKYf!XO8dCqHWq@GYc)n(?t$-aeWm?1bz;uZ+4#Mu72Z!D zDN|Tvz!9nUC}OZCCOkA`;ZhT*E9~O+w<CCEP#9&O9)MpCda|2iqOg0%R=hK80W0>i zrP8$#P~R%$h=2Ofx1)X-9odO5^*ccE`xk-sngO7oA13wUmAI<&L-F2KeblIVAgmf$ z0*fCwQs?bnke|PVtx5}p(8u9qlP~rC7rR2r8$I5!^&ZUYcN8Xl7zxSBfiQJlEO~h; z;fWr*g)p-%z(KR%-zrNq-M^lXcaK5Mfa{R{b|dSS#8Q{z{xHg_BP&yesB+H-TVMYr zyYG>ruWuQ&cF^VdS1n=Ohaot{;Uf0Cn*uwBOu>YGzPNCY4IWu^lJ1PlMCU0p&^Nz8 z1fz*?Rbpld-bx(hv=)sQ9;M`u+xTnE5jg!F#bMqZ@I|0KZk+v9tQ?*M8%EmT2c655 z*uOg;h*84w{M}&OaW&jneBB}WSZC<^MRF2mREUjDE#$wrLMYi62^nXGaI~3@<N(R1 zgZs~m|0Jj9!>8leMPC7Tk8mR8$YvT@ngppc4SB+cO?Z23d)VC4kNT$-)5;@>;yJDU zaQDMc_@{ImR62(6g`c}|Vg3QUtY$@x_d4S2U868LxENZlZDv1{5Hfoi1EZvTQbF)a z(J<SbZ(r{t1bdGs!@p7veLykk<*M`Chd0V!n_dx%?e&GAMa{5&_!_vdr6Wb$9`6t^ z=BHS3&<CyW4nxI|BwS!z2P$zR`HjV0vEXHY^32*wmz?@ze409DJ)6p_szdpK(nM^o zkEYrki$H#2Hhmk{j<e=(lQ$g7rkU>@De%fSx_vF4t{X2V=N=B!<Me%DZnF)4v(aal z(CO?lts~AiXqD0ArS$3ZdhB`TBJJ0*g|`kO1WUO-vzWEzzWY-hI<_`KoG0?Tl3cP6 zsivm^@gNTh;i)a9xTj#Duv7CbJ(Z=>k0X7suJ9?H%}}BR&gaSVL6~U%yOjRvDuFyC zT-;`-fLEM1@SW&f>bKPijU$s_n#Uc{Cn=ik_ei<@hB<IDF_z9d)(RexUHFElDfug= zVqsGR9IC2>)W`08^fyEDs#8LBb1t>+=tW~ksq@s@1@x=nBxH?fhn=%u(1PGyymiV! znEyl{cSj}2O+O5W?F*)3lZ%Jw6skpKimDuHy#lIq$MMY|by@${FDW8+67-TdKMpos z#k>7#DKKy*|H)F5#X9!qlSf_f+3)SV&S4+T_YB41qjg}T#5;A-_2sFz&%!u~0j2fo z5?ClmpRKQRgl~D0pJ4hP7Ik{jmagC7_j6OM@JfX3;y=jF3guoO1XPVkz>wehs62Qq zJ$iqOif^x>Fs&1Cl&8r3_1w^Wl(NLMRHV=u<M@-*M=UAWDLmaA4pzOpW5<_!I4Jyu z+*;|ls3xeNEq<Jewl^vq=Jj-@Ce?iC<aUQDwl@l`s}FPNbvL@=*c&`sOi(X26MjZ7 zhVX}Z@baA>{Ie*)-#e2iXjfl8XkI7wt{;zU-%I)1ZH8=D8ODL9j9B%Mhw$NxiTKqs zi?jw-K#JQ7T6elPJzS9~6g;&?r-c(mKMQp}zD-YTFxTXVBW2{NV2+A+Y<R%Kn^5dk z59zB+Fw!&+=Djz^g;|p4B=ZgVwH*{UpYBU8->*{pH({{m^#G`83*yd!v*1KroWwg< z$H6XhS<t%!rPkL3qaCWK<rGU{5fPAX{FW9C&Y(X*ld&>-9D9DV<%`pOAa|s-{H2ou zq^$L(kL!DJN^(9axJ>1xQvNXhoG!<1-zct7838xG$jG^!8g<l;;{kGQ+>+l)dVjh= zug)WQm1$4x9aAB?9!tW5%DyOzse`E#oGGcIo(^wcf(;d2_}k}5+~TxRaxq%S7x-E7 zU5#WcIK7_JBTB?uque>TW<U0RGEz3=iy2yUoW!XeA3<Kv98$J$qv)70;rSQ=mkhi@ ztXl~Vxz0TG#2~E52I_P!0S;`+rfa$iIBvS+(c9bt7d2AJ^>~38RNWs>t(NYiQ6hc) zZ9(>XST^YAaQfmp08RZnvBj8AaB})l=oM;0%}ETi)DKYfr=Ar3LF#LNH{^>q{J30l zsmO0_6Yqu`=ja~3ymx6PMVX|5%KCWN6L}3sAFGmAIID>5O!h*)<N`E$-;qrw?}Jr} zUqx2E2I=2MbJ{#Rm?k+LH@zOprN&AaWj_$o#&u=K=)bh1y#c1L+AVG`-35DeMF_2W z2Ny3rho@FsXs^&tbnSnPA@Y+rWOywp`2K{nc{^yLTaz%kF$V^^`O}7`H2SR>Dy)(D zOWfQZ*pgdFn_Z-A&T4J`K42;jkNHkXxoSMKla#A{wSvnJXoyx-aggvh5w05K)3>07 zxKeIMb0R-cZdN)CI5QCM&9jyKA9~oYc_QsH*@DBCwbAEFWp>D)j_W*?v8qglml_Vz z8kX|Gu3K^ZnK|5cSMsqPpTh&6rwXg$o5V@#tKp({B^S2spqDec^ZeOE@n@Pc^bOq& zcb=~iTr+my<E?dYbn$vV85Ar|@e7B0-K~VJ35NXGdL3nrI1COq*HD+qGEA_mBr}|e zj&DtH{Y_gTKlB?ZjJZ!U%T2|pFWcixF97p8MgCBG438;UK*=VF)$g?cwv1WFTc706 zv`bIGZCE~(bxVb##deb4U>d))c}MS>l{m{XfS$ezhKmuSphU-(E&DFP>Id`aVCr@1 zp0O28E@<<g<CU<u<ugs3QY9D;Z^!GN9iwgSE)i%c;V<1D^h&!O{_{QsnY(A;-Ru7N zy;~$yc;CklHy3eLwLARUWI*5Ta`;)`NO{Jcd3@Quz0`+xLDjM-3OZef|8kOH`MDQV zzo{1(Y}_MHn7?04dlAaLEklIt^(5SS*qw`_-Q;`fCvtT5T6{FogL4BWV9T#+u|UT~ z<|CbVb`Bhc<Nmb?w`y!<yC3yOjpTe#A94-`?$!sBoJz50`@M4O`eOPLuFg)5OITB< zAAdZw1gruoDb;H!PBC`lK0FsXJlabU=OrGAmBa?Fk+HGmN8yo177eOt1O3&bxPO<q z)bMs0beijhw`~K(u;8A;pVwz$etZYfG5;itpSoA{veu!$o4Sf4GKRu2+j-!1eHW6l z2%Rb>3aaPT@Kt~wqWM-(Y1$@kY3<4neY5GU15tXXVtC~>o@PcXq3OIg@NnQsVc)(v zuu<a_x)u-O-(8$IwckV-@5ADXUwZ&t`$GHWtHI^VA7~s_2o?n+;d?uMR4nKK^9zV( z<md9Sg>k~Owm{LL&K<R+oc@DLCs8ZLf=-Q{iMe-m*#1)*e6Wjyah3h>-@$m=@Gz5= z>fTbC#ag-*pjtNnxIXOl*v=|4$+LCXkU#1N2@58khPj`L$zG}jZF%fW3v^Yn*C$>4 z_E!OlPEA42nW{W{c>-l_NuWQK#{`d8TC8_DTzF(s0xAo4u-}eIj)-@I`BBd)e(WUv z9&F3Cmvg|#s*o1-^umJ~7idO}A%6U|i{|}ofF19rv&C?h@~iV$Z~aM98dD0d8ft~( z&a=5ba+^$$ay^#iM*Lnmg(k#?NwW?cpvYr4syoI}g>n!6)BB9@{;WMmw?)a4>Q-ax zrX=cV`G{U?CWE!-BeELrfT?GcM4JZ|{CQ0t9IAay77t>%YuN{JQMxjH44+P?-8Mpe zYbbqdlrp8iCd+R`4#Www6!5dwNm#e@Ih<_kMyjc|gb&d<vXxU$2p9S!Q>srjXuVki zVSCc#E;~Y~gXt)|u_+dFd5EA;S0;Q3OB6drj{xykF0Oy6%hf4M$-A+jwD4kIRyJM_ zy7Dc!`1cQrQ15{I4cDRDyhQFa!V0FXQDWb~aX82=g{*v(a8c3~;grpCN>Nf_2X7T_ zc$tOqYXr8Nt3sAHyWq|K>aq~2?>h0H0ZfzncaEcUc<+xTv|`nASl4v~?~eZ`>Y$65 zw%L@F4b{ab8rOu>4X0sRv?1G0Y|px0k7!UzSIk<~4PThXOP{NYS;t^1wj6L`7rnJ$ zmKqE_@0+v!$CJ?4b|YWxdy}pdo|cbwN`NiTd-447XM|^-`7mjK4f?38DyNn#@$tb1 z7~?e&?WEkdOxl0wU2daiy{bTa##J)gXGQoqO_=|50lE*i1?{w0{xC@3h)FAO%YsU= z*Pdj)y*ZNRT!<6<P1p(A8>W))(;jRmmeHmUT4>axH{N`m3jJH9es}2{an`=B7^$3r z*Rn0J^!7@+cJzUe^x+;wk30z3^b!s$EJoMbu6SNLdw)t;#goiSz^h{|9yja8@*ZzR ziNORFrqW(MuL?F8?|{SmyJ7phNYquy7IeK6;L(St;`Ik-pbrjZ>7K;i>HT1N<FxXI zHRHH`)O&{mp1n|Zs0&>^E@fLjDoNbv5}Nzyw9xzJARPL3zL;oMO)f_bImdn#1-B{l z&$v+d8Qc?!KfVG_srTI}Ue0GvtMJ?oC!ovtEvP)Y0W?%%Me|)@fQA58Z;I&tj(2eB zO%-S<*23j^M_||PJg7f?29p~+#nRvZpii(SrVi|gEiZq;u(j*J-MfJ1&rcJBy#q+8 zOCa`cixNbGzHl;d16qIUAn_LtlG|o!PG5Wu%d5s=o4y@r#Tg)L#<Kh3f&4iyPFVES zka}AmpeeB%G1LArnBPA`YHfW))y<D&<AT;u#fnhc<k<<QztAh6`&oyhf(P*2JZVnS zKpmr0!a?!wAfEAcHB23<A@8_oAb$x66Ao5pp~vAMv^lJXM_wKP&w+1gmRuQ^7iz<( z%%fyJsXe+YY=`0lEBMB)_UNE&#-FF8!HaXvwAiu>UvZ1Tz&@qeF=R3wdS=c6k7cwh zWHl;|Os1ODr(mrk<-c2ILDk$>;P2ZX?X?te;KK`|<?I~HN{9lj3$c*W$&4XKkrn2+ zbCc`_-Ts>bn^L9q-fcJ!nfwY=y-x|>RIXCMmrJ60v<6wtDHjLsjuVdNkL0z91(Z?{ z$hNxivJDLjc!i!EOeZeoUj0ANnf*hhS&rABG};OyzUmCGt1xDpp?2&WxtO<I`9%4a z@zPA&I&s~|T{OSC7v;>B>VFrv@-2mx^y%qy7~-1+V`@fY?wHY3^m#Fucbm+uo4rYY zK%3m;rZUb}zeR}~L&U!}brd!)6MmdH#bQ5w;d`DHK3ebwlAb8R#q;{uZNyC)5H&_P zJ?@=wu-j2keB(~4$5w$`+;y?KO_Pm|0Pno<N<OxGlH_^5%73dYFxqq}Jglyl|Nbh& zg9A>|8|RDA{JmIMbjnd&KQBr!-Wq}1q)glSnO7wr;S3(K+z5^vZ$s~#ba1*l9`A38 zrf=WQ!mze7apoXXxlxcIYfst)Z(eKT$}j1poqYthJeonOPrJbrp@^1u+0Mu6o8d#s zWAU6&E%cse#*1Z-z*Zy1;hDE13|950tdM+~;*ie2c4_17#vt+9-^Zek>NM2fa+Ow1 z?MpM?9p@dFLcnR;75V6ahJu-6I7hEf6{^d+^Jb`{6&~T7JE|vXZjk!^*9UX#f-K1J zwBoJ77Q!HV9sbot@>f}{#^1}X^UBP%wBGIp^l;uH)~{O!ip{HF{~uQzJ7Svb?hPZF zt2l}D^ObqmfyeT;f$sESfgzIaSDAkGY=?PCG5pQp1}IlV(mm-c(8(i$?qB<b=68K* z!t!cSHD^5;6>k-j4*8JNL{+|4&<k_79*3^^Gg)EhcCq{B$HKhlgXpi+^9dT?l^UHY zXr%TSjv5!wZvG~i-$Mz<E;~v+;{p0^?1%&NV(92LZN6B4iEg;m(%I#)D1R~<2liFK zzUM=+ZQ(c^xvCK)#54AO-AMgUt^OZA?g+zJG)Vsm3a)B=Ke9#4+ZZlP8=K0>mjXGY zNtaU=s&SZitC()5h{vV<#;VQ{FuQ*+*{JP;S>|=ZLCqSFc4JttFcF#)`b)F;&CvT) zt3$kl4xh`L%+Fi*Qd`V&3e&$SSSfbL-D|&!ZAJ^AQWnNG`+UTRshvQ1;}*I;D41@Y z)54pM6_o#UESx#8gO#fTY5QDV2-|-__*t7s&)+6upY%AmYHCh;U*`$(>yvpzVU#Sv z`>Zgp*pzB=hhh6+Kk3-bFpBG<#qEm|r5x-Y=rSV#@2+_-to|9myPfXQi4=*qWMzs8 z=LMXU9zv7UuZpSzXY#EneQ<EU;W%;sb#U%l&z5KVh>dHbz{Yz$lg@8Al|BROB)0fI zOHaJgcmjfd?-vFpXG@&nowVbP5C1%|M{xYAOEs5G@rlYC`Xpr&WA>heBg5O_yMjkz z=!3^p<)I_|=;TDISG!1T#Kq{>y#y3HujKRe8-f~h;Y-$1Y2Pe|@V5=Zby*<n-#-`+ z#P{WIiDB}E<z4YW@=Mw3;gZ+)(L3_o{YJQSIEy`&j^u$I1?cU%OZ3i-l;7;EM*CX^ zgLd9cvAe=~I<_&Krmyz^@kSDS|Fs6IkD9Q*^#A`xnz!#{W`|Es_e7H-9gcbF%o(?@ z3HJx2aPQqi<VpS3imjp9uv4miZR$FOCw`v;3qQYv`H4w<%vlCn@#c_#XN3V%Ojw;4 zK!=s5WkI)6FjitckIjrI_jv!0rfP@F9g92A@u*fYRyK)6<L}hrls;a!Za^o!JXq}8 zhnE~tgk0xYF#qsBDAY-&?G2B}b&o)Ey(iJ-;t4eNfRu+{89}Z&-N0aZ8_a26Msu#K z@wW8Oq^+1LczdbBVzmbpwEaEgcgq#_t^Fu?Ru#&OFPx-@6*p;9l{3mVbY#~D1>%F$ z5#Tq$lj=X6;0xdUh1-R*v2?v5Pft`6*JoCMw}}#C_ZQ-k^u?I(Z6W^zhA>&N5dI^3 zpqOpl_+Rsf4gGKPYya0g%m3eg{*Tw+Gcu*KLObc1OOO97`{DS%XFvYi{GQ0Q$`aek zgnOGtf@XZU>`{p|IfpOfP3ymb;XpIgJL^zpGI1_%>6=W4n<WN_&?>tX90T1>?SZ7k z_fSFhoW1oOE$^?7Lng@R<@CIAh2!ICbM0dOGx&yhJyM%*)thoykMq!&yq~R~=*nhG zOjQH531Bj06CE!yh7lHqf?ta<uaWvhHA7nA`h^d4Dnp0e>IcAte=B(9)2l+etZ1<O z>?N!#?8z~?$H=#ND7CDU7#WJTY|&vC9&5cW9=kFLj-VQ7y!|Df)O;s)Xkz<93-d z?UL_&xf6zZ>fq?>RS=pt6)&=v;PmCZ)bH&BsjlhN?{x#!$VJiTVox-hmIIIGt;W6U zv$3Vf7>g*C^78*dcl*9jn7xv`AN><&&zsLL!mPn)?{Ky|d`Mm(?@zDHoXEvSnVpRK z(dq-C^u9+YvAt0YEk3at^9vU8gsd0{7zL=(=Q~}G?9b{wI`M@EBD|v~pj-Hg$~I}j zv0kDqWqcl8GLyqP#W8%%WdslIUnuS!C9P4fmZHt?(Sm8uCH(hGC~p2egY?Q&_=@L# zn)_X!n`>M_{j-3IlfCi%NENjDByor5#lq0^2*LibGW$;`qbGr#p{Y5ae&5q9_jQ)s zv9Bg!cIhAC*wF(pi@T%YO-0^j`i0I{UZSyQHEHceRet<<EY5qGFU<5iK~tW((`@$y z9uXc!J;ryzxMCL^T{(^~WV{i7eGTG!Llhuj_yn$~x5a<E+@K}KlG--hhTIJUag3D` zeR?jj0V+<2!!ov`*WT{TN>kX<B!;?kCz_RE&89z{a7%qpepi*u%kq~{c%cPoR}P{3 z3B%FjlNmg}cv6Ve8deshZcGE~yFkXtQ)0nKb$YFpK<>ZJ^Ksqf*dj}3Bd@)zb0CtB zhZ&b&nmeDJ_EmsxLXPmKaxX?!{e?HPw^PX`Eu4`joppmUs6L}7tBhO-mZ_iNaH-^x zXg`EX?Ceo}+Css**c#VA-yt`7eivH(-OHaX5aF0wC~tGhhMIAu!j0k1!l_+y2$^XG z8P3OHV9;V}ey)aJW)Gv7c4@TBY@=XyXCe%oswACProw?K%KX8rHw!s+RQbV?);wqi z&tN?c=&p`+dN;&tv+bqXt2FSCT({{8gV8r(4<G2)4Z;q4;OM*ca5~)`9`sjbtL}Ta zeNjB*D@~x)KBK7b12bB(WCWY^J21Q^)sM1wnZeb#1ws#llVCk`J)10yWxMF%*!MvM zUK#X-Hl-JH+&f)9<e>m(_wEJzKU=BWj2D9Q6Bd^bmb&=u#)}KHtT>Seaga=g(k?~x z*&HEWQ1YYWk;Z%{L!bM|OU0#!&0we0!ydijr9AG*Y*w|>L_;GLte$rnlLGfZx{WG1 zM{WV-Yr$Z!Nte`at>qi11Mv3UaQOOkILyqHc(v=hqvH65Y<(~kk6w_o)zUTm^vEdj zoQlctgk*_(U+Ro6Ja@8%V+Y1-u6%R-Xw3ZFB;5VzOTQF1v$drneqN}FfiN0_iwwE8 zHcNc6wis@0FQRExwmh)*zTn(#8$P-kN$a01l4`iS(ZK#DypKqdg&gWnmAaYG$IyfC zk3Pc-0`}qIxep}%rJMNhUw@7mx*TlIU4wjEcZtL8hcl8lkn-_DipaI6k&j2R@}?SU zX|oWou5$&WZ#ClRC?C;jb0V~5^<$;%cuFmi@uHpggqQ2O^Q2$y9BX@q#>VZ(PN6Aq ztxyy7)@yTnA0Jv@+e)&=BlN7Tj6%PpWAoq{SY2trW*gF}+xLg$*>o6ggpNb`U0*1i z@deLy9}nldYon`|BTjmHhq6D^@V^`X$gU(2`oBYbC8}b~O@F#}L5416&N!i54L4mc zfSAjJxVN*BU>)O*&xQoRjM>2YTfWN6LmJ?&$s1bOb&Bjq{eO7NoZ;${?etl$z=0-7 zbS_vMmug;tt;5D(Op*%vWSHa5b<apt|3!`xkGTD-DU@921SJW|G`+zX?_Se_(tU?0 z^|A$bP_v{1j~!v<yJ#A{*$L|nlIXX@D7kdiQ|fui2$N5N&E-6?bdx2I*&KmgZ!O@! zq)+rWUlHF=Ql!&UV!{9FbZmFi0H^y7hP(a?(P^9H6nlR{c7BRI%+h=X`H}Y^pty?c zoHEF}xtI!C63Hs9J<dtF1W!gN;hUv<h0EddDPho5s5^61;^>%SyGx^RTa`4UGQfv4 zl}<s`m9yefjhFOqRkTPICcNjzd^ow=6bE<=;Z?aw{O#324s=oE(tdHc?8p^yZ{KDp zb?he1uWke%t#9CF;>g$BCyA4Py%5zK+F>s#V{kJ<1tYG9;*8Guq|*2UX1**I%afC7 z`GY#~{M!!fJwO*HFItKB4&0>ZqrEt?XO__3_YDl%V+@5so$<(RcM72+&}Yyqa{94? z1M2Qd{I5cIcSBCoCtQVAql09DMzmV#pitOy05!DzF*016ryo7YBc~5y=Oan9*+2u# z0&O`wz)-Ap--HV^O>lzuUugYs1ioDCjF#)ouw9-3mK8^e_m|hAgmZ)2RYM?0W1BD| zI0RSP55V~B6ngQ#j_!Fq2G=M@8n|GW!+@|46#ancqf$4#9G(wnl-E<n!HtmfNfjK+ zR4`7Z8|w_+NVz3m82WY%9QV?|4i3{fZ~u3)%{W*7dvLjsrmIEoe>8~KQh(F?M+|rW z`9N#KcOmIp5xf|s%EH<@>a*|!Ev*>?nIlxu=c^LUxf6skgJBeLRD}56QFN<=o0Lal zp+{S_(9=twEl<Znc;b0c=kHg#-zAE3#&zKllTXU}udBeONd+>K57Tk}?!8d3uLIkj zNf$P?hOyGJ8W3B|U~84c3Yn*krXh;#vXvotwI$Z{UkEcQx{=wnGGS6>sl>G(i4{}y zxr5z6?AvuPX2EuuR-Omes=b99Ze7taKOF<&EYZTs1MA{e$Q)mr^TR{6f^2yd&C&P^ zdhZ8<-R@FKzJ3sjPwW%khppu5-@9cq)^*@<zuO?tz84y2nsJCx75$Ld7^dPV{yF_T z;l$p2POUxs7!yeSOs4YVolB`v`rGTb_T&Q1VKjXEC|Z$TC0N-$5N-S}L+SPaNDFfn zH(3mTD@B^LVUz=ReC9`^V@p6O{Aamqr+4IY<-54(Zz^rrXn<Mjh2mM4)2Q=iF1vpX zXZi9il-=WrIN^soT=tv=OY{fum!W;|lOfZ&gk|E$A-No-J(xmQDnf1jZXEe`v3SSF zO@O2{A>-J6+MqoE-~Movf8F>E#?`s-AQu<vnLQeBWm>?=jxk{Jp;GLi)Q)Z%+yXt{ zjTB;{33ivlxHR8Pf<JALPGPI1`)4;j8<`5<v@Z#ZnzoDnT1~Vhstvl;-lnR|5uEO9 zDvC|ZAWgdoN_=MsaAUpfZG5ZbxHIK3cf7?c%cEp5qaSLiZ5G>Bjp1H<r=fvw7WzCZ zp_!ovh*o@q*J{t;#Yjcm(zQRP9=S<<-;<CNs*Y)u1F-INHkiL8@#^>`+}bGMqFV`= zr8%8em=uf7euqh;-C}4rN6L>a>BJK+#?#Z>6VPX0KmOi95xXB=#C8_Se2q(}$Cf5B znev6~maRN-Ne(EkcZa6@%k(tZ9Ov3<v(5tnwJm1z+I6ew)AvRp`c?_cFQjv5?HVfn zc#zQb9S#0FhV>?l#gVP|NW(Qjyl3o3JrvIHF@sc6Ny-7coa?NsXUzIijq%O*r^1e3 z!_Z93hsP>epf@|2Hd*dunb&E_W08T@E9ddI<|bNZZHo3TdAKHE1onNM16jLj;W}jF z!|dZyo;r$pJwFAGExGjb=P{UzN*vpof~ymJM474NuDH$OjQ3;d>D%36!}$Gl_`y85 zE%`g!s+2Kds3)7PJpu1#T>|ITe&Q3CJ#?fj5su`H1#=HS3Q4*TA1)sg`v={2=;c`s z$@&puqnQ!jsa4?S`1z>N<pF$sF%x%;Swhs7Lw=jLLRNTZjFxy;pEOU>t|2yTsMMX3 zPAhY{)T^F&=NlaVM=&s5728xT`OAxN!K3&!y-H6bTRW1iZtMYT`aFhJDNE4kk}(A5 z>T%INH@wmFH|^7qn6B9uVdlzInL$t$%^Gt;nz1p4)s7z_zQ`91K6K`W)-SYf%2%*` zqRx|?g5Ym%8O<Nj3Ku>}zHf`S;*ORI7_{cEu+pv%=f-v5A*qcp&7_nzx*w+PFAK4~ z|7dYS$GN=rM?E|;^McN^{DePQ+3;H-h;HwFEyfqk7Tm4`;l=852+-ZZwi#hG+oT(W zM_Zz<iIEUrc0ne4?m<IRw}8s)-iR5o)Hy?9z$Qe};frx%%JpVZgz?zfJcJG!tf4t~ zZ_|_UZ^guIH^i$tMz~`_tk|vTzM#})pV0T=3pi0b0Qy^(!OqqN?00B1?cs~C<%V>w z8@7b^+INI-B_&XtrYV>kmxwPu8uMD^rF?JKT70p5CG86uMro#UQV6x=sPGfQYw^4A z>O?=(w6Wu=#!ZwIeL=prT5`z8SYgk>`n+HJpV(O`4c2?a!OlLjIO_3hlI1*?81WH8 zQ~O+U8*YbNZd>x3JCiZvxej!D&=n6F+spEMh4JJ!1*Dj>oCh{+2bJ^*e0TE_n%aJ+ zICz^Se4j9ewJ%4*@hdy2=;ugmF})!SFUVnePYE9rHW_9`vGByj8MQV<^Cz{_qHnqi zjTy8`*gGjhxY_SI^dDqQ-lPaqUx(sl^BijUFrIsVSO>8IH^^<)5P8`1REZ~;N{L(D zFuMN+sQmm8Cfxl<|6Kmjhp$@%@sk5PKC?sPcg-}WWj`HRyn;2m^yK#2*KmZ*V&1>g zi%aGNLS{uL>TJ3ini3ZYCJVi}`^I^w-EM@;;baj!{QXC4n|PUK=c}R5_S<mANK^9C z`Op<T0}d2-qfMR*D@_l^lnsmd@s4_E++0nKGY3+_>WO5k-;=JyP9W8j*(j$H;W+0C z-S+pufM?s-VApQAcWpUzNsXe&Z_goM=r}ere+aX>^+t__TcGo2XEac6rq%5(fvstb zaB6=kjMd7OxjIPuzWaN5hTAH*=I~coxYU(g14c0qZ2-@#bY9Wp7|Fwv%fBY}=PW%n zT(01aid|iV!V<Y)y6ZMAsMso+);Eis=mPhR`|++|0WC)zCe77t;?6<saojOWRBvw% zTca#U*<&b<H5pFhQ%1n~h3dFWI~JOQZh=ykqr4@qmaK=`<Dx_7gt$=#)Gx)J!Ix$C zQ%m7(ye118X=w8}8i#!9gsVRd#gJ7;VDO0HxM7(MtgE+#-|LFWW5Q-Sx~>Ob)}F_! zx@6N>eHBo$-Yofxq*|gVxp+_dQIqV0ux069p6*fwR*Hw=r&SfroNPf!G1uYd>iuH0 zeFq+8dPWR+J{<np9;7_KBl70mSHQHc7`i^+PGK8+@~c))$~uG;A?mSkN|pCKIYVb3 z*;BdG5bR&9$>jY+oc(wb9XaC6eFpVm+x;)ed51K2pY#an@pszWQyEh_NFK-qS7gt+ z><~KG{-t-gfS0v({y)k)^SGL}|NloqMUqq`p#h~?^M0==ilRAVvm!)<NJ1*jq);MM ziVR6^QD?uGAwx-nF_KD|B6G-;@9uu?y^rd2{eJg*|M>jzJv~;Z;XKz`d!N_dd#!!m z>-8@B>O_cs)+UTnd4Zp_dV&A?Z?M6!H|#Wd%toa5A%2IF;8A5JThX9Nf*gHNR7Z!{ zHu=MW_X*^=Z3R2a&x@RqtRSA+b@(MPj*1M?!>PV?xbu-A*3RokQWI2&zI!qXY3Y)> zA}t#By3+W*_ZWI@#wa6q_k66LF%4=~ePYy~t;O$QrX<^bD9KCULG&ktbt?-Eo1&MI z&}~AbJvWAZ+Gqwa;4z$&s)S>S(@{0A0_u($(6R&L$lS$Ch<0=aGlugeU1_rdtBsCe zsWO4<{yR`}sF87S;aOUK<p+u>aAVOci%=;oh1~v}N}WF_k<X_`5^d-0=#ZRD1f-)$ z#NZ@)`=}UJmG%P>g-zJDXd}t$D+ZqjjK{X*nWTMQ7W>Mp7k#>JC}^$khw<MHp!k^t z@976iW}n_RbO@Ve{32P88d+a9j(dFxBnr3T))Uf%GqWPjJLRc&>NB!1XEu?NUqoA; zXENctZi2$&B-nJ;156|iF|#8qF@THh9$?J*%A85Zw;N3G>7=L7F5*ne2M-~GGGg&* znIe%_IRO&VBk<#n6)?^52`~4}40y3V6$8c?!z=Fzq~ZHWo~2qdSnjXD%0pbftsOgw zvD<LkJU*0u{;-WX{J1}<`*sn0R0U|Vv^wgkijfw#m$1>1^HdOxhEw}(abfWfwA~vG zmG@XoH2B8$lUxigPHAKs=g}Z~&w@MWETS^c8Zj_kn9E-yMd@67=6>T;s;oGIs&Sr4 z&EH+g3CF=iQu`?v-&0FQINjpi)DdLAo*GKdeZ2xx_74D=DZ{C7_E9W3lu4F+=5nx& z*ax<sTG3Ca4R1aF0s4#j;Y!a7tcjWs`Pj>cZnBHQOivlAC~yg6Lby0lxh>qf*E{f7 zb0a;zBm_K;AlAE-LF{H)6dICD<jmrVyuMNLYU?(5uAu;ESjM`o&V?tlbxG!+Z1!cO z1CRzyD(h@U@2wAoli}iUw0<~kPq>UDf&!Rnm!r9SnTMG{F^Ac!l7e)#iy6(nuSJ}~ z&oX)a&*1ED++1+-P;%hfYN}J62i8BTnBRn(L0nmsOm}l5&F}T;vi|ZgT&@z{SUQpA zPo$`t0T=mwcNg_#`qD+dlGHk@1aC|oNmGsyvT5E&2>!B;Rgi9mHM(s@-+rqwUgB>E zMPm<vmh&Z;ja*Kio9|HdWhU%<pU>_$^<j$l29Xr*Gp;~u7a|awz}OsmiznB%p~vh; zbX4>U_*rn=@JHc6Z18`_j1Ul^sn?TXz{Mb{Hcplid=dla6)i~9_mgNIk<WYkDwi25 zo<KAgn9}9|Me?1C(X>Ba0gt@`>5rSvbXsp)OrJ4<I4lsLK~vJvQ?wP-3YyWp-3>z> ztI=_C4D!;hz>>xj7{}!(2_4`{V$Hu{hLa8Jp5OzH10v}#b_wGql)~10Jj6a*ZbGgE z?<DJDs&Vl7xit6UC+4l`Vb<_S0M@vAL-T|}oG*A4tyVk0?K$@3W-+0|rt?6ZT}IEW zse_Go4Y|na5b7<bK=w3<Qt=X5Xs@jSu_80_aYPu(A1uOLF%@zwSdjXesgTeqs&q%6 z-L$E42O5^|1hq+VOp|gHGuy<J2>YF-H*RQxz41p-o0!eI&thSgHJ2kJJ<ce5*;0BZ zOc+uO)X59W{-9nW$mKF|r*pLWb2+?NobYK7gbSC#Y0+r}4k?rS=Pof12F{>I4O=0^ zSp#pnZ(&)D4BYA$L35nMN#gTyL}Y3e%$(5(?L+&}+RZ{_#2Q^Hd&ilJ73XqHW+}j< zF+Sj07|K{JGa$-~wb7({E|jOrfL~@1s>@cR`Nz$;wNQ)lHgY*z=G)=(E1BGQaR)gs zlt%r-q{!Z%6)-~L5lQw}ry+?ualgDEb*T>}h3i}3mHc*eZ9hug?ZaWTgFpS4?}19X zO-y^rB<im!NirkSsBrNLdh*z48rhcz>9vH)&b@~h@KaH|Vm{e^%z`wko6v%D-(Z*N zewySp7<W7~2Fs)q5OCokF0Q(X7EP@nB(x95{x*k3-&jcE73^XDy$BFWQ2=B8GNSUy ziHw&TOoE0^Ag?q0N!63j7+(B{jT{ugzLuR&9W8Dn3|IkIl2nM$lXU9qFp4fNai+>o zj}YFDY_v3=LbRr7uw|1BV0}<76Y9`{lH7Pe?tm59Iqxc#PE4i`r|E&~EOVIQ;6g@J z&L)~K;^DpMX8b;H9h`qVo*X$OOX@ywzUVvU*liZ8(Ee#KaW}E1yX>>^!7*Q+*Fhon zQjH5;-Z&GZBYv>O8kt!5eK48Ioge3Yyu`MMX3+h^522V`Bm09fBML5gbfLikxU<ZP zPO6Y*H0~9^Xt1Jo{ifn;#cOD;)Pia`4e0&bZD@L80aY8DptM1lEEtIJ;o@nWBj}3Z z85dd2DSb$e!azF6-J4n!^rlMZRpC=rJenqRexSXVFpYg$QL60)OqPwqg4u#Zi^~OT zesdB%vicNWRx>AO{AJLKyEmhteH_E9igD9cZ_<1DHqd_NP1UdIlPME-qv$vj>SvS8 z<rVJFU9aWPKTec9fR#A%=VqYcmPSv{)Uadwijmm*J>Z&u4R;hx!`|PD;lZsGRu~gt zX1J$ueMk;AXH24@<R_MYsfDd?;+et>RdQGOCr^IJ6+r6^DAOj0A{R~)b<HN&F?STv zER99kpE5Ki{4-PTc^mAv*MYLu9x#003|msdiJ`kH$VdrLOPi5s)nH83@5SJ%*AX~% z`y5&m$1-k#;#8sEE#{r8B=hRNFkF+JPhP&(A>W?NpnG&i61|#8YU90!X<0gqn8-Jw z<grGa0^8^w{}oKbPanE!PhYHGvJoY!Z?l6$%HgfY9j0IN4W{gL5Wbbw!Q;n!gY39$ z>JSr+DwDIoCoC9N>iS@Us1)jWgu=UZn~d{QVll@ofhfj`(lFUvT={)1(cSzIjQXaa z@ID?)=oLc6UkFi|7rrFt;d0t~PZvTo4xrV>Jcx8GhboH*ynek9w^fcI7y2oIhH5zG zRLZgM1)WIye0AKlHl4(`0Nt5z4ikqzVqKI*fmMtX&aB&t4=z81>oXVPGR_ZePkW59 zOT_|uZi5v`c#{ElTgTw3+nk?n-7sRGq)3PDABh)Q#L#FR%LZOhqH6=Uki@yh^v!ll z%%9Z&WUoHl&XQxUH<=NagTk0<BTp3P#$Y6OKkfdhGt4#7N&*JeV6j(<yYJ-89M>O+ zOMD{nU<T*SB4$suKWxG8^+3PWwP4(VTqq;lS~{C@&{gX2V1CCKdrETg)QPNN(`ryI z*okXvHEHUqOm<kz1!y_8217Om!jUWmdekBrbO(zW_xV;#{eQMH=e*L;X^|wne>9ln zhV|p_^POXRZ%U-5zDwc$vnx<M-KZ$>qX}isiDRpBHdXB}MGp50!BMG|XnZ;Wd;f?d z(WBQwe7Fofo)$>via3$#c30l)qDFjPJAx|5%c7CCHT_)M%Iv=L5%x<wW)@}40YR%< z^eUG(Dt~wlE)m}Z9uMY|Z0BMaCb^t=2}nSTxe9qmu3~kj1eHD?VYI#dHChB+gG}KS z;3aGi4j&qsZ+D!j=LtJ{>f>her7RLY^K$S)xEWQ2T(CWC!#pvqGBydYqXX(R=ub;) zTBCCp?qxSH3xAl<S8D=zhYyS=!K$yR+p8S#|M;?~R<W9kdznH@qlU2FrQ@mhhg}$^ zA_Z#ZA24uSGfY1*mLy+YOnWUVLESuMSm+xDSt*CXVZnCx<~up+_W1_x56?0_7%2j- zVal+fR)o&sj`cekb|f=#FHOI0Lq)$5@*>NDd`X>2%gs+gFO9KerPxe*^;{`4OFjta zX0F3yhSKP=FNjIi34oC4(@A~1z2WhLlZn(p&Nn!r1XiB3pbPxYpw`x_xUIYv#2>km z=;l+*rj!8amD8K^&)mW`%cxR|iU_j8tsdt~rPH%J?qY6^CYe(sN8g?w32Kk0(~1TA zVe^{ZAXmBz>>RJ4o3<k}@gf&{p3=_fH)o-`s32Q0TbQbD&cP#Y)}Z!rC;lA7t@U`5 z3wHaI!L5Bbu5ePK(Vz9;h3ih3SP?-dk-a#x!3PvP6FFT)F&ed6kX$53q=kjZjIC)< zX1JZTx}!pQI&&at<Yw|t=NwZ#ONk7-lF0^NDq|fuU$^;=@>sY27+v&R0)|;AFx!MT z^F-&Qk?q2-Sd!yy-1KcX-5cI&>{VqBam!Dm#GUtGSaJun73YD=p2=+gNLe^Jcoh9j zQWx@%Z$gbUGg_>;93*K4CP|+`X(=v8_R|O$>M2ehl?&04sxh?w$}t%A^afkKHxCJ$ zjcVt%!qlYixXb%2w_Z4fSl=^**)H=T|H(awak8U*#y>XPbN@0jt}9?;ranZ9m4fa- zP14XPMU-3%p)tUhojpL5WRwmf1z#fRr16q4Ok0-@Y)ODoUW6X%{f_DPDhvuuH0Zd} z<3@o#_T)(;%SQdk#CZpb(G)MClTRy`i?1Kf3HHR;d7qe1(<wO@(vOOpZee#%jv>>g z3IFb&azA99$p13_<NtX69G!oiKZo=GF!^tmj`))+yN8e3%!QwHlz$k1@~{1SxhDUs zkB;q6{d(l$X08_<<)3Z;#DDD{R3Gn3=d{N_p4Vk#xylWMy9Be#cy_$Xe0f;ZJ_RQ@ zOr-v{PjT`2(Ny!J6w6Z?O9yWc#~a6^*fVwNr2nIA)ROSV_p`;Qzr-Ti|3M?$?`B_` zHhMCxkGu!z-#o}5^)_&9ZN=$}@<8eEU3PbyGa37&g{R7WHY+QWZ!9PvMl!Fuf|k|? z2oJI*Z}yC(QyL#J)m{-;+~{reO)MPm-d{z-Kbg`^DvDUI<wBh0@^}HIiR>OOKiy+4 zKbzKxCpg+I3HM}J&?USes`B{@Gmmb9c8xM%hBYvExqGL%Az^6aZVfN(7vjAI1sD=o z1Vz}2rtNZY{E$95o}9!KK2XO*C6?`TS`FWati)I!zzN)XQ`2?*sey?txiq((;f0K( zG6!6-d6h7J4L=TBL#MD|xsNf88{nwFcmkd)9-{L4<s_9?3ROHY5_*0x$x<#elGeOV z#Jr<OQGA_o{Ok(OQ;Q)V+JtFTf()4;ZBDk7^(B>K?CEo@>+H!d^5kb_79$oVNPX26 zAX-U~7LVRQml0($*{lgxtt-O9dI4(FeuWHmxQGj?W6<>RJNBtEw+3;(5Q$lmLL6J~ za(S%#l1M*mR_N^nl&Kwv<%7hD!{)nCQ5FsjYJ+&5SyR|C^@VI%kUwq9yn%a+y_s=R zFN{A+)xpX7QS@Sl2n`gJAS)kwqNen6cI*#%RQ#07^qKMwqt!)7;uQrl<|NDb7QV;% zPonT%Y%V+g=6u#ssV~hNP=LkRqiD6P7kCEBK_R!UO@8qM7$F@4QO=TdzRxX^Ej1I~ z=Bv={sZn$)=dD$9`UK64E<i)CZFGuL3Jh@wLD8uxc(*2&E|NS5>0a(6RL<GhWxE`4 zxgklT4W8h`Qx>qvB^CPz+mV$gcaY}02f_bYH1yH9hs*ZOz-uLSsFmPp_)b#?bu9&H z#cB;wGd2W|EtyPqRpqk1ZuOx*W8GQ1n`#Jd=|peiIcU-1JTsSax!&V?7o8A3Mb`*d z!IHg(#BR9?iM+4`J*8qumdp=~@G+ugxnro?pq(Vx-jc=){E9;Z4}#Urd33+G3Y+-D zopHR-Xmo7*1ajL_kOU^qp%pE?>C#hy+<N<JCiu~ISg54~Gme<S*vzxw5*Y#|<4nnf z$(p<+7E**)l7sK+_Q6osXQ(8#0o3C9!68z>uCdRDi|=+~sPaL&QCo)BzD*V55^ms! z=<x*23AJvTj0eAGVY`GRy~259&f<Ilh6`zsD$Z|Xv$Hkuigc*2^aJwYrZMUNAQE3M z(}Uo<JeV@+3&`txP)+aEbZmJ5P2A=H&qbpc?GQaK2eA?r_9+Es({w|(XnCj>7b6ch zbNOn@^@y#64BXu(i`N8iK*^_6ym$H;){6V{Y)dY}<Me*?&G>xgJ{r-p%s_Tl$wE>W z@*YLQCXpKtH?Y;$%W+j83nmBhjrLY;gZWY0U}!}!?EPAg_pUy~Gi#PIXC66|q3K&` z{g3{{ELemt>rG(JHWykvG9J^W3Xy47caZ4e&ZM<=BS^;&q+25u>7m3tY&T?}Ia`2E z^h_iY=~~3}AU8is?ZupzyaWNC5g)a!gSF4=nXwykA*wD37mqW8yi4|Yr+g{-;@Jld zH)TSQXcatF+l4`Gwxm_MoLhtG#4df41va)3wC#dE-S<WVinkPE-p)|CJPC2!s+XWU zWHlU#GB>uanaot5%!anl<7hAM+ZYuq2CB!_L8R$c#(RV;)@hdG>%)#vrqcx2W<^Vz z9%A+kFWO|ot&^Mj4s}c=NQK=kBPX#>Y{3N&@Q4;8#aZF7LzN}UAIISHJ_*EERFRYq z5ux*I9>S10>#5G8vvk+HExZbaD<D0gicRY)Myt6zyhWBD*c$sSjAmjs^;Gph_hE<G zG~of@^63`Oqpt|OAKC&*HEO84(S>Xexd7f`4=}MU6i4g%&~-{~WLiuvD|LwLUy==T zcbqDdn&(RAUNfLx)`oaQe<dxM>q;wa`cm1T?+~l9i_J6XO>4M0_*a?3^s|>9b%+zA z!*}NwW?d_TfRF;%m;M~?mQ*uCTO-l;Il+b#8?jRCCG>i?j|$Hcpuy{GpxNJwjC37M zPoF*w*I9(hmnpEXvI;Wi`qK}#dsvCOK6Hjx2u>ZxkeK!YJikYWyxu;69Wa$;{9cYG zj~gW5{DUf3?*Z_9l`J`<4lqJg8e=Px$Y_64xF50`4YqE?l(q~!lRpu}gA1WuCXwhi zt|QGRlZf`&RB#zSi-^n)!0X56qfmPqh)4#|iJM+BdUk%`tn`4{+smHDl&I2ta|s%y zBaZe)aWGNs8gGsLMZ8((&y+|>V&SsEFw1=a2^WjPoa0*{Uf7F1)Y(c_f6*s4WqpW} zq8nYEa}>tn7Up)#ShA%y7KNh<nUEVZY0yGLJo=f-`<J_e5nieU<NPXV?CJY>_RDzM z>%lEZ3m*ZeU+}<=H<V7QjHP{iSAkst;N@v)r1WJtYP<UKV$QVSq3E%+wN%L141$3z zD*@|<bf#ncvHP(9kl!8aIM=#=IR5yr`}cM9UvEL&nn~Q)siXXV_dmeff7N2Y<^MMS zsE$ZS4{q#V<kXvMy`%hn4@L`UbdI1_|J{;rmAC(558uV>JyvtQaa-(VZmRlE%=EPM zbhLDJG&Kzj^>noiboBHMv?gjAOw`si)YH<@(bUn=(l^xC(w^AS%N=`*6vuHr>?pQY zy*fH8(Ekso!Xb_E&f&uZ1Ukx)j(>D{bn_8ZU$NA;dj)6S)Q0{MLoP793w(#GvQq`> z3;q0;d8~H(;|=IukICAV&42Dnxi0nCan*LNXVq#ye~(4pZrt8mw0xQO^3~m|3Wug) zeYq++ie#6nZg#E8GjO%HpTC#4fA<<j8jEfH^T-(7rG{CuT{`poTdCO{S2$)SFVNwr z$d#^ml>$Gm)gA9VFY1cha&KX$UGaPC+THQa(~55Rcz#@`JN}E-RrqmEU|j|Ei`G^7 zaoz5?Uvg5`pPs7SIzNFQ*XxdtzWuaU&v+HXkL!2Gf6@7w{J6nCz^(al!|wP>OE~ps z=OwzR&enz>*W#01lZS;_&wQ0z4?LglI&Xcq_E+@4|5A9}_?12Ie<`nL*VN8Cz}CM9 zo=<w6y-d5df4(Q4PkX(yqTT<zAJvtYT-8JSeB$f8o~A1wpV$M>r@ocrF9!A8zWA;l z_+QGeTl@Jv@O=8~eB^iI3;20$KK=c@e&Wyc!1L+v7ww<zf&T~ki!bbf|4V;=UJJk2 z7WD7=_=zv-f#=g-=X~g0_n&xK4?Lg#ezE@^^uY7!?-zV+4?Lg#I_J~s+W+{6J@9<` z>+FZ!l^1{A1J9?wU-ZA7pV#>(`uo`f|Cj#$-{+rrZi!s?>#h!;{{H^_Bhae{o=<<B z*F$#Qe~vxzeER$Q^H0_29@^*AU+4SrUE8-+=z;$S`l~YQf#=iT-yi=~Q+nX}^!NAr zshZjY&!@l6^P#TWUuDkEPvp~I=lMui-qxlE{xALgdH(4<o9N1m+jimqW(eJRoIkqr z<x?P`E8p`B#MY{BUv7-``(vb_+FuoD{_)eF&$0vl^9KXFj&Xm-l%j9Hj%Np#|NP$Z U-PTH|WArVc$$fRa{y(q%A6_ihy#N3J literal 0 HcmV?d00001 diff --git a/tests/fixtures/tl_checkpoints/default/meta.json b/tests/fixtures/tl_checkpoints/default/meta.json new file mode 100644 index 0000000000..9f93edf6b9 --- /dev/null +++ b/tests/fixtures/tl_checkpoints/default/meta.json @@ -0,0 +1,12 @@ +{ + "d_model": 32, + "d_head": 16, + "n_heads": 2, + "n_layers": 2, + "n_ctx": 8, + "d_vocab": 16, + "d_mlp": 64, + "act_fn": "gelu", + "normalization_type": "LN", + "seed": 0 +} diff --git a/tests/fixtures/tl_checkpoints/default/reference.pt b/tests/fixtures/tl_checkpoints/default/reference.pt new file mode 100644 index 0000000000000000000000000000000000000000..130ae8029f40de030baee4a10f3c1e4db3c0f108 GIT binary patch literal 38053 zcmbrl2UHZ@^5{#>D3X+*<RAi~Bw@NXh>8eeK$K)cL{JeNz<>eCAeb;9q98#q63oJM z6$VV06Nm~Z<_Ic^nK$40&bjM+@2&s6>#o<c_Uc*C{o6IwyY}7{dXkT{jD&=Og2eyl zP?u1Xhz*|?9vdDV7Cv-tXnd&c;)PM39x4*S|8*outrkjH2Fi(VkB?ax9vvr?42zG6 z4U4p$wIqIiRGiYR*zh?^=10w)6)*m0Ozf<s!<B?mmX?9?;ydERSB6G}3#I?QMQMCY zbi|au|E46A36vCHVkeXhlop@E7B3OX`N+fz<tO_{_(=N5WC#^}WPBtugo<m!!eXML z!o%X{$3({|O^lr@7J2UI`C;)&LZ!ieLgjd&il0#RUm`}uM9hzm6ZVM|OAz^=UOl5? zLgW8ew|)QB?J%KQq@0g?ys+QDyYo@Y6sr5U`>6RyW(qadM*c<Cj2CM83AO(vU%;%% z!v23_BmYJW7Y>M&^OaDD7wU)^mOfJAkD-rRyinKDS3<^DLh3J8yihOeUo8E2;Xps( zpntP`ga-d$*$WN-%`*BY%lIEGlfPM}@j^2{q4~dA(}WiPU^xga|IHfwPnOj`SVR71 zS;q@){DijuW=#|h{RhiYX!mc{u>ZwU^D&GU4*w6!@UP!!|2NAaUg+p2bov)-j#yvL z|6n-@NBpgF7cuHz+7heW^)HqErE<aF2rga-enR$dgpbhe9|&im``?I>|Ap}Q4`S5c zh|%%FF@8eNe<P*|$NmE`Lg@83V%&cry#Isn`5Q4lUg+y5obYeNMB&7LAY6o#{zgpx z8)29s^b-TbLv_mExBJHnr}_z}EwP>CBd@TguTLU~NBCcD(*J0TIuhJ}8KdpInAk<3 z@w1kO$Hs{Vy{E^Bf7s-IndkrU>#)CMINA9q^)F-ltNqV1=u0^Mw+#O_-dUkh^CO}c zg-6GGdJOx&7sJWnF9F<^313TzJO8Ww&jM&jz<&uaCm}vO<}YLS^l<sV@Ap57iJBiD z9~J(W4C(Mx=dWq(ul7I7p!Sak@z<<4)b9U}{#C@i{dN53{GW;VFHQV){3V&h$o-Or z;_rX8|D9yU|D~OONjA*WW6=LWHGBB;l%ao>zD03#prW^4Za{MV^``Ij$>wwGCp=DL zTY9In(vJZwPnn06Ss88$sTwfq#(B2ut3-V~?qch1nb)6tZd^Zkl!ROG;YaoUTAS)K zj&7-+d?B;mKJX*O_H3jHmBDTc)~eB~#rkZY`*j-Tkim}WxRIye2+g(EuGbnmwcdKO z4V|rdTtB{jNc|<BE%iaWS^d~vXG(fx${wf4(Rechwyo-xTaAJmo4PQENt`<2;>9gt zc2QC^_NYz0LQgYwru?WsoISLDZ_S?izO|dlui2P3H7dCsS;p!$FYIS|SMIy1FC9g7 z60UA@?v7--%$xqw@4b-$=l?bK#pC~fr(ci%w|<9vda(b4e*ZJh{-^VZew$!%jvjn# zpUOJZd?ChbAuDb_2<KZ9=#a%R6jepkEiRK}K6M-FX8GR|T)nav0<FVoFgt*Ds%d<@ z<rO&fwiv5*W}t$?HBsHFJlJOX2+pf&(N_&eN;#RzKW;jVNBi`^tm>m&o`oR`OB}?8 z4JilBM~`94>d{nE)r!KwpZH0;bTGmD392^wut7VU>9yisex%h`yqs>01=a;@`p#Tv zUn)WE#hX~t=OtuZdKTvtKEmC_*>oz_nDlN`VV8L|O}i#T>Yi7??e_!hkO+pGfhO!r zm$!f>1+t@Oi^$@{D!#=rj82u8z)K%p+CO|goWKS6`^!E&bXp5D!m4qB)&qV_j}qJS zxdFTy+pw$MlD{J}ndV+`q94Hr;PB($IOLrc75yAaX=x37V);G@zMI9i+|!4i<#!-I z)P=6Bn2681RQVNeMl%0E9h&lCF--sT1h)sz5xuymK!<$nI6nOeyL$T#4pKFwkE?It zb)RQ&p<oQN5C+17$7&=wI>P1i{CwV8B8IbYoQ4N<uH%WBIehXIdp7^ibFOL3OU!*~ z0Ao_ZIEmV$Y{Fw3I@O}V7Ff>4X*Z?$knlnHd8sNdsb2tU?<MK`8jcoS_oTJ$@l2^L z9rr$b%iYY?BIOe%%-g0MFB{~OOj{@@hI(^xlfU604FeoDGYMLrDY2nX-Pw-s?V#h| z%36(&(c_XOq-OU{H0!`PW-z^iZ9AS1hN0D5%IG{ebN&GSe67ZAUSi}~IMMZY!XCJs z?n_sny5iN8TJB8qG)hkR0I@C_EG58|wVlpmuiY{^qq>89&lE`(S@8&sn`hE&=VY1@ za~ZzfKTK_R+ywoG=JM-@#WTUJJ#5PcB|6=C9rCtzgQAi#_gOu{^=Y;bMTBK@LtEXj zKIt>ItWO80h$C?F>~DT~q%x|g9LJlX@4$IpIp&6C^Ox5z!wnDK!sgZEX{c=kzr4hT ztNnfeg4*VR(E1bxeIJic%nfMx>8+&p!j<>BRCKENu^FuznagGlyvF|y458eEdhChY zTY<#I7#h1j9oyn+1P0_rPY)NvTfbWvux|%jl>Jh`eNKUs@7{t@dI>N1;g2_$reXGS z0kvyg0ducxy0miu+i;*i<<FCX%`X~Yu+0vbXy*$(#{1xd<Uzsa*kj;5;xjnM_2RSB z%1mX?Pf*I2V1DmpsgxH08~z@>IwwQLehGM8VgzZqPN+2@5|!S3=W<M4*|rF6TDd!l zF7Mq5@fY>k#iPsU!*6|xFx$r8^Ei)74+BYa^`IfvMPgGLJ7<^%A>Ibi*7kz4zrBjf zwd}@X&uTEe#h9Ls9-HybjxtP6fW=vLHY58bcID;rPaDob@%8VZubaZI1O(IGO_tp1 zKjjpcyAw{toCnv#fuugR4^{WO35mUlP)eoPIsP;#jL)ZQPJiI%TtyUHHS;=dMw9QK zCdqp{*yRsitXlake5~g=_58EA(R>LdAJC)i!X8-Vdk_oTmq2ZvJ*k{F;YTc1=Plw7 z!a|i~?8&gBxRTj4_*NEF1YIEg$1#HGflqm*ln(K{u#>j?jRd)3W9swZDAeBljoagr zh>pL(-#C?C+MR)$$IifIS6lu;AaVB|Cki@e>;sKK3eY%AkC`prjxn!p;<ZH|_yqO_ zOSWc!;`uqyVd6o-GLPV}<{eNe(dFkzmcYXMKBQ%p#8s-5309?+VQ{Go3%0X?!B*F} zrz6JV(c?03`qe6MPu7P4UtjWtD>c~cR3WY|-XQAT)`!xUyx~rsSqw^(ZRz)-0)G3K z!FZ=U14g~50O!ZD{I$J<S<8zM+UEX9AR1m@H}=s)7Ie2C)NE>kCGrMzX39~{VXHUQ zYonm$mJw;^Ux7%!Gic(G%e{_X0Nyo-!gU`7Ha|zhBkiwz+0;4g`AliLd3_Eo`Tkh& zrM)l94%xvD9J<37^*+Xo;w095E*EcVOLIU)*b#UM(|#iE8gm&YdS>FVt!Md&%7@UT zV$GiJIDu2w_2-|CI0}QlY+@#|L%{gu4se~j9iJr5<Gkl*@^ili!0R>*y6mUVerTJs z;B%Mw!@D)ONiUsA{`FB@`$`c_j!fVdg{3gJ;BEAw$Q4F?Hl<s4?b+#fA!PBuhMjqr zgzZi*1a1Q_^HHX4u)g;Qv`aOzJvl4jv_}E^P<Dz%2HHTc+J1D+=AdA90cTyci3=<X zq!r&R1^&alpmNtHf!~J+cv!p?7Mp(ty@mE{3T}smmWyfY%_7*n#+x?lZKOLZhqEuz zMc8!DkUyw-7o4oB_?qii@!q^l2wh|Z;}?x&yC+z%PeLhv#^MC1?JG}H{{+#Wuo!Mo z&~*0m<P@0jVFC+Ko&xSoaqMNVIy*Y(7^p{=Gj;X3EVRuYcNPS)W4`^^f+3Tr;N(TY z=kd`rSJ$7SZ+nAZ*b;vF@+Q1oVZnN2C$o~4=UhJ(L#j*KLVHXKSYg{cepOW_H2K=H zKShpEw>6w87yN;-1{c}8BYF6)t(;7+D$``6wPZ5e3nfa9g6x`0VB63PHxJ3t`QQt9 zZ}|zRzO|0&eVYwQIufj(i6JRR*w6|&9~Q5;2`;~DfZbV!tWVx+kUn>q_mVTfsbOaD zC9)sep*}>smc_EthslKd1ms!&8Xv^&U<0;H1dk76K})uoA3pFDZxmz4jE5W3@aJiG zfC(sQh(7CkJA*T6(t|mpvM5}&2=i(spuoifRa?%$?WjPeIXhHv+;$Vvk$a$VX9O#B zI?BIjGDV*mC(!n1U#tjt4OO`Y^kGvI?yCF%8M_rJ|0}_a^A|AglO<+_XR;!7LprUn zmp`)E3rE;}fI$KUX56uxyfGGTNvdJ?4G+pXVS}?fibzVv4%dc#;{7YnL;s#_{K5M} zS%Swgwy3okD^@M1J`$end)RKKHMorPQ@RCbm-(|VoBXJKYZS>{YQitBI!s+Qf=nMA z2iJgE)PD94_cZno)LH)$Eow~0wZRr_!(qnE=j+qnDIy%#HJFtSnFC*B4CzZ}H|Xrk zMD_8tX#aK^7ctPDId?o@atGF-XQc`x86D@TIG%9mNcJT(muWte$MS7@)T8?fqaH}G zXE7!y5N6@{9qX9Paq$`&T!EY1m1)3RNwlyXPn?@B+c!BDH1n^+Pn-hzTAy%L-}RLK zCyhM~iAAloxxD<C$>e6W(WUw<uzO?jG0Lt;<k;)aW;s@{&BjBS-iU7ku5cO-&D4QU zvyjuOYQckUX7DXF4m|U$S?dTFes_>9^IN@^v9Q@p=k_Nq@WE|?TpG{Sr_1xbDNiu* zm?pb&;TQy5w}cOVCD1SEKE6Av$PMXh%=*2)?^+cpCs;S*7Fxf4#Y?LE#EEwu*n^Bu zILAr~qUUv^VwNP!ww8f)-y67C{|@-zUx2NOa&&pzTmDT>4qVUA0K4EGEX|lhLhBQ} zk(L5`bMX*AA*h1ydL-v|_w5Si`l|_lZ`sRzzv>Kg=AVF|ky+?s`kfOBV$geQ8U<X~ zB??pXhy6)Gq8Vi=C^LRHIh^xnLxSUI`b%?G_$7rYyq!axS>Ej9OarLmMxvx|1Kt<8 zGMN>d;H$?D9HzXCyH>A4BUVYls^sH{sf*bXx0l?ru`2B1q#YQl7{|i1zam$$l_s4X zMUwIvBBe#IV6<QWq&~g~Z~M-Jv2LZ{xHO*678%0!)BV}ounan6b{A*I{sNgh4{_wT zg>1Cfe%k3}h%47<fu&fV(>%i2hNlUbq&ywG*L33r&wZlmpfK*wT4mNH-;T=f2eZe< z5wvd58(2A72nX7wa8a-;CYUGF*d3X`t&yOeHEz7=owq=5&#^4?0#q3h#MTDHk(22$ zy1d+uR4X*`OMo|f{B{z@pYr6cYbE3Ipo!2Jn@I-`_uzn9UG{j(OwP;p0)}fuz=IGi zsMz#^6Q-Ty4OE&yb8tVbJ@$;xxVDj&MV-KX&uhT&M;Sl$_Id28@M5XMfxl&Q26dCu zA^Tw@Dp-|4sf|1xjLyR2h5c!L!cy2=Gn;;LLG0bKe)PxYL){bMGyK**nKjLmgY)Wh z>OS~P75#Z@#JrBkL#AsK#f{~dUak^MpPWwPW0%m3r|NV@JsGn$KEQSEW~^+}bu8Cx zggXgCspeP-|4`-=pS{PFNsoPsuMT#=ICE8cpKr^iDc<AWM;gNM^u;K7O%^8fyUTt0 zA;g{U56~IOVHDe@OOI<J&_=_6TxMC2R34zTM=(T;Rbwi<44K(OQ+CQW94uz9gJ-iR zu(YsZ8aPrLJtx)S!zT+VZ?O&J=w-vIC$e01s2Z*4>c_<{&SSm{Ey%X+6aJcL;rcYz zjSq@4qmSlW!TaMN-~~>UlGFk}T7UCu+9%-2(3#Zw>@t2>*v)y*ECQ!WEh>DTO$HM+ zpx7~*W_GN^c_Z^_XUux0d@u`NzX)b`JW}|Xg5TiUep8^<mV<Lj#AE!lI@WYKvh%I^ z@J0|Q_NUvpXZxS>B{O%x8<jmsPLKGNPm;Mim-fKfb|L#@+RDwA-c4H9t1$YThU=`) zx-{X;esEivg|+Hc{FlJ}IQY*-7FZc5svLM7?!VWex$8Cgj6GH?>l2Tfi=7y^BA88j z;moT{dk-qyC)j*-GVbheL7%fqF-+kaxIKvBJBM1*4oweA+?h#{7wxIYa|<(QE9A_K zb8)H88_?(#FwgDtML`~pY@bMiny%i$gyICa|EDkgnVZ3<cn@ZgZyj6<k2<sKq1#!l z`*QlwHi)|C>C)z{iEv~<749zBO-2uVS;GKDa+=`6CGR~V-jCLTj*b;{M-8Q&?rSl2 zL8idC?j)+M&cr{vf}zmh3hq*uVebbX;5N*t2EkGtu*jGLjZY;RJ<283*OT$g&4U8n zuAf{;{9^35GMElXALYlt{f^OhtC^n7S^V4vY_Iegu(5v+bJ!Zz6VrlL_iw@R1dbB> z?c?4ROT&&*P1@a)!3sZIV53u1ppqX#`u#;<f3Fc`&PcG6rbetoU59OO^?<fS0hxDb zV&2MZ6fVujhwah$OC=MYzT1y4OV!vhk5aaBZxX9JG?gVA52l}!w9qm(n}W~eaaK>% z*y2^+@m$_279`rjZ#!zqeC-%ps$oGUi9^|%8(A!_Y(GDE%68m#`zK7l7z}Rf6_{k? zNNj3Qr~da{;rrnSe4JG*X&?H{eewT=yUp(4?PeEtzDk!LG;$ZMy<(1Kb8`i*o$Z`+ zy$<b<QpA*B#;~&L8${)q<Du_Kq6FO~EKt8255GAEn|{^e5|b$`|GfkXl=`sk_q~~( zk1ptKSD>%u>G-<E8@<YF@q)1~DYWZTs_9?~YkG|b{Pn3iUV<v5#Xi{2VWbr7LPOlo z;)v1a^!TD3O&h1m^5>0bZ{Ipo?xI}Iq<I{!cHIfripvEhrWWkn*aD2u=oEcW7>jGd z3h_z8au_+ljZQ6<U{BQa=~n@9hq@2I%_@I9HuoE3R$8)yM@CXMZ$w{Y6S1(hp5Jg~ zGYj3NMnP;~{rUkTXyg<NR(p3GeQ|f=g;9G@zTAj5eyc}rV;7dFK7=iMWtsMu{kXlO zKTe(FM|--~(eAWTd@$x9i`timH_y}wDxYr`8HDVIXW!03$%t7jr+NzB`}~}{GPxV7 zz2(VW_5sLgmGc8<STji{O@5+c06koJ4o?r4X0xm2(){k}Xt`}9Q}&l)DaIGUSvZ(9 zzwM^4rx`0usRe@((OB_eCtqE40<J5Ru}uw%l&RegOM4l-sT#ob^*9fS%MDnuoDgDe zJ;p^x3t`|BQ+jpn4kx$93&*J2gJ0n_u4jk~E!Ers1%W3alFLHZ$ujg?)3;7E#S=eV z^TTszWY{rf7ie<e;MCW3U^i7AE)I;Lfm%^i+}95`in?+0tS@luqcaR?zRLW+_os-m zHS|rg1q6F#sm#F(gZ7?9+fM`7uJh4Q=!)oYR=meuzJ>4Xh-8mq64^uhjTp617b6~g z=Os@Gs3@cb48j@v;yjFRObTU--Rzk6k{}%QJP@a!awntH7HrgGdpKL(pVd!p2OG;Y zf&1g1u;FbU|0?}GL`-hRAIBx(MnNGe$1P;S+-Yd%+r}HE=~L2h4LYY^En1ebA8_0= zRNim}-%XLA+KIJX?T?X=leitK-podMsh3dFtxL{%GhoiV=DPLjW-O~LPqbZOGkdc0 zJ0GZ#%+09Yf|64kaClQ8)VU0XFqyj$mii6bPg_t9>|)2$g=n|am!f1nA!4QqQy<t1 zEq;%%@X|$04m!ejp81IO{o}wv^%|1D6SW_D3bV9NV9unzEbZkn2<#ZdW(A1ja9`6Q zNZXQ6i*grucv-NWJLLqqHa&IU6pC=$`(se_A`9Nh-Qy?9u3`3<fO+`p;SkG89Q-kd z`pCS1+A#;YGVAd;+-;jkswM!}^xE=UHaCL(X2RW3-|*OzDJ)>oAhh0F2!VTpXkACW zAjQ`PTc=tRK3)ydxrgED1Xq-J6G`b;3@MTi!<e8P+~OZhtBj(l`E?rhHe0c>7G)}a zoqz%L6WEH~9$+87mN&E7%&Y_3(4%b#FO=BBarfldnutA^b?qt4v^JqxD!Np1HiJ8z zB1;YdTd6uik7Blr#gND<ytsJ>pQEQhN4qZK)-lo0sX2yu*qC7E#R**3*&q<%J8oz2 zOgzwHg1?98;(aL<S|48l@<%>ndQvak-rb+}2CZk7j|S2+Crv8aWi1HW=*rTyZ}Mgm zS7E|Eb3R)moqv))n|~H@f!*oJhi7H(_;|$`RQBDA5uH)o@da~nciaGKdpHColma2p zcsrBW<4IRsDnLc;y`W@5Bh*is$M-*eik~sY1xaZkOJtGktxo`YD{sX6V{hUjIe*eO zx+d7z@fzgj-@-T<H^>}x7YoLmVn;q2up37wGAAnq@P45~_c{mS?+LlAuWUMK<*QR` z$y=zI89*BQTKJAv27IPm54P<NuG60Xi))yn#2bv4#riLjEN)COFRGtTN-bwuVE<TR zgJjvSEBf>^Wd%O17z47iw_ta2I+On1!Z+w2#)X<`^r1llziychDf7PJ<FWd9$S<4o z_SlR1Yfo`!dZvkJO%&TY_Yf|c`=nOknlWTw$mB+Kh~tGt?hsa`AZp&}gtM3JhAx2> zZPj}ZUWdlPHWOWDv!E5z52};y5X3Cq2-bAI1WOLjrh?zI>DO#R=^5!bdP=WoS@|aj z_v@iiPqNu<<uF(s)5fhi)`zlYZRY*g5qJKZJltGi%#E9P8rMo~qp4GkL7BBcUuhA} z*>wPe6%A3+^&7Z6e}uh>wfJX(HOnge!z;W^!;l$Aco)|-?B%gml-HjC8f)i~8#@oS z%GaUe`EP#I6C>7t?j)KUF$I{c5|MiuF3I1{tG4x_w_d4W>{yOwqusdA;pQ}<pA+ln z+k^81_1K#13TlfF<y7Sc@e4zL;O}vbIAK^AEN|Mw?Of;!dZV)8fz})Dx~>In3!V<G zZ&c{@oNF-XZVz5O;6_Ti;+Uv=60BHuo;O_U&DTCtVGHiNvDD8YtW6x3EIOYL{qi0` z^W*W%vwI=@Dlp=!=mrD{UcuzA5_EjXb*vG`biPeF#5wjK1Ow;&!b26c81zYkq_Z!G z-hZ49MO9Li_>-VVe=MuJ_YoK87BJ@-rReZdf~tk8_~znOY&DIf*&C!NNvOa@XQ$ws zj1anP-o^K<Pp0SECX|vlnw)NY=B6v9;=Qt7=wwQ)D9wRBjNU;8@d73aQvr)J?Oa7l zB0Jt&1#|MOD6hE}j{Tg=K*E&y@2W<xOl{V9L5d<KKE$b(OSp#7;rL18DjIxxD!TRa z3(vY$NcCi)puO6KUGCWn7o{enH**HHr_Z?oKR4pmwyC(JV=>FHx&wZ)1?b&07_|&% z!urMG{7A<h99=S<t6emLtOso6lgBRR+vfIV;U{;%UDpwG@=Y5|9kdnkxjlv0%*HDb z8_8|<dR%0!&q{?KK<_kQ*M<IUt>J76oAenKrH*oKefu(X$2`ciIt?o`9Jo<se>i4z z41-5;tT0&*<m2m6YRYjITJZvW<Tmp6`gie~Kg_8mRHoj9*P-X{r_lZ7gfHcnKz^@1 zKG3n}Bdv7U#T(WvYEcQ^zv#;}&s1`gujFzOpN6x;pDM9NYzr336RCK<3v0ciLK2GG z*bHy+I>lMiR+*i6<whTP7@frmrbfYob5_t1DFtaRrD!)d(lzO&K7H&{1=_zX*;PAv zHu;4k>Pg74n?BR|iH01T)#n>*bUFlvdds*``Ze6!%7fs(A&abi2JtG_{Mpf#G+cKe z1Q$#kN$Xi98(68%5`M<v`1(PVcj+v=G~AB&C*HxYJ3qi3lXS)wN6;s?;cR5iJPItl z!N1p)C&x4E`S)SvoJ-4Q!B4e)tmaWRr?#qyHIA}peNNPXMqdeZ{_>ffX`REn&N3jI zc|JHW^ayEBDCHGgmB6s^2d?pd2J>GelXbc^8z*0idd^u?zg-1pcV)n$+G5UCLxmFS z*7B|Io`XcE4mCQqb528EgUGIxFZfu13fsNe%sKMpyCxHd%=E{+kLsjp^1`*i*MKsz zOmTxzGK+bUhEw~cqGV<rH^l!5%xKyJA7+ea;Yu6$M+G1GPp!4MZ@>Tu^Sup?jx~6w zHww4akEDS*(^)}dUwjhs1|DXq)15tv%=lvt3-Bry6#P03XS6>FuJ6%cIbP$qgdq;- zhGuNUqi?*FgbJJNW>4ofCsFrpPf}@0B16X_{-&uZyec0HRfjz>{S4sCLDQMc@O5x; z^?97aZ0Jn20sAr8in?>HDI$C_ojBH?ytm)Scnl;#**NM}mEgSgeFTGF3s~x>Vbogg z#@f}Yct`dOeOg{X{{|^|GW8<<QrLt8RxU<Cc%cZjwbAy}c52#xg->qEcfIm616Ce$ zX3zFLhf%HvYv;)^8s?);z4@D%VaYDAFWrPUGdRdJi6O71lUz7o%6!VBz;KHRGksvi z0_=;RzM?<nwF|H#J&D%ntDsMP3RdQ=21A)*&ge}c)s*Vd^gVMxd&qjO{=z7+pX9>o zMnrNmnq=wznB9Eyr&3V9IT}lAIe~gcECwsBV>N61`5TkR(FXf+!5Q@qzMrxYot{%5 zI)1^2xqRG@lTM9c$p&g{)!bfKvTGl|&*LDQR^-gbnV+O@*7e-8pak~x!UMdVD$A|< z*axP5{>j@so68oP%wkoG*TeJhCa{aH#=B+luB#V?k$yrc_Iv#n%EWPqzUC?TqH!JU zHt4~m*O}DzpxU+V`B{{@eh^j}rh-!TLEh4MFbfO`#q?)-SZ^B%2YQcU^d!W?JG%IJ zQd8OS^U2UEDuC3@XW>jsBD~Q>-pta4%v*(Q$%=e|N%A?^K4t}%9=Vtv^$w?_g~QoG zrGadh-#gsx@e!{Mv&ZSljc{@4doY*i5H(A9FxB>t=xw|o3zOz?1tlBtvx_TjlVC9Z z<PiwE%J`^Pnf%%pvb5UTl?oHfSq3v_6%~o7ddLY5#+C7h_Dgbc64}^M9nW3`Y=QbN zW%~H`3cq0GdFIn*h$A~R`L$BoY<0hl><-?CRW)uD@qRV`O1$sW%u%DRPgnUv6(eEK zyhu2evY3fnUHFd0w_seTGPs*ofs(WWGw<!esqGz@TjI+`7#`vVn^m$<ry*44--wEi zhjE!f7M9*$z-$}_&@Aaps+`iueERv}ZQ~w(N`VyFPA`W%*Kzd3?hSfL|G?Mcea04t z!PFGo5B<kv;fXCzxebk%=*aj0Yzc58uQ?CE<(MoDGwcD=^QGu#9m*`vyK`%weTB^? z!C*G<3%BskV>mP;ACpX~>&9H(P9vP><J`ItY<2b~XiAwxvch9f*x*lNZs$Q|S2dr~ zM~Z$NyM}?UR>ROqhHT|YP4@Fje|%G|OBFd5=$CEI6y=tJaM)7Z+a|{*?R2C^dv;Q_ z&J}t*a0=bJ6bw$e7EEXEAa>G!CG>q!&9`n}LlZV>v4=|!iJWdUbA66zk<BoD$mR@5 zbKYZg%a9?<?ilW``U98R*=zn)+#xRT_`j+@u>b#Ge-Js&4D^2w;;rr0ikvmRpuE=| zG#@A7@~iw6_N)k`sH!7;*Q8xAYP>DmWPKF8ykBtJJ(sXac@IEpWFaRh^$=^T3)tg? z6t<IhrVV#_JmfxIq)}?bnmh~Ps;Vzdv-V{zDKbp5LQ8bAT!!_O`!Jc5K2V@5OQxfy z;0^Qut(@!p)!*tY(!BtG7w#84E?$YNHHXrqU-~3zR|ol{-{ZMyeQ22SYF;UFI1Oyk zXG;3Nuw#NRbGy`lohH+0TS6!IBlick_#VIm9e4O2{kK7Bwl-8RS%@=(bl9?*WNy`Y zGnYx3qv8D%5zfsP!sY8dFj`>*D=RXi3+7r7n(0W6D(ApufCMQU`>>R4^DrbY2jb=q zBpVq!nsDe0taa3^|JoeF+l~NQov1~ZJ2EkK#UkoI@iQ36?jhLPj5>`%@%4Vp>QWWn zn-K>MIu5io-HH{@QpM$&L)f5$9w^txh@IXR!&N=4;WxhyVAjrFSQM>`YU|C|><Nne zqdDr#{#FVNEl_|Vdw1Z8JLyc)GMj68Rmfs*<iQN};k@>T6-@t8Bp(|)7dABp;n>Fm z+3zD(*mm<VHZ;iKn`V7#)vzax#35AkdNaSnV<oHcP^XvCRv249AM3sDadl^ogM)HD z%bn2%;X^-T$ldWce(p=u>)Z(m-xNuyA`{BxML6P<GcD`0ueN)vEi+ni2;r_Wi^|vz znG1dJlI%rT^(9IW5E;S_)ThvfovKvmFrNQ>Dw!+->QFVqf)4E3#Opjc4jb+r#aD-t z8M-*IweND6s%Rh1kYe;zwh2}r3BrbJQEcJCe!$vnaltZG@HyOz?u{J++ZAzafNmz8 z@?VcrU#ih+sS0j)w>%Y9R-yIkmwfkmTR2#k1n)JPAt?GaTJL$o&oY%{!v;R&B61h7 zfYO<GL_dofCYeKj>JN=+6LHPWYpi2#EfyIb!1{eAtjsYN<7H|vM7NMvzIOtB+t1Y5 zPH%!!$;&Ca_hj9erpq{7Wf^^T9)|bpcCu#8U))ch94uLW1pYW%;i_tV_>d!BTi5Ty zDI)@*J9{jpT`q#t>AP7<b1R;c9nC^ZH$mseP>OzF!cq!ei_$L?LEEF@ByoE!UTyx3 zhif#!uxS~qefAqIwSR!g(U<U}Y7&3?R14}Ik>z4t<!OkaJk9xOE*`Un?BH7BCfDDD zrmC~t*JWMk9hZbAL*m4-_nlmrWgphnkVtBU`FK9jjio)>M4K<4!a=2jNgF?7VDJw5 zmgB(B^LWln|11Uu;q*fg$elF3jt~7)aQ~I*<o94JDb(2Ds`GWov`)b5<rgt?s5!|N zU4Yh~nWBxams7u`Rv1+^lf~>^&Dx6|V!)yx>{?|_5^`lQ@<bXn#Pq_WIirYsSqxtu z_JG;1H@uYnR7~Lza~vgUNTwvpKU@uwU1R8+)qI-e&x2pLInMgMow;9q!>`>vANlKP z%ubviQ8}T)4iw$xwf&F4OZ8i9O`5m{VL~hPPgsK;zL}KXu@}yJI#Yh<cgV-hqQN(f zvBkmz8fWRCU3iN)H>bpnZV>N9Y;*ZblS(lF%~_b5BIF9jv(X2iz{K`;CUK&S4xRdg zox9zbYLX-v%vWVMb2sps!#;3I-`2qFM|t35>`8S-HG+1M!6gS}@#p&u=sz`_c#kmt zWr-FRsq6Cn5@dL{ndMyX6AM<bu>}>Ij9GK~L~2*o!AVXIp#1=_Z^;6R{G^Y!uIsbS zWg}=yp9GOf!d=krnvYwmMxa7R87?fULB%q4ZhE>rD%oAYH8Vn4{)l<_Y}7Z9|2T=+ zDk>qle*~BIRQ!>97c0P*ri3o%!tCCnja(vGTvwsxVV|)Q&w<*dQy3N?gD$U*aeT&E z{1UT)E6n%|E)T6i@ZJqxbnVCV%u&4OSaIArwE(YZ{l-9}^|<+ph_|##hXnz0>|#NG z_U_jie$HZ!{cOmF;5bv7<MJFK>l*y-_#s$5SX>vS=}O*x${=-{D=X3;LsSz)amI)F z{nK*jXZ{K@Hz?-Li?T@^$mFUQ+~jY1&4+B)-MBa)iKRVLXPTwCFzT=!H+<K3@bRBb zC!Eiq)k-bunR<)sALPV_I`^Qrq%?J0O<}bzy|8GiEZuSs;8*Tl$G=o*fGgE1%<*Iq zsE3^9J`71^-Q!PiM@O8)$M}_d=s1?S7kaXraRnqB;?3UL$BF%ojWDM!g10<iPF^z{ z`7LsTY211XcHD3;*mmsWY%=1ZzwJzDb~ucw8lT`spM0)6zfq7fEEXi(j>EX^B=%v} zquUM+bq-8oaca*vo%?z;dfZVK@@4~$4I6;Xvh(=H2U^%@=YU_npg7*|j!k{~u<=9B zxilDUq}&IpEaZeCi+@{yFT<{ab7TeIEK`NUzB<7j<7$+%w8e($LD-lW4BI@)_|P8~ zY=HAYCc5YX$5ZBk(T@haDQ!z1g!-UhctCW%LxmP5J%=r=7PKT!SJb%cH11sU27Ef> z$Yktu!IFRhT+cTn@Cw$asI|7>I98oKe6p6G_a>HqjxtO$dOPN<TL&8V#PRq(TiHyB zvpDtCR_<(qD<}0N0AkIWMIFQMqORD#pDldtn!4^hf8*FyRJ`{c@2;{Uw@LvhsjATf zD<_)kx15rBXBgveLZh$-de*qpix9-dRT_vg$GP~2g;4p2p`P3xh!@I|=eY|wq}xE$ zeI^tXCPtxX_9$@aeE`#TTCq=EL6~2-mVfs)4r^nV(bn40&~|13ZJzUklTy!OzKOD+ zAl-+CR`+LPC2mmAt7m9B(3InH$5Cg2EjhGrVWaZiL9V4a#^P9(urrSBm(*Zy1_0@} z-shw%b!e^4CN`2U<FAJe;acJ~Swm`;pyy`;f9{<hT^W%Hje`;Oe?)`B{f!X6wu*VT zN>WquQoJi)FBrbE5Z6ZaWy$Ue^j^xDy*iiy>WjzIv`f0oq4FxbV;hFMmRqoy(&OpP zRYf*vr35YUyysdw{VO>5%d>X*P&mn5V8d!Ru_fP&(YW#>cdWC7ZuB=N6>Sx!H}D*Q z^GvR3&Igz-lgCcYoC9{st?*(`G_HK+MX8sHP^HEj-`_}KnupX$WpW2-z87$I_D-yC zSU<`g;!4lOvU#k!!JQK7!tVDMdF4+=Aal(NKl<*+8_n%tKemPcZhad1j(Z2c?4Chw z>P>h)<1_ayZ5YMh&4-I8It7W7Uc(Qqb$rO{QXI^^g_KAgb|A3`em!+&PKPSl?~ok4 z|Lq>!Y<q{?jxtW0Rp8st!LEbnt^v9Dm#{({`~2B#Em$<SMPR+hhz^c<BN}1WiQO?r z@VVzGIC(jrJC`LOr&J|YFt`iy8;vP&-&?qJ{}{-d?uX=lR(RNbGCVVDLKEFoIuzRr z@g*8$seFf9@?M3yt<}M;HG;LJ6NKNkLHD=cq4Upv?46>8w^m%n)mn8lDXAWoU2fyk z4&H@f{wGoENwp}g)0~_##rs;jK6FDTfT?S*fKb`x;8xKLBg!XJp<bzIWr8C-@af=& z8q~p(R(qV?9fF%jZa|4IE9lgyNLW7a1G>x@0j-=TS%0|59iKh`E%v41@qIgJbKzIi zeYlI?Z0JC<z<}-9VlBwop~Q5z-$v0+Sq|<N;S@bbuA@r{9XM%lJiZR5A5zAsl|;Cw z1jlCp#<rglEFbd>EUqksJGT2EUal{NKays0`WKnMrZ1}9U&1@=-Xs!yZ{lpst#IXP zBf3?h4I|sF$oTU@pfy2IU?+tM77?gn*@dr`9ubX`_ytfZNrg{NfKKv6(2iP$2eoz! z`du4DiRxQ9e<P0l@eaY=;&}XPBWWfP`<R;$`V+5t<m1$aOK|+MJ$c+~$L52F`06W* zNH3&Z&>d$2_LHqZdj3XMU^1JG`dF}yHihi|JY~+|&m=yT3E1t8Pf#W{5eFUAWSV~d zRAH<}88c$=M0_PaGf!ay3xw>8&uMn@y%VSAe3~ziDZ^hE!??q4>9pmy78!pX2V?mb zu4hUmEILyT_9^*%<Us@emgQTR(OL_=`;?h4e+lb6?Qz3r31;nc22;*g!@_BO*u}j( zeVN?Nod}HPj+OpGv(*OF9X$%3<eh@5{aM&Cte8KfK7tNSZsn)AA7Wh|wzzmh1B^}d z!#n-&;*Bg@-ea?vbK0LwwmI>mm5N325Y5c@$<YY)9Rj<mERuMu#y+d-F+0mrjNffe z2bu%1>EUU}+h9Rcgtst$lsf6pJPU_alSmXLO|=T1Ok-X=+hNs)3CaDCJ}E%-c5l%9 zrb=}M4`E%$eV9F9DrRL$vJB&5JbPy_XJ{x5YtLCSjlLzYWd1oUT6rF9<{ib8lm})C z`}v5V3;eoAeZ^~WIX3K12Zj57d~?WRw%79-w@PAT?T_FTcw7A(j*Iu6+2uAEv*{t% znb8-e!avq!nt707;6yfYNdR<zF=eu)HsG~Xl5P$0r_%AkSo6b^DQhHC(SY%y5SM1m z^AqBd7Xi5XRK8&Ai+<!7yO~zlnlhF9PjUI5QJB*?kmOT=b{)BZTa9mWqh#OUYjMsb zvr&UK*B-(#ilH<yVKyaZord|*PqFyLX!gl}4*KhiCheWi`6HpGls9fR*EXOX*4-V) zN{9P`nt1-GDqY4D-tB`!NxLB4#f-XZ3($gTv;EJ5*b+@`R%zW0v%jUl1LqHDq<@b0 zzwH5Gg{L^tXj7WjxCOeh2ZAt*W9u$yvN6xBS?B~ia5$&QKF+_-na#Kf2l_kUZ89bc z8%M0#b)Iz<cu`MRF(xWGfv<uRo4@M^%wJgq11x;%w!Y|vhY{XPDn5gx3!g%*b~lXh z8OL7FUrVxz+OY3?H<gEpb88AAY<ox_Qiu}!$a|lPb4R8ycVQJAzS_v;b&Y3D2Rz`& z?p=IGpV5Mg*E0mKM~U$Lh@))jb0g~eNCdTUY4G#r9IjuxExWz&2p3<!h-SV}gC|!< z)5-Q)c(nKtf2G&K#njo172#SeZ@I$fDZ8+Sg#*}wGJjMyRwDI&U+{3+Wl&BU#Qe;* zU}|g@W?QB*S<5u8&+M(7?<^g5^nw+tJT_oQQjXALy=fFImyRw|S8>}Gy}$~`+jJ;n zH+U~MXV=5e^Qwj$$-_RD`Ks!%F(Ynql~TUk_;p#_m-rzVKJFxL-2Qw-jH(2k`k>49 zPM=A`P7GouDUZ;8wLUyPCq)jbM$G-GDlG_iVeR>uxbct?1>8!*8)N*S4-6yU2^FGa z5u@1}i6pQwb79Z(8RT4Ahg-ha;=}L$?6gNUjT_|xvI7;!cFk}Sm8FT-tcj#L$(nM9 zd(*xeb@Ev~pHE3?f~b0N4efw~Fx69BQxLnAN-NSSx6+yxYs-V?$!0idy?`2aY$2!G z*<gMkg=?&wSo>yxB8&d<5m%loWw*OJAyLAevDMOSh?G7nf8s-_*Nd5!fe~|*S_f>c zv)EsFhu7><*@Vo2IP|Lum0ODIG)DUJCrTn|$3_9>Zoi4Vbun%Y83B)$KIH3;+{W#O z7T~>%V=`*JxZvY${N9;}3qnpq(Z_M@@!%i0#4aCCx8{QF=LfuR$OlZ=RtxDTOhoV6 zBglWhxW4ek8h)>mJvH9SfVZ48jmvElw957e{Vr2Bjr8e9surbRV=U&u7C6*UimC4l zx#KytsC!?R)=XVbD;{O?cP3sInD#HjLDRO-GWjHOyl9Sp+{&T0LdfEFi=UrECIm=T z<NYfSx$cOCXq;8dFEtv)pIkVW6|xm@_sVBHC9xHKrkl{MKI7=jr;NHvB|8$veSv*W z+i+@SH|}3OgsJsJa;gg|aBiLq_oY6OYr2`l2QS%*ZK@|=y~+W>o|^aIKOum7w(T_R z*bvQLUQncGZ}(EZjT!xXa-GY6FUO@{)nZ%P%lV%7lTd$(CJX;)O`UdjFikcCZ-uUc z22mCaHG9Tcm3YvbiV9F*d(ot6G?`BtNA}k~;PUWV+~qNxI*bp4^lc-$Z4*YEZ8+_k zZ9@x`yy(%hW9TTZ%MG2R$`pRY&<!gQ{&ri#)_hc8wc|Wdb(opJNn<r?J}ja=owjIP zU5c8KE%>P-fP`C&;jo4tEjT+uFzWFZuqq!;>phNgcXS)-t{$62WrdPtamSu~K5b(I z_ngN8FMQd{dk!?E^Aq|SX7W$!qxtcM&U7i%fw`EB71x3(a#y2r;KJ^3`dx4o#+xUh z<KZH<^our}mgqwgn#S}=qCfv4|1rj;CF4g|6_S!p;SW|$qru`Fms(vp#AQ6io;$X5 z|MNIj-KhcVM-E|p-83jK*T?T)doeUZo=r&)<{|^eK*!w#{<!cge7!N8o(w1v1#axi z+&gVptKEKDud;)g-l&F$K3{N9ND*FJ;KSET4kSm#!!&EJKTO}co=@;`5`?@u2_q-; z$K%a+;qkF0{DYb8g6zd9RJE@K$1i>ZdxtNeUm3UXuu3<4Oxr|>N$xNx`5vCU_Xv)y ztH*&`&qM9}4G=9Wq}GW#wD0^uoMiq5&VTU5B|D!B(i+6?Oi3p&6UC9xa5$5DW|WTa zbT08N>-D%)FE_T&&6oCv8&O5zeGE{mWShpMVOg*_Dc^Cx6RA>6XZ${Dy=uZc>_3Xd z`(}wGPi=<~A5F3~kYk?5q}ZfyZfx=}C3agZ+n=Z~Oj&#ha=Kla+|dx0eI=beRymA+ z-Yr5~vEQL(s>Sv<&tn2Bb5<6kNax&}`R^G=I7?qi<cCz^Yr#NPaaI*-?xwOy?&)m4 z|4$sY$R9eb)hI9UFy8R*gv?^ZS5p?&t*|mB3Her>u+B}iD(({8@zms6@7S}Q!PV3| zd?x9Co5G}3Gtsd4HT*fQfF&Vwx&EdC?(m>C_?A%3{dv6uR&K~4iPA#W@8uuY&-=x( zqXDTXH!GWj3qD|ZnhsNZD$N>RrSa>3#Bld@wo$>gkytM|2tIFqkDVTdxc#0uf39CC zusHb}p0@;W&J&~AiEGJx*!nr7vF9yRsG8yRW0xSQTApkt+Os7yjnV8oV(IsjAmda6 z9nDG@(oYull@vqtiA$hy=s9E^o+x^HHwTB??nimuRvdTl2$&DK3NGvWP}>F#GF#(| z3o@!;_189TeuoBmELaXIpVVlXuGsc_X5hvh1Nf1yIe7EbBnsc~1R9O1ad7NJ(i=RB zqAO>yk!qc2wMmv5FQ#J8y(EH!*D%TSEXt1gg>_dIq3L!vK40yPUDG3B#Wf|C|3RAG ze#-*Elt`*o&LGd?8<;aU9cP8kXKtDT>{T8}%SVmlEML#(V)~!vt)I9NRISJGC~K$? z=d`5UlTaXUO7k|yk>%`zT;1?o)<O=<V90h-{UgUtjZ|Q3UGw>qQ|rKBhX5)SPl0#x zL9SxMXx6uSF$+x=*MDim(XN}~T*xb)>&eLE``d=Wyq<EZOng<_csow8VfiJr*n1I2 zzcymZaXdsgTi~)!0(RugMQZbtCKWR^(rfo()3RT2L4_J1>$wVl4c>>w;@I}0+cqrZ z-Eh2rgLjqwu^)#%Yy=I@#~_(6gc(2FnEGgI{`9*$aC=o2{xor9CX?)OXi+M7wGZW5 zipue`?OW{gOO2K6Oos~rCWMcYF!v6j<&SRs6FrF4?HI%Aq9tg+IeT<>RAd_0O<|5i zC0&+20S%s}+*%AG?XsJ2c7ZZHU1E$IyEf4#l?*}m{VCKqdoWG#c7ZSRDzHQBqom14 z(EDGC7+^4kJzcAZ@>3OIg7`fXpJy4Uk~WRbZc4!_^QG+Y;hps5N+pD>sDhirOzGlK zIdWb#o&4>NaT_M?LXnoO>$aMu@HNzz+*WyF;9e`%*xSQhw(R6Dk8x!Oz7@jHl|NB; z@C*31`!|N}S7+fSA>h~*0MWk<$@_aIes;5kDd)<ts1az(_G0dd<y>Yl-v~o1>Y#UB z5>qmW6>YIetE(B4OGOENXtaU_n7;F7IU|cif6nKzoIc3kUvwJ=Pg@6jx9?~E2d8mY zccsFM&pEs*mqIUo$q9zbY-8G=bHK7l#4oY8!x*1qR8sR5&BPCD=sQg|^XEladgUSC z8>dL62cB}5{8F%ErM7rZuZE}MczfUX@t9y^#VqxfP|i_z=qex1tawS+6{j~s#pnY} zm}bwk7ISFYbOlpKZG;D7MzHD$op7~BTd?Jd2Ib9BrzT}Bd}pFa%QKir>4-c92kYX# zz2Us-t6aEq;5gn+mgS#LP@=q6HM&}w3VnCjLZVVLKC`g{i6SF%7(}9VrA}=4<$aX3 z=?=g6+%C|y{{~lc2V?R37cfA&8^eAqVk)1evUexF$h_qS?uu4o0fki<)f9m1q_)HU z=k9!_e*?z(D`QgAYBv0871}D8ut$DB;d;$M=w5Jv?6@p2RP2F|&MNGo<SVqXpA63R zp_m~tNEDl~3UAI8zXMUC!kJkeU;{(4;QodQ;ypqzm)&O(+a0L|r-z=SUk~@<qrROu z%CsNG8jhyb>#a#@$}4a-w?hrB`OMu#iG^-+po+CtR3Ui{f|94Os#+P8zu$wG7aZiy z?YqrIRa}M6g*A{mK#z`%H0KuPONvO4in@u(EW3C&jZ6K;T~Zr_mz%{t>W5r1bI26G z^SP5wYxHH>xw(A0g*96-@+jDb&w@Rgl5AJ;1g1S&4(`1JZmZ=^7`D0sg=K@7Mz$?W zC_RR&GCEkt?7iHN{quRvM}}<Kfo|?XUu9sv{n@G4JE+pE4!rrv7(Z?_d>RyhiVHiq z&ibRcbCVj$)JxNjz_WZ(#eDc2I+G&S6oK=i=}h(45b_#!0zS_4q$uwgHWTytv#qnx zX30!e^6dq`>su$7WJHkG(_T1X*Na*=E$K}{Ap7}dIGudE8w_vP;4kk?diQNT9C$n$ z@3`1A!%sIwKbGgQ0r*{1BV5A<xb<P0`iV?NDUX(qILJA;hT+MV?qC$?#1i!ya8yzq z+Rs_Y<|nAa)iI&)rbL#>uiA*$U&xSwWC}T}b-_4sjZH%24hqTa!UYRj(1peE2jWvW zHtq>*Hrzy$o(*8;uhL;^A8!`1s~G&!*3(9*Q)vD|T!XDDq`Q&x*}C*+{9#Q8Og3nT z1VwX}RHFy^b~?D_hzweZ93XdT8XFb+04J}U!K6=K;Cki7{-i*P)5=T5`>X4i#pO?E znUe-j2L3;!orhnIf7r*HG^Cx;qKuRzlIq;om$V3xLS__MMP_8RG&D(zlG0FQH0j*e zB_l2TvMNMIBzr`Z^4!nA@I3#(Ij`4!UiZ1i=ks}ABl*Y5gW2BQ^{`o4@Xq&h+?1=g zV8<2%`ZjG8GkO-!SKP}&&y}g@SX0Fv{GbQt2AI;D2S#k@vyGhntQc^2mqP{3)y(nD za2%>w2Zc$IOn2=I&e7=v#0;&5)xT~r_qq-ktR;AGE_}36YnehRzcShJ4GPe4cs~<~ z`nfCNV^Jc%O+09jkIn0(*=+HQ^~`_GBW&DY#g;991Hn6wV8VTK>}ptxUe*hsxbY6O z95#THQ?%GXt{Dbx)y7-zd!aik0R#VD<E&ik!CCztuhWr%rcMj7Lg-P3`bp5eP#0R( z$m5Oj<ycT+Q#q<{6FZ&vpi<kc9gK{2g84cpGHI9&jvGh8{M2sl)-O+f#^Oj|^B+UC zYBcC6DADD}2y)BuqUl;w5o7Wp(eMOq`8J3>uG>bE+S{RZb2pbZ?*dB?)1vcd6Sz&! ze!!7Ero3VB0ABBr4o*ujqFZw|@Ny4!F{}U9)54!pB;_1POa1(?IJ^rw5&$D~b<tqv z5iZV18isjHCWU=HaNubU7@fCddmq;T4IF^WQ}UU=?sUF-?NM4)9fx<8J96$zFT?mR zg>YotA6Pp&4*WDr@k4+!zrFu78hzZ4>#xp%=0_?_*+GQ1QRXQ3ARTw?v4xHK1tc@t z04?vgihDv5arM?U%qL%k<vtmJkI9Zn_0~gGxeK`$^}?qb8}{J03snz}rcd)7a7Jhi zs?Yofb^{97sK<@m*)L+WuuvjZk3+1-=OFuBZ_IYsr82pR1ht3tdGWbr?DvzIH2C*Z z*q>qyLwlCdgJ%bzWUw`R_d13>P}s_A?5^V9J+WZJhsTner!9Ybpf>G|xeG_Go&jxf zG}|b13$(uZRDLX(&VPK>!P#2M@c-*S`1-Kh=>OzDu=xM`4^}kJ!28<&@!U=on(;oA zHXXQ!r(0B5=uRnE^e>k*Pl+qd^{nx&bcZ<f&LG-&d<xk<lB78m1E}0Yi3%Ho@z;!J zux?2eDv!*?O&9%Gm`uFDA@BqD;y<DU?Zp)Bor)uNg^)0&)8KG(I(|u($=xXD&&Ju& zCSxaxyL5nxEuM?+KhY%drYmgPnNi%sfnD61<WqS5CdW23NYJ6_`*3%%Eo&>>1M%vo zQEHSfMdr<gK!-RuX*0O;ng3aw?K6p&j+o4DRY=gjq*74neGZ{^*{r8BnTlp~K+~rQ zY-N88?pha%FUPlF_2hrNTc0BrIA}hG-YcaI<Fkamp_TisS}W|_zroGE09f*MC{0kZ zWF3b~@p$(wKK7_I8(gB#j(%=|U3=}w;Qd(KC}jjH4Ley`pgM?;Z-KAYsW@dKqa=l= zpzA+RWO3~-zHR8mE%GIB?#^+N%q&3DC6l-_kwaMV-Z{+q`AoWdr<|*I(MGvaD^bJF z@7SC@olBcrgC9#LW5|kmj4gGBq}3L@r*#r{VYxoDS~rj&^&&=U9|5=b;dpzHFKJoi z@Uxyyz|CqSnX^G6HadmDRkaDA|LQuGwkne-T;LlRd$BG9ed^Bq!^^kMfZonR7^nLU zzhyX5NZ=55{r!0SJ3JTK`%kiGjt6b-IzPu79~Q8Gxi#WmnVB4piG`X7b8HBG0I@e} zaZ~CjJYBFE!ftB9!TASZ>4)FkN{JJ=vuQG&{gD7V6OC{fH<|YDzsc`deU;xdaxG52 zd<UAG7s1B(r~GR_8&<hXfyFe%(Wo*($0(17CZ*wccT7CaUhe`+VJMw>A{IX_I)x6+ z_F!t5P0>GgQ{jz|+@__Ph)3#iOGGDrJn6>@|DEDL&uYaE_dD>fVh(K!XvG%WK*7H_ zn(mbPK-=w~yv(q-oJY@4cH8+pAK2Lhi!<-T?m0v7_n9{~x<Vby*-(cE?%Gie=D@ox zr-l8_cur;Td0s&&n=S4x;_a05X!o71OmtM5CZ{Vwa?%v>DZOl5wksYMY?Ma@$t>(X zKOUEPN5k~24(?h_7rgXRge4UYY-`j7l$&nB!qhd%K_!A7?r*|lI<hRb<TOG}BhE}N z6Ae8&kW$+1QK3B$OHOMDx@8DT9@@n8`jfc-W<L_CzRAVt_dBt6p)}MzQJ~mZ!R!Cw zG%R_Wgn7AHEY4My>D@2Ku>Ik5bcYK|7fpopvr=sGp%}dG`3kJ$lPT-&QyQ}(kQ|qu zhdBdc+04(XY{SY8{DQDzZkN;)wr%2RZi~o?<=Iq<R8}0oS&5r*z07CymDxeo^F2h{ ze!qct50HDi{XE?M{6X}}d@!5hugmPuA7FPzK1Z*~GpVkBD4YLEip<ouqvnPK^tp8! zyIwpTP3lx()aw<bZX_+*`GX-Z<WZXECE^|K9A3U-L@Hf3QCaRH$~1Xme_#&?Rv&Kr z#5I_ll8+6^>HJT79zUiccXdt-S$!PK8Xk9Im!3a0=NDl1i4cJqP>&7{x|kThAC6wl zB&XIAE=BO(TrW1j4gNm#@<cIv+!4W>jlPBBwiLmU6`J^b&v2U8{S;HioCITDkNGdK z;N-p1vAw1UjZ=?vv-5|rF`?%mlluW}kv3H3JDqwRJyE|p33aE8W5JObX!R?at30v_ zv_9oR>|iCBEK-C09$!%*EE!hy#<FFNj#M;#3QHXlfUdP}Y~`yPVk(<JwLS0QdS?io zi7OCQ-K@avlXLh&)9<)5J`IiY6mWL+JYE@Q*(6_dfTM$-iCuEbAX8u>*a{4e!N*Jh zoYj~@hYM*JUxGs~)^f(~+1#u#h44D77E05PiMDps!Mc6gBwICy?|wI!>YazMiA%2V z-Jr#i`X;c9eU8j3@}t;j-Zj>%e+2!l%i*Na5x#T!S=`k129#!hLQ(2*8han%#+1{n zpm7GQ8SX@L-Bmy_QIg4W(xho?%a+WYimQF{;9Zs;YYA{=7lJ*pc8xKG%=hCzcvs-} z$rCAVT_w9$ybcaLkbuqn5BLyu6g5*vqrO;?vYS^j@#<~l>pu@?8tlSFK{IGe=ML2W zOYC@i5)N3fkyFbX#vd@9g_~x*uYB0_02*z*X-Pv9tW4NICvWM&$-TGG{me1(P8y8Q zP3CcKoiFh7!Lz_d3%kDJA+S;-gs-mKg32*%T)WFr3N<XiukS-(rJ(<JCoIA*J<_D+ zHjHd^hOs&0USaJVXZrHrATECI0NV35l!|CEUbU;nyE>1+HdBd>I{ch_m_3Yjx!=W- z^XK`c{d%bNESw^fLivDUkDz+Z7S?i4lIhB6a=m*k$;h%Ff44WH?I3sD5v0l97bwtq zePar(F=lcz=72-=Ey(B_$d0OCBaK_ZOnY8CH(|Oo>ZHzRr);KEjqMLC-D|~k>tkV< zA5Tkj<_f*EEsP3PVlL*txdFusVEZ#WO8BKjSx4t0hDGC$vEHy$&6j08QKkPzT;!H$ z<cZFn6R~lp4p8|g2lR0&$Duw)Aooi)`|?bOw950L@u4(X9UF?TQVwE%u|MqHGKq~f zF(#{tPq9Bg9*u{+fd`>++|Bf0O!}jPM<Wot<_r?a+=v&RUp9VnmE&8xlR3>h{^0XQ zf_a8r#B9q<zDXt?GPTZ=QkONqYmp>rKQIRCt>LJY9mB6KKMjh1n(@i`V6HA*5`NGd z*kRwtg&56bAGfvha}?##U{4Snf2hk6R?Oi1MoO^}${ZUsHX6so30##Dc`&~^i9}zH zv#jv*Y==i2+@5s`(=7k-F<Ob>{yu;<HobyBRi^C3=mHjBb020L6r=c=Bt-2=VQ-Jl zg>UO;!)@mrC@bBKw%IqKXQ3rrpOi#S+r8j>eLwumEJVxYdGMd<dLeUFjZOok#R@03 zqS`baUOvl<lDtPz&MO&wSN9v8_dBp?7k{b<-$=h&<tfMdEoKzo<N7Qm>A$t_F~Yf- z>$)yWpD(pw<nvDc?Lrr(G{grognZB_#TY)PcPqMX52kH1&G6l&225L@4+|AIs&F@^ zhHY)wf6a?>gJM`^i!8}a%O^!wE9TWK=w7;u=t)2?87-SatCF+@=2Rv6neAc{(^pad z5?$`S#Y*yAaE$Xanu>Nd>v65L44y9x0r}u{(6F{0HLM<jh0X{TSFT9X<H9j8b{t(= zTrR3VaSF}OOaS}QTI3!-oJu~;Kws^<*pPGxF3gW&5d(r));M1b_%fW&ThfEe)K%EG z@|Enee=1h*%Yyo&x%})1MUwcmlBT}8kKQ)nq<eofy2<I1(u8?<`Rzrvd9fYrkC%bC zeXZbqJdtbMtwD7Xz1%8=!5}jL@ze=9`aZ4++iEwHOLri<7uAk~WEs+Z!r7U<;{1yD zHW@P}^Gnk!VcA<t8WAqb2JiZh*G=<dR=(!k3O`#>dFyy?mEkT{<iCv%{qqoSyO;90 zNkMo(Dh0Q%Ed+m)p{%HNJO-~lNuFDaVN?8CnDfbpDU5FuuQ^r-C3Am5=0*{F`TI7n zGfkCg&Go{FH(_l3kR{}IvmP(?x^mUW?^fQ~9t|>EY^nV~9yDp(gI0}XHum&cp`LkQ zM*VL%J9Uw;FDS);NM4j=b&p@(^B)~FmSp_jAQt#7^~wR$TyFN)L&%NPVj;r}A@iIA zJGkK({!I*m-;T{V?c!uUZAnq(m}n!kU)&29emd}h9_y%YYaLwu_#ISJjzMs075BF* z57)_jhZ~n>vI(2Mh*l?EM8EYDQO)}->`61`ozoUEKa&wOs9uAeJUIYE+?KJr>P&3A z;wB1mJOl2BqFL$Xa2B@VKa`d_K`~xDUUL0a$>lwSBfqq0<6n8!q`V)R4NNgWb{;#r zHU=LY{0;$_MGFkwD97Y2_UTQbyx8Yp;Zgy&mi^-0O5&((O*3SquEDzR>g<WZQg+$! z9o9H;R8}h%vbRdix~T`X^Mk=1_fg-*0Q|i^g1-J2&bPnRV{+?DMFm|saBW&X{1^O+ zuX?UbiuJnq^|}nH4~WJ9O*g`eK78@B+c@q>98A6blFwM7MEB!VSoB5>++sF^^&K4z zDe2QedXgH2R4wF8KQ6%!LT0|f*OL7a`l`hz_33z`DpPqWMKaTKxpxkGVPyUZ$dFfM z+r2A6?N|(GCVmq7w+VExP>t=f%;4ge-h-;xN>CMiN6!|kGSS#tens?YF#Dne9Y031 zf{$z1sI)2!8lVbiZXe>89N3CECAaw6-;XhBhys;;OXDX_$pF`I8+Ojok9%b|lFYy6 z;K3isOe(|x{XQ&#t;#ZNp=bzszD?z9$C7A5#zN|17I-dO0&euI!pVDU*!ZvO*w-)> z!YDVmq?yO79j-tL311fcS<LNpjS?xISz$fKZZ7cCKl47uf5G~kkW>36jSH=Qa{kwZ zIx%@C4t?$gu8qP>G5jIOd^Dx-tPgN?(m(Lk*aWB3N@0*}5R35ph)T}8srK=2SR5?J z+IRc`qxfa~@^?Gg({uko``b@=yn3-P_nb%9HRo~OaY@!*@&O)6`EvS>MYQ8?J&qLo z>fUVwS;hvARnB!pM`st5Rx>1DaDgWyw&LmjGdRD|kXqalaPq<`-Y{b+?eI5aQ7cE& zHbYakFz=^m*x~i$H^812B}^pWvYBkAojJ87CPK-$YHoDsF6dX$gN=`Cao^r@n7Z~U z3=w?Vz2}^%ENT@yB6*JunOq2`%szo*)>6#-vITbMg!AY3j$_VCUW(V|3x3B23)U8E z$L5}V4vyzf@dL;Hfgi&D{O0DjxT~)YZ|t*z3vuP5C715w@w)xsToKDAq|c;?2a~ZO z_%&R=v4yOk_|m}zP5f5Dv$KY!v9A|(*vPXDczUxAgXT4GAUqu#eFjcD`+gp`Nj(yG zRIFqr-7DzjjAU-#g;*{o_C4I#o+UnNy%jFqPvW2bR-w%57fc^s;l)X*oO9|3lJY5F z8y`2Iyqi$^KMI_W=zc8Y;=p;Cpfz~Jz!~|mtoqR`7RC<IO1(LJZ`x^?nDQ7`9xBHr zsg>~hVJ0ds6x<q1TSZ;p_lxG+#PNQbE-a&^oJlL_u(S6QD5^OUt>%Q&PkDxqFQl>q z{5Afm>o_uXs>HsAt!zT@H7@%5A+9ZIKW?3zj<Kzgutiw`H*^x1%}&CLf(|f3N}rW% zx5Qs}JlUR*0c@BkQ>^?$4>kzih24A<$rL-Vx<~Wq+44g$>$^98E(@iU3SH);vlr%9 zjb@<|2jO7cOMIN1jW6@PLATHpwoUTJ0p6uxc>Xn*I_GmD=FK*_s52|`{g|on*v9vG zF!LEH{3l#dq3xCf*v%bRq3CraZP&NN1trR?M`IEk?_a_01Z<(NsqR=fdLV26@*n#w zDMw8ue(cG#J7{PS!{#cQau%EvI{`<ztjt67;`>(mz&i2uTuI706pt6wp5v?F_00QY zFe}@#8a_^RU{XG7=$k<_w1?itX*rQN{E#%eZZ?NPC#JJeGKZmM?p3^W=RbIn6iMAn ze&M+{ft2Dg1-_-P#v_N6g=c?>i|#p&aY7DuUP>tSaZ9LM>jjr4%3vC@GEAg4l!zb0 z5?97zo>nMJv)7_JA1O5Dyl|!CQO+VR7H9T!a~fvr&^9myE#qo2_s<H_`{zjc--F=f z92s^va3sc+sWO=XvuQxX5!QV(4lA$*^|lOPj|RWSph*+ih+TVWbM_Z_aLb5^M<l_q z7mhYVwM5L{GM2ZOS_A5{{Fy>XCJe3k&8g42$ju%ehV|5iiT;<l)@Vn*B62e<iIZk+ z*{URAb`*L>@1h=ubNJ=idbEGNl5hF5PUt@~_{(S1SYFgA)aWW@LNkB{|5@-<$JpoN z1+aP7O)Tx-h+CwE4Be+FYTTNHm;5fE>9Y{-<FMOI_4Hx-Q?G&})7|0Xrx4s0VFUv_ zoyc)tC;vUpi+=TcV$6eJk`4X^E#3<#)ASs-=fV~UK9T~%!wslF{RwZr&Jo-%m(ap% z7fEV|5xrTh!@L_z`I`zZ>|aF`CqF}s`wo1BLp8Txl-*H`%6kv}27x5HoQ8J{+hF9J zPHd_ZF)80s^y9S^C3#h{Em>;p;4~TZ*HL6gV{1{aHv(>@mcZ~8iZr{=3I<CDfvSBM z+!HbhkCeg%=36{uq_>G{KX&nDbGNd~x8iYi=5yZl>>`qR?@KCUuCU?FS}cJV*h$O( zaC=)dSxcQJd;DV<lS!8*+YSdPs4C%o{Y1jE*u$Dt)?&fJDX@Uave%2unU<O&<tpC5 zO{Kb2_E(-Zr{3Utr3Z@gT4h*Nq7B=a^AuLxCwl88hGMB6NS-u?N!i3>lAjMBIVcWu z3Z!ApsU}!ZtH1`<E0V3oT#?FG`Kw{KY^d~LCODktSW%x0+kILG3+K=0wrE)4gX|}8 zHGUtvb0U@dTrOe}2gLjp`5EM%vYONu29U|O$;f3(;_e(Ps?_V}|7=!aJA5+fO5Pa; za+)NaFoIWkSO|`$M=^Q$E&k!HY}jMBoYKz5(cA(@PPF4a&PzWf<Vc2-->o7{E;-N7 z?o6Z?JvS+L!%loIlZi4hYBXTqVNg`(q0QHv_UlE{1V1A-S}-9l{Jnq~UAZat)tpJc zTbHm8_ER~p;gZb%;6Tj$8jiARpZK54qu{K6B2;PSu{XhmP<zmp6)#%PRBG2Tv1<+1 zR;95sDwU#)5;b&f?BVU-8sU^fZnUW?l&NZ`!Sd#J7<Azjj=ZlU9=p?u`77_D4Oe<_ zSbM#om8i3q=7+f8$7TGDFBe$qsWEit$sI)V6w()d!zt@c+0_+q@LaVDw?A$gI4*62 zTQxP%drc0XG(<qo)iTr``h|-ec^35gD!J9&uV8WS8;~)MfgNjGY!<wG3Wpusp{2`< zmgTu~72AvXZ%-J7w=G7?z%EQpyb2?2SEF)}BP<H}20^pd(43MywqT<J3x9qa9Tuc> zKQG^g8^JL!WRe>Dt)IzeZ#3dJ&YVCWB7#t6%t_XFX#<_w8_tcqa{{;gdyKyhrqH|* z8g%)fJ@s}3gMrjTD6}%7v0rSct)~UOhCD~5x$gAxrZk)4-H0ZylaSNT0-NAX*c=%~ z(Z&wAE_n>QuCoP;Eh|KZr#kr$K||>NOa&Zh8YV7lWh`vVD{F^=Ygx)bUAEC)nPv=_ zBeL_^1a<>+!R*0t*83nH!h#df$w-oz)=Y3LSxD8b#?<@699P)f#kKpD*p;bm=$~)L zZgmy&FUbqCLi*7CaUjGDGAHY*Pzq$Hu~Fh7CtYv_i=;An6*!FJ4ySRQ2V_`uwI(}X zR1LRc4bfrhMpnp4@cqi);NPb#di!!awCCKUn7}yheB%Z<-!y?X6eqJh=bvyk%#r%9 ziRofH5C2Au#N2c4<Xe?YP2Ia8_t!EmV$N!E8-I$*Pvpa!Bd<6=fitu9ay9M|d^KBL zT5x`58m2EkOaUJ9u;Aw{Xk4a4m!}>^1N$djrG6eSb><Z}<Wv~;N>pCy2ph#3?{0#v zdRsxSM2!Uvm7&sa+eKqn&7v*ux1zFyo9Lx%0+>B3hNVu&Xr;akGY?i~&oyqahoOhW z8kcoh|IMK&J1S4)cVj9{ycUdJy;IoW{aP$gR;Upfg-qtV4hyvi!h%I^Tzk<3CSQAm z%vL3G*ET3JGspF8OuWEN+#bf~pUFlNpM+P@!`Ub;p@*7pN9vx88Jt#PC;N)fr|zxz zqQv=1pFl1ATsoFi_Qhaz{y%)Dm&`Y{-@{u02+9&85S2#L+ddWAd1VuP^e{#HG)1<8 zJYd77D4OuLlVj7~!NOhxm}0-1KXLjtpOhTOo``qh*|1GyD765dz8@ssmlZ{igH)kr ziyaod^JUl0now%^5L#_@6VEkWV;AEMnR0_Zje4lVYz9@pj8$G3D3{KQh4;p-><8FP zDCKTuj$~IjN2c%9Ec*M(n{z#P0rs3rgoKJ1M#4OzEq54?P7lHRePfYUC%_)_Q0~2d z0eBichvWV(WIeMHru2leemh0BcJz82b9NXV%XrAOTaKWCIvdGBTY+=URRcqT19jW! z1g`BCbiLWK^rU|_NClX%ELAITFY@8i^?Wh+t`fCI#o)qm1F5HDIW1Z8Rj8j~c;xp@ zIJsmVYioZC_pc6MXKUg>q`M6k%LuHfiN(-VevEeRNoHOra_P9Bm!Ft^4W{+&h71=V zxxIgFluS5MPjut;OU9FmNdit-I-iMjdbwW;T3~<kI7G-r!6UIYd#F4I#<*^R&BtEB zn&9ntV7C>?8oMwWpMpbg-oV{y<LTp-Q*i#KJ{exN5?7bkaV@bntmIKD7qzSdj|!@F z-Pi9h_=pzW`li6DdhIwxfz{k3eVJm)h5u#G#c;MK0xk`n54mGcQ^lDKGQB>C`TT1W zVg5WO5s^serUpW8r!84JShJ7wgc>=d495H1gdDRl?zb0lUt^D=Oy4y4V{c3*O9N2) zVKr^;x8r9%Du7ov7NAPLGM!m1XfOFbRQ%sn?(x}eFl)0MbuC}OTRu-^^TvepkIa9; z;*{~&^46H$)zxNC8i(RM!*t9UGKkBY8;DW57QCLpY4{fH$ZCYXSUp*ZvLA-y^#)%W z^L_zZ<P7Ft%)N-ELx<9MN`X_i-=nO`OvWt_r^hE#s3a>z*y-!C{Nqk!I<$k!mASxX z3G=wO%PQLM+Ja+--oxqP!@+dIDnUCESUW<-Bs1}e$b3WsjyrXSk1`Vi3VA<qE!nW% z|E96SyIi=}76Z6Vs_JyBupcZMa$t4WNN$nF8f;Eo1(L5z1Z^$)%IUG`f}LR_yfU7Q zwsjFuSF6q%ZF|IrI$q<YtVrzn9EsOoPoSAQ)?)OEO1|!$F7q~;%Dj5yX!N)1cxmJ| ztp8WdJs6<I99lBKa{3%ntMEj*cw^#LX|aXZ8qi|-23k>U!;Vbc!y2!|BfVuHFGlu! zhbNhD`3mh-aUAb$PIf<=!8^rD96sU-*lU&XTi(|QY^cX@)B8BOTxx;2_9Ixf`$^E3 zlwgv54e;{t8Ls`ch%P)dV6*S$gTxpacF^OpX!FzSxLLlFQ}#Rqj(!SkU0@wopgIf; z<T~MbS17NMtwerNKk)dPOdMJl#JedvK<!3P%1xcZx2z~bMS+30q-z;#E_u!uY}-Vh zP7iQFt~Ff`8bnVM=d=4$<yrKL41v!&2Yv64prwnpFz+W1L_e?E&~~>sd|t(2Dy;p+ zckqv(e6uGi)jh=(Wk)$>kY}?@+d%43JZ)Mk?5l1J#EQmHe)CRkdVXUB`;i#~!`~~y zyEn&h%Fa}LSS*dFijR<FY&01&DXfSnWiJP`iH#Jr=#JZ5-V>LCOJf8YdWA#tg@xF) z{1Y5sID?(vYeM^d4#8~mHq=ySqE{0iL6<=}6gft*6`L0_t6u|H`(`(`w|F8=I;?@U zN6JOMo6f?d`^7^3Z8dk=E}hNTai4oOssZundelQpY6|n97w3nt6T-W_&2%kOdf185 z1NyB4qB`)yy_;B~y#RZUgtDF$W_ZNt5}E6k^DAv{iVRjJ;PAa+aJC_VO;S)|>*RzU zf-mCNeEW<2CnQ*Rb~e0E?}Bx0HGKKPVw{~;f?<c0QMTIyCXO6QS<WkQSn46%5wis5 zl})0ng9~Bc29By`9A=tTH(|WnG5&1$2i~aTFE^;|DV|H5N?p%`!E8+l4i1fhMu8ui z7`%!id+ga`Z8h$;>kYUtF_W{bDCImIRN--i7gl@~u~#J*@#~QmOwE4)k^7@*qn0)} zk4^;FmCv||KZ-?T>=lJtF&^EGCD>BI_tx?(*?Na>5}LoUWS@5$iX}RSu+3@-#1C6c z-Z}1IfB6P893{atuk=BeTRC2d?dKZ=|JUN_*WnqAViO`}<B4NxZ0wLoob4UTFW4Q5 zQ<HvKPa2oVU!3)p1_r#ww~?pCQriVz$bt-3HOPWVzB1;w-r5iInr4wgy%_Cw-vg-G z!M{uSPvF|0hn7-Zs5+U2pQlVAgXWRIPA9SaH>&KIPc$vda>FOebJ_p;5BglM-1tBF z52pP8{Ri8AMA7PfJ4t4#8e4KvAQ1@sYq{2m{P=Z2U?o|Ew(W}4D!EK-aKxCsNWFr3 z%PrZdWIvj4&xO5f9L%0|6@!OeAZ>r9i>Hk_Ha>R*eXxk6hpRcVD0+rJmmfxLt4V~Z z@p$p-Fm~R|g1wE*#E|RBc+J|A{jSRs{kheT^`Dg}aQqIIX(SJ0GribgPf0Y_Zs1mo zaN;8Ntb!#C##9xzh8;8c#%~w2RL5iW;=>-{{8{A=XxuwU;J8KMgy+lf13jp`crgTD z{~pG+XRAP3-CyyHZ69F%Sq188Fk>zM<>J2orgDiV4ah)dF3I-GU_fX&mr!a#SwC#3 z|GtDx--0&G+j5Ov=!~K1D^Ib_LEG5+#R};E<S-e%zKOH{U8dre88m-QEIXU^9sIV> zVW(P!{e46=R9H&T;+M-Hd1)M0?T&@96C80!Mi{t8II_T?hoW)`DLNvW0lPlE!8!Np z;F7#7>e?6L8Ive{7O+|L;bJ(pjITqppEtxSFG<l(z5o)}hQsFL6Y2iE3i>lsOidI2 z;?plDxjTN*qW`TAi-ykQp18!~?)lnGFL@v)OfsR5=`%5Wst%M09`|VPX3#KDrVzhi z@#p6!#b359Awy*?vdPxMqjzLzYk576($g2B?J;b;rWuiRy!egXd2nu>PjZ7c@V2KF zc&Xm&T-%lR5I$an@m?`haZr_fC%oYkSInmEztT|uq5*!{>I5BpIQaA@z~|$a(SOz( z3_f`m+>IBJ^9WUp-fqRGuRY7h6zVXGwhcCJogr9VoWm|E%TR7hBE0NQgGp}{$g1ZM zpRzm&_AL$LAJv^f*Gn;6o?JD^COYFtE{J(}4`eRkb+Gh%6l<I9EnZ_;4jwP{@#LRt zHh<)eSl!Z0uH?l%oEC(5|L9?KlrY3A{;}NN32Mynq9*gSiiP(k1L$gC9{%3^0G{p} zK#SkiapQtRIlH$rY4xW@nr|$}w}+)c_>xsr_#z%Or*C8NVU{?p#sn41TyVSYbS|9} zSmp<2@J2I$Ijm`fFFhZ4bs;akYfv!OCRD)3=tUGz9>{j&>0nu=9_w4OjfU`ZSZV8R zyguz8F27ocmn^<x>f10{By|`@e2?SH4K=W}ekxRN|AF>P-oTV@!LRE0idR1|0zWpN z0kan-VCtWXWly`|q;dd1^7UCvxo<?-KTZpI-%ae%Sxr9Hq7(<Z^@3)^XzZ800Hy0p zIJw3S>|ei{Kk{(}nLZNungv_w{5oB>=WZ2P>PawL=Lk@JQHhRg4x-#k8%%qtOm|Ox z!Xkm|ms2<yt!p;J)C5-={m_^;+I+*()uFKO`X^Lgrp8X?+p=Sxce%(ubFmjC;#Zqw zk;$i_boS|IUao2oOZO8HkMB0}O-KL0-hCG6F|!SNtA5}n<q+oa{5kjZ%^hCK$OdFI zBH8>rRhBySFPwN?gKHbdQ+?xCXmia2e(f;$Ech|DFIa?=XFY|gG1f3ddL_Hx|54~G zr!ebPVFJNNmqz5)h;IAFgW=wRWHGf8(;iuo^s_3Qdgcv(!15%UDBma!7Uti;{by0H zry1(oP3Zp6&p6iZ0(5UyrmZz<cy!xOk>`gkY?4F>?Cw>Cy>oKt$m@~p2Xu48pH71M z&U=_Huz9Ea6udF7wy_xs<KgVMHi&mQ!Bb5u_w~bUW_TnUTo!+VTXmz!<X1k|Y`8~Y z!&}k8_cP(}omM_GKou;t?b++wuGVIU)Twr<JT)wC0JRDo-ZfB%y>32*!8UW~{Nh%O zzLtP16y>M@iXk+zpTG20Q}o9xp1t>2!FJ|EQFrhfkTE<1U5!k%cj$I-YZh)K_*&4s zYBkQ`Fb{4|b?DKIlW_mcbau((ruevhHZGb{2MbILXxpzY@$?*lxwo+uM;ZRWAxmE1 z!#(YIDEK5Mj@QHC(jjb?@on68Wje00EylbF9A&57g9qofv!WI!{61<bcU#dHX6+LC z{O$X3ShO-*I?E3{bceFy0X66<Wb4dQ?hCWXb2JN@M*8EUA#KGToH(OE<gJv*jcYgu z)d{lLW_AEPE0=>_)NbaxQnhl~+hep;;1+&S8jS91%;5b26Bg`!9p*1orH-pnV3+<K z?u37bFuAvQxZV}N?@DLG1-*Ev;|nHP^ulKH>386kU59JN45#(#3NRq)r;WnqomdpM zh`&?$3p@7pircol!{|FB=#|MIZl>jYL2JziRC@(JS43=$@VSl$M^oGEc2UrIKPdkf zEBchF%C{@$;^2c5sLfd&YA&erYkZz^|Kz@ax?ux0nzRE~pik}^iQLuUVazolpZAp5 zO*w~MVeijFSpN0|tvz%CKJ7dM-9mOE#*5Lse<@hq%2Co9Jz8~P7fpD79X~BO3MQ{S z*xx@HsOe-&j($<l<s1XCS$iRV%R_XU917-g92hJHYKyue<Tpa_l#o|namt##6+8#G zmY6fm9s6-?;9|&2v__W=YQhaFQ@W(v%?*?cVUyJTK+$bFlbWtYK?+i!@AH?dR31R` zrB3i<sVnIU@6NU00F+HTLRV+~6g6rnp{u~I7+iJ+uJAL3U7i7Ra<OFx$KM6Ts+YK- zYaSV2UPUXn$-wE)N4QwU;ViLW5{(QQPIV?z`3FD4DPms$tr|F-72ll->RAinj#{kf z$rms3nlO@H$zErQ7YEP>`9XM3=+VAS_{4os8Aj6uZg<a+4V?CU1DZU?ANj%@h!FAx z>MLcL>;8khdqxC}bT_B1KKdAyqRhGvw8QyRLs|DuSJE7tfI)2ssYaN2^oB`sq49U{ zZ$u^p(n{7XV~;XD?ks4W4Vs8EDZfgU^{Lx~#h7x=v^}5w@O!~8<9zV>>Th_oQ;Ie_ zdBc`@uX*DWukfpz8ui&JQIp&q$o}KVG!HLh`2xq*zh0MK2H)lyYoEdxH4T1Bo)Zl4 zJq9o5R>H4tSuz&p{#ks9jnx7>K5*1>dVNKQ4T(GhiYjv$_k9GFx9oxXk^`)Mjw)p} z1>uvU3W6U^;O7i1W!Y});L)^1F!8kF`^TOXKOQ)i{d!Q3KGU+eQ{@_b$DK(uY~UhL zd%gy=`;5t~d<(w#smM>BW<g(nHlxl%N!Hq)%?<grfqB{b;m}d7;w7%CeD<|Ayqk6t z_sbr^rVDknvUV^_><?lRGLkI5<17vwp-k@Uq-m1gCfX?V4ki{PuyYN8to!9QKH}Il zklGv!E=6GwTXP-Pd)c#{|4pRnf1YFS;!Zq&<^xQ=%2&=2_Uvs>=7IFUZ@kk$6)LWD z!Q9=`A=xvIdLK1|q)`MLBUYe5?Q*DVNDx2ls(@uhhN29QjcoIW`TVR$e(++sDsx^l zg4%?gzS!g(KQ3F~5Pv!f8mA-CPVXm|`$LNgWhGe0=oWmSc}N^|aV)IKp2il1s#4_4 za_B3y<xFK&z<kPQbckDkTPGY8%e(!BDY_fLM|CFprAV?v)}feK=gjU{KNFx<&p7)7 z8MrB7n4pDiXS0$E+4kog%G*6a?+*^_+s<H$`SX#>E%B$zt-EM+$0cM*PB6N59sFI< z%*SVKfRLC+plT>&3_8tOOPV#A`456`v%bUn9$R``B_Tp3ecBXo7%cLyR_4C`f-i9; zE!VS$saHm^)_}$A+<rSY`gjy?8*ao7|J0^1w^Htm<tYeyQw#<RPQv2r;Y|Kv0=$o4 zF!R-H%$PlqrKpc!2OD3(?hgi3_rZw5Q@&vPyGZC6`coWuRTGa`%%Tk^oJBJ)=YhS) zN%6AB&a5G!k!$&Vo^w(kOBe4-2~7Ta?#G6^kg9VKC;ewZqgo2^W#}pBD)k`wZL7Kc zg(GNO))ea89)qVZNAdExig?eG2iJqbZJc_0*qM4?ti0BmrH7sqw|CmmLkVMuygHlq z?#<@=D>|{@y&+fEvY(cZ7))=|g?*V`2m~D-Bih&TLEJPigek9mMk6-_(jZcx#ycI5 zv+JjL(ZWw))jp34wJd@kD+e>#8Otzy%t#tBZWQ}^?Kdi@?SgsV7QosxT{cp<m{c>} zfZ8;!+Sp9DK)YrHgQbHx{{mGs_*9EmBucnd&wB8c!6>}h9F5C&4r10n<*>^)f_^jr z+})aj<EsV;w{s+EM$9RmYz~5N;#KjG=XIcRIg3xs?+3YQow&&81b_VIA4sU40|RDv zp@ilcJ~r2yYZ^C~?LU^rBn-;%UZn?pm?J|08*ZY@0b`nHs6}cvV=>{&Ie4^lER8<! zn|o5GL5JR65pvGKw5T<Rey0bZT&N5cZFL}-&p?AFUKHQ+%E6+bYdowCV)cJFa9*+t z=-@L4YLk(I$P;$#=-e0l{RnN|F4&5-YzkrHg`02}1s`Ce;GOFDej1FEXS12j2f_Pm zJ|^cjbN9-;uz7bN7jU4PvzqA0T3-1;l4TXjCaFNozJyAh_p?c&WGMBN%dj)suVKJ& zDJI={n^wz>CA;yDQRcM2b&kn2DsX)Sa>pY;u3OMlpR2Q!^rOthEM4^6Z3kQJr-fTD zJ>isP7I7ONs?*7z0DepE*vws-z;KWhyPNYE1{Ljxmh~55mgFi{ACixS6Hl?k{bQL# z{&2dl9R*u9?FEnMKsYOSj=yjH0lQw-ab<euczC%AUt!w`5-Kxcjofp71FwK4kD}>* zjy9ExV)=^u??rvro6uj#E4@;6Mf=}FA?U<PW?-_O&3`nR5@c3mUfM+zc`WC`&n4qK zVPDmML5F`aFOy}3HDRIJeCklG<gyEmndeqTVV*UCufvlC=0ySv9yJRW?bT%Zr*GlO zqOE+;Y7O=(VLfzVEBZ&)^6}T2;l8lHSn)9n72?`4$yt^P<8>%z#Y(ZA-ZIM4>E^Cg z_u&|ub5LuvkB&7+vw|T~Y+!GO_-<YZ+hP+0As+>v89#)r^_qc6>*B#+th&f_+-rDY z5P~D8=D->!b!uM6(V4@7Cj7Y_3z|3J#oItr1)ll8>@t-V9cx?Lp6af8gY@T0X_v z1I4Rn^D>#G_<%QJX5WPVXKn(#*gOw3nv(eNQdzF@ToR)51u$=sW-sn^Ri2yZO|v~p z1)tU{aF}dB67#q4QnPNNSaKB{`YgjZ-_fjcodpj67*BH#U4<<V<ng<70q1sU6a{tL zQ<CvTZex)PtAFqWG!ky%&VLu+%H3wDDLjXYjS?7E{0O(r3gFWw=E3*(BJ#Ot%&h9d zsU}~EWHzaioue92L#@D+5_aPj*{tM~0Szl;6yT*pw}m-OeCPu@|EI+GC1-H+(mpJV zZG=hQ$Dv6!f}5CR#`*R9fUzeP>B`&h;BxgaR8LBSX%)_R^otJrGj%L|zc(F59n0eG zriNPgwq4~%Sx;ay1*V|4<05)?e*x*|4q(+TqnMJDK3ixlJX4o$E}_J)a+%XGm{2!@ zj+@2cs>++76nh&JzDcp!`OPBJG&{!ci^l#JkGaLCcHw4Ad+xlh1dcNwOn>}BE8n~y z36iENq`9+-vq}Do#Q|x!@6C7a{-`u?Ymwwi<@9OfD<jrtz8?5Fx~xJm5;MiBe8~0F z+|{Sn;NO@eDp@>;#mUb_H|tR_&ix6*XnYm;Ln%1Z<~c~1A0UgI6#iRt9zU+PpVL_T zj+-5CFLF9A=ygsxytKazbKUC#K8G3+mp{YhBegl(#V<i|Ogc!A1k>)>zrZq{GL6 zk);RDd*FiAY@GPDRwkSBLh!%NnasTxX+Y!V^L&P$6@0C`4$CV$(R=y=Tu<7}%=H*( zxSj{S3zK+H^CPscA{c#E{}%g(_`+pXRq}ML=H|J)<s{nAP~lWF9QLmOtVblkzQtZF zX?h-L4)Z0$A%5&rVxaJ|je#Gw)5u`ydK&GLB0~L4?l;H7sKM7jXZ0}VR3VS)dEGdt zI)IYvY}qDfNv7@m9WJZOvqG;Jz`b>tq>{r@J5n)6!VTYf{m1;9gp6;KF2(OoU_RfX z@!g1J*f-)24w_cT1=m#w`~w>ZK9DZ>d5Unpz+vg!t54@fXj0Rnbd=D#$2+N9#!qpB zU|`QW+V(@x3-%rs?@QSZhYVtAWZO3k7_<=f{khFw<U??;{8Rp@u}~)$#<Eu{li9Zc z!gu;g($X&yZ0fLd_VeTmtg!FrGL8&ps=ZG|S7uakAF9lmOVdDJ_gxKF6`KWNuNz>P zd^;?^ngk1K-at&EA^T5jBeU0$!!xRR9D6Z^E)_)ZzUp%9@vl$tKxHyjE_)za@a+ux zo*cwEefkGW$Iiidf3ENqLGG9&?a1mb20`2QSez5QhmWc<#d+VYs53f_k82(T<=zPb z*Y`dDeDZjFWxD`9*7tL&ALp_C%f?drFJ~O(y&oHuLMZE>2Hx;p17{S^!Cg;5k2v=q zq&ntOyZ;re6>5A-`!L$^Tajxi3dPr!1?Uwk#WIW!Kwazwm=JXXCkcMnr1TEximHIP z2Y;Z{-J5CG*WmPI20f8qA$jmU{K<4_)RZ9Tbq|K=>^661s44qT{2K$kV`yif27QaM zrTRaQpv}@5y1t9S<X$SL8YzQ18TYt=-&W-RbP&6zR|c2Y_n@i82xe4f2c-`?xGz>w zXkc?2jeiA0nC2(+sjY^4+fQMou>UOkZp3!H_NApY%jnJLJ-lDJD~pfU<R{%e%2>J~ zsRh5q&Hf(Dr6Hj5YiP6R;j~PwRrKTXZ%v~QvF7;jbq;8{AI2~HOsOk84GcF2;Cc&h zIwE||g2E{Nhp^8dU4D<7EO7iHcU{Bvl1`+wV-P#{XDd5&Y8Ah}a}=wq*h>C?XQ9`l zM;JG>OB}O8@LgyR0v)$hS~K(n?6k>cle)9uyrPJ8ijCRi`FpYCj~O%_RAo!zb)ah) z!QrUo&^bXDp60K^$iDI1k9V#7)l9*6tC>MMmuvClnlQ8wfTIUIiuu=r6Y;`HH_SSs zME19Nex%N9*m7HrL=%6b+3p%>xv`a9yBNz2IT1!S5A~_}@*0s-MjKZ+qzi4A9mb1K zH?T|5&7ip81$^2X0Xv5~!OTNq5UI?A($+!nXZ&V-7k!Wg3ioVin=(DwQHpEzQ!wYg z3;Q7O$!4rt#dq9J!F^Aw@b^Cha`8~0q@>Asefb9)$+OR3{pLkt`QRgHJE;=}`^3@& zJ-``TX28Gj>G(25hBYT7V!CZEc;64eGButL_PK{v3$iK$s@p+UK8IaOOvk&8R_vbt zcgTA+nx&0B0s-Ilv&9`1WE;8=roF2H+4u46_WU5IYE`Fc-b-oaqOGDs>&{?HnmIGR zroseXXQh??IbIx)&j&|;$Ks6=Z20^X+!klTk|kxCui+c$+F;6*WD?k)0#z1cSH&b; zE}>7R8}_a>gzUu$lrQ*e=X!hc4jX+~hHMjFcx%ZPlNXlXh^Flm9Whn6iI<<To2(ZO zh4CNFX#c|~h-z?T7WWJIl{$05sN0&qEq)7HP0vLyB}dY);k($M9d1;s`U13+1`zi& z5?&_d(J9GB_y~{jz`+6(PRigqp-+{T+X`vxWBB<^I%HkHfuw4(*qYJXXi9dwO-jH< z{H2$~#%wwVR-cQosWy_5KUQ&HnkTTo4noerLIP}p=D^v^4e(ldDYxRA1{2?s$NBpI z@Y3s}%)?HB$zNN9d))1qo%JDBqmv=rF8+w`uIHecr564kJdq#WIUW~%7>4J>9fHU1 z4wMy02sv&A>R2=kQf92c&HGbn-p@qRS1+S)&yIqJuOF2akK=bP{f*DB=Yh+Q3$XIe zaZJ%Gq`19%SX-|Q3w0ORWdo(yhnd36CG17UXJz4*O^H}I?H724PK2uQGUELYmDteB z(GaLogxas;c&~0%SS<YzFHBd3w=F$<nZph$-KGf#JRgdkdJo~P>cOOzdzc%me*kx1 z{sw)kMeI{+8r#^b4tI<UDQ^8X3=ti}qS+DDq;>(<n#^YH)`Q7wqB|uCUaWj2Nmz9y znxc1f@b_xdA$yl6xOFL#!)|Bt3o?Zw(+g<mdXbLB&Bw6Ud7?!tc5xQ1Ytc^VH5Dd) z<cC!)WDZ{z!lEBPVe}DgcDwc!*B4sNFFE!JJPUN-gLXUg?onam<wuI6G?SUk4=c)? zS;4tl&Lz_a(UivTqnzPSL3*?y9f}wBzGX#RjdV11>X@_H1GMP5#1b}CXC)<6gz zJ;v8NF7llRv`BNMCL1j>rBKViHZ$h0pm@y`R%v5HI@5--$gvs%%V!gI9nqtiPB$UW zVj>GZ6~g71?d3|=n*#r35$=0sflh4~$jh1G-)}OsV4Xa>@%J5ykDow$t?5iob_%P? zmLbzSy{Ngi3U6+y#Yrl5F#U2pe(#$IvqyCDf8^J(T?sefp-U$Ic0A6%o}4T;v{i@R z1Y_L!A(U2~SV;w|odgb*DXf1uO1LHB4@Qr!bBl{(`KiOT@keDHmhPO4pGKUge}@F0 z<Gg9YT@y?8=%5xmDP0N@2g~TH(^SY<tVZ|C1b)iYPX6_qI2t3|1eb5^=e|rap~0>R z!VQ)e&=X=ryG`<OVZ&^i-SHMA_IZ+?kb~BEbQs=GdW!r(HRkML4y(+9*&8bnH{<ty z(oFB>JxoQcTRs>0wg7=OGoRUyb79AS*NT=8o&xKI`&91I79^*?l70o9;Emf3zzyX^ zkh8)QPd}>@JRCfy_}d=SE7tNW;`N9OWl5wg4@H#)-1pp2a#skWH~Ke41J=4j*R~jT zt#AtasCo{LN?yQeMoNP6%vf#Q4AfiefGY2%QE~h-s;&>9j~TX{%-0xpD&i9VYwmnD zc*+hqo&1h>yc5A<%71aI?GSc!S-@t~EBH+{7Khk|^2^`f=0{yJBImvg{>WH?rK??p zmxLQ(o7@+(u=U&dsRtL6XLJEX&bb6KM-yR=zBZ0q>B0Y-dl&iERh+2vIq1kYz)8gj zc<ZLamPeam--=k$Z`WeeeLGQejS7i2NrUx3IcB^;oh^}Z6mGB>v8yLjnaKJo@W-7% z+G7&S5N@9bm^}g~hf0i>S<Do_4q?w62BDkUWU@<Z5zXv9PJz=ORGwru%r(b`j;}ro zrVp-ieWr@EYmXtV5$dGO`AmGT5W)OP4Y5U{4raz#u>H-hNO^#de_rDr?U_SG%Osgr z-31t)Xh8n))gYewjGv;|2PS8Jf^!MaE2xiS7ltR|50#5h);1h`#)`1<v~VXhaXa}< z`w6i*Gg<JI3jXOY4XnFLT-CK2-19FSdtXe&j>*H=9T^`Otuz%zdCsDW!@5)$-H1o6 zg?-uMGO^2*`Dl%gL38~G(TAd+u=VpL%vCdFM}E80eW>Qcb}yk3odTnN%Rla@-AULS zeFnzJc(B1H?>OtPHmoh>Ck#8e76%qr;9G|<%2%FC`GG~8Y?K2?x;vv+rV0Idy-KWC zxCHhYU551mj!Yo(Q?c)V@c6a?x2xbO|3k6|CaFh&c#ji%*x?DIqX&}Gmm&;(T`SH` z3TJuVarkzfAI+LQ3{|d<=Qou1qI}H&b~UUNWuMi;zKmq{xw?syG0vpqe`aJAQizYz z9s)M~MdS8?V0dx~mf2rm^6w{b(jQz|!S+&=Z}f+Yrx);dZRfMf*V5ovRF5U@<C$4$ zDj9r##`hUKN2_(mL8&Kzjh*)!^gYJVp>7Q_+I$cX?D-}>aC<0Z+sJYrGwRv0o95(j zS{+ZHszBlDCo`!Ra)e4B!A`c24evNaHizZdEOR5_9)iF&8`B5%RbG_xGMCwFNZ>C0 zO3tg^nwj@K!1yU8Z0OzAEB`YcRj0jnwT`xJZVh&uYP#&qS*Pv$dvW$Y?kkh`JTsqX zqrK|yp4&|CZFnE)*sYzfY7^>OYi9@CM|^7r&pyk8sdif>_4cveRoUnC=#1@&<-B_{ z-k9#qyu)P6@Z+KFO8Fo=-U)hk$<tW&r2zL0Tsb>qZ;)#A-nFX^+kP^s-<zpqzwh#m z&b=zVE_>ad`0m@Vl*#to*06ocFHW)h=5ulH<s^CF9N9GMo#_wv=6(*|SMaQF&(f>O zdm=6@x82ASw#O%)Y42~ljJ=}3P1}8Y-GG;zb?>{gQOU0K%K5!tc2C-uxLkGLo*bz? zm&DfXRokxvUVk9uzQcj6^#^8@uRmZI;Q+M$BcgR0F9RbmAb^;p`!CE#!=KXRLs9r! zjE4V!hQH-#_|q@^1sNO}ic-^3i&FEFQ}xSIi;6Sz^W2<l8KNiw#$2ErcyV5OC=<{t zAdI^NRA4Yba)Dk-Vo9QYacWU!VoqjNVhPYS@tG-ZPC6tx$iULX$impd$iURx#K6qb z+}Pa2$k5Q(z`)$V(A?bE(8%1<!T_iM<jV8_!8IB{7lLqrHzUZg3>+LFl7XR~-@DG1 znSlX>g&;ymBqH2Ej@B#4&Oz6|{r#P0XcfR9iKLT(p%-W(Lcd;Geo<~>Njx<C(T%Y$ zFMJ6LNEsAk1X)mwDK5z`N=#3U2ZmjGUT$h$3A#zGwvQ#*fF^*jFp5c&P)tgyEJ@8T zN=Yq3H-N!$C8GjRBM1usF#?FOB0B{fjRxqtuR4Ywh1H3ux;J3zHbmEre4`YqZUJ@- zyN%FwBcBI_s(XSuhHhhY-N>ssQFRM=VCXhM*Nxm3L)ASY14Fkdx^Co_1gdU<P7K{< z#Ot2Wg`wM=c-;a$7`lP^2{r%VcKL)}4BeLKx{=EdK}13WvIP2(b>ql2$QfG!Mf)Ta z?bwT)0B<%n9jGojW?i^`SOmamZeYFv0gDL`8mJGL7#SFxr$G3i>>uFG3d(T63?sk* IQV&rJ0B<;GPXGV_ literal 0 HcmV?d00001 diff --git a/tests/fixtures/tl_checkpoints/gated_rms/checkpoint.pt b/tests/fixtures/tl_checkpoints/gated_rms/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..be6f2f9587294aa9da0e2c263449c55e439177d6 GIT binary patch literal 104501 zcmd>_XIK==w)X)+qDoXzBq|vsr|DHFDk7M`1Q?K{q6B9c%z=zx0wY0CK@mX|6DrfI z%mLHJh!F&H4v3h2dj|JD!#Vq&=iGZg+z&54(^Gc;>sM>7?pj?{-80&+le~<Kl9J57 z{AtN_lL?;@89r-vTvV*Ee?+J-)OPl)Xm2l7nc#o?CCY_|$3;g+h6|(OV&hdt2_hl| zkr9KV!i6e)8H)hEta+d^hmMR1i;S?H5Hf@>7cPtwgwL=INf1Uw$E$=0BEu4*q9a0t zk+Jb{f{-~5D*R67=7EY_rFc#mIz5sv-(E$<J31~@IHrB43SS{mmebhrI|p{+zQboH z@D)eM$O-vM{(drk>VAgld}TjFKiPD?O6Cl)C8|QcS^&RGU^h-ZJ1#!tPm{PZzv~RH z%^C7D+L`wJZZkNRtdOrRW+{nT8tp92c9xcquN}bG3GB+T!lL8CXT{sv*+$3O+s@^8 z?|^dP_n4vRXDH<N6q5|o__}_wY5ZR8l-@$VUI4$(KT{kyihc)*Bj2E%V)&b4B&Ha* zQ%r>Xz5)Dxff~}bgbIbRT)6%DrX4^|d^53U@<P73m?XzJrJljJ;Nr$bOg4jW*^aal z@~s2-Hh+s8#kcK%bmsT}jkIe=a-q-Q+l!H0^fUMl?MO!<-zk9a{5Qxj&PtaKNEg0q zySr|`iRpZIv3==$k9HVO$Y%oh@VBs0e9{5t%J*!C4fq|qf#TSCwSxu;`GW)aL;eOd zA;gdG-2vprANt!-AF0?4`zv<7?MOc%e|P|Y#NQ&v^G9|-y7Nc<Mvj(3`u~LtXh)6_ z^2Y}7$Ndd5jI(n5Ur0Ib3b421PiXfy@UP$nwFfV_oi<U(pA^8K{I|66{3#u2_WY1` z+SI>jq3yJ=c3QZQ9}&Qh4AlIWt1l)reinaPJMMP}_KJ&({;U5^50u#Jz@NdjAS2{Q ziCf_QrSoU{$%;?*to9b7h5VQRer%xjzqBxH=*Uq4L-=tWO?KqZ=73!1<#Rv_i*|@W z$d3=;3p+X;6Fr;jBT=#ZgpLF!{v7dXQ4;d!ifuCV8<5VQ=QluX)BJYa0wI540Dn<O zuiJ5P2}1tjjyz}nlHa_gV%`98K1poPCrRzRWFbE#fS($uCGBH7Z+fUOlAqQQ=)zC` z4b1q<z-8?QX0`*fg#6_J{1yM|Z<yHMl^t=e{8e1U1_=49CFZ5^v&CH`r=7G$$X^@4 zU-z%BhJ}b-UEdMq#^2C>y=;^imd4K&qw?BO`9gj{0DqHozO(<GH48gp-1$ZAn9aX2 zTf~^H?U-UAza)Ua?VmB?w7I<l#=(wX+K$=r8&f96>}<!B3;7iR{9V!^vbX&!Z&r3B z+4Fb*?x}mksNadR>aU)<x1G69$lo8pKk&EAQT&4)nGXEw-^@epOiAWE{8#2Y($1_A z@<joB_HUTt%!wVDj(png@zLMNbpEmS{(8Kfc0$NM8NfgFx3p3G(;aC}{4?#ev%lka zP8`45cG7tv|3U!&;@^<Ox${y-k~9DEZ(skAieKGd@w?K_yei~h3*cY>TjqHFjgCwg z{>|UadMW0uznHh%nRkTzy8--rf5Q}K&-)#juKb2}j~oAr--Gt}HMP^4h5UyB{6~LF z8_$2-k><vK(oTE&7wuU)?Rh)xg^>R;fd5LmbN#*sCE4@!Z(Mt-ba3at;c#54d@IID zQsuk$RQbLg_d&>S3E+Q}PL;oLzf)yv2m2iD_@6i+mnz#h;Ge1TvylHKfd93lhkvEY zZygEt{O{s(z@^F`Vw)tX@@IP{`X#YRAj6H)gaTP^8z7M5wjF)_D`yHiamPBc9R%{? zeG<Gtfh&qz5GPMTXYN0e{3%f6O6~0^aV4QZncD^kRJg5lF15G)M=lkpatAv$<S0-R z?`v<U3s?Np6+u_-KN4>Q-MCVFL+V^fDA3@x0Rl~K`>%l#=U9Ojcc^1CP6BQ5K5?cM z=x`;8J86RMT#@Tmf*#y=dka0el2D+_Z36_oxb44&Op<#Ay}4r@n{XEBiT8;|oPs`F zN#aPFK%Xm$n=s(M+nX@tN<x7Vw+#>&bK5`_?#>e(8!{~_HZ<CHuE2!b*AeX^=quhQ zLHFZIztN^#QH(a@zT45}TuCUf;I;t*OKuye%Apft{~T_Q6IgLOJF;B`*5ZBK36K*C zY`9|k2~bZL*mD1o^cX>ZuGHRu9aj<x?73}#z=7L#Xdo<PoWPOW*|7mPfs=TjqMy1@ z;LMf8eDySe3s)2y@5+6*v)#CoP~gsO0|Xx2Hes=V$9+lWFd;FK!cg%vNWgHr#B&M( za0T-~&OaFe;fjj|o?@|h?lLD*5YNqs1OvDoezNg_3fw+P2m}KqLN9JR^Us6g;LnJQ zo7Fxi5e$+j21_f%bEv;2ErKBufw#0kJQI;1{-GKwQTa%#I&-S_`HEne<fyOoQSl`3 z&)JN?PjY;?^l>$*<Kp|aV1z_7Qd-mFpER}-qvFFuqay{QB=XVHa=pKm$Hoa_LZhP= zL`DewC3OO%>->`$4uUZf%~)y8KbhemZN@l>e7v;$pUn6zmoj65q>gyT`PWJ7B5k^O z+#m>&h=Qd>|2k-u#AB9VqC`9CKhW|8lO@_I|ABUnAVi{_`X6W)2tp;=Flnv+KReYP zII(*n@!@fT$aq1xq*jD<tzLg$Yc^LcQV=VMl+>6eT|@WpYjFI8XrW-bq{0m83LQ>5 zm(2xH63t9$jgB-_oblT`gdkEdOCpYz7I!%1(&CWlsMyF*fgnawCsw*n_rGr-E?y{z zlc;A)t2>w@F+~za?%u$chz0*%%;)YRvlE1Zc!^l}@5QrzixVW`IsaZf=eKySL_AMg z+~Ina*vy4Sd^?*j(JqkIc8JD5{NrW?f`t<GB5C!%Mng;%ix*48OQgjex~~MyT^Ugk zA+x!2xl|%g{7>XDQ4xY9i9A_a-c$NCO4G$BGDV_KmDWq;MSDS-<aoODaj9!lGM^D- zNHoi&HBuh6&v_*6w&%r6i9AbME_HQE$)ys~a!H*P(slmH3<tqViDs3w=AX>ykQY}= z<k`~le=_4=^J0#q&Kl`DQhBj`@F`d;5v`LJNnK~ZN2ro`%o41ZXgB-^TE1YTM4S5` zXy*v>B-;G{K)XOtAkl7;)=K3?X?vyfVxgo~k#sGoyeM7k&%C%<Qe%sB4XL~+UE_CN z+$yP1EM1|)N#}Z$phTkCCasa`dhH`&Nvy?vX}d&RDlP7C%Kt7e?vT_elddC`38Whk zcc+~ab-A><gE<mY{>+ON67jBoFa9$xR!YRX|GoInytqdquKM@lKl9>ViFlv1xWn}- zvH5pi+%M4{kk)pH#y|WMr@Dg@b+xqmU!(D7UOXfbAC?w(=)My4Kl0)ciM-}Nk&E-9 zNFryY<x+XkzWv_$n=U?)DA7}Cy*MxaIcE?Yl^j1NeO$bn_2+Tv0oZYg^n|oj+;m5& zRIWQI(Vvpmi-*|1=M;j|lEY`D4~y6M|D1A2!o=Od&PrtGq-E;=EEAujT8ZwwlrAAY z%yzV&qSDvRF-gxjOU0kj|K)R&UNY*^&rNKni3hX7klzymZ!b0J$0xz|V(_82_r$jy z8S&@82KSM1>Hz-zcqD`yDNm1$iHsF`d)akh|Klmme_e8M5*y$-m;YS68Y}+%*9K_I zFwzEu%@;<-iJ$j)d%69$+x^p`=qRBuI#O&y8a&h$&rigk|Jnu(8J@Ha;^|m_J8v)7 z|EU2#l})HklPO(os)0dc7ot{OD}=|KgZo`vG1WYeF52w@qgq^Oh{zWb7dybMn_0{> zo+FX3F(T^F4L3CpqW5wWXrBHy(7pB;2J<VS!ZDOqvO`eOI-S@Jy$wIt?!@@!bL`;k z>&!OCVdUbGNb*=&ncNHhh_kEr;*{xYco+As!aa|>!KZyopdT-pdARWiq%U6z(OqL$ zo7Bgkcfkw|#<cKG#BGFC`%1~jLu2XNGm*65!wJl|_#G5(^dw=*mzk<)1*~`-h7G}m z5bz`d<sb<!FRq8@UX#ew51$ARC({du9jL*{6cUD;(Y|pTK5$nc3il03{|g>OR((8m zet3i&Qp`lF5HBcfnMIY{H)HmfEOa=MjgM1$(MJhJWb5Z*Rx#u`T=NZN_y0Hp<1T%` z^PckVyKlC@@*~r+CZjufGG`Rv(ykbAxRf09Tt?T;ej*}cdr<vU6?!qefJb*;f}4S{ z#O=#q`g!9}xSz8Dmb)XBsZ5}bF7kNQ(ipxdb%P>P2Zp&Y2<{f7K%c-=2rv1Hi+r15 zyGv(kmi~hs?q5ZReK2Q-EG-ndgsYNtFJI`(SD+#%Id)|50em$w9Uqx4Cy#BS&};c> zhE|#rxx)&i;fW8KbmtfL*(t!`y`E#(l@hpowhG3$XMiDDNn}3i(gsUM_QdUU=BV;6 zY%p=6dsjxYOLpnd<+{#zKBOxoPwS4QJu{h`qr1STQgh;JVFusdg+S}uXROlZW$Yj2 z4b0b)M$E{|q#GhOGG5;=;E0hf@M1zWQ|}%EC%dW<+w^WU;-D=oE1Sn=Zd51p_}SPt z`yLSO{jl+cGaNH*V_s`7XFpyX4LhDWW0i{@Rb8SAvJSp<gO@IQd)+9UVG>4TVn<-m zDm|hSkjsQMhLgo(n&F0%9)*i_IN|45;#6(`H}sFgX&H5@1lH8T-iB&)FUA$g`LKQO zEGW&10G<73n2R+(V99V(RA|v6)23RpN3Ha!Z1-@8x8DV2bw}Y-l`VbyW)FDGFT#|K z7G&hpQ07CjDsc*H0sabSN}qKl34>Q-%O!VOJywOBZgMAOcaEX)?h<&o!xTD|=Zj3v z>eA!csqEG!Z!}sSPL}_;&-~Dt1oIE<U{iJ-hD+~^(PqN{dLwNcn|2@{T{djQJ*i*7 zS|CHi-+lnOR#S3JX(^SH8Bf=FE}`lR?U*_TJt|l~ns^x|vISZ@pwrKj?4TfRc4TOG zA`Iw69FJt;Is5NuR<(oCJeNrv`-VZCuv)aj=ov<&ug2j~{+Ka$CrqAUMiY9JvTHUq zv3&=6vxh$nBlDYff$Ht$XsR(5m)H!T`w|n#!&U3a^)y9Xbnha3{<Q-x&g>1|I_s%N zMLw8l6yUqQUs3T`kVv`Gf*gH64d%K&VP8d>QQtUM6j&*d6Fp7osasni*#9wPwA^5K zyuXIm8a>$6ue(sbg&K3Ia3wC={SrScc@1B}4xw=(j~LXCA))*BqrY)CVjQ>t-y|Nu zrRiVUzL_i0<@G~o^PSI>SoWqRCK>FDAzF0OA{#2dCJXL%aiFTxz3Ie^5Hh1VPqciA z0C#oO#)3oZ*-^i~i!8IEVQC)|@=0$6sd%kH>KAE{o(2ms<AWo;s^CF9ox6Z=Qy#?a z>MSy7-b!mnj3sw6?da(ZHPG3ot7j+EY&JvN3hc`!L#XR!+|kmF<~{c!rA;fy#QPq! zR==FOURqC&?dwE7+t#9I(@OH;wgDxgO%Qi~3c2ashg2?gB2NBkjD6xva(;L^+?~u2 z;}_rI`jhU=<d2O|y;zMFuaKc@!ridf!GWl|HU*4RvSGd7Pte;K3fULN5cj4ma9x~_ z57urGZ3&@pcBm7oohimTr4?A?=0=a_*RsPFZ-Kq>SJ{zQ6)}F6CA@n51X{IR$)Nd) zr0iQEc$Q3|cfL7sx#lM9?G!+w7i)VSdTT_cJ`JN!_~+4SQYl3HI+FfrM-XP-$E0Lc zczpR7b0TjmeP}a`R4>lKdnBHm^3kS^{e;AMf*mbbbsv^YvLPS&m+{BD9;|iQX0pjE ziO_|2YuXyRkolp#=!Fr9r1qo{s!lV+dCzY_z~mYXx~@#EeEL96@(K9Tl1BcRYfW70 z%klnxRdRK0Fx44XLT-PsCr9&Fpx2ZXIDc>unKN-AOxUpwd8zemNu>c~n=fIeI*x|m zk$3QElRR7q*TT7fL=d%=UC_7GhR#)w6{RF46EC4I?H=)oj61Lm!wzL)W85OL@sC$* z^e|W45zHg`bza1?>H#x0Kb$xon2foKzNAN%H?#%Mpxt_vqyD2xn2^61<}MClpO5l{ zqvSU0v2!FHytx8`kC;)%!`pdt2YzLWcFNOV4?{^w@FNN-#ppb6K91@<5Z&H~P^)Q5 zBA=P|WZk)GL~n33om$(8jhs>q@3s2Cirr`MdT)LD7~+XTpI><M=MGT%9sr-6hcoqF zK+QK?#5JCjmD{6686O>bZ=ML0Zro$%Wd8wP*R$!k1-t1}r=N^aVGjMg8A$EA*;G98 z(C2kcmk|$UxzE2Fc?|rQk%zsv7yM609_=HbyXHmMm|Bf<>JDL2zb+)RcMD^elEu?Z zpG%HAUPXVE2e@VTE?A)LOh-NGM;;m9gp`HOWZKb0T%q<Fm{+fG+o92*eKm?&_t;NM zR_x$4@Vs%k?qnQ(z7soV{6|nZcN*%>wi1_eK5c4U3-UYrp}me5v{YM=TLnC_Urv^& zJXndPS1f4Yk283++fKUaaS-#d&kJ_LOJ#QOt{h_DV-U7oW8k`60c>wtMO0_JV<Yyg z6pgsDhMBx&gvT89C&1jE1=(Hw@e~DObzGH(cKHQHvt!u7I;YUTDFy=~X3{>r*Mg;E z8QG$%01?yk@Z8()nDwoX$9M}D>Z;a<cIkcx)EDp|eS`s)Xac$D-^#4Z>;olko$2d` zr|HLOCAjyHKF)f#mC5y63sEVCP^{aP{yHQ>ez=5_&Sk(pi||FA)E7*Nni<Z#T*0Ip z?;+>>1`xelJyFSE4BT}(On$H@sLA~*GGgR0kKB>!bpQS?WZA)`7~#?bqx7G`i?N0@ z?pPTqUUMI-vNN#<e>GZPe$3tp*+QmGEMhwgwCJrpmznOz*5QNsCCL9NPtWwcjW<3` zWKE+D$h0#+;)*TMVeBLP9C3<OUr<Pm3hF>Fs2}vHkcV+M-O0O}p>%iACa52%Oo!UN zU@lwx;r`o&@ZK(jbh%v#>n{$%N7HA*(ebjxYHa{5Z?huX6KybdNGWNZ=Luuy6_OXR zPSobi0pi47#F6=?<g~|TlyiIq_wQtr;Y*j`CffzL{L3;d-^##^pK8>+=Tq3JT8bO1 z-!K!3+8A9I6*9*EI+U^jbdS+GI;-C$u5<uauM^U%7hb`o%1~S_?@q>OtcOm`!{FlZ z?$q{%3LV@gPfXUVg!hZOl9Yk!B;u+$EzR1Au?OqfX^xIGL|{cMW;e5ER}Z9<-FgzA zL)!G)GE-zPr;)2smGqrOG#Fet#hN?pgem$x$tNoha<Au1s(+{fAEF&?{<$4SC!YuZ z7!7#hX9SOiOl0j`qsaD_73`!(!|12ZooL{7A8fX|2v3)<<h4Y#;peWKpm|X)#y%N9 zGfsq%oc+<T?esdD?w<=AH&;R{FB-Qyr@);LVL1M4KGAvfhF$YHoOLT$0WNPZ!swi{ zyy_K8u=(mq9C>9SiA+$UQ*C<DtKAfE@&{+CFm@{(&8udHvfAXXyaTZfbv_c)IfV)~ zoP+LZHe`GyAqJ<1;nMrIq+qKE6PD(Juu7Q@npi;7swP8iPz)Nqx`8DJ;|Mvl0UprK zD9%476@v_9xVP}cpZ{L|arj^79~qfM&;QGh*#9#ddKo5i?u$SFJ^vm5Klk71f9=1U zMmEe|-WxtQjw5eVM?jp<EV8As0#4tbPj{IeK+*2$HM2`(iC@jCnvk(qco#41fI#a= zYC+1;u1hK_G`|3cUv9xY-2FyL>5`~s(FR!G_a2<?qC-DvIn(4rDeQxrdvSl)HVE0X zpV?q$NWz!s6ML)epndNFtX({qZryz!`4%78(VKK}zHbwDIqygG3+m}}<sIxm%THLF zW{vsQ`D9W-4m8e@p^aOzNn%R^H7-7ZVTJc_^Oh`nSYS+h|FIk2neL(EFUeC)?+f7h z;|?~<1Vde*32A-j%cG+M$^H{X)U0|Dd)qmj9^Sqcp8EBoI~}5+8fW5<)-v36Ob64$ z_ux#OJM54)6|$}66!@HPz;})2>@|6R8gapeehn^%y$^n%)hiuZ^sPTlO+Cdf*<J?0 z*O!sCH}s%w-ZjV#b*C2=jKW9ny08mg3?yR%b?KNlvtiPQCfpbtCVFyKiSF{VW7xDN za`Ea5)bC<Q-!Hy`m;D~WnfxKdj2{Sh9;j2<!PDJaqH<YlnK;JGc|4ZuUdHMv%h{wc zjwI^mW9H_Nr<n890EQ%oGcrf^laUW>=;7O1WTyF49ADRoofxT)EpxiCvU>TT{#urP zS<29u%ic7zQAku8(s0M!mrPxb4ppr-A-*=-vDP4$$~T09a;Ps8@BbO~wG7aHN+R5T zq(b^X96<8EG=lEf`{cgy0s3HT0#&zrB?>7YMhqsElJy63!7y|WlRS6>96w!--=3?J zx^skj7mo7yF@GD>rj4K%A9~=$<Ri@S`tdYr{u>auYmwv$9;D&e2J+l9gE6Y9VB5yX zk{PA<(71jIo$8uIC&$&o=UaPe!!=KyM*kdkrM-~wu52S~SE<lrZ!g1!bss@l#h7W) zobK^3%a2YE&tm%D_r#-#EqHrn8n{f~2Zv7lVCT(HMK!g9SQq*VT&HcvobW8RcI8}L zb>}6lSv;Kfx1G+;+iJrc`BDx+4by;ceHeqj496x@1L|;W9aVqg!TOvlI=tmUKe}jO z4w=&D68j@?BF(AjO`1Gk@?_4&(V;ujupz#RXFxsa!@XPJWxy4jP?kqxvYzsomSi~e z>LnPZZDo01$70=_RLq*kqm4Qjz|<#;o-61{R+aalxzpre&687LVUq`=>_$MFaT&ak zt>CQ@8~|Up7I2OKg^!M@618pLKqXg(1iVt9#Vikq!)x?;>kp+nW#IW%BS_70L7kB^ zQ02uJW_e!^vVMAZx^Qzet=&-o!n1nh?Ebm*%?~{~z2AEFhSzDFvlnP5<|t@!u~>35 zm7Fw8g^9if(9rONalE>S$ua+kTfFx`-z$Xl*6mFuzp|t0Ce>heLX%9+dW!Eh<ggD< zorEoyzkpt^WO89bFx`=D&Mf}9oyO-BKy}<{@YowjHHUVkdo=1`$*(0)OpEdD@MEAf zJeOW_`3c`5l#%Pz#OGi?>VNAPmA#QiYTx*fJ*qF^{ZW?D%sqjtO%rHRd2hOr-v*0D zRA6Ca0vy@kNY##+ux_(8Su<e;%u-9jwsbohzjZ3LxUvjNgU(RB2XVYffe%@g<Yq1} z6wr+U1EKR4W7_r3emHXd2X2f{r1anm{DI@>Q@i6(ci=eGdf2je0x5Ij!4lrv$z`CW zuLS4qdy{^%H)7ngI=mF~hMiAd;MR5NpnN(EnoYcDu>3vPt9=dBw)SGfWVgbsTYgl> zGLflNFX1gpDZ$`kc@k`A0~VH-n1^mdasNSiIQDE23`o+0o}Zqwg$uRF)D%80-m*&c zYkgOmmhgf(bbL0b_}kJSG5PGqRtvngJsk!;DFxRD3hbpF7UcGmiFEyddpwcD(VC(6 zMv<WF8c>ye6A}~+=<zZ88K-r=^k{eF-M(T(yXRhj83D)9#4Cq+9y=3!s}T7s-t%m} z4TgK&Kd~j_!pP$(ooL<VFq-h?0k5^O8_AlON6L3yV~c)0!1OJNr2b?M)^+d104>7i zz;l@T4RO<uS{UV>f%fZ8u+uB=!c8@6@-VL&$F1nWK6Kj;`mNc-M8OJ-pXPx_#71md zGL7+#%3vctO@QYOTC_Gmk9_TJN`g<GWA|>>Vn#o8rHaq@W9BnuG}$+ji3v|8p26$s zn<5Vw)Y6w;x$a1gy_!hP?%0syuM)A*<q6NTPc0kWw*glE+6Rqt=gGF^3*eYnK6z7e zn9K;YfnVx7(Ibn2{Hghj_3msYuq2Q!_-xJ_YwrV<o3eQUZ>GcDEpuSDX$$n8<w(Zh zMwn$jo35)Xg3U{P>6+fF>9vIpq_tBK-n?PRR%l-b7t7sj)#ZzLV_F7;#u&lyn1N*T zNHg+*FUL-vJs*yAQ>5d52GO74ag2V@B=YUh7#R6xB$=Q(1_s=WCr^Vl$$tF<pc%WJ zXlh1~&;~~=$PXk3Mre?kR-<YDp|iY};juKL*H{{R)fWQ76WB@fZenez8EI4SCtGhn zW;E0cX-(o<y3Hh?6gIqK7wyh~n<H$=&mw22Sr<uE^MArngR|t-z76=QVLR=6QI+}| zWm1!=J}9$wKPW6c2ezl`p>9`adOG+F-k4Vnd#<b?y+2QdL|qx8VPeRwAKB0ao&AVV zIU8zUor2BF3`y4w&!N-Fy{u1X0~{CL4_aqvkUUK*?p`J!#dnh^mhq_f(dT$akVks1 z9R*%*hJublJ?qftFl!WNM~oeesl($`EGIk~WTi*CT}@|9ZuW+-LCa`lmm=J7L<aKR z&9KYu<8U=PkZ4a0<sG!mM!N3?XkBw7g)aNqCpY_|-{fkv{niajCp?GUIR^Ai_D$SW z`3BNAE7ROh6egcOgYh5Caam*rDbh5g$CP%k`_}lNo824G=P426=FQX>1#m@H9kc%M zqRXmnaB6cAm6Nx_%<vEF*viw;qisD~ajQR>?{$F0+^@&dMe}G^8E^6>d^6FpC}9Fr zuE2@8V@Ydv0Bu|sO*@~viLD;GL{nip?R)1ScuWYPjVFFG4+TG=#`?P`=6n)n2Ah#p zdkHa((xW@Zh;Z0D3sP(q2CedjwDs*r&@IbA&EZGT@#T1CdLKvP+I)v}E?<G(m1>Y^ zbdaT6gcSP^B(0%2MEj8<ZeQP<w)J|3(RXCXqc{`f@t5K7ycI<LAa@T9F2(Ets<h`z zSv0d9P8rW$q|9Fc+PRnE8;*fooe#LE+e(`DGnG7?C_tUe99D6NKlQX+?Y`#(kQ+mC zG1{(3<os(a32`nZYmEDo-fo|HOyPL!pP>sMLXgqfeH$x0`@!dwc<|m}P42t7vzvo# zNx<SvLc*sK-K!s%z&lrYol{xnXqqDXE4c}m9MC2g&K!UVm(Af#z*f)*x`nS!C^J^w zjETncTOPY-bmpy?d<CtaKVxOpzTv3rPUKGd2Mn{6gV<>wQF)mx$+DJ*6<<y<g0ao; zW^6v*SME$}hrMK9EME?nbJM{txDAWb!zkannl;i<A}`MFVn+s*vhVJ7_PqXb0rB{L z6MwAT!F;*s3Sm*z5HxTZy7&FU@Og3QyDpVZII~F<u09raCI*Qnmn5V7@Xgfe<XB=A z98V`bH6?|u$wcX87=63Um%N{10K1uiC_Ca5-V%8b`32eV$tw@-Rp&C7j;c|&MRKqx z=^$dtY?9#llzB8%jhr2whoQ>xBr@w0GNtS2=o5petYW%ICFU6n=JkY>2WR1Bw`nlc zvlyJ`2-z%=A#6O>gS-q+r@Q)H$EkwvAb;&H4*Wce4EEVc3w#W5;Zhwi=i)iuE0U~w zI3E*LCxP$Mk9fwrOtdE`ocWolO5Q0pqUvi4^1ygHU7`O177pe^d7~W01bg6o(<C}H zF9VpRGPIz|lkI!$CD4~A$uiS?RC5a=nG@ovOWy;ucAg#WQmT!u6MV^om!lc>us3sA zCkf{Tje_%n3|g_b4SOExMINl3!uZ&p!APy?aA%?plx9C+_^F3j1GSr=ZJ~ij4m@Jh zFRi9?qpPv(aTOSTEn&x9J&o^5eMpJ}uvcu3W3Qw%$htcNl`Ko3*hZ07#4f{wg+1uX z`Ey`R)l~Y82_moNYS5oHZ)%$O+}cZ{Ke;)rGo028t9j!$PW1Do5%Jll2pJyHG=3;U zdgrLXB>yxzT#!H~Kh&hhHIs1J>N~h%KtEEFeHpiFpNDJn`_rleTiLtvAK0vIeMzUG z5Aj(=GYm8BLSN_FlJUwnnAbB5;b7Wql)a<?BQ>rwAHMRj;B`4YE^AK(4ZY}ts_AH> zWkB6S%&6Q3z)oJlFny>xQQKrl`rYkI4%<e8+0+&AXzECk8oq`08Q2}YN7vxprdf2u zY#UhKI|~*yDKL9N)#-wF8jN7}1~Oun8MUqXfZs=%c{~(&vO&@P=zG(3;QL-5Se^?_ zPP`3Y@Bd)cyH`U~|0(qTqgs43>m%bkr3hRqb!g$^ENU=H3${4N(kaagaoWIKS`fFA zs8%e)=TCykHLqlL3hxJaG}iId8<u0(R_+=<rioSWoXP3?x$uG)NXiB@FpqXVWVcSq zgBNPskh<Js7d9m^*UoK&6ODZGq3?ZWYNyRq=kjih{jB8?($b5LJiZe=7cRpin!DN7 zz@2FEb2SO93=~!NxeT{n>(Ypo+HCqZOS0?(i`uhY2(uuVjDF(Es*QgQYRm^%bI~6Q zdYI9cWyKh-bO}7~#IbMto6|gPFS?{4gU&eXNQ=DJ5`%_9rk`;R&e44VTEBS2dt;O+ z$jh0OiDc-_i&t>|mich&XE*vYBAre4wIDNII(ZcCcO{oYH<BX*=Fv9|`rLYBFS=&k z64=*sH*U_~OpWf0Ag6jNQ<stMOwx{h+<5c|=;~U+$LRjFU_d4cW@hk=YYw6M;tc$` zDHsY3F5o6jdGfkXIkRf=9^lQ<1+(-pIR8+V(0e(w%X5D`URS~E_3j%pQ8*i$FIdp> zPW##6FTY^y^*yAw%?WI20J5Xgaj<cG4Pj&{X^Xp!mbb3J!TAhbqEW`Y+R_Q~inZzH zwscbX<_sB}tOk{=71iq@0>>NYQU16LIn>vPG;8XTRUTf@u!KiVo3(Mn!Ys-9Q(W%G zQtqV%8S&@8xBlY%zg~Y3v}DrIMF~`%8v_MZ{efQG_)qDc9cw*35!_@eao{6WdQWzu z$T-K6JjgwQ`jcG9mJKn~_R2`|tgb7$|8XA#4q8Ct@AttHONLl)H>59}meK2>40Wo! zkFArku(z87#oX0cbySz^?(amNEZdC9CpO@5_Ym^Awv_kf%uhW3rVCwQEhL-GmBDOt z80i`!i}t-QFp~|3G8yxyLeK?EdTdn$S!ea0jZfq5zw6G6vI5iDif&J#ZlM|;Xv##} zhKcx+UaP69O2&tubxC}&I^@-U7kLR@!srSW`t?G8a_f)nxabd8W^Iu%HI^Si6@JR& z+>~l2`+ybQ+TuZfUX}6q5zvUGamUG?=PT%dDO*TvqJYeppn|jSWKr{nr_k@$VY=^@ z7abk3l2mN{05S2y$(DQ5;X=k~IOHNjCp1lj4Z*AM*u0fsX6ubT3RA&9!<#Hfyw0nZ zk)t_0FPQWCF%G{{3wxCnu+QLfEVIhQ`*UM?FRRk=mUS)mZ#^lRvR95KusdOGWIDv= z+tI6|4$&_@B6`#AJKlX;#9WG5&Z|AAMaK6W#oQUW66cNXP4qV?Vzz@7eYMF4(_Qso zfBHjQ9$pXH#@%RgOp@qLLy_pMAc&fD>rOq2yW<*et%hHH9{cJW5S{83#9F66rJYub z9uL|LzV}8`CABDaV2KJV_w5AJc;p47Tk~*r*a~`RsRoU<eax<%>__82<)J~9F}~#w zg{N#fO#7J)Z}Jc0>|u{Fspv9{wH!}<4K;9iyc@eIvVvVvu1B02qddkuPsY>xwvnoC z@^t&Hwb0a-2M$kEsN2^xcH^Y=uqZf{y-{0+{(Dz2rAntkVXZG3F^MEFT#<}SuZ7?b znWWJ#TomC_4S`JtSoGz%#}{RDQX9OP+5g}QdL|-XU6Y00GA4Lr_DW`<ttK(4(jg&k zE8&GzXL@u&DSnQ<26q>ArW2mkG8RcG%%CSeH1u^H9c`(^K1$7l^q{G<{K0C_86Y65 zQ(e&WtQD#r9EtIL1~8i#ZY;mVi#7KGG9;o7-hO?_YOORQbGW(hx$HymYWa9Nw|W5) zmg?cb&HCg=kbw4Jhm!;MF5(H#UpVP#Iqr4(fVodn>3F#;F#NEJtv1odd*@x@bbJdA z4tfmEZ5J@byP4H0FvM5&WzheD71+$)jtB3yK~cB4tkJ^?+{mr{7q^svQm0wuMuiT$ z(&+#yj`;>U8U665!X7v<-HK7Fdx}42gt9rWCR3XmlhAx8pYERChs?ix3|#bO$Uxr= z(0EXT-VsYtsmTNLn!3@;TV7-35O>^G?uhPZW5G4spZ2?MNoRVz#{;1$u;|2V>^4!8 zY}ql8tP8o!Ec;<E3ZrZBoyP{A)$5+L;_e$(>6jYX6oYi^vzhG8HD6%iA}0*=X@qab zT5wjkWD?lWz}$U&iIp?=0D0|YWOS(p$#wk>1rN_+WSuoVU-u3g{Y!z3)P*-C+9W<; zJUR}$%e|A~4m~<eAy<FCf^GesiTl)4yaj!z;r6q<i_xpWWT7H;a;?F<8*a4I{bT4_ z_LyDbQbg>O>qJRz=5)d03e^8v59c3Q(W^b*pxK~3&=%W`^3Q7G8bK>B<YgRjkV%Gl z-!x$1@NG2bp%H0;Hm28I2RQ%y3U1==P0p=7X<4&?c&S)Jg+(K*9$CQBv-g;HFa3x~ zPBDy}@EXq4_M=vxb};oO^SQZ^8(sRs2eK~RV>i#$0GHl_$-|5O?)}rW=(%8JdSSu^ z&^)Ba`Y+HU59_yJlE-kmd%`_jemol|t18i*un$s}{bct((cygwTTNaBP9_Q4GHF{< z1jw6|!N)qnTi7!m#?*6b1kp}()M-s-NEQoY?&{GSUPW-VYyjCCcuJH%xERMf*Fu1m zF%^9JC>pTs9@NjghkZ?2ut(4XygvUCrX>~OT5Emm)hU?_v%H9cBLgsE;65z1WoU8U z6}Yx5o>bl%iXZyAG8a{&Vc4ARBs+dF>Mrj_f``RGV4t33U+1&jdlD+N|Hi8jk<@_w z6FsSc^>WCYJRj}6cJjiztYs`N?1IzT3fS0x353*40{zT+BzlTQ&BQ0`XmI8<`nF3~ z92?OeUM#U9N#Q48^f(Rr^k^mw+VlY~rGJ1_r6-tm-XA~A*+hDcbAg`T4~T5#0}scN zXE3I?79&i1(HUAQ(0P5Uhe~V$R;G?;FV%d)r;EOc8snbf@=J!a+3E}9<1!i-tlR;p z*$mT;@JIy5^<LVKHu^o{CGL)a>R&5)uQzM3kGgHgu1jrcqpud6-J``uOuNhcQhEzo zCKs^I>JczI4d__ywan38sl-2f2OA<Yk8aEIhlQ<aSpB4cMy3_O>x43BYx1Nk!U!Gp zYa^b%$I$f=`gH1^In?&W349&22CSL`$@edXs55jR^^VDekG?Bl<<^C;I_^3SbxZ+! zB?gQq0By`Xg6}3J<CgkHn7qZEJV{-SXM*gBj&L!WEtmkMYu#~VlqMZnX+!t+X=4;+ zlZk^?45*G7K;#B=r->?ZU@+}FQ`4<8RX#8j?gabOK3v~9o-`K~@^a|WVXeG6?JnqF z6-2upEQ2Gg58WPROoomeNS0b(2GwIt81-=!wLCnPP7%mM$(tNzrD`v-cBcb1O72B# ztz6k_t?4vl(Of!Ju@~8Q*%h?5j)O~@D|vU`hS4xvBigKRf~Zz?rZ1J%@Je49^4|6} z^HN=x4&cVEUwcF`y{{Tm$KkV)E#C$i^Se^5DGJ1Y@ltkdVFooCYft&p3@~$JH`2D` z5$xX5le8uHQysHxOl(|A&ra=5^mXN!l+~B;d&Xv1K&Oy4`N1gvbu3A=@IWikX1e2; z2Kk{i7@SP28JkBtNK4EEb|Nzk8$#dXn&)yfc4#=nje5vh7Bu6#F`D$ppf2<#H|{O| z;!Sk2CXyXJ;$imrKC~(6B2#znE|_X+v(BYMp>y;)Xc|!ipV}0t<)bGsjNR|y7BGli z(03Ajctnr%SXKtA>ca{1!H`zpnh)pqFX7(1(V$yzCgPnnD(JHz5I!m%AjM;*!wt{1 zU=`xV{xmBR-BvUspRS$9X`Wk|E!EoW(@PFiS8+UOHbg-0AC|O#bsRouRb?GLo#?yP zdeplvOYS`?W_o;zB4Gn#uxH<Uq9A__w)l7>Ud}s(ixqP4=AK$Q<y=>?_GcoIk(VW_ zpH`rvVK+K<dME0jKa0+kdj@tpv&pUt3rJg&fX!HU9OPn?U}R+~tUP-HXM_zV34hqp z0bd&M+l1%1yX+-69%XBWf6IWzJENeJ;(K<eqB`AIGZMGY8vq+ZR?%-a>Os~#gP4j` z=z`wWP<tU;bp7KYm{@7TD-4`TVqcDChuw&Q2a`02Z-gOjtmP4r)h^becnxWKy#};P zmf;}%R%UxkcUrC>L!S1#h1Yb_M2S^q5K-(&#;0h|Wj@vL<G?`1MnN6yo!{V)RRPGi zT`E!@^BtV~M8Px-AB@>3OVZp^aBZzGx#WHy=;Hg#;3b7PD_fV!?~5nHHk6b21_qS} zUBmE~L&*DtB)a0uD`xxt+4S(eIkex?y-3y%g?{Iz!}rPc?CPyiki6msXqXg%>vMZ@ zE6<(wpRES(hkbw<UkB3L$7Fce#emM5n*~lgj@E2{_!gUR3Z0}s7+jC^CHLk|AiEY1 zBK`6+*@5ZiB&)SIO&xQ9DRbEZiI4YzaX=AFIFU}2uV=%H3<5sQepu*dM>c91lBIRc zFz=-?t$k@u(>K1wN6(hQ*PgAS1xIx-$7vXiD)8m`94>{yfkmQ;w|&Wl>^kPw=iSUu zEi+nmS&o~No@ZL3E<>)~QgrykiT1s<6Pr@Dz{dlDR9O(pEG{>s7F(U^^Y|55ayXM! z-mZ#QTv+g5%B>Bb9}Ee(S4G_--N~kuU7|<NJ?M2AOIUW)k1kwT%>F#|9CyAjVQOzJ zrjrc2(kGj^_W|^iAu-F8x9I6h(allGq+8^DY817As!<hMcj+l?o6{;9KkhZSJsQQN zxKu*Rl&(a<Ya*tb8c{=wzU1BU�m~2S&XQfXKW)#E5%haCU$(ZPY&M;W5Ao2h}4O z2X|#=@6<r!*XQtv%zkF-{jb<;+!s&PFULs<YQ()&2|q?>(3T4Tm-!pf`dDXf?O2w2 zt=Pg+5AGe{wMRug8froP@K$#1j-R09`5eca7qI!KzCiZr;n3OdBg*KMu`9Q`GdC?p zkj3lrh>Y<;yiyZLUk;b2bE8h-$R(C^lu380>0yT1Z+F3s1T)%i$!F%yL2a7$<OulA zO`_xPCDPBE=Au%HJgwvpq4IBls@YYEu7qvF%EaR=L?)8+-=ml?g#fzr{t((IF9*vC z29Y%*9<Wz4db5L)+{mq2$;6s_CR(*spVT?5r%ykWfaL~1;#0pA!r$${4cqIPD+j}{ ze%=CR?vggf&CZ+LYMut`U5=r`dUaT_D7!}Qg&&pK-;;i=mM3NL$8l~iInwF*MH;GP zMh98nM){K2?%S+9=}!L}pp>5hN^Q%qSA!PWxM>X;*?$wSVT_Q3#&k#i-aAY;`SHxm z>sqv^6(A<)lE;WC*#y<($mMOfL9KEz+?r7Z!(^wD^T|7~+-?h5yV#7#?BL${=$#31 zvlc=i_xzy3q?mr-x4@jHTIQg>J!VZ(XAcd04l?RK5TVq-MzJbrbz?cbx~(^@=B;E8 zU46m(apES<J|d6J8vZ!=b5BStm_m%LW{}Z09BH<ED3<0`Aun(elfG*MKD%yAe(up@ zAB@^ewx-_1a?R27sYVS`Ty9B1_^RA|)(YPB+JLiHWs{`7!*KjU9b!;&28$~BY+|T3 zY0jPjAMqZ}UUrUMeY_s7zT<Jv=C-2Bsz<oqSAmwV)}t#XPZ15$pGddrwK2y}|3FiZ zU2x8P5nXqo6WQ5AjwpUB6kRS&CUG8#ko<}={%j8t8RmuSr>_QMGcBHt#Y1>toQy`U z+aO}77OkJo(6TH$V)*6}?yQf(s*6Bfqd()X*ipDp--}fC4y5<LwV>m{bL>X<Koo`g zvGSV_;5F8q^ncHde@0}(gV<3Ze*eSh`V-mz+4_Ua|9bs_TlM%q{S*6N__gdrB4>s8 z^WR&4asB_?f4Bd&|1*a51HB*mthHUH$W^Nq6@9Lu=`b1h@7teY+k!wEy?Y<~ZuBM? zG~AYCTki)S-zUsQ?*uY>!yS+tSjfoA-Nhq&^2vkw$)tdFrK_&9xNE>9kyf!0_fB&m zT<kJ}j<+5`ZYRqV*-{<Rq3!adZMz?lPwoo&y%cEQL1U0x^#Yycm)VOyG|7ws`S_!7 zC-1?Qg}7L|KOOyDkILHBK<?nzcyfGKYOlJORhi*H``qT<^Vj>1%_Bz;&vU2nt;u-0 ze*RnLYtC1^J)#`To3F87d#s1ztnRQUVHQpa(j{}Nl9)xO`?-(K7!0qQL>Q69huX_+ zFj&crloa)&XH0b<G{c!XtDOY*o-$O`*pDQypN11REQk1rKGa6uj*i@Q95S7?kAAA3 z$lAJbLzN{uw6-||Qx?R~9-~^oKw%q&b@ixwp3mt6h~>H6cw=%roYHlo>(VUAmXIzu zFT;xHS9oFPu14h8`Z#9y!zy;o^9jV-)d!1Wd!hQueq`!MW%gc}CULxyO#A06fz^&Y ztiF~;WX-df+s_J#;ExS3S<`{-{$>HuyElUs2qGZ+d=L(O(1-lkXNe7U5Af6}d3;f? zNAGJnQmrLcwCed9HqUDzsq)gKPh%}na5M^!`rKe@jvoXk)m)M@xd9^kx8TI<!*O`T zQ|$e=0Oo&IrYfZwuw7AvZXaCf+^%IuJ`S}dMhkW!TvsL0=^G(qmLHx|I17tfqj?i% zOeEz;lj*90F0{~TINNeKiJDEQ!7k}$w0u)GtJ`!CR^8Z-&vqpdbax_|ua=W8qB5K; zN9ZSoo3MCa5T3deO=eYS0BNwrnRB~<-`-z1;CwUBc0oMp*(-w{9=j69JyoZR<w}{& z9~Eg)<!-cI{FME8+7>Em65+L0Jp{!*N9%1b*pR-m#J<m6W_r#{GNE`1?$cXFPmMN( z9<&FXPaTCz>n@Szh$C2JSdK@_Oh}1y4hrR~aAL1QR`o_Tj%YkyV>{_498Q`?V}Bj0 z8FI509n|L17FT<`RZ~FfwZAjp{FdX^dHdj}t0gYlqX%!6KZZdo%W#YvH-E?)N>gi# z;8@ybl3afuPbv&1p~czo_I)Ugy<<X>^Ph{-&J;nzJqIdtH4`t^|G>Rf+F*EdE;;h( z2by>P3MTuX!q?rS*<*)qWAA+mjKD*YS{W+Ruy3Z^HET#JGP(QJ(K@)f`vmi8?mP62 zPec=|c<$Lx0TXWCmApH}{XV2H7f&zoB&qkZ>6+TZs9$VByJHIm2ItYw%bnP1UXR&M z-?jjONcxo*$Q<f>8Sjow#+?@?(SSSas8W>;E;?O<+#)4BpLZ5B`kPXPqBC&+TZU-$ z^LbQbjwMF#o<ibwEGCUbcX2{Y5WZVvO=UWlz`*KMdMfT0+zT5-nWtNzm0LUO_x%Mc z=Qs|N8N}tzveYU=78UO<h8gdM(36%?G-NCb0Uu2<<i|!b;NlB5b8{5;?rbWt<DSo{ zRjZNmqU&t;vHReu<`uFum3s$f<bCKde<?PP$e?M>JK(gpE6siT1#)qX$fC{|Z<~3+ z`4C;Si@eP}8&+WkpHiT=Y;)Liql<9@nv!sJWys+TCxa`RU{qrxk*O}ByAJ=vx0^jl zmqb}Gi0VS>a#pe0_HP)K&r4zIy$#@J>`iNos(6i59uq1Q@Z0NEICfklWxc}Lr(1Qf zNV6BKF<+kboU)zy)nrEUSKmhEo5rL*Z4_-()y2^+PQbnHjor4+q%%I~;g!pJWKD@1 zUE6iO$YlO?=>9GW*Y0*hrREZxRaAw_C7R46?)Nq-c4u(u<WQ39HVq#Q`V5NiM-y9R z6{G{+gL`8Ne$Ba#r7(hy37yA;+r31a&P%A-Wi>i4yag-qB&eS|jNudH(f!#0hD|?# zt#PZE!t@q!ziSP=*Pig?-A+u)7{q!H)j{)=e7vCZ0|Sj#;+hL0*4#1;W=`l#&gS<Z zuf88=!)7z&+o>!Fj_*su+#e$>y97U)zw#DaaPQ@5dr;r5C6Kb-gB0lvp_E(0h&SHL z?i|0Ie#>1zO%1j%r$t$m-Lsq7GqaAZ^NE5i?)~lA6B0@4BTb@RoCAaQ+A$8BzJTA@ zskGYlI9e{$p>5-?Fg=1?NPpKh^p)*In=dAlBksQ-W}E`OGGGF`aK{SvsoE*Hut$wJ zA1VUPiN}~XRw?A;@M>ni+ev(YpP0MOL&<<bZ&DYZPZcKml9!J05OZWTgw;%E&C5-x z&tzwIZD$KQj9X7WXt)Dxo68uRbRqPxodWevdoe}p1N_l7m-(1`o|kMd02$ANFl-|w z!xUGdXC8yP<)cZw`Xfg7R&P3Z*nTqc#VQ;c-V^H;rm^Sm=-_!fCv0s+d|EsJZ+7iU zhW9_|e#&Sy&AHQsOsqB}!k78@H2fmC&M0N;<#(g~Cl|P8yazj*+v2H7L3n;iFs%0~ zVMD)~k)Ey<M0D024kk|nqpzp1u9Gc&!`A~P!*bE-W;Hr1@iDBu&;6!rLod<!O~<fc z=?n0C8&6G!KISD%=*hHwHUghuJsO>93(i9|$=#+*cG?R8`xxbkcI-x6zG4Mv-LRwM zyRIWsWKQ6?!|Rw6^F0{3rU@YES1)RIxQ@LBN8(iebB~l2r`bOaTtwv?U-0@OOX^w4 z1C=i7^p2$qjR=@WlUP?6GS-9+!YXK6I)FZzh<JXH7NYz?MtHXnDt{8}-FX`bx!*5( zpFD$B9}PqwkB5TNsAv>T9R%*b?!fp0OY-4e5at$Uvaepo;}OAJy6(teXgJ=Jt_l0f z$Z0MkBbF#|>+xM_|2;j(P?<ky(6dL_w@+V&$r(o9=G#)I#<gV7hF6edZi)gNO6C{D zlby0!<V8=Qx*oTfPL;Ye(<Yk?WJ}n~;a1FTp*A^{vW(aE?G$_RRRF!<mI3E25cR&s zg43<lAk5rNeDBNBn@Mx<y5dou!@@$$jP6E~1}M?ja<1fAMLKBC9!|%f>qVR@FOqAv z;kap@8JW^)IDK(ZndmQ+p$XnMJdRBI1WscWNuy#Y9AeH8`y<&T;mZ~@u6)lNc)OMU z(ZiIgbyp+3`<w)Dox<D<djpf?H;}_q!oW`TK0MhLiwht5(3EpUs8;2Rum4CU+PgHV zntwBBzveM^jxMBIxCYI!@}Q47+q@S2!5rrIg3YhbvZ^19K>m^sz8|p@|KQd$9f#g# zzgQoGZo^)|ce_V$B&7}>Pi|p8r`l8D^;|ex{g$_6^mF*Cvx1%YycjK*mykR|my|Cl zg6|JqiOa4^@?+w1y!H78)HS?9Ca;9)PD=4*OR$GU#8T)id<qLJmEc>wH7_ROHqUyS z5v>^VLgd!(Eq;vKhmXAn!J*n*=HxOSbxBbn`4;aW_q;I;<kq&%-8uk@eRo2Vh9&MD z;17@b-9(dKDRh_M7YMg%QFGO6Ou}n5`q5ewJnv5@4QUi2uiD^%mtWxR&z<;dj1FE| zP>YLoYUt?1qcFF&flaNr4)$XYp-$5tQR-V$>XN~YYwfzyKXfM$&F%{zRAC-?mezyY zc7IyfyI8bvzBAnMYi9Zz)Ih?0N1Xa`BCZ*@3T0Xs(8Ggfz`SX1(0#HS+-JP0^_#QI z!AU*QtSl7|mgUhkg`cq3-A(KoLnj&n24ve>Yu@ra71C?tRTLE{a6dFE!ZE#_ndWyY z=)`ma=YuO?(k@kuUPvi!+lt)J15wa;m^W|8BQU!#3$EFgfv|Ho8hNi1>8y8_jMW~2 z>bDYDr_I?S-j|z<jj1IrTx>+IZ0!yM8!f4E%PgQvgCO5d4(FRqM=kSr_-xKT(J-0s z0L8Mju&Ek!lSV=J=($*-vze!HNuMs!T*r(xV#rV5iMUyN2zhSQiO2{ZFe5|1;U%wJ z9Czv*9ISPuUN;)CzG4@<=RyqaJ#jnlW4sAC`ddP$sMRFjWGXf4YDQMu6p~xhR2iqA zquCU~BUk?)cW)j|WgPW=izXyPrjkk_Bo!*0Ywtu<h(rUWNoiD+=8`EYQ-~x(D3zk9 zh;!{-X{LG5JkUskCe7)&?)$&zdGGgm-}T&oylcJIvix_}xt!}<zrDZv`~B?X25P@7 zoO&J9M(qi{T+~;CV*{2_MOcZHU5-M-tWa!qKZTX=hY1?PPl;LW3+c~=K%s2JCQkdS z!+pPvh0)@D;k!=>1fSjuHc^@4{DZy48)k1I;BgiF+pmh_#EVqzYC}n16wuQBG)0{| z42vdp#S8mHZuR~uR7_bS94q)kCL4P5*Co!-kZ}S^4{WB!A-lyxT6TQM`?2WbcnCi_ zSxH*NSr{EYfo|#DrfZw6M5k2gbx!$mzw}|^D3#svFj#`7`#Z9o)=t^r(#@>!Rs+9i z8Q@^E0t(AB<>I?jsQ%$8$VlqXKA|@#%vp<jPOpJ7wFs6kR^qD8uGlRo40l>Qp*4}3 z#GgCEk{xcK{X>nbvmQc1<2{(EKan=awZquHyQ$`uxzMMN5^OnRhTYWjVa0+ol)LU6 z3=BF-m7D=4oezlf{LYIBkJP2JcrTqjun9Wfn;_nu8iM;=uL$cElB<6CN5R{}FX6aU zdv4u3kd|(HD73_?lj7V@)o~_HtUSdFy;e+yudT+|p<p14S=o+n4DjUw4}UuH%N13- zMRKmbhkUC2UCNjcN-JJXrVS@DW!qnA^3Y|eyw=JXyWD$DtJ|C@zQvF`MFVG-pQr78 zuM5r{nyE=Onc`}@acb2ea#5beUTbD@_~uivV99gZ{c03`_MJ_>-AA$Ru9sr@EMv|X zJ5zX~{}d8#kHrG(aiAeRKS~Q$Vdr=I;ZQ_2gxQ<$*QzY)kGgo^r5~=)*2NObuQ2mR z6g(LIfsA^d5q)nuL14}aK|acuC)K9G$E}7Cx>>-4i`wY&!V+hB4hGvZ+W2X~J;5a4 zIuz^K(oOEm{Ra-E((H5iF>4He|G1mNRffU1&MLSd`xh)&mkav+-K)31`Uel^xuIfM zEGy+ahbrB#VCOy-n-*;04$8W)|L0fU8|Xj^ou}fCsa?7AVyQ>o_nh2RjbYBBQYgDz zD`b51!1`h*D9_6l8@rB@UAP)6YjT#;d%L5!@}&{0KaxY$@)-DieYT+a)CzAdDi^{| z2J`e+8qjcg6jwg2f=3~b#JYdB_Qu0aFqgK_-urc8hN?ZDU8Ilyp}&N>LD~PA{$&5( z)*k|{JJC&pYof41gPoeAc~STcD!iwG(JK^T(!af|vs+$0q?bLtQhF_qzS4ykW)0<$ zciOQ>Wk;?UpvJ}5Bk7ydW0>c6iqwtv(xM~57}Gvk%E1JKZ~0HzHiO9=ADBsfRz~s1 z7T~V2LpZCf19m=BAs$R}=0z5xIjJn1%Wa>?Za&aq`JyTuvENv@-RY|^Cu292o)vIG zodR!iTu-YrM&i@rHIUq`kQ9v#d2`V?SnHAmxei^cABP+y_aJLgDb5DZS1R!O^kbm$ z`w2ul7vQhz3@&kc4fXG>aC%z;t(=!g4VL%loXtPc=dY`<w##^mzHy8fSmuNAyGOzg z&5P2R`w7neT?Kw0_1Q{o7{1<mjIzF;7ZX#Ju<Oy@nEK%!tX${JeVe6>r=l5X)UCkd zYrBCwYbkuR&!nM)k<(Qkf?<f4%=YwEdR6zEmMR~GLszo6UETp2=w~hL->iq_>paly z$r!$Rr9!CnG@#DM>|}K-KGWR-M`7o<3-s=o4MhcbA^N*P`Yc;<hJCtlcv^3?o7agU z^9XG=NCBVbSi0P0G3(my5yw8Xq9s~J=+-xlu8)p^lUi2LyRn9kJyK^`tdy^@n2lfi z_U7+-KSkw7PVl>_7|abn(We|&j#{gSHO-dvr^jA+(UyykT{k*hb$dc*-cG=OdoRd; zw;v;rX(C*R8$xx_w;=J%MOu_;Ooa!QK+IVk*f>5M{NMf%rYr2C74<fJ@XI#nIoOQ4 z3pTuA!&xzG)=6=Z(Oj}QA#pd{Cc(nwhvM^K2drMHf(i9WY<yg*V=9uNUabdRHBBb> z`JUhplE>hIT>h|RH@Vy$1p_55JpRjSE<W>4Smdumlu}De<C^GQZZH=A+bw<=`-ooq zUV(p=9=vSTBf2+oEr(Z|@Regh@bq%4*uMKKVahLkyzEvgu5GG^$$2+nwTB-4*#FYO zFru3FEVxABSDpC+?SVH-3+dEjOF^S+si>k>fRn$Mh|X%gc=eS~l%*=MO|}|jqz{$v z?o~ijS0=-Rh03JTE}y=aTGG_OcyP>rEu6ma6&hx%f?uTzhHg1bogHm4rkf7CXvFdD z4fV9WX9r9?T1ar=x-{=RF4Ir##5-P$B9#|w>8RxQnA#jg?KUmKUTx{ZHTOF*&6j&A zzIg>*oTvns9;k3)q7&G>Ed;+;=~T2gACtU0V6U4M6tf|gQ^PzlTQ(T74=SRK<Qc1( z(Fk_R8JvIhA)5xQWmo@F@aUL`V?Jo&g6Rvy2{GluO2whLY;d8lR5lum9I9m+0pT<@ zZ3)e9|A7{_4`ciBQ)J72yo5KmNO-lp6fS>wD{CCm6^Dix;;7Paykhi(X4{P6OKsA# ztx=H&X)Pz61>yYR(Fm+5??D4DX@c?d0Pbd{BwO(XNvydGcfHc+ig1WdTrp#fuV+cU z^AT!aKZDxV{sKwr6P6F2Lm4~vQC&v1*g8t2cbO!d^hjX4cm1*MeiMD|6~cG-9iW0; zQBr25mRwv6DJ^*eq@K*<(T|P_J0$I`ro1mL2nphbUFCTHb(}cJ<UE-#ErFB(9eT2+ z2YY>gNIOh(!9whXArou`<=NTv;z9{oWM&EO`}EK>`Viy^U*PFx2R^>ok$=0+px)=w z$#AGSMsCg_yKnJAb;?Z8eZLnHyQ+bWOba$l`A8};88GvAB2K;T$|a6NF;j09d0+Iw z>5XUPeB6pJ{%VGrrYPQ@bU=3MY$YwX*&`MY{7fs7calYs3b~*264ha>L&gyoNbUMq z?z#6k<Vl&Fkx~Y%>-GTvZd$1F+LH~+%V5*9xq^jnfiTvz7@p@}gk#y;WudPx!Mybb z+~Jgm`29^+u65JH!G2ZZchJT3zgC#D-WBaOzmuDJoyOn2Qz*o~0&>k##3sjswCL+g zP;-Azvdk>*f0N+M&_X<L-3jLO7|r8+H9$439d;0uSZCx&^cy#vW(5_&8)@!#Z<RM5 zj+{Xk=U8ym_+asEU?qLF8O%xZs_{noJP5y~087L#@ODco>13KvZ@DTL+?|f{S<85F zh!>6NyOJhFIPub^FzWq>G3!M-b)2wJ&?@RKh7TM|i^eur->$y}*GC4jUtK*+-?o5r z&-a4db?3=<|91Hb?n+MvcnLmD&*;O(gMcR1oLH_0)3u|-bC;HqdcsrTg=Z>9_d7rz zo1<X5)c=3qHi=q(DY2GMcXsI69X-q&>7s`lw_NKYB(LkpYhFcj2~Vbz&gbZA&pR+O zPYsQ?J`rvgbjPp0SLtYJsp#L<i*z5y^5*ntaaH#_aBj|0ymzA=8g|wZey<zGX2aU( z$BXMUvWqW;Md)Dj0TnLoZNbqOEU>ea2e`zahn&BiFtyuh);=GJ23{`&D@P^jnK>SJ zJ2>)%kzeT8Iy*G1O@!{jBKz+dN0Gfpf^oDOdJg#^bS$3$%O5-Qwr^^jpE{06^1|r# z4+MX$#hCv<ldttTBKT<+$qpWpq51A`u6XZ4L8B{3KPUzEwiIB?<DRTru@A1_R${yD z`qa2%Bkd~>fptr*vHt)Iwj2DA+V&-rMfaC*D>_Lyn;l8%KYLPY9KmdlF0%G#lBNG& z0loL`EI#_4A?REQfuNTPI3wl=6%5N0>)R(oo^C0teYKak5bfCDmIc^{#*$h=f;g+9 z5LAEOr3a;v!lmqX@I|6*gpK+uM464jcgtRg9;(XJcTEIj-8RH+0Zu}&ks|g{7qClz ziHVaW<@$~)!;q8KENjWa{Mb?qo00^V$L^+`!~Tg0x@q9syowjrH^R?T196wh0ZhJd z6Pz~6N&dJUY+17dU!{(NPxIa3vfCawerz?3EI13lBtKP+G(#S}d^UWpZG(S##WYM} zuUs2ApZ?uFN25C`$yIiRlGccxqH_LhP7gHZJ&o<@&7~jYw!sDCJwv!Mb|HUzq|AHl zUr|o^4dL&wc6@DaGsU^x6~5MV;16Z@X!DaM@zq36RMQKBoStQ5tePP1`5j8$%OiQ& zm_hWWtd4fh-v<*_1g`Y8;JRf`sqOS^-W!pC)%QAZ=Mno@)!Pnd-<9fK!%6&L)o(VN zI+SOo8%UYwY6>2-5)~Y0a+{x_&}=)MXH3{G1e*;fXNUPTSE)Uf7Ds_{q~zb7`+~IX zZi8*lKA2RY%1Y+3k|)fZ%O+RIYIp6XLHn&>l!-3;Cimc@@11C|!BwhD-vo!pZ^5{Z z5twhjm{zs)5R3eN(bR4l_^Dz#o(Pe2@Adgmo4QwYk5grZ_tSZJ<4p>5h-Jf@Cgjsu zVs2P@(TP__aLHt6*pS>FlGZ<hz^pXk`f6>yr0`pqsnQkNcO=@qt22K#ucxOMm$2ve zwRmI83+mDViElEw=lz!Nt88}2ak3Hpv#Vk1t6|(HwgYxuc}+Cj8H{#|hX?_|BV`qj zEQOi<R$@uWGBNt+ZMy7xOx&9uLE(x!X!+b?2pOP{C66pAa&9iq2rY+2$#cQueGsZx zK9$eeUJOUawL;!P88-a5EcP6siMr!vQ{2lKoUiA{!Dnl!?6<dYF6(ObmF4l!e(6Yl z5ncrK+Be{lb_Vt@JSd%KQz)nQ2OJzeiAzk5(W=d&EZy#gIPKRp-e}Pd#Xk|a_ETn6 z_`tn_`^QZr80lhE_kNIf$OShp*iL`aBH)MXT^e!3M%?LFQf(S<Mx!SGhQqBc;@T<m z_;2VXIQi}~Xzth!k(sB2KVOSzUi;5*rfdvaEoza?N<Tuu^9Pew;6Yfk(?WFHISGRY z^x-bG+L)W$k)nL2;-zzW^t8%H7U8-dd^g48u@kWvv)~#jDejUydm@#2f2$UXZbQm9 zU0(P{8SB+Iz}>zBX;lX=Or4uRw>Ex;Rg}*Y`uXsl0k7z9uc2I&_ylY{E8+aqZ=%o9 zBz`*QF63m+p-Z2;;e)>Zc%t7Mx-eSc;}_**mac~O^}onqUnKa_dj7j`75$hW#~-i7 ziZ2>^VdweBWCy<Pfzu=Q!L`Un@zfJ_R;@Lpk2USNTgP}>rQ^eNBuFfOe3{Hsl3;jE zgP0Sb#y68RFn*ynEgj^9e^X6hN46s<S!;3BsfohCcYgF%%52mv9)>@qx@vN6Z_Y~7 zM2!YTZtu8Pc;m7TjHKLhj<O~$53B~Q?Fpch_Fk&rtaxLw7OotYBP9FZfK!RppegB& zk0)!Qtp7zZAifX=wWz`CFD7{4-5fODd5R)BYQp}@n?%3xP}+0!ym;})ecGa@!pA@D z6bBE@0q<A`JmeZIG&&pckdJ$4<ChFnjOt6lZ~Y)ty**Bp>G6zLnZn5aEVIg)$Y0Tx z4izZCnO`%>X59s}{5TIk#%M5Y@qsd(B2jB=B}sTojQk)MR(NlbsqPQ3H+3EdqT>fK z$l?#!ANoWgpOk2#U8@jsS~@3eR*?Ra+2DO$>M45MhW76Ua%}!vIBES47Hcnp!kx#U zONR)I3w}pxZmap?{U0znvNOI2`w3>rQ^jd-R^Y=!|G?l=E8L$oS?YUA$$L&Id1bZ3 z7f0X19mT~$Z`TqIyIM;|(!4tGX(!BCAYk=4S8{dpBqgnWycj&;L7z}6Y}-%culM77 zKHJD<;wiCTjz5Qm48kqbO?X+qfjF_KRn~p$d=Bn7iYILw%!`kY!7<K5_-R@i95p{D zm_)CHHjQ4e@cu<wzpes?&wU7b(wz49AvZq0WhSPyyMcN(#jtzOdvMM7r=pgnuzF9d zSh~&}-TWHlbN5N}jyhX>n&^z<a-V=}>29%8|DW(>MPE9*<Q1*_dx_4hw}Zn;6*9lF zo0N5F1GrTtqE+@7j=N<;b&=1Z=FC#If3TQ0PN)|{CCzRQ?!=ErdZN+6Ix1Y!6XEV0 z2#?LC>p`7H9BlRy7IoW9VU^SI==T6_aLN$YA5Ih!5}V=7@_c!!eJGUOOcx*j(BQmt z&#*T%(h=)S!7Z~7D+V3Fh4<@7*+)A1-$^;M_%=E&B!S!112kw#0_<1rkLT`;#TeYg z(|dV{zjqe?r~l>otkKT@&-#Owg5&>{KP90rh5r_SO0t~%fBN(P-v9DnUcUt3`v2qg z|0j*?fB9Vg%j=f_K>xk#|G)PeX-T)_|FZvY&=3i@^FQ-zCD+ow|M(0a>_0bL!WSw0 z$M65{HUB4q?0?c*he!~c|8suqnf|W@Pn{}ZO0c^Soukd4-YyoVc6<#x4NikcxedEU z458gGQ>nj6938bWmsm3?TpDJD(Yr<>u5#n6MG`wKUIv}_q{Fc3L*RJ)CUmyg3NzdF zI5BYYOc=BEH{6uwjG_4z!t!%tp+Um{Z}yAkT_115Cas~IQ9KYsep$hTJvp%ch?el6 z!iG2R`a)d}goC}e4c3*m!OGmrU{=!z(HB3HLA)lnI`<H4Ka9hIn!U1N!QJW9;calw zbfs)=rz_%zqwccw4)LOy!vrz$p&uG+f2Cl}XooqM4aGY3ZQ_soa*CK0C%owC!#`Jd zh51tEagm`a&+IjZKSik09Qhw1SGgRnE%wHn>JQ<)R4-H%Y{62C2heg>RhHYS18#rR z2IG(Zqo)xe6q=;R@s9Jcdu}Z`ESf5qOLP4iPi<MsO&=Wb<D)#-J07B<mvi^#L{xFC zguAm(LtE>9;JK$o$5jJyYUOw8x!#$zyGmY@zT4oyxp*u()C$U}<Ei}A8JhJx7zRH7 zLwjdL)47%FA+=WG%{uKATJFoqt+@qa6<Z*C5@F|Kd1A}%t<ZbuLhe?w56_>euFjs^ zmCal?%6;|E2+_(};;&97I8>T<2f1H^T`u#f(ES6<m=}VX>vzJcnd52ZZw1yq)CDwG zexMnzjl^zItKdvZ4OI`;Ma^Cs7`i419`rgX#Dsz@+S>{ZbyVTWDKlJGyNN&8#Z>>1 zorXP+b`Y<eO}Yb9xF;5n!A&E0eR?Xlc)GEZS0?_OpvMw%049lV;Ce_LG~VehjD8&_ z-<wq~)qrW@L$meVq51*rK6|Qq?5}v(I9CIw=WQgbsY5YqmogTgGGc`}hN#$m1Khc> zU!0pc6!bKj#0J|W?C`h&7BsKp%du{7e8&_N)i%S>PbZ=Ks5US@-7NT98?$QQKwRSK zDoeS4i`KpC#50G@;<XQ7(2$yrw0^tD3ob^o(+g!xKbS=!5si>P=ALM`=O+62m+{ZT z%G5nIm|Myc@!2TA+wU4_YeFw{KQM&jmnVzveUd=WRD&x=8DepEC_YJ@1DAu2la03I zSFv)T^AQYVEsN>&7FXe0pVyH1=rc@;^5FW!6k3v@!{t-bq2y;WD{UymL(ATfuKi@L zQBjaA5yrTB=?!w9s=^CD>0!S<sW|=NDBk}w0Gk&)Bb~+=?)243oY!^?R#u*-&lQfm zv+HM>@}eEoCB?F;wR)Sd@wJp=8a)V7Zf=veohag4gW@<k<f`~B&jZX8GezZj!E9?; z#->Z#;G}Us@_P^u=|k;k<Nh<?`|_~ZMKKj7*6xJ3Qx@bHb5VFuHHvrb+YDAK)M<j- zZ?HET4&P`Z(ytFNtb8?XJf6>ADiJ%2DgVI-LF>UK9Jx>%=8PRiv+oD+)8aUC%rXG; z8dJV+yondy%V!hQTh#OWM2vAzME6Mrq_yr7oZO$sud40~M~@BWC+03ZG4r_aK*JT5 zoIMJPj|FziS;!Z3tVEC3!{NoI+rp>;&iv%`1F`ULdsLNgBGZAx>2&iO!7}*|XfH6} z=VK4U)bBZL*7BGR<Q9vge%NE=@hZq0t<S?|XrZ^1A-mslg!gt&g*$cqxc@H;{?s7~ zu1p`pgV&sf(Onj>_`rcRuk;sO7M4p~*0XeEeFKkCzRRI0f9T}Wa}sNJB24&rkop-k zi8l&Q(Y8ww7-}c69e5D-bxXkomcIP;0da6fPrOy(0^wugpm<@um^;<Tu6msj&3ZbU z1E<IEi=+c`r!_-_Yb)nsW6~dSoxTZvYS4k6&*!1HiX|KQtbsJUdo;PP85r&C!4D;W zfcqyocqGq&>6PA4kwjIizj*S1{LQfEd~aGcWj%L2Z%#>xY1mGEDOMFJ;msa1#hm3b zUVM3p5One?-Tz}prH>!NvQlGAX>kPY<>`WB_5yHUy#%&~R+4{@dU77Cil1*teZ|>T zG;-=?SfMIImNE-kjjyO<>u@-9*M%>nU7`M(Bj9&TDZ~c75hv^_Cqw%>xGSVle%4w( zpEHGvI%rbENi|-%K8JfLFMuOPa%w&^f^UAf2IDn1U|mdSJW`TLxwk{`@27b*VO6&9 zMpIjCP<lh(=9P*0pSO!!)8fQMIw!zJy}fW!x1H=-qM>kUbQ?Vm(`Ugp9g3`{h+%#f zG)GT^m+jERZ2yg@J=~eq(#F8eZ9Vayf4UHEZwhV0nq`|*Tq*FmGUq<M%~w8Hz{2m2 z9Jw+@`1^i1FOpaZ?Oz@wB|~K#5>+keuFOL#Z4><6Ne6HI(ZkDjrE-PE4*W@528mnK zX!!=<e<|IC#%(3ykbZ^YD$7oS^YCY&V`%{=2Fw@3Tv};E=^BXeSPliYNe~#(j(ffD zMz4?5iRq&c$oC6YykXdB{P@@nThHt8&w<@Zf&^p6hNCC^!V1YmVhp^k&}7Sn-7r1u zvHXlv1V_cFK>yO!xNGhzb}t%)*qSGEUD8Qd)v=U*)UJ}gb4pw{s|KoaUck7|dDKg` zmg>zD`CDcq%<!xe)^EuWBHo-L+meA`q%%tV`d|XRtM0*{HIn%CUKL&xUqlssT*X%w zK5TFD6>b#w#W_>Daos~X#H~6ht~>f1j8Z-Db(03a=%vl=y>7tkAQjj=t{t4%T#8@E zU80l8M$qok9{S#~fR_E7Eu^Nl$I>a1XLge%MkZ^cYQ|x4_ZkzD_jBNX*HqaxMI8?( z>0tGbAnETf%2&u_y`dKP-TD*6CL6G0&^*DUt1>Fq4}*ml`{A`xeZHrl#FYz22xVV7 zi#>|m=-I4#a5*`jpH3;G^DRTELR^UEpXcH@a)e9rYB*QCkDgK)?>^TIe_ARbPc234 z=CTB*c5vpy9<M;OEP>sYd-=?p4MOZ5FKl|JO6kW3)2<$yaMq7ZaYe{oVd<0{*_G{< ze78A<ie`7<<N*cHdf+#W_B7+`2Zzbp{dp<e-Ijund*x&QxFJH{!V^^e_X>qn#t9l9 zi@9P@Bwtp(0bUb>abaO!TsSU)r%$gHMn?CCd%ZQV;r<{L7uaF&hgM<gARY96uZ`C_ zjTcXDkn)|LYW!~;L5}dAYA;@g3snZNR<4VkybbWx@B%7(q>nc@9+uy=dnwkj4%WTD zCwGz9bytpkgu={`q!Z@ELsqs5Wx0B&u%dyS`&@$litVxM%XA)fqy);$I%B2YQEJwo zkG3-&I9$#yg;5g}!H=Wx_0INucg!mGU3XG=Rd>fhQ{uOL?Xn-{Nc@;0O%v+VO#%HX zr}DeKTZGfkI&sn4X5n192OoOv&vq9*;O@U`&{OX-v`&%E<;RiyR7Hdn7o}{`zb)AN z(P6si?!f-LGI6e{<Z<-AFHAoBTUg{Jz0apNC8pH?EM0L6{47j_Q;)9Dr!@wsTD(%8 z7jDddpZ8@QT`Shvbqm(g7vR_^vG}JnKT3(>4YkEIF8q+(c0yO^dTb0jnjPl|3j=J? ze-F{GuEXn)dvx&GJ8|tdeLgZS61^NR$g18A<@e*gxmlw(eTj7C)!XyQBw_$}3<rK* zI~y-gvqyg}fjy1!6mlyH7OV)w(O!cg)qg76s*0pocA9>K64dS~r3=G<((H4wFrZwd zdABFPJS_$2aQZFm-BSk*4e7$qo=?Ox>1MP+R|EBnv*=UCHh6L=L%4M*1rp|dfaOk} z=yB4YB~K1k-_RkSYZ*dDkuEfzw!>8mB4CQjK`Jd#!H~YQA!OlZ(&3djZ{j|hRuoD< z<|%`c+=IuRp2ar~#fXW&Qu*ywYyRk+Pk}?9(cs7$80M^rYcfLQmu>|^(*t8To16>T z^LwHHH6x6hkmoS%@gZ?*su?d;O{Dtz7-9JJIKf9v;MS`hFl+EDDOab9cWw<}r!zWu z@V+0JUUL=41kFO<pcs&M5U@yEArO@6NS)nMDB;IH$lcZ|<WCubsj07EU21*xcLg_m z`6x&j*I^%6uQL;M!X}8dEiYko_ZH!Vat$o{X@QVxjdvtH_1eEu8XjtcYhrw`dBs3h zJlO(&dtDSY7LLQpqEL9>Z>P-CGLNhJZk6kwe<QR=4zevR&Yb&hCOAl}ME}a6Rbwn= z82VR(n`-L8DdH1_cQ`L3dlk^R7aRF~P&5vhu7b9UK2W4-EGpSW3T9WQf_l>ideU73 zV^q5E`^be@IC44|cRfm$ZzDne@~iM+>KF(y3}7_>K?g?l5=tf~!xIBJsH!dm&9vP- z&8L~zu+ZT~;{XnxY>ziJ&QRl*dfC<P=CnipTD+wCNxW(h&FayK;8j~fBYKSB>yp2) z24B<sX-BF2mL-O@Gr~pLJ^5{CW1RDN0UjMAG41~>z+Ow|^H_(M;^Xb%95Z&eVD`Q% zSJz5xmy>gZ?^&HN_jm~mb{|GJwN!DE>T0q0!E)*QIe?lbs0!zTBoB{E1tjb{B37T1 zm@Xl5p?$_!3b=Bdz7ATzKRR#4gyB`Rr8p9&zRQ7!^+jac-ke6<IYFy0PlxM1b>y4v zA{y%k;f>jyIqLB~2$^_ORFT=U>Th%WJf|H#v?_*EH6=7rqa!Ei&!g)X-qNjxMpBpj ze><hKs_Wta*^kN^)E_aILnG|T>u&@W+h3$z?K|P1Q6^~R-jj<sOSt5iA`}KBQrjy{ zoG`5;WyR}q`@Q4f@3!X@g<aU_!)w`<Z`Z+MrW2<^DK+0|q326}Q#@vJmrhHer)(F< zJU@wNFD>Ss3%BDN$@lSQu2fqztMSFT{_t&!#O$%U=@3?XRJh}{3Fg(+QS!`!>ehyK zd^SgiHO*65TT-x!W;~a5^zK6I&h6k2OC&#hu^N6c{=_qnAnPcba`d-8tabA$txtBq z%{u3){D+*Hx+S8kdlHY`nhlQG5~p9w2m5OpP(gQ3RDTx8&Tp<mpEINI$dy|(!|b0J z^;iSPh*5aO><dH;i=t?&lQh_W2YNS;WR;lCJi6OC(J5M=cTTCLw=X+GoxE7=_B?_Q zCPsp)(pk78dn+7Ci3HJHs?Sf~hF9N?)4{_dgfFdDV8;!@a>E7?jueq@%^2R2LtIk) zLpUnUjSoD{6vIy4g514TP&TBB(nr3PJ&RZ)_})t)-5Y5zDmxa=oLrAqgOVwKqApIl zm<=|L(tGH-6(TiiV1Hpe^t<~7PAc_<b1#(PdS(D>DV!7`X_;s=db9lFr7kr3UOUzg zw8mBE(uK)}X=2KXzBta5DR;mW?$f%P`K2OiJ3Di$<XO1ZPzD!vTk|kU|J?PWN=zJQ z%%e8tb8FiGRyXd7M`tCYp+_gMIyV<-%2#o{*LZNO*(d4L@5Fgw##s5vgid@Jh;5!* zxa0FG@KkN3;g?U-rN3p=>X{=yn$d;F_nQhqw^exi;(Jw1Ei%@u>k1t&Wx@RY9{f-< z6i@2u;k>#5D4TJWdLA<buNH}YwIq%wE?7d*zawz*x3M^D&3$r_RnVD+3+V42$z9%M z!S}_f5NQ{Tle*PXc4BW{liQmn^^1keC3}VZ%Tpl5CXJS+>(S8jY8bZ3pL)6NpxR~T z+@K-(lU?#?;dE{MG3OvQT1<hzYaPXa;bG9P(^+iZ@KWljDyWC*EF6<9)hjR?)^BP* zVxUC~_Kb*<#hE3DKlC+4$Kh+l{>t8<eOsHejXw!Rn`Q|qRzYxV!vnGDx<2l#OQY^S z!Q`!_LH1pH;`qyFg`1Nr$u?l3IJAB?&64<5;Um6NgJA^KJXWKWDU0E;>3ktSU?Hl< zcVxqlqwxmXab*WZxb;qvd+u9>+xm#)TP@)FN0!{TArs@QbE#MO6z({B4xTyvfP5w! z(E_PQSJ3T4=Z-fBSIi~u%HS?&x6~3Rn-!A#gc4kH(2nCqY^F(PEx6^(HcaW$0h>Cc z^14^O`I&t*KbbNQ4*9##w~QZjN;8Rq4~I}^^yF*m+wk_o@AP*07QtlAHR$sBF~zmd z2W^E&?$9_F<C?74bjCGuU!TGTGr!Wy#$A-%@Qo689HW$~M(CE@o0}bW^NQzkzGN^M z)qZNwu?dmjv(%rSTe@TPUTx8RbRkqJEarnvP9Q!=#<=IFVD-@L{Ow5|%x<pXz*EWW zeseA81@ER{uWsBtBL_~^%D9KoS8~~)gwha**EMWG{iIkLv#F8RH5?b4y`oqp?;Jc* zVp8$8XX{)AjL~eCxpkV3*Uzic#CfB@^79*E-Q;T6;-SLB&yPXtIb}3yKrWm+)|ck& z+6S2pZ@^Avr0l^iMJ!mLje!DEqU&UuX)~9%s*MC+OG8Rgh;_K&`5k^g`viR?PiVWm z0KUI1lBVYMmAv(7Vw$=oo;7h7j*JgLI_u58*WW-uz#n@2K;q-h?jx!y2axAmfgi^> zVmtE@^xi0!uI&B-juW<vE1cXYZH>7&IM<wf8k~eBcMItFyn9vW_s&A`wH56D+XEdg zM?wFFolv;)CNx+dCB@Lez*)V>ZbUZCvtC6zep^b+t+%kKsV`pcW(|KlT`@9j1o$P2 z_(?Y%;xcsb)tO8jP~M;8uHA;QtHQCYs*F57YLQ`kcRrPT6$W3-VD~<N@}vZKl-QXQ zTz8QFA5ZF&9S%O9vfx~v8%Bp+r0X`R<gH|ZuL4)f?L8`C+nzk?y}ARrc}K%aKNSo* zA!Xx}1eS+rQPa{rV&CU)=~2uQaZd3>Sh4`gA;W>1=PJ|OCv9T1&2lhLtAlxAL3n+b zG9KITkS1v<35!gxi&G+J<DUatX@Gh(DA+iIF#fqX@6voQI@T4ZTnXffwkk~C8vNFA z57nD>$F7k(P_cJ7WkwS}b+n+TORRWz=seirG#J%=I%2mq0i;^mBCdVq36HYo;truF zyJ>w9hizCU?M=wz%VAI8{anSWZ_~a~;HX0s{I`blGFn7)S(@yUZ><;~(~i$)cA$Ih zM$1oD7lMPrelf$dC-SAcP&zAu13c|GY`v5CWWiPOhZxNffgR+>Hv+2e+Qc(L<aD{? zAk>T=2~XWskvkuy@HaiMyKNjlnckmkGgI-<K}TM=LGsb5B#O&6tQWgoETO?Z0@o;A zfL?{VJbze>*e%5rReD8}qW)k^FKL3GT{CgWwSE}$-WlDU(&@?{q{H<pT+-!<gQrD2 z+_jqs?pKTi^+hkBsjDHH?uf^MiM!!M{!dwBL={!dO2o1?int<8ky~R|($9Bsxc0a) z<kbh_Bg-vRop+Z;H!bGRo1A&e9c>P<7Ad^{3;t1|<gie6Kg_O)z(-qdk=eFpm~>lF zNDS%*4;x&V?s~zMizDRM8=|XSijz=lUw@v{F9x!t`P{qf1Nm{I8gJRzn@^T?;Py>j zQ6YRGJ@4_GUi8^asmu4!uTg#AwUk%XlxoQ@Whvsh=nhaj+K>)iIWEqLZKePXTe{bK zyeI*E#pyyJSA2?qFFS_dw^c`kvOrTF`sN{x@KZyLGqrTySgP4uVmUlpq)8#B;_+P% z$>3HOJgmsD-XsZcEmpzk0Ug16%}`i%@F;||dSUgMQb^n%Pf?Q^s7m4-zA?=w*WTlB z;Z0?@W!Z_N^o_B|Y`hrcvxz@@E26sOy9>V6gARH&L7P#K{8Ie|FhCd1=`@miJrc3- z%tu&itIm<Z&*|ptwUnl)2g(s1w0O%7zOHuyvK;*RN6Qbf&lfLfeQ=GGhiH?YekM)4 zQct_aucCdD4`qvYU+lj>lPU(p(LTNF^zipB`gpe&=E$lc-uOElwtoqBUvA<Soqp7l zV{!EjZ93OzN_!Wn!S-*<@J=6RYMq$Gb5G5OS%r&)1J<vFn~PFWf6oU1#l2LOAU%gJ zCEz*7ku2M?3*N4>CI`o3)#)ZByy*BSNoUvRk;awew_~`hNq(Pt-W)3U+}XrCG+adc z?8%s>cMsMUO~9kaFObqKWjNrSA`GmT({Q)vFm`*mq`MEH<Fnea_K}z1T2c(ghx{-j ztp&E9kA+^-RB)+N0c716h}&MOa{A&D+889AmsjUfNOP6ASltvay!b8KH*f<Kn3v0y zeE-qnY+X41-I<ryeg-o`i9x(;N%h~Tx1vI*wDV2xhBOy-!$*UM)7i)Rm@qLxl+$X# z=KN@!;-Z5)ekl_L&Jq__b-=ti7sLdAZDG$U2PmEu%`;|{Vdah6ARA`~&1nN9%`}eY z2lpbc6?Js+nIp`w@6HED%)-cuyJVE(E_PoWEx+bv$J_4LlhZOuvsUVfw;dGam)bIT z#$09Sw?nFD4vpqcA=2K#*u}7Z>`hR9_f8%^s$A@x8-rKpo&>D{1j6p__?HRpK8uhW zYD`9xwj~(;FO|DSB-62|tAh4LUGzNWjXIU*#bAx4G%aN=y|YoJV~?)D_?M4CyHe_* zPiTu5hX_1XEPyq;ev5h=lR!36z<@FLgyoXYtfzFJcm5qs4h@}n+pmp~6FeI(g<0UQ zz{SF)C0Vfh(=oxrwG+HMkc#iy$C9&mGld+RFFtFzExUZ-n(SfHW>(I0!ND!rvKu>F zq<skY=)jIyZ1u;Tj9nWjVVDaXNiBnef6A%bhgjTJLh|X?cjMk&mSFn!2K1M5&nxi= zbV;9s`72!b)8KwAE18IQE`E~@pK${8>^#uSuOlA7UU;SFeOf<521C^oAnIB>US6q# zpSSE}rTykoKb=6Mjoo-?>_m!<R70Hs%kje~KQ^EIfwY@#VOgFgPYmhJx9=a5J==DV zwC;bTm0gR3?7t=4Uj7rt9#)6mJ{bb;zXyHW1U~db84aq=fpJYN*mPb&Hc}6GbbKj} zQt3dUVO@CJ>CN;$h4JEYMQI+Fj%R*k({~MfNz2gWk+XH#JpD8UhhB!w?pieaPA<9L zQsn!CY@v(QX__tVv9PI9V2`=R)G*2xW-L^ad@&9z>WstT>ziO^f(~vN<OS1ix5|2Z zJK(T~d31lnWzpHF4^EsP!Q;Mc7Ngq^a_2Z5+`W8|ylLkiOxD~d@!@-7^@1^S_2-iX z+u-|x&b{NJnqDxa+sk15f=HVCwjQ3Xu!YAfquIsL2`{%^tF{Y~o@r8+cb5MtI2^9T z8`}*-L(fehe_+P;(H>ZlYmVt@2JAk?2tLNF!TrNrVay+IR6M$a-eh#;k2=b%J#-On z9bkeL+XzSZ>4miTGfe)Yi|YdH1k(W~>^sU1k9gdpl`F5&-+k-gb<tUDSJ_0*7W^Zt zH?`G0KHMe$_aDj5?>>;nJhV0>npFB$cKx~`pZoO}OlKbyI>>iIXZMSA>vb}3={8ST ztRrC0&f5j=q<6A{ZE5Hhy^}XMB9AQ4r}|$lV0+>=tonPFwC=p8poD&W?Pn5IraI7` zWz(zg#|>gp=_(mo?1F&!j})vZ-TOPsS#xv^oOdy%+{B*P;5UF%d(6PKPmAE_C08Lg z>L$zxvxk4rWxPXTe-78P!kJZnM15&@#Li(#60=UKIZoUcg&_t!qphQOddVv>V$OIT z;5(U&9Fu9+rj@YpK_f*Ti=(C=SLyJyG5ARD0DJE3gdU~#e9G(|ooH%;7hdbIV;@y` zuN5KiL>t^A>A-=TM&Kv&&0OSukaSx6;k-{Bag})pFMoXiF6;aPy|XD23(g#0ZWjTn zZSlL+e$jsC2=4Pp+6xf&i|mH>qQJ~R`28b<POjVslkQFDU-#7E-#%67)cponua&f^ zm@7ic&rUe)U;;()MLJmVo;SAi#BhBBX!DE03VT3Pha>PNNf-Or#9{kJ3n4yv9QPd& z#&YLa(2jQ!_YJNRqtr)Gh>HnU{|bW@EiJ<AKTn}%L=f~En+tPYUQ=J4P<R=&5pGMK zrh4xR@za(AR5ri@-`v)s121i$p{grx<G--hH<x0Sl{nwTgpWOr;zO;$c=m#fOwDuH zccJ7DlJ*fUUc3)0b9BHfrXM7X^Wcy@Q*h{V6I^MupZvdlrG2(H#f2MUAbD4okTup6 zZ?*0aO?no9k8wI5U)@HsP4^+b)sOaG^x$otIrKTBk(|z(;=tt9$QCh_G{XhP|M?{x zZaxp|f;`c{_8Wbc_&4=~%fy$tBmdJsqgb?cra~Xdcp&}$@4N^9g?~l@*#7tY6Y<@0 zA@ijge_v8rwPr?(?CfDl2dmjA{#86kUBb+H$dyPsS1_K6F2zBc!z7yhemLGv=#1-q zLcqnWFHRT|$#d>#^1P9TaDI8TI3#Hp?+RWZ<zU9q?mfv=eQr0lro9&}<IV`0W0d%| z(^m-hnl8<F<0&-65FFq9mUXI45LYW+tiEXdiyCx)N@r^ixs$a9+Yi>kAHKcuyZHnT z-cwHjJyyVKHCMdg>PVK}S4GuF3V45fcWj#f0Hp~%4qN66fm4TJ!tV1lZQWRkah5#F zAKbC(bR(roIqZV>M?l@6kca2S!Tpp;5F@>xin3`OHO!P2)F<$W{#y|oj!8nk3yx0i z&$b!uS)3CosE$r2qml!BXiE^CYPF!cigvvA{7DK?@a4W|dtmS`U5QPd2G;R;l0NA` zgU{IV<b+F-7em086K)DaE)RwDBm-e`-}U(2Nlw}4Ot9>SB|q{R$zGFRL%eX4ni}G{ zdfIuOxX&K%#Z@|(hD!SjTqWLBz;AG`-cLJ^?h&s)N)tEfkC0m(2%*x|<9Vm`I{BK| zhZL(DCvWJH0gie#ta5T1jE}36LD5;tco&S?8*8ZDooSph-donX{*>%H58;Sz-N|)R zoOq>gC*j@3Wjye=DSnpf^-j%7V)ol0RMuY%m4TbsTy~nmm6oz&nHDBm4rQfYQCKik z;yOBM;+gV=Fy&b!K2#qdzJ78R7EbI3R*H8haQS2Ll;>&Lk*q5;Ib#hL`Cf%>0f*pV z$4z+hnLTPh?umYG(Nr9>N~Uym2<=#XTK3)SF1<R{6=O$7v5#s4)cBe63C(@vU!G6B zjr!4!k|H5VuMgI5Z4-8@SWC~9ZaneZGLS#>1f#wup)O%PH~&3B-6db^=#y)p?OZNp zZi;|0GY*i&=@cQh`J^y@za|#uZ<VzMnBt-$XExd158dZ>#uK+tcoG^#`&-)K*jrC% z`nwC@_u2|g-l_1@UYDWct{1fK$YC08-AEs#-aq#GdeYlD6OJ9U<u@iHVOLFGYPa29 z+_Lf-bTF27`kbrgMBkUBmQgGC$Zv^9_s&GCE#IO4y~}W9!Bx3>eFE(f^GP}CG`v<{ z&7LWbgjcFjbYb%}(A^Qp+cgdNTWL3*dUrc&E%f8FE7QbZpWZ{{0X1%~_zB#L)nUcn zW-_|kk?ORE$iJ^wmU3M6puVw9C^$ZxgWr!3Ri~)X!=|D5uWxtmaAhZKjaW|h8%t<h z)L@?fM*~%M{sQwZx#Vr~h7@}DrXR}+=}Af#T%=b?PmeCb^BqmO;OJ*quP`33I_C*} zPAjtZ5;dHpXNR{|%tgfwlSsY%46N@zlGSHA$vuvl(cCmM?w!#9bH5gd=K>|K<{Jl0 zG91WB_nK(ruT%(_Fo*0SMvHzi1bI<g*?;h5ir-lPhaZoJUWYEj?@Q0&IJn~qCq34A ze_VWaSQ9VcV0LlI!Rxhh*xBI(ELmwPo>=A~dUsT2tI%D7y)@f4OKML|CBxvPwjV6% z`<?nt`wkTrJ80yzNz~NTDs(s*Mpq_v$GPE$c>b~|TvGl<R4Yq{w%#4VRDPRu{5SLG zvmH6*^K9N`bya@t-aoLJ98JETB!XT^2CouQ!E!)2XWbdX<%c@ZsS%U8_~?Gh&|QT; zj_FW+vK_aC*9dv*9IMrAc0tOQDY#-v1gA}2OqrwNIAHD=QQ_BYYWy{k6KrgxGs^(4 z3=G9-`quoeqYc)!X`=b*a5Tv�x`BFnfpxH%?N)0kbq&y~U1}CM=Mdi_>A*7)6{` zAu%#t?ZDM}G+vQ-N*;mE)x(Oi;oHL=Vy#q<Hbt7^ijTjj<BR~9{Zomr7^ZXkzJZW# z^imkO(G1NBt!YVojriK<5c%}6fcJ&H(bqX2inVHl6@S!u_m92&y1M}zg+$}S9tp6# z@fbf|5`ulUeW7)2lUQNuF1l}Q%=aX4yYhtIV&K;8VEyqA82M<Rw^X;U-QOQd&i7~E zfw55a*$kT!yNS4>5O<fIf|_>OVqS9;&KqeblucU)tN&!f7u}h%9@f%1(CP}_m;TY) z_aA7t?IJoFyN?ulZWQi3S%g8A1Mtz*M9%M&Eq*c-(P6(ow3UxW|8y&wx_T2gPB_Jm zpvDhO?n}P)WU{)|hh>dpP-B56OABzQH10TUj`tC!zsv#G3439xPmy3%nj(C^8b=9Z zwop!w-n<}62X!v}1y@%mHhJkyvh-fq&1*DIsAvX_$2JZ_tJNVj{vAA=vJ4(6TmV=T zAfA+X)JlP|@F4xPgZYV>(!R=AJ}nyH!Sx++W3rR%r@|h5Xww`2&f1Gl)+NIF<pY@; z9|_05t8v<^JaPVeP0sP2LtdFxg4^DBIX+yAcRc&>y8FrU&w3@WJzv1p`&IbceWv95 z%6R_TPWtq768if+;3p2!{dYp<u=lEzi?Uk;8{Eg>EbX57Cg>+k%H2&8w4Dz3+JR#Y zbOFWXkQJ)H2^FclZ|z`+SNbB}ukJ?^q+E-qr4~Q?-HR1|NqaYY*<rg28DKv1FEv)^ zqFI(5_IuF-3tb;U`xZM`E|Yc%_n0b<{5ug<-&^3dS);{u<c@tuO01GifNl?zaHByu z-g$dh9JW#oc2(txTSGbsf8C=wIzJUV1Qx>O6CJ3~EgeJj2g$x%ze@5$4rm(GA8a?a z(VuOVyiU&*&pO><XWb9t9wi5G(w!yox4+QRoM`d2_CU(AzCb$OHz@D(4|wRCgtqWo z9JT5o-O*gezHUc^p)W1z!IJ$rd-)A|b9Sq6drtzdH#rO6t|!aw*UzA0=L}j<ZV3;j z$B<Rx{ptfzo?x6}gx!Lpx$RLc87$EUy~rYbx_K!do#M@h*Yx2_WlPatN7@6{!x=~T zd5T(rUa~Fw9r@p!>4LZCf$G~|z6;iY+bC$}P^b-;yz=u?p@WSyOI3}AMztb(;;GMz z4@|+yZvTY-e@EdCOGm1j(+uDGYq9HRQ`8*XpZ_eHMJvp<l2^~P>L6Cejjw}woYbE` z{&SiPYUk5TovB=L;x6@GydS#fm~glAx4|!ZJZ33%!p?2Vyd&TSC}p(>ZMj=%uA7vJ zHjS5kQ@%yHtp~wT<uBEpx*(pm*amzv6supX=2b)c3&!(OaL>9CFtfWVx0rO{e_<ne zRO4&W`M8B}<m?vlOJ;@8!|OQ|O#ce8l{5M0(k*n=OpEn5ok5MCJ8+wO9r%U^qSw=R zv~*1uUa?+Vwr9E%7ndh-=&`j}SQm{#gB$Zn2idH7(jLGfFTC@npLk^ZR$5kLgx|Hz zS#gIsj`|t{5n_xixgj0SpW09N{X-yZ!&!*w=Lxze^f9J?D)zp<7%jS{&>OW8XuaVQ zq>da1zW>JY9aB5bv;0dr_Q^EN$Ck%`8pMyE3`32b=`gu&kMO~CzbJQog=1Zc#pLf+ z6p(fR9ytgO2iFbdZl6@(YQQd0f5!|+kmh}t#zk<Ldybe8HI9!4=95n7eA%uUrc}~0 z1&;O}#QI>(Ge_No=Ut;QB&VJXTu%!Q`kLHrc00P?*edNb-bIsqz6dvV4-#LGzXc8J zBBWi`w!%=)txzy+7R%2R!l+CyGT!Tib)$4~{e=E7IOHI{>Cz|$Yk8o)HA04+9?aaH z1XmWnCpBeF={qasAo^Z}-uIt_ZKrg4HBb{Dm)xMfXb7HPW`Ulu6IoA`q28HU!b{U9 z!q(DJ;>wX1#nozKFz?rFY`rm`%l?djH61M}JKRLPr`&}cdZfVP*=4YKayu~(w$uC- z`h5F+8NC)%c(<OOI8E9WHnL<oeg5u<L8G_O*q(oZnswQuEtqXuqs8&hzls}vMAEd~ zd8lURDSS&QB`uSUEWhbV3H^?fSMDL$5PF`@-Z(?$LkqxalZ=awTBGGYU6PM40&$+P z@H3-7(W5EicWL+E+iucQ<`?Q5qkfDQJvmGzv)yHmbK9fY2#M8jXtU_Aq9^Q{+!ary zUIVp}lGfc@@^^YUlJzmkliFH~<ya0r=ST8oD|?>zG>D3Cn^9{I5Z%t|ps$bA|5;{1 z1vNm-24jpf3KW~x--WdD2G~&OO_|3o!7GUy8@`}`PFk13j*NcXB-<xGGYBHTuF`B* zNuae87TT}V)rLP(rb%9Cz+u~+INwhOQ`_6h1J<_V2$$|?7xM^uBrhP3_exaXuwS_0 z{7^pWY8P}&_(yG(5>vLKhVJ&;DfarEgvYuLqRkr9Mf=OI;gMD<+-x_Q0@CYfyrUj8 zeW?Q#j~C$Da)S(K^?-5Lg6V8WD`7&uow)Wea6z#RA8$XBf|Sze<62{QJtkG+S~<h< zt`CKGMv_;+*OV6)c!`=vt$1n2<@mM77D&1@Q-r!+{Oj-#9N?Z$zk(O=<0)JD{@)kk zzL5%e&3_6sOd(WpD-&LxsH6|R<~S=Pl-ru#)0)LPIAXjl^B-%hE;|OT>)Z&g`eT%{ zx3Jtk5&s8yZ~l!{{QdnKGixwpCQ}k6L*hF7g-Swcphy~2ispGPb1IpVsiGvLl9UqH z*_%d@N+~62K$-_qG=I<g{txcY`@7b??sZ%1`jw8W%Xyu>pU=m`w)>P=spgZ&!7Ufu z4s8?lvngQ3MkQ!5&=<HznPT@NIWQxr9~DmUq~1?s*<f5n59`%vy2NIdJNy*hoqnNm zQv4s#n0r^Wl*!VbzwzAphA0^FD*{!z60nm3Scu0`^i0p74+lup>F|e}p?Q!ieEAY? zjICl99`3`8LqBnHXdd5-Y0x!JnjU}dfRNr;NVyh{N^hTVQo7Ml8m>b-!)@tKM>5^G z-;5iFEN5Nrf!OE}&R<G-$7xTCW5bR`;A}@HT7G2}S-wBXjcZyYTDU-(P5OEcI?p8I zxQZ8|{(A0g(NPbSyI(|Ie)>%E@Kx}htjwDFO{a{t&0z4v922Evna=S#hz(GnMV8($ zbIW$#DfOl}%SVk~smhQtlV<A9Kk#PDF4jNe1ju}KV%IYau%Y2D44&SGDQp+{89#Hn z&@hNjfCa0Im1jATj6&-aV5XuaclXRRrt)4%JR;=-3?DfJ#`GIPZt3yVJLNKFSLM(x z)x}KDXfQZF9>Sv32o`^n;s-_j!FvPi`5vV#WSMnvQcjEgePxKRL|K&c*qV+9O=F|P zaoo6t6S&{)=E8g;o!s3VDLN?_HrdKRUqT)gAMU{yR~OO6ifeE(u8}{MY00XZr?MIK zvQ+-|5DR;j!C%{442wT<EKR!r+d{<jwR;Hfr!|;v3E+;n<q>rIWh0jAThi;zip;U^ z0cwYi;FnnmtdF%9Nq)~Wm=hU=8N+>0;0;m#x_n4i-vOuG9>HqiZ0BelPP;@)h&^jT z<>C-Fa??H>b8H|VTy&dXU#`U}G{=(h7<1TH91F3pjD$Ro0h?wg#geYO!M)ZaY;iHq zC3N~R3)LAyp3Q@uF4%)6VH3ePbTL)3LCjH34q9rbu?3Shah{by__8+#jOQt#=Ivmr z%zcS#-uLH@<Sh{V9Oqy|vp(D&X+TXR&59)-;6^VMnyw&8a~z5=a#<O6UJC*HnaWr) zUI7OSovpI6-CUT<Xtv<B3VD1yBlOSavLLv{741txi7Rq!i1imy{kT;u<f1*9HHtZ_ zX`S3Y@jw>%)0j=VGn|!$88VYK{b1Cw9&EFH%6F}>;#Ia+Ltljog^gOw*Pc>$KCv&B z>fL<lOpF>Ce0YS>*~+AO-hg)8>J}Ul-e4Z{g3q|TiGBwsqYcZ)OL{%15jBErgwN#8 znF5PQE|jUYKgOd~0%tMkAnKZrg_(tcy!V$V_^9v2H`&Z#HCKhtwyYUcj)<nzdxvnV z|2NUF<#ssp@P0t+l`zBl8!p-L6dvuhWs_!)2KnWlFf~4duf4OIvkOl}qpHW;vC$X# z$7cu8B%879&PZ$YGn#_S+Et<C_(&SmG8`7ig@MFIS6F?v2Bv);g@O0a@&~>4aE7aW z*_%B#aAxErczP}aD{gy>PsLy2hG}WDK#?S5?>|d_+azIfe_fU}GMHp98}lN$eVD5F zo?oRk#(7)6uP_)7vJR)G+^Wcb*dpZtJ@Fp6YQ8zYEbk*IWZ3Zsn)i#+jay)N!(Xl` zrGQTgQXmKR6aFUJ(!^WN?0LC0f7Eh4^K6J>2Y2knHRBCvc<pmoA@>P9SC3`iMmpkI zhXQytZ8%+><4Glzi%92qE3ft@h72`_^0J+wOnJp*Nc{T%qt`fab`r1QwD(LXJ7&nf zhgj1WkKs5xy&h%_`-Tc<)md+c9BGUkj4G*4Fm7rD=dwAHRmIMvL8WSJ+2z%2s^t+d z8mLXy^J3V!^~)$Nw_EgU&>5Vb+{(2Iebk-yi)l)7Is}Qb@R`7&cdxQVw-<SEFZMBB zy`n>Ms*m_}ncBiK`4w6!sIv6&b$sb@DRyG)d%kLzC(T{4lQcdLVHV>&>783I4AeM> z4xV?pGv5kfOz~?`xP~UXH)1=@5}w6E+x~&qz%le;SR1^FspoP}U4d!el<D?RKkn<J z4%F;tMe8>!k?wdaCfO+AJawTAuCcpPX&^ra&R><HJ3dbAsBSS-?2Ki$XXC^Veh;QA zuMOy*QXDt`y9I93{|0ZDsxmL%0Ju}FL90U_L9s<KnMNK$sYh>weDHeFiZ?&tuGKae zU*(OTFHInw&~QE~ZYcBBvt!Lamw{@>BQES}56pC11$VRB@Wt2dtjSW2m)bAOTo0Ur zx$D0`m)&W2wptFh8u_pg9edjNbqj<vd6E>DM$)R1^!n%@Ja7F1d{yV8W{@414d=M8 z>G!cEYdL0|J%C=9?U+)m6#E@x%_Oa#@ezqi@ObnFdQiOw<RwpIU~eW)ZQ0JQ9?SvP z8RKB9=R;_pvJV&TcBKoyt8rbS9E&y|$Zf8<2-f$H^E;Mx;rkCy;BR;}vzu?s55E>b za>wS9$&*r6cRP#so?}L%I&J8ttrh%u_5(T}9OoXM+Jl|$t65gnXKava7bi$`VSBqN z`(_e};`T7w(w$3r`U_Dea5}`BtivzZgO5GRP)EN4&bK6R{HnE7u9_#dJ}3dRN2X9< z$QAf|GmTb0XyL~sjOS0SY~e4D2_Om2FCxuUCqC}UYrZ?D4VUSAV8QG>?C)>D#7l); zji!iS;cLf|J$&h@x<4&fm_Z+=N7FTNA=C-~SB6fA;Iy7U2jAbqxvJ$TyLI^?x>dWf zzb$R3b>}j?eHeygH`ig`MSaZaSPB!03hBz_{wzRkJX-E|A{FbYw0&3?SmxF+-DXp0 zOS#OY_Kl~Kky#MBs}TckA7MOK!Sdf$LCY#fPW8!ffvp#Z8;pLUK}R9?E-{?#3sh#a zH)%8fjLopHpBWu5t%Xm5=U}(>NJ?H1iZR&{OkH5s=Ls&hH&1VH!xW~_qn{t3$X1D| zO9fGfMj$rqR%Q#FJfLEdE}guz1+(PGgXo(gE0)n`u{nO&fBsrdVRSNt>gs|<t&k^N z(2WH#+IV_H1WP%*PWX&H0B%q;8us&q=>K>?rFLP5`cZt=m@FQrk-^fR?x3)ht2o2& zYH;@F1_%i;q?O)Z(AnlR*H}Fb{A*^iQ3Fn)Xl*&K^7aCI*yh1{1g}|P;3C`>s*G8M zhhb>bM(FBv0u6U*2)}v<4{W&2y|uMxpH{3Tck2Tz<oYjO_f7%@j4cN>*G|4R`j7Zp z!cz>seL%Fh$AeC2%dspoO}4i|nL@o@a+})KaRPneetoaQK|!VbkSjhE^gbNt)g_{q zvMFae^AzqoxttG_On~<DXTV198oe$nfEyny=;`JF!Vx2=-)}?Ku{D+R-m(jWtp>3r z6}_M^FP>slm$GcX$^6bx!IN5X9QxaQfuJ2abYy_MKpHGyEjfeO>^W^H+cB5=l=J9? z<a?p>mX4uE=g@=Qf?M7~iUogO35uKJX^YfD2){5DKP;HX>y$mjI_pwcG)jgaq>{|# zS$_pHgSB`fDUZMKO-<lLwnGg65%yG1qlJ2V*oSC6W+j;oLGAg_Ts48*J7Vc>^d##4 zd!%^o`{asqcU_qF{1k5VtbfiUHU&U>w-RkAT?XbQaZqX#$8uUPaq_+kNZGD{HZ}`0 z?WeQJ?bQidF)5Fm+Ttzxdn=66vmUXB5{tRV4(Zr>q6HrQolV9M9WhKx;88t$3pICl zu_;Zm^ff<)y|j3VK`l41v|XDO_Pyk<EsWtKv@XM+pJmR$HfC&7%psuzX@x^m+MRFz zmc?ToQrzaB99lnILID?k<JUg{cqz<*{Z*TS&}u^yJpFKjygW-<AH_5pFT(pt)A_+8 zRiNiq3Jmrug(-oroE^jCAbiRb8laIW8hOVFPTB|_?xZtJPRW94f0bjSO@nFib`{=o zbvf+g8u0FrGVreA=(d3k>Hkz?*WRVU_3)3#CADDfl`6daVF~5RCb6>DEnM>cQ}F%Z zLh>E*0O0Oml5L*Ft*N(XYfXTq&K-=r);h=^poCc~b9vj})#6TJ4wUZg#4nyxg^iMt zbShSgew;o>wiC4}q#=tol|RL>0ZHiUyo3e(VJM|~nA5zx4lm6yMuj=<=yup0u04E) zCpTQ@<98Iojofee$9n)tH^#%YYA0GC5zNmW_=)TG`hc}M@<NYgC<g3RVHvp*RQEBQ z?z$Q<w;)w|R3neaG<Ne+^Pl5y|1((7Q^URNxhdKuGX-}<i23;-@b={_S*t=EyXx<U zi%#eYuDmqLUKc~Fa<-A9nGBoyHxBMi>J?2mo6Z8e>fzbxEf9M{8~bf)#M7Hg@T$}| z95~@C%n?n1wAwV$=8bQ7ZvSYuTjDlW1-nsrVKgpI*~@}I4T6O;H9%wIQntcJoxUZ^ zVu|H@$?WqzKG;fKaDm#B&c|GM)wq!K(mq1*T#jnrEr%@Eh14%l;J8T+q{uyk*qAWJ zuJG;7QLprw#gu>i_enOe>Ek>o_$*0QQ9fLP$5;HA+>g&Ie;}3=?!(SnA+YV84%HQ& zhF7-};NoAwL-XP#Od9wPWm^Z*l;bCOdD9%u?QSJIJ!3EaV5y+^sS_g<Gf7(00!CF7 zag+Xj#Icua@woRLQC3v~+Z?_ZYG>F{OZox2u%LmvD0c<Lf0cRZyk6KgWIR<LxQ%me zYQV}z4RB*r1Xie+F;#<Ru5{*IuBKKE_{f1+b!QgccHIemPG3R4<u;tVxQ$l6Ex<w# zFSPj)$4X{f(O=fZ=jUg^G1E!3&%~7#@AwG(w8u~+y}d$hkTCxj>A=btLZ<zv3-*tQ z!RA|%%;j?@H8+2O#SWA3m(VX6`5}dUHr)U}j>K@;{q1;}ol<n-YYgbjvWJgtwQ%B` zh@GpiMunW682S7f^Ku+TuPz<Ma`~lje^dcTJiWuYEQ`c%?FXo(=mczW)FvGv<3D9j zDy<Eiz|NfVW3qk?xTZB8y+)|wY`MMsYlAHmU%3Zd+XmD9h$@k>-~w*<isovB9^C$d z@6dnuCUAb3h_ZS<EIEO(gGUasS3_>_Uh6yHYHTtNP8`ICxs2ow^=sfVdV<-%;x|y5 z^A1<4O3_od*LcOg7F!cLz*J<(uJ5e?r-*!5kfBMUO(`sB##%bA=mr%E0{1=tAxN~k zW6KdaGV|!=uGx-cd!L6w>yh~kZ_eS$AQa{fEC-+PWa#|XAN6&w!GL4^8JCredUZ*3 zR<i`pYgppNx*l$2l?0m;uo|PwJm~zVeynPD4PSeA6MU0ZqKFe)uyRK*NF}JV#s!CI z()@h>QkW5~JhOq`$9U3y^9V>ZZ03zz1#@A=hD*;@o3KHBlUat&U9Q>MktsGcW1EIL zGsZ0DUF*(Tg)@oXc5QmEo5WQ53%QQ8SD?S`Cv+>1W8cPy!}>X^`DHa)%-8(|w9L5z zv+oJ>Rq1UYdRYjAV_U^ZD=wq2nTkmMV**5E&tWQ3<|zC5Ck~k?xY57+(7Mg5d99fu zdb7(3`pag~v`h<5)6k#xZ+4@D4FlQ1MJ_DF&7WNeb^?#`d*aPU!^m~xI)08+BaCuZ z5*(J=IDAGr9P_dP`L(?)A=Ci3XSq>tv;;jfuwpsR`P|bM4v#)==TZkph$h))a}N#( z=j7X~aL>UilzP{QT018*=}+3sNBbgQJEslqnMh(*)_r`CuSIh{&t>@+1(wuJW%e~l z=s&23asP`SI_%1}|6%?>5ZL^8`GYkZGqL6QO8l)H2kjq<@b$3)tZeKtwyiLNRi2)~ zB8`X9uc-pZeyy-GyqwNiz8t_-Z~uW;)3>u(;#5BEyak(V!`Mc_zh$y>Bn!X0o2|<$ z;)nTWVA}m(5L6QgE(!8XGSm&5t_`BWkDcL1=@WjP<yz7_{fGPJ_ZyFxHRAn8!W`y| z4sYOgnD*Q<$Gk-;BInP~Imaq(Dq609QNN90+nGLCo^Fn(e?*A4=xksC!ZUQ~yD~`r zU4a`+d|Bp42^1-+vW&-GOxIfnbTj1XdwwjwZT7;6`4xD>Scl}F>ru4nFk0C37Ek)= zQOSA<Dv%b@^IxM#G0=&IyI#RD<IU+sjlgU7RAHHmC$abM9Vul+3TN`j6L&Ztg4;*) zMY*O!*;S8&xJ2!<_>;T`hA+&<ms>W2o0bb*+9<(Z4$!0D2Z=k~a}pZP_+i<iK8P!} zV5iQx(O!NCeUsgZ*{xN4;;sEG`0xOl#q_HZw8oH|?@(6pz>~hYy6|h3A4j?TA(Yft zCEUNiV6I9Fq!!9D&2L4R(KZ-oESOHmzwDzUF~{(U`zf~kL^?KHt`rr&$`BjQE`pA} zE08;89*ZyW#fM*Cakr-RK#7+exyn9)ej53_{v6?1aI`u<MZuq1wq3<)TWL1`%pwZw z5&WuYZcNEfibWaMfa98Bq~3RgzE?Ar9aRB_V^(6pr$d5v;3C|S&tu8g6ev#fIc)sL z;N2N5uK(EUuyeB^JKApzth)CCSDeoV{g<Zn=5{03FMJ}p53+;l*|)jg;ZC$sJqr#7 zltU=D8=a@g&>!{5m15rs`04g^yn0!Nl_@zvlRXEQzV8E@8H1oke--I#ET^OWr(=@1 z2lvnW2KTxgVfdqZ=J#VTEy)X~KFMYf70Obc{Y0Eqcm+pvYq7)ES3<TkqP@VYY}lO2 ze{Kt9Pgm^}c7#c|e3=d|dGdpoyd<KW+09@WB2<SRNAYzL!ECjQ4fEPC3&*_*z@SU6 zG^Bbc8~4HvuH+A9Rnwk>wMC4`^~Enpe4oz0iTwymrai)+7bM~C!E97ow~VbxnTa-& zpYcOt^eDnsjjrmIh&Syj0`%-arNmqK!B>JRrc`hhKiwdHX9k>kHy`DsUPEq=4mqX^ zd*sEBDia2ovE6y;;tYAg8+hmkAD|Y=%_&Jm$r;zswkaDborb|enFp{ix(}aM52biG z%r3;PL7PL9X}RnKSTa|c4buMy&C{P^_RSiMoOO=>e7Otl2CoBqmD_^cb~HUd-463K zFJk=E{w(Ho83eQ$uzCL8%;0-0%+j>rW0t#$#!ei{4jt+zO0n*(?32&Iw2x(w)43Zy z^n1upkqu{dH-U|vtc$}fig8$1JgLfbLWTP&F3)Nb+Pb8PrOx_e_`eZ+>i#;g+fR5z z@O_oN^kx1+4%ezM8v+Vv(Z04SQPgB>Y@K06_+kf0r<6jwkHC0-7fP|Wj3|^}h^uDB zW2#>uZ6C6d9=(kbSbml)uUUzXzTJZURX%L%5rL^1vWGXb+|R56p5fSM!}&E5$2sny zup3@-9CzPthq+cJG*4NFa<A;-s-tAd-v0oVEYYP^sUA2xROqPg&*I~C<>~yF8a&{> z5<aWDv$57D7+2%NeYr9V#Q1?b6gU@8Hk;s|;X3$ON|_SY7l7QkuNWKg5AGisOog)& zn8j0l>KLs~Ift!8vxNRztY!mmCQ%PQ56$_#YO(yw%=vuBk{hhCHxoMYT=B)$%cwND z5SM&j&Rtl#2#>7OqGv6`Q9>~Q0*o`5#PJDq%c%gA2YeLe`qV*{?_z%Ng-iS#!E;QC z%h*m9%HDhX3k>WeVW!Z4EBg77-jLg(q_(%vFYF$!lW~DKg9muf{SrIZWytQHpTb64 z%7a&@Ha+~TkAHkpSby1A(8wG_tpYRb>|B3RJJHOyy)oqD`t{<oBY~BgVZXU+a};^Q zNwQe=O_Hs1Kgx@%f=IFX3JVy#mY9Jo`+ZA~enoA?_5yd1ou7(5k+DqrM>Bs-uN0T5 z$Iz#168Jqe2%;AE;R_EvJUx9c=QXwv_4ZuiF89t9Q}}XrV3E+*SoE?&{<bmfy&=rU z+djeC99LL)Mqd2r&}f{$=?HugS<(UBk1+AHC#0F^FzcnQ7<+0E=?q8Qt+RwRUC+hb z()o1o&wTnlpHO;EEROg6C*G9b4I$HeY23@b?7q@M*s<yv7ha}HyXWoa{q_-ey-yAr zwi<JuQ>t-~R2t2gVGK&F8Tw0$alzq}7^q-`lFohL^y(@8+gX8seXQ8-?7zJH`xu-( z=N#|k9L`>swW6G!52y(qU>9~BMkw8ZwpV}naW99k!9uQd(Gp)^vWi5mdAK1ngI9T` zO7AB|gRw(CnvEB_i6Q3XGjKE;IJp<Y0(4pU-U52IKA2PKXTUED{)vA)>(FP^LfG7N zoIA8^GU$%m3r{rOad&ivQd(dTw7yfOw+n8A!Gm6`Iq5=*Ixbk{8Ub53UFVJVc<~h- z%53Rl7Z&|>HhcErJ>*=^gn{Y8{P@KrHlb%3{608@KSOt6mgo&k`z}EjhTp-n%2uq; z_cZ4)*Z}kw|Hjh=6*#L~f~5D}5P$3nf}Ar_wDT81ubv01d?;|~QVuf5ImgicwFH%{ zQNedL_1J0}O7jz?C}NE~w{mY3zS}jMYR$j!y$O-@O4Ed*(#O;2yI;8=#b|t(_YXcZ zMV1p|PoKtTk>Ps5p}J5RhF*To6-4c17yg}r1?iTQ{^%c+{aVC8!j$<PF2RX$nyl`I z6fK$3f-@{OaM#9%V7FR58g{pf@BRA53nhJ0DbE%?FL7eEy@gOCH4VL(BMfMN#cBOY z!UNA{;D)x<Y?oytOqV@~UQNSL!)PuftPbJb9C~qlZV*?oVhmYn9pEE9R`bsm^=BdF zS@6Jl43)oo1~Uu}AilDr+1B&%){-Q0nV*0wtOO#^nopox4fy58V7A9-J}sR36&0k; zbI<zsXM-HlA<nWIw(YX##^wFx*pM;|9LKTjNL`RyFPtxZFR<W(PVnxR#6KMTg;)P6 zuw#N{s!VuodiBwl9&aXmEw=$O|JmUaZ96{HQk&J>wPMRx<l^I+$xQunF*og23b&-& zmX&rF<5}UeFjQ#@9Sw70t+$j(LLrUK@e=M+oCO__IfS?FszS@k-R$6u<?!UHCA5V~ zL5$Nev{@AD9PwI@x>V1A=5GsDZzIR1bvmG~L_gNx9mG#D;@CXZK1dpU8jN%|apUyP za_@_UEQHWAu<|zGm2dm8^UX2Bd}=l>o#I9bteEK+4`N$>t;0!G29$pD3cNPTz{gV> z@%y1q&}b6N*y<(J?PAN^;ulju_Fev?jvP5$PT)T-%;%hhy}_>mC)nAid$|GIb6DLt zJEnT^EU5LDK*w)i+2z&+yz@LmvR>>h>~+tPrq3~6-dPch>V9IlUk8MBMv|4l-|&<Z zcukJGsVYMm=6~5G_>7Kn&T7g6e|!($`r#EweAcEqhv(er;cr1~)5;(0I*9TaUTp3H zIhq_Ehr{RkVS3jfQa9;zJ~-KscI`IB#37MvRc8#&7#NL`ah2R~zn3tl={S6vGl_*L zCh|`YcJbY<6?j5R3l>hk4|NV_@$|ptm{#RR`r1M4U|oNFIr|;7>>fmo#}$}yS3L8d zcvN)ocQstr>=xZQuEydgdU9Ka+oKDbu`y5kcqs{GHqFJ3uI`VZp8FF>xhaB-9CG*u zQ&V`8?*V5@Ct&Plz}E&rOvZK})a<y9zRa2~ml(32(=4ec#fp}MOrwisgUKu7F|Nk| z66JYPkBS60@kAFG{$9$WyGK!Lz6*Ok;0*7;I?%hhQ^>GO!OIyn_**_1wYIHBQAoBJ zH8pX>r3`8+y2VE}Wjf#bwF|bDIkJuug5%BkRK?<cj7E76qJNpm%qUmrP9ICghFu)Q znXICTP32q&e~fwOF9)Mk6K49vlKI=^KvltDN`EfGw%7;?*HcFCswgZ@-vLH4N4X*I zvgzzGT?#tB05pdua8);i{rnauRyih=n<LEe9=jjmA9WuCrH1i%tb!8_+O-w~75A~T z;ePzxX`Ym5moK_JsEr?}G=!=b928%;;mw@7ig4;BcNQsdNVYHf2OAEb;7^P_#b)L> z^6SjYsn4p4>zK8LwcmJxwb8QN_AXVJ@%0yP-LZ(RHkrrHtWJPeAx&VjvIHOGt#{tB zLYM_@Ifet@zK1;L<<#Fi3Onoe!4bn=jCd<_2cMKUKYMirrS6=9?MBg{xc3xqVLXfl z%nrub4qdDo5eg^&oyV0^5lgea@QbBpunX5Cp;dekqW52c%gsCCoeuJ57EWZ|x`u7o znkh1gyb2laTe;ZK)%5hAEuGJ{Wy=)x+2QFQ@W|LMtRH2EL6LP(v+*OCOSFj}NsMJG z&%4mexCpZ&7IO!4lklsPGo?u|m{fiaX4Nx(`I|U?Pp2&HuyUsCo%w7RGiL<_J5lBI zXgIYlk3U@`$*q&vi)|(A*&F{<sQRKL@bzx-OSfHT-p`ECtxcWZBc;i93`}B;_*mfI zxzLi2JNP%kxlcWQ0DbAM=T8^7!SThRa4Bjv6FWQcZL9BrXRs2unw|kgX?bS;uNP-L z7xwnKli3)f)7&t#Vir7lIOX}(p@KsxZZh1B#~v?b*7jO7Pdbi@ee0O_!0CA3xR>`m zC`BWJggv*jC%v?JhZCiL;#=XIk!nATngR!+pZjjSnA*-I*4?CYll-ySe>6>8@C2O7 zWNDO9FPL6GhSRNrnZ<QiZcoQ|*l!XDX8PZ_Wq)75={cDgVNz1*UYkK<9K&!?<rubO zZ!$DRO(oeiWsrT%kKFI4L-CgqK1!JX{4BeT0dID|$f-tbTe&*>b!#xbE775X_@Ov` zuQ^lbw-MHi+K7eE`mw2p9O&usLsX)Di(csa(!HC3Fgj%@(_Uo2%Kf%M|IQM=H6xsS zk~LV%#?#`_cOP-8=QPNAlpgHmj7WX)3v}5fLl!-&IQ6D9epi_+S9v-O(W?YTJ(gxK zZueB4wpl3jg~|lZavQkY8I#1+RlJnvH55zEhuBXtj9V`BLzdX1&W9wL99spe8Wpfx zx`^{FHKmB&@swtHmRnlv!|EEJfO_%`-0-giD(*ajnv<tdsX+o`&OOAno}2hAn*!*5 zC!)n?E!mjbT~t#j<gAwsq;V4l5#72eIN{C-&wt0*`H#k=eUj0p1$tB;GaALQ@6qe7 z3gc&;z?Ff2@MJ;**e*N>O|o&EP1;Cq#m}!WG)IXlI=jKAYA;;3%>tKlFWfKmVSYFc zrS5yKU@C+>?_@?h{d!)-n>vkP6Yr+O!U;2}_5O4+%<spp`<OBn4?{M?NpKST^m57P zS5(gS(1sDUMs#rGKAc~94O9~9F}X|NwG}=RS!a!7{PuYK`$FLK9Nma3M~~;u=u6<R zQL6M~MRet>cLMv#dH`u`sOFs0f8)7LS-AaGH+SDO3w$3-a%J*{G^EXf{TUSs{3Lx= zuCyDEhzIgfSMs>37AIKUkS03sugnq^CZn&DDGc*_0{hgz2>!4Poap=vBu4Ed`{NmW z*P{Y{*ss5wddM5jJ88Vg<Df9F^f=B-ua;rcw)w!~*apO)Rty@V%enc#1SRvsAVCsL z_q7Ie)>)7qzYi=%2jG-OAH2?niQ6@gFozcczkZS(_fDh^4J*&^*#=|aOYId1s(gzJ zU8iFx>9Ud24uJZ!Ghk3+%g-B?LfgwDadB{;czM)vxHNDe&6{?eo8r^SNpzf`la3=% z`(F__870GZ{{<|~wE#4<my^lh73^c`R^e?khp%ofWE>buW<D7rG(5ufalF8oxeR*2 z+RUR|0S_1S;-u@FD81H=E%TCOx?bIINljoRF4zaStrpV;9A}xYGx4~DFTPoDm#uDc zVS}3VDQR;uTig|oZ;WQ+AEO_r>~fNetS!gQA<huF^RU3<Eyk&WN9pZ0Lpp7wK}|Cc zqlD%?-ebTe{FtZ=azB^Q+OO|WcH3U@_KbB9Yn(tsguj)WlxM*9ANBlMJ_@%fwD9{a z1<(A91lBe;optpSe&%vX3j8d=9JLR#Z#gfpeEeT7J4KZZ{M90=7+cM~uO7vGn&f!> zH#J;!LN3I#-vVug4hX7BgXuM|V4rXgysNpCjn|XM69Wr4_QHWmi{kj@YVxf4`$uRT zU`Lg+8%5K*PT=w!WzOT{KL{K;38(z1;L9WYFim;_t34Y5&(|g3q{uD&o@#5H(mjUW z#wYTLkCfr!!ersT@{WIIXN7HU(=j0QFE{YR6t-jbP&)kG3r!d9zy_5l%KfK~SC=n> z6H2Gy&b-}t<@8<1oRCi)t1IxP&<T9pp-t=il(@&m(bzt^2p1$sv24qoP@7N!BlcWH z+mlg{cK9{)>?w!D#vf4Tw~*=9)u3xSgP*&<K)UKZ{KoXj)FA?X`9*>&tLIK=S+l$1 zKHR!+A8ptwa9a1dQQeP+@O-ov^mL11_`OVS;BFbz%f82L>Kj9=Ta?*7g9~se^e0-| z8!?Lu<Dl%pYwq)yJ!tG)kCxvfAx1;!72doK_tqW7%D_SFf{>YC-@cpzYi84{Ph0pE z7pJkLBn{rSem`S}O=wVLC$3x_z<h3Ps{9iDNc6zv2;Nj$!4=+cq4x=+@Im`=(D2)f zpSN36&#o*mS-A;A?H5vt@O`GA6f(W{&w<&+dz_u)W!$~-GKNZekji>xcKXK}7JGC) zANtmm)t0ZJ)xSN3zUf0u)anuMo1;O}y2_yEn@Nkb4#5WJd}iC53ugr8;9IdJvzxjN z&;J++P2mICtRy|?(I(itCkWn-(1(`7CAj;K75DYcQz0)eaPu{?N$=84%vls8FioV` z&VX}#yJ{+y<oIH4iVBUd=lLOe-mt1(o<uf%IC66hJifYyT|S$@4L%e@&JPUf(WONq zkL>5%$-zAW`*1IwZ3$zg@sB`h`V08DCJr{}c)-M1F^C3CfwHH{@WW~)zKIWKTZMTl zt?f@w)|X+3VFn(*@59~;e!;Qx=ku@YGjMxLHTL~8CZ7OBN=viDD?#s_B~P_N=*pR5 zg~$|ivwaJyixX&s0pQqGW8vQ}SA3Zv!yYB4;$gRZSa^REUKqslk&Ex)nCZEdo33|& ztio|tntB-TG>l>QR(C@|n;FX*ngW};b};|f<>VGU16<zJfb6>@RzEcYs-LQn%fdjK zJ9CXFcF75RoHdGBULL>%-)`j?!_&NYQz0K2-;L*%N-&+N8MroaI7^q5Wy?)oK~I=9 zQ;|t#KZ*vjedDT`gik3hKH`hNLQLS8e=-%Cyurx}=ke}K7qe{HCM@Y3&HQNrUc4Gl z>ue@qrhXHzFm^LJ&Cmj?4<l*EgFUe4)&ypMzlfi!HyJE?o%nijCulZ36TOriLf>^Z zvLEYx>E^%}psCW2xR%}UGOd7)N;bd;XvUr4MJS9kaf!ttURr()WQFeIr#9)4Q(YKI z)#S28W^2jeSch}Qrlt7ZAdQ(XI}Kw#6=T!Q-IV^Jn*01{1pDnSctY$Yz&T<PoH`N) z?fnC}IbG^Zd_w`J8vYaZ;QLv?I7Oy#c_wc08^^{u#j+Z`Y+S1G0pDCXjw45FVxOuF zZ}!#-XTBHm7UI|NBJDO@D3V}p^A+j!Ol^VVF$Y)f$fPOXQpr&50(G_Shk)fP=)$>S z{D#0je0HTk$Xb`c+}j5+!{8((Zrj41|B_+Reu8^YPKv#sxCDOZZUd{_TwJv*6;HZ+ zhk4O9P;Dh6-tj<%X<dqktpkctw>^<x&^r+Pr5|94>p<vy{FA@nzMjg~YQWBU55yk7 zV)4dxRT`APmm6xh6E|P#f<M6`_VH;JTlz~4Zd;g8V(40o5*@%|?>K52RDvPHy@foR zDlM?_qqL(<P^cma^DE*he*J6y-p#{sY~wue?NK84&0e%3!WxRLOVDK6Svrt76=T{9 zL^J1X<m{h@;JAU6oTAMKUb}h*bN@U8W`6w!W+}R?{$?BZC;B=+>%d2tSEL8;bvxkK zmI2I4VTgE-Mmm%ED&+4cmUGiaPbTZec*^3p({Y^^kTx@+*rXCDyHL#4NXOG#y;01& zpC&z%n8md8=2CKb6#w0}8Qa&N<=^hqB#pTm%uHlW(W8GmkDWS)l7!4lrL!~XxoEN7 zL)Gc{=VjQFVn7o;u0f)`4U0S)#T_5HjXNJ|4gBYsxV_CDJ)W1)0xyQYyJTqk5(Rek z_Zt)+JcQ#lU75VB1FJqJL)N!{p+-nGUR!k&Z3m14*GqNS{l^Bpjo$J<6qc}!$yecl z&k^jKaFB1eOBb8Csll&gOWg22n&uyxOGUvRbS=voLLZn4v#ix%@$d@ge=dP{)X~MS zm9<#5!45wfouPlRD`D0Y7h$e8nmr8HWI57hAQ66nsyrMa+kX(<zo0-Jj&J$)SBYd^ za-LIo`j`9cFq~AU37p`v7w|L6f;JB?#2L4|$@_IDNNk@cWIA`C`oq2O&b9^l@IlNg zU=++B8OdIa5piStc96#5UOvED#CjF-k$=94B#upG<A?dMgMBweL8=ZAD(px6r0q%G za4vn1IK*2%-w9Xy&xGT1=3!oIEsVd$b4q>V@o;$vKPSn6$V8Sz{S~0NvWV->k0w9G z7<y%RP1G;M4|>+_W0y}lunz-I!+yyUbg@vuKw#`<;#f2YamN8~T<BcVY`QKmPd{Y4 zaWY@_v7>RN{P)RInX1Ej$V-32Pq-b&_FeqW1&>2m-(wFett+r=U;++yi{^vg)$^vM z7UcCOn@<@kxUhALu~gVmE%WndF`?^t$8dj|7heRsCzXQC{#2M`sEfnq2JrtT-$DNA zd`|TC8R#k8f*hqd==9ZNLGdH;&zuA@?9gPc%ip5Lq5&jYCJjz<^2{<!jm?soAj|+Q zSXE9Y6FF4@f6xP@18iCL*K4q8<U{aquf!x7f2Q<hFl%*JM&ChpG%oA0XyUJfwAHn- zGKV>{X~&)EVDKrhZmi<|SS!)SEhe-`xKGNQIfCyL<JgKa6MQUD3lkIV*^WokkO}~s zzg^}YZka^Ivn82kZ3*b48q?~e>mYV)<sFp%!0;2_!0SBEE2<4+B|542b--D;@LUHL z4;5iUUar6~Tt_QhzCptAi7e8goNxKAj<r?9RbQ^bE&q1muNRK^+D@C@mRSsDDvn?} z&y&jc>eI>i2Hfv7o^5EpAoi)4icZ)J8lgs__r>2}&8Je#A7sK(`uylVT<2pp&myC@ zW}vs~AGd#84y=qn0p>CROm+Ah&gqMgiOu*1+BqR8cdlH>9L7*#|H(qnteBJC;|`L3 zUbx`MaQe|cUu<x47Hqe;1fiQIFyV=i&Mm(S&Gp9I#-bMftK?6xRf_}h77zB|^*k_( zmm`(W#TeawQ+zCK7b{qph@DGTkf*mc4ydx?!!G_pg&H9%5K|^(O>V;W>~!|&dJ`vO zd4$scjifPAC-Gs{1Hh)=XxSkLCONb4!uS%V@NNVr{eBuNT33b&4Xfd7-gN$s+f-KB zE-kQQ>hQdu6&qQWNygo+{2$|IIA+N~Q2Duu4V}^lh5_aj+pA6%E5mW;mM-zmdM!BS zEXxIqtz)yVjUx9vHOxC&j*+;T4X+bCU@9MAoa{-a^E#HC_sTQRQ5M4UxGlAr|AD&d z1(fkJpN&_Sz>S8L+=4nMHtJ6!COMpET6dmSSecDQw!;l}O^9IUw(n(*vg!2a-XT(Y zkji&D=5WJL{^47sK65H<R;>7l88^dy2V;T<@JXgJIb9B67qmx^Y^y1aZL5cyg(?{O z!2!dbN`XY*ODHm&&Qy{|u!TameUae(cyK!tr<={j;*(dQ+ddq_j9sYl@g6iy7>^zu zlj&@}6x_KmlM3%0VBaU*!^Q=MY-&NesA9=WT=CVHR(B@y`41QJ0S$#vsyvgQG*1eD zu~5_yq>G6{LJ*I{)6-I8w&B5D>=Ey$;6gLvm+SENHP4`VmDT_9mkM<f{(qQ1aQtuc z2Lg5B|LL#)<Uayp(^VuSg<pg4`rqVFjQPL)_y3>k&;EJhN`!Fv|9SmyUcVsF{qOQG z!cTSiKl%F4_1{R>G5VifzwQ4I{+|D<um8XH_dLn-LnUKH8Z_`5md4bAQiK7UZkCGC zYj@*bi)bcm5yPp@Kfq0%r_Ijauten-hU{F_IeMWxlLGt2qSK7+T-u6GEO59_r*|Cz zug&HH`|CQdVw6N<?bfo%DgrTX%ssAHYBD!z-)`>P`r#PjS&m5=ug0uWk)TVTbXZ}~ zTpD%JfSE)+MY|n(@IvTg*sBa-uI(zcG{lKL&y2&Q(?iJrUJTxKpAM=piYEIMh|89Y zXW<eNU~T5aUS%?fzbW)>epFz~4?k8twuC&#If1OcJdF^#cjCMlTK`}QsZ6z^6k9Jk zA#{MecZBg#QBAPCDhJ!NPQi=`LZ)f$0XkL?ODV-xv|3Y6;Ez3ma;v4nxjU6cSIh_V zlTln<@sx^pS_*9C&o10{^%%P^^viZixH2a2Jcmo^vHX|b6n*C?(=Z&u9HjOE+v5oH zpMJpGHqp!{P9H~pSEhUmAt&ZGnZKADN?A!FOvz|K-s&hGm^}ubZfxNz&)vriqoLrn ziDNPY{^8QD`}pJYPFy;>8gjZk*^6O6af3}JR=1|Wh_6rh$+JJ<mb40py=Wr-@O%mR z6-^?pGn_9}w4=IvyWl<NNS-OrM6I%eLGOzxn@M`~Gg^aUZ!@;)Nh+Mab_}CGW^)(f zD^TaL4u#K1psi2i_{J%<BGbWnXfQLCHpxYhLybB9b;*Z{f;DX25#jx_j{|?H5`28C zh3i?e42^dm<u?u)$CoejVA*UdJh=50FG(Cg?;sPpr|L<UyLVL<E85VSb>HB`=ro*B z+=E3shO+^^p`6On0$h|X!+opT$u%`Z@PQi+;4_trkf3~0bo}f`@bmHKI?}2kD{&=z zeM5MLd0$AG)@JnU<sB~bV?QpoUW27R&*ytTPDMRmbr#ZPMW1bKV5aOYycfJ3u8DWU zNV5*kGIuP!D<}YYR){7|<H>xgC)wTpB+Nf6@bFmSjAUF2()WkZed~q9jR>K`^Q~#A z;zWAdQHBniFGBEC6(;|472UNI;~$rB7TzV#Dm*8k$|y6@Xf@$EsU=5vW*mXWCC5-b zv>Ce#{Ao?9F_fz5($XtqMB`qhf@QwobRB!1Yt*?`Szk7l^0FmqXrmo@cc(G^<JVEE zb25AV(4KrhcjIKEIR0hTN`8`&Bi#(PXHF&_xFA=7t6v@uH;#nRpM&RNl6eF=l;*IF z-!!3GLX{-cjj3B=FyEQ^0@uYvVwbZrNl8cXr;2CNFk#0&pfVrU?P|x~#u4=Rt0yb@ ztOf~g!x>*W6Y}%*@W=Om7`#M|`Njrvq5kgB_FxNtVa*lze%F>>YUPRplKL~(&(^Hf zric=hvzY1K5@_-Mh6b|*2JBLAzDiP`9281vUZEcZ9Z2A}c#js%ep3!^K7;YXqX+P! zYy<yf?sL)J)lqcjL^DoW{SFFkm(uTD_pnsC2fAXCX=j8h7(_nA@`q2MY+n`XAGi(` zVTrI(b`7;o(WVpEPvKPaZ*cw7WZZD*l_=)g7CJ4xg_$V0!L?Gs@iZhBKWN|Nn-g@o z=!q`ugv(?q3K>EL0fIASKru^pkHNe^b5d%w$BWTYOncG^YOOcn?TgOi(G&B;l9w`I zwzoQsFzm-Blu5CveJ*U;C`EQ(__zICz7V5U--P%cXV&lhY_|7SEPJ6`ihn<>z!B1_ zq+zPTiXJUyB1?0Yw@QJox<2B6>^jF;OqN7`crm^e>9c|>Dsc8eG@I%g%fkGA;iwgU z@Y!krr3aMaT|c3-dKB@E@3P9RmZl^j*NQ&-T*TYg-Gs&o>RfB19Xk|QLjP>%l3t%L zlU9jCqoV?+^MX9)&R)a~HWhKD2G5{xO9}V)Z5C`xj2E8Cv)RDcf1SS;$<YF>Xo1tR zm)0!(g!wVrOrc$xU3>E%AHXW^v343AyzPcnk_PZ~|3~~h)(A5m3i%zqV$sm@Kk%yA zpL3kDl3lzV$uCS;Kx)U|LxGAJ-YL5Y5hZdoVu~HxFxME(ejpzEQ4TVr&qCWHMVviQ z7Ea_Ig_Rd?g4*d<u)B1Mxcxyq+KwnfIh|JYe0UDbhu4Esf+{^rR3o$S$+&da8QAgt z85h>3Mq`(578uS0Xp@fc+3%f$Nm*LFn{zxiT$)NDi7%mUNC^&GJB4(I&7+mY^O)O! z&uEz}OLaBT*!xhBFKu}XQ%$d+tov`QtXF`h`vUWMhZlYc3Wcq=6<OvdX?owc8$`aL zRH3wsCLFzs@rz<{UT_$5Q5WGqC4JgF&Xcov8^)~~T+Lg(bRjsCfFaARpg`EEOSwj% zNX}Gn8>}OX`KP!_+Z5JJ_RMg22C4k*$1b_avpvq4eEEz@Fw7D`vBD+riaf;?B#vkO zOIEYsNFjf!wvG-r2)kIJZqd6djvqW?AuR6Ar{bM&D(dd96D4lGi9-u(aQxdLOlcht zOB{#drfwnYf4PR9O_wHRvjL?0d?K5<_YF5ITMc9<Y{%cjPN1>52CTR*_{Kii;^Vu# zv-D5Fsn}8nY7<_7Wab)}^HbOjjkn^fKQzMq?Yr@pi32m4YKJ3pqG96mkz8|5K7JkX z9#sW?Q|_TyxZ!U?_%s4j8VN0a_Tb-@2COp6omH-sAg!x*=<1-r)b5zV0*PX(mA(kq zCYW-2a29FiHNcgnO3*I&CzHM;Q?l|dQO{#vs+&KIe7v0C+hU=+ovR5ka!csrZw2%> z^kwaPbWv`G0{A?!X5JmUP&sBMT}h6@Gv*svY3U*QcB>d>Z#@GIqfDu0WIuA;9z=dN zWnAKv!zk7m;hc7MBYY2@OfK6eU_hZItNYi>)mnV!Yu%mM$-Zp(we1(`4C{oxBY!Zs zXb=k#`0oy1{9)xEBl7wYhhJSrfbZ2j%&7xP%{a=vv{=N3h6$dgf=c+eFM=r=t`(<R z$5ftmPobPGsx)4HsKB4|V)1S{;=k9^S-dLpk5}A>VKetZVMY=2J2jK5KO7C6U*mZd zE{ZyT_Y>L5q%qB}@nDf7<~P_0t!VEu%02rX&4f2=<Og*&_g4*Uyw$@0Tc;p!7}~j; z)1$C$o2KwLy#(5;qNxAJ^|-~_l3D0(p!oBy@Fm}tS@M$3TdR|xVEjq8CdQ6wtme?P z=@v$hOM)lvV_1pLXQ=Ph6s6u$qx1!Xs7Xl!KbR=c=3PvzcutN219k93VF<7CCIuQ# zUcmd2vV6ObBBi$up!#Fc&_8Pg>{NV&9o9A=ku!ws4M@E2*l1>3dxCZ+H}b2m9tIt| zKB!L_hDSekf|j(vp#HgnDR<9cAIc|^dGlR7yi$?*XP?34P5!t~Dg%mMx$<#-*KnPm z5=J!bV7B#VaD==Gdpi9W+&Oy+dY0ZG8*VokDfB{@qcUree1q0@)4;JR7<Wk+h}Z7g zjtz^N@l>udXJ&bl>CfH`j}v`_GeRJ@S9JwD5~=~!Bd^l$mO^~m|1*v=9f)g<#?y`j zE0Xek1CHi4sHPFdT%8nIaGE_8?6IT*$=fh1(wCj7kU_b}y;!^S6nFK+eQtR{J$znv z7NWIusm#rsTb3y)CQ&r%?2Kf4j~*e<=sxb|00XRj6ica}Qpn6cjwwDnMAd5jnPy4~ zA3M~FZFM^jBSPlEadk;{_^1!l9N!NfegN)(#UU8AqX5_B2`s3+BiNQ>Ww?D;o3J-1 z<bD=~@#;^F*rt;`+>QQ9z$Oo7m)>RxKITgB;-}$y&+*W0;ExK++PKeE=kZYT0FtSa z7Tge5_=tiq_!>NymW1bk<BA}r@_RT<9CZ=8=1!pHUaQz#%;c}M&O_@Bb6IX*Cx5u_ zGnnjJLK^M=;G)ex)M&7vcUuD3uXnao{$A*R-#?4Lz2fLYUjm$bF&-P8?3ht^gZSs> zbf$$r#Anxp3my+urmnY>$tb4N<}s%@d*_8%{@N9W1dL`ob+6&Lh)T3uu#AOmQGt5* zV0f1+%jC8v;hjzyGL(!W#{pl!Q^@#j3C*I}abIxh(q?pG>-dxFqd4aI680M<)6@<v zX8tA?W~h3xC5Ml~^q2%nlDdTEono+4Swjy(!`Qyq4!%^~UU1Gohb;=`EaI##WZGzB z>Ny#-6x&0}#uzqk?Gv1~Z4Q$zzrp>J8$xG9Qk+J5G(O%@$%fW;qeXlSywo4UH}_X% z+cww1%Kk|#;4jBHRNa8}tBk43#gtjJCh_I>a&g|=Oq@_t&4s@=fYbe~=~bfz(`sGH zjrZIKetz<(s1eL2ywX7}rCK<dwwvjPyx=By9D;pX*CF`(4dz$-8dNof3}DFz=RuDh zDC7GPb}&p4Uhmz(M54c3#jc?!QTSY}ti0H{J<Xf>j}2w37d^xV!4ow*=oLh6NWtX$ zqp;^z2rh7%4(A$f!{fcikmDe@p}0q&yhayqy!!>cxyiWo_hoL(v^wxoyT|Lj&PHpG z8CWiyQM6V_(7k9Mn%%(T)r&z`bl$nr^v^PuSI}6gJF){TEH=QXB_1^VmMctHY6?>` zd$}9m=ka6xcLST+4A%$7gMp$7UD_Q-zQ-4ktEMC3zCuVfIYg_vlv#7_T9VXV2Txb_ za#>SK*x?vWI&&(STh{s&Qnpz0CXxMkgNJ(Pl59aYCWZ0xjT_mRyP-7Wn-ocTZKc2! zEAZT|9(bJ$7^km~#uHPxL<?!q4zMG|?LT2>%W<$cGn#E{t^tzkhd~*IY_+~Ce?4SB z&A*<Aw}U2deu0<3>hnoR8TJE0%o1URMj5`}<i)T1n}-&{-XpY1@Kik<!1}w3&~48s zly5wY>$kYU(!wH=F*C-|_n(S?Mx|o#nni4J;Q*HZq#riZI41S04yrHukYDjH_*mo2 z8vA_cx@x@Ok#fhe(KV<x@gI!qSHw)48@N-S#b|G@LIVS0+0Vt{>{Fd3TR$$7$=eXz z+-t~-PtRt3PbQLTUkmKWumr82v#GIlC!AMxVsF~_u|~x;y!z&9{>>A6rlXTUzVqDp zdO2O%w(kz4RGk1_aXecpa|1NH7FT{a@5+B@d(F9xmf`<1e=&C4w)lVMPqzK9{kb+7 z%{VvqBF@UJg!TtVu>XvqOf&GQsHb~}XsUA}ze2-@Wk0^ir2jwi-aH!1`1|`6%FrNV zlSEVsg@oMK-YF>!Dl}-2A`PN6M*~8IB4Z+<B$ZSo!+q^tN;GL6NTsAS&!bY$^?Uw$ z&ibwOoVA|sA7`C&tYs~KUH9a=KcCOu@BMnI>*LW|yEyv(cCw$imA|PW)gMp8&0?k4 z;5Lvg##T_<l@&NRyi$n$93?bHZ<6>9duivx?XX-!otC^}7&0M+PDuLzL&ZK=yvmlo z*Lq{{mJZlMmL=EtVgO5|+QoV?nw5*3@zUMN{Af-TjQi|M?@J<hcWG}NtG@xJob87Z z3ZW1h_mt{W_tVouzR>%K1+298r4GKQz_{=kSd2d;$j}#;PSQqut4)-pUhfc)9*zr* zB&U0MS6*e?0jpM@fn(3M^Qu0!<X)_Stvc3_{?`j@*DdFdX;bM)KNW2La2?;bQ{{Wb zf%srVEgARSfvz1b1RFsSGaymO$%^79pI7i3d_~V(+i^}*JRR41Oikg7(f4gQmMoty zxm27{ap3~~)HfEIBd*ej-0jpeN(s*mnaB~Ad(lKW2I^eT(8=2CQg3=Y|Csrm3Z94X zZqH%xY43bWh|-YW`;$U!YdXbAIsD1HBe+eN$v^a-2zz9isMB5<Wm?^t#s1RF^iImt zi@-gj_4v|4MY0flXzrLq!6t4exwQTebcQUVks-UuHtr%F{N=?4|Hkm4&tb4{qB6#W z7*Sk_CMtKBz#Xn6;EyVabxk#7u)G7_?edJmtS!-Stu#x!{{!5vHbuE%3M4-n<Ir7C zhC#MF#nFljKzm#es&C1H?xjBk?QtiB2|Xj}GJmC{pfcfM>=?0hyVMtrQ^LmmnyfG+ z5nB7L<yPkc`tWEmjea&)toyZy^-gDsW%*i|7oANyUr!;#ZKS@}ZA89=_&)ux!~m_L zQ-A$wxssIOeHYC)R;19$z~f}`Xp8W+$5qrk5W~MNcO|2}Q{lwBEwpl*DRl52%VRda z5<kcJ@b|yowBxqqp9%jCb-wPLWl<mmA72jP3A>@^*1mjL`+;b+Xbem(E9PmHCs=W{ zDL<dDkG@wd#47bk_^&itP#Yzujho*>R82LQI3-ea-fQ^VH-u$nd#Kj95sW6jqI;KQ zs2E_vU!K`>icbYD&(Xrr5y}*#-w_jcUL@7u+n_qF7<zhj<Oyx|&_yW>G)GHL#|OrE zxARttei;v$dmH5!-+mQKTvuRObv(__dMvsfozBXy1GsDda_o6u4|j>3ux8FLVZ%dR zth=O(^<R3R@?Iq#+2Ra`&lZaTfimg62*&$e{pj$tVc?GK@!51M)YIz72RojprKfsJ z+z~ZimUdqFt)wE$d#H@jNe<|r+W=m-m|ytFp-8b6Qmy-=qC-5T1TGY}cZ#Fj!%DC~ z${e^~R7aJ|9eJdVtE}rswKI{`4ty#!3!D!KcuaD@u0NnpN2W{>mh0Hl?fnnnO#DWy z%}5j8pO)da&2q6^Z4^(HSpQnn*0I^Ap(N~YN9%L#xx(PD_-k2LT)j|o$K~ZCs7g5` zNoLgb&Jh@6kw~dMtHnFjl8?Y?4(~Y{$F7IR2(s0$Y4YA|I+Nd%1FMfwYH^`B;Z+hp zX|3XeOV-e1<t$R(p~W3G#z04H5gG%mc#}aa4-PcNel>^0X+Pc3w7g0lpzFdvAI`)# zqwNHrp6xIwRE4rWZl(5G@5FC&qTy&z5}eh|!{^~g;9}@VESkO;yIx#`a<>|~cy<rw zcdd|R7Hg5)jaG5=3sV{v<-tqOMxbV|Juv6~OA0%lO-8r$<pb8(W01yLUQ*slJ(@2| zdnIjrY854f*O!RrKOD!j?EYN)pq9vLH}{b~!(odp@Ql}UD!9;9*c3Mc#>{Gj>Y5t( zU8zbBu55$cGbPlk`v+mW(NQpHs}Sb@Xo4BPpM&zC9kAN3&cXd<1H?E_g}SdkJUefy zP`au}{PX~MYvT;E4f#rGNoT-l<a}x7eT?Kq{sdv;7VyO4JaqSW#;uR9lC%3>;ak~N zI3K<Px>;-C&puf=!QWK$cNxrYwuO;$|9#kYatUW|*eV#+X3+9~_4GY-H%~Uy;j+-t z{JSL_`YPUmBle~|;DZA<w$_nPx5w1kbt*rtQo>=rH^}T+3JHC3z#)7o-QOO`v4fmx zQEGoYr@x$vY)fUv*{{SmVN#ydMV(YEBIP9y5hIs34R=<N+&e10(SNiCkLoZ{=Cp9B z#9%lGLvE+z@7wVZ8NQ3gnzm!qbAd6%)A+)}LHzrR6?r*aBfm|ZvD~haf(|)h_17Zt zDf>W<#7vu79|Ai%S@H0*5iAic>4w4`LFsTg9aGE_yFv^NjM*c++N_MR7j&`k*afKG zX-v*`{&+-C5dUg?f`9LF_{Gyz(41SvJ3``w!W&DV@ZMlvQk05$<G;bt$T9r4QqCuu zMfhi8L<b9|a=_VCzV~B29Q;08*fw!KdzfeQ>5N10JfTSll;&JllwF`;sjjwSQXNgn z+CzJ1#PB*#HE{n{4L4@%bD3QX^&R~{sOXa?D&{u{-LfO;w?ajEOQZ?jxV98l7_0z; zVl52ouFR)Ct&$CxH;$LTUO^fP9<rzHcfpWHMKEh@GSBU!j8@?q_*myW-ie5k>y-7z zzg69-y-A)d@Vp&ZR)&+$?_t<wlOBe&m)3~PBdGjYA0uqS=<sw8q50TgRJ)kKL*^w3 zl}kF}kTHv~fBY`Ev?@|Ol)s-?z7LvWd!mWn3tBtHiM72E`yS|w``V7t!b>mYClm@R z7KZ52`%?qB>&6{);m|*NX^<-3YraX<>j*Ry42e4T;}?=Qa!vVCc<X6FqxW<~FZP5b zOQU)4i&p}UcnQ;f_l04j*NYhku8JwCari*KmX1a)Wn)EmXn5U8zA-17>%%mmX1Nm` zdl`V0N6k2GYd4;6UquD?D)B_TF=|}t!zOq1(V<f*jGE^|A*y>txpd!nlzah)!KZ|( zEF&xz#$X?x`?5bxzJgo9aR@F*f?cIMkUPXuFVz@IblgI>+6EBM-vz-|5yI=B!{BZD z7}A3#@o<+LFswBK|2lO<zkZ9U|Ir?toOwrRwl!oGeSdcDr7pM~)B<B^4*lxb4D$OS z?dvDB=Ldf$fZ{qc%+a)msmB%ydkq5U;I+>DFnR|~8>qsqEpvG0%#YIg8A%C0t6<;E z$=KNZ0&bn@fJbX2hfD93Fhg0IRkxHf?x&M^O>ipuWE|vl>AalbSP3KA)<fnbVATzO z96Fl`texZ`_9-^!u4cPv@T@5))Bi1eSJ#uWL+P+hH5%^9d*L08iO}C|DJ)BFf(7BL zX!Ck|Za-)eviWZ6UUi<<?=k1M<=IeJ)rXDC?By3uUlQteI$-hLG$DF+3rXf6y7cig zbV<<T>QCx;_P3MJv2iutQ!0~qsM7bc_Y64ND)Du?Oo4*~4sdCHCR?2AgbV*k138B# zqr$c%F0c!MgRe%it@ChvJ6T#IyOn_X!Yare5-I%jVd3M>L{e@W0l!8MVzXK6Na@Z6 zUiQ~Xbh&#Nn$EjZ*FzedKfi|F9$Lsn*Ut#`M_0nQWvcvjj=O04I1MNF-zwg<`VKR8 zn^WD3L3piqFMM#LJ54s;OS#=T33;v|6y4iKH0XN(KE+DD0%>2Yo!XiA-`Ps%t^{!Z z*Y0GK+eLihdV)@M@6P7D8?vuTdG4+*D9qW)_4{^nan5d-`Jy)-N*~J>-CKl%%Exis zn0)Y^G><pA)lvWMH_35pPp}v~kFV8Av%UpdoR#!IW@WgG24>fa(L=TZ<$WVRcEH~f zrzmFaB;lD&2VtqEHdi0{3pQ7BVg6SmVY<!&x}P==+C3}A#v`%i2L|kw${GI9G-xP| zl$;NjB#zXLk*)HmmS=P_XFIjNm-5!n2D8g*KZ^CL5HG##jlQOK=+mmo{XU(elSV7) z^1lnh?G9S#T$c&9juTm{)SFb}2eB|u52saLA)7f%*sI6^6D))AMtMB(3j{Se;p5fb zY_<F&G@p$VL|-d*`gR|DciYRi8kWOoy%KTx>l$e`y&kH3(`nMlI&d9ri2JARgFfvP zuw9$v?v2S8nxDz|_?^Bu;o2ck=&y{So@KIS4d-Z?+ABfBJ0Hdbs^g-NOTuBz9?)0y z6+HeLA?ob!%z@EgD1AW|b-xrQdUSM#i~inxFm0Gv=T$--7xv_tUuWa};>Y6Ql}p)s z>}_&CIGoRgb>fDkDR|3H4P&2V!mR+wy>-iwXH8#@z7KB8zMXO4RUXg9Jc-A1<l-l> zMZ62Amw8j?OAX{zk|=0^8jiDQ1jW1YymXc=8=hC8(i;)tvNgT<@p(i1lC=YRzU~Mw zpC{9>HEDFGNQttG61d&YSRRCmRJ!dHKJCybH&xf;S`SyzTT0<gy0MLneYV2=<J0Kt zoOh5uZ4?%6Fyl=NqhNwnBk5|RtjY2&eC>N0j*W>%uVvHF{(A>(Ugm)tiY(bWMu#pY zoR$SFJqp%RUfgc+eBpr8UL3Xhmhi~r3P~ajG9X*N7wO4Q3cF#3bZ@V;@Wak`UXfCV zzr)u>x6qrLRaD%|oqi`oV5`><N-#aiR=rP)b4OOm`p(@&JvT(c(JQ;qTD>zaQkC`) z;xTc-r$6*JLjiy6-w&_%euYJiHR9=MMKobgF-1mcQ2QUAU}<E;IpgP2kF+RSy<;X! zF0tk_U5<dt5`oW-ib378RbcLsEFRtZMl@~tBXnwPpn@bj{`xo^hAb$iE)hH6#)$op z6h4o)w~og8URuIcxASn^GE1;6JtcTMYeM}tA1eJQ!=~aB^f951(hl8*?VDoRU#}O8 z@0SE_a~}zoUy5Y?M|YIg3Uit|NC9U_b?>@Isl!(Xq>$BfTYSI9Sg!D@8!pq@#bS>c z?3+6kMwgvOV-p3`EpLOb9;c~%=U?%PRR5jfcn%(c2@c*ifijY{a6q^1G{HAQbYCAq zb}8S7TMtYUPmFuPD(jxni|q&GimRl$kb5Sc?PP=Png)q0syD%8iBG70Sx!#tZvxb; z7GLhZPKO!_q3%>~IJ+;0-VYnbeeW9q9!SAM&owc5VJy$i@t_A9uKzQC71l68{(o41 z82X>*U!>5@|ARmOS^me`=Ktrf|3CH*)s0MX$nR1BTHE^a6SE_*@z^_>C7o$LEIbBk zHu)@{(Foht=)=Ofsnl&(F1)F=!HY?|!LQ;zJWM=CN@pEdU85bQ7afr6|Ma3Dzv05m zKMUFA$QKxKv6S|=b>xOi9#k^xvG~{Pn|N5aFUtGG!WZwIwEn3ZZ}C1(W6#RLIcXhl zZ>K8O&;J1*->G5g=yilcwD7u;s$jJ>i6@O7B#sKNmCn2E__c9=elva+>N?s$Q*$-> zPq(5cug}6D6Nx!@JDC<FOu_@#dZE(rT*$GQOx;IDP;>N4n)C61Fs$5N*r;Fw58lQL z?yQKfliavTTHi(vd<8yzZBeJWs~8@-7qsSn18db1S`pU{&8z3)n^(46zuc5V|8>Je z6D=~G8qSL*JR}G4DHM<13bu*Q!0E>vKC6>KMJ@l}w*OO6MPJ&R)0W|#P3OSa-g<o7 zBAPl@3<2-VDD>(t)rmG8p&FAr(8)}n%^n8e7S~v4(4R>w-A3@N{+p!QX%s*G>A~r9 zPf_P=TkN)F2YqS(7LGXFfSt9PRC{MOb{kiM^0|k|&oEbTTiXsfN>>bN%cKL=Ec!-T z^XK`x=>Mrbr!FU&HheABchP}|Nx__uxRi(8O#$^Rdr0aZg-=I+r4=t~!M>{(ws?+$ zKI(Jj?UFiTb<Gg=J=IP0^l6VTr9PFu$2sv(FLO5Sr^pR0$<U&%4ac*bg&WI~(5c>o zB0Rl7wSx}1+#d$D_5Z{R#`Tam?jzJ5-NTJt?XaT90?~I#J8&0Ug-_SwL4WK^X|6U5 zPLJLvM#dTASL0|H8>hqx4U*f<PY#DI=L*LIXG4%(JJ{O1mR5X<5Puq|(z~hU64ybE z{0j@PG;234I#UYqTRM{C+Gp~hPkOB4_`>12V2tXMrt-L3`O>*oVkP)@;rx^NG$(W~ zYePCap4&o8GiAKwpM~&Z%q;$0RE#OpGw@k|7apQo2qPY+u$!wi?{mHcuBu%ZN_=lf zui*eFt{;M>-HU0q<q1?eKT<B9RN*gA`eIu9j!+eT3l^sIXU}?lOsed|u6w^wz`-km z(oS{$r5#N7?)SiuYm$FrzY<@unF61xE`dBg8M^&A4t9T@Q_|-*G%`*)vxg>&<!3r! z*T~~k{;?}o{CNV;(*0n)u^|_I{R|fxWjJ$}8@~N%%uA&Cw8onjcx8>iPr@J4MdwwV zx+<3HMn9CVaqy?jKQiU}|K1TB?FyxH;5A`~M=S>T_|ulabnGSGgRIa9_+mU4r&xRx z=dM=4OGbmmnk*fNDjZBBmdK&s;aD&qCG`Up_wc;;zhGrRf8P8moC;ocb_lWAPkXN} zVc(iw5Kt4z1FeU1^}Hxn`>QWbe?1%HKiT2Ete@1b^9j*snk$!$yT%!%Nhq`a4rlgi z@H_SO;<hLDIDNu=Iyg`Tr@V3$?q`V*SzAgOwS(AnisVjMI}_e4)j?;4EaB##B=O!Y z$;&;)1m=0<O8(*u(mEW&?=)ufoz&I1`9>7iZ@5bQxfjljnS!D9+u=`L0@zt?rHnmZ zLes}`_~2~>g(GaFJd8h|>s&1!aNh)V?ao0r<%?vrV=B6pw#Tb^KgAqZHJE5`4SkJo z!r1I(Xj*855z4zMY+f@t-YOAK_t^!qji)KK(nw6-`wzC>+7HnKF2kBNH(|xW=^S>a zD<AUtDdb#wfPEZPaMh)L_^u>Vw&P)wcyQuf8Zv1+1)r22@1za4bbClArFm)1h34pf z=9JLXs*jE{YXvh0SM-!-!e?sTq4^Jtd0}y(%(5a$w!m&XmL?Ij1-o;$+6U+nq{4|2 z_L%59h8^eb07s*V7~S5IDft3N*S&*TUpC{@XChsB+82A=+z)>|?!sq3Mf5SbL<J5N z5VgyR<sp9J0^6zNb+-o|uP?^O!d!9m;Hg-+<2;l<namA-3wX1QA-rfwgSeYLsOP^3 z9-?(b4(d}RACm^39DfWu=lj4yv_=Kx_H6rgvbf~SUwKM*Q(mLnQ%qf`&Z;5HxnsY} zU};t%ByQXd*7tRV?Pt;`@TCd5*eybri>_D|I|F7Xmx^8zmoX&Wgr8~k<K>k{$tdP0 zsPq9eIbn)xr8>RR;hoga<dK8RUlp(&_*HDGnN7czujf;b4M6?haIE>=mE8&rSt?4w z4-0{39N8mxD!54f9XG*)nTm8Fy9^YsmeG{d7j(j*9nbhzgey|bdFY>hTv{UK^Ua$C zoAw%XPe`Kkt=*whgv3~jU&8**{UGmrC4UJRz^6a%64km*1n+_U@K*UOQXH3ycdmRE zik307S1{m(_3?bUVhp#B-2lzrx2R;}2n=uQO(WNjf`a}R$g%b=3@TN_7LOS?QjF#L zfI7;{_or0`x;$rFKBt!$NNh)I`lDS;5wq>(Z{N!JYk+_@Bd_7QA0v2i=T21kK33cv zWXf0CeW#5NvuIT29tzCALE(Q5@X^7?P!Kr*kM-_}ueQ(Ux3Bw&*$2~M+55Y+>%IXx z9)2S3{}6(H`;JOA?VD0ve3evv+ktB|2ea}NT@1XhB8V*taHuUAHtV*-#7?Ec62Bhg z-n<ik$6f->5oNHk#GIpI*9f=Y4?!Kf1lVylSbRIEgiRitfMeqev80~{j9+z<oI^Xw zyY=WUd6=i+Q;Gi>xxy1Hd<=L-;x1H*JjU5?TyW;fPB6#p5jDUb^bAnpuj6-!2_53- z^Uf1csUY>Fq}rMDmILyc+6{7*B^@eqnkPetwLZ}Cb3LpO-Lc}DKVA#=r3c5yW4kr+ zXgw~GY8C)*zg#Z7&1#R{F#~y<l%0%PXpD|S)%cv=ApWJ&0|LzBAkN1Jx*TMQ)pAMp z`?NA|7&4Ju9?pSI!ey~0wVEbK0g~ahM=9ln0bVvJhEaFA;;?{i^r_2V@$-}*;r%{k z(O3$}R35u0yxk%({QL$}knK_MjH#jG=}i?ALw{27oV#GPrA<8XCK7y?SCZ?7e43_q zoabygO^Ip}<4hw>nnlt9zvdKd^0`YZYRzHjEiEpW_`8Zu?eI~}Gst~l&CkUFQg1wf zZ9M8AXNV>0q&AS?UP+bDPY6oadr(`4QVJS3mB)9Eq)xt#!WPdn@QND6mg4WQ`p{+Z zMWqsqjrvSr^-4T_L<5#eY<SZa7t|d%nCHk6a9Cm-Rrg88bC;_?y|N#TIg><NBSxTF z-vnHE>IO}e_Hn;$%mppi8jQTu1K-{JO8aW-#oQya@q2Geyq7Ujrm(mdN2cAS$RV1T z_&}e9i_bt^VJ8RQisV&+;goY~5Pmt}$!?BG!mjOG@b)l2RvchUW$PlLp;gM^{+v(W zj`(6sR42YTU_T}7TMXK32Z2J*aH*HA#MPZ2h<C5_M2*_}!s?Nwu;jiYb>8L$1qDml zvaCo5dlW%71ybL9i7TYO(c|q~?!vqQM_|&2k&vPs08`h;k(ZYe9`CVB2sa4@4x9!5 zR$HL)zF<DqUGkLsUxSRd8(FtBj=CIM2%{`JvNC0gDtG5&>+8Q{_dQDV@hOMa4!XSH ziUo}OV2x88F5rMWsj$O(3MTIL!9}}m@bKajbbDMDI!~E_J_Us$7)*pK5_3%OR^n)< zb!fQg2&H`7%3o^_!>R8m4)gASF9Pgw<Ls|u)$nB4FwzD;=v<<tf!%q3tP)lf>;l7% zYvB5lYYr(#J44qmlG7}+QfzwGLJOBv3Z;9aAoH{}$C&6y4uJwXu<xAsPjUi2cruP% zdMe<q5l*BW)l4JHk|AxTK2P|t32%*U51U&C(7^N(T6H){JgYSj?tJ(O|CDZlO2-gB z|8o~ED%g*g)GVp#ZbzKGa}=fol|akY&FpIwLMAU`VU(2bDGXXA>gSm9t!w>+An);{ z|5wU!4=N$OJawM?;CjVt<I7@+eNQ29aWe!DUkm3$J5uDW@eclDeu|X`=A+e}VW=3A zjDCjopb|fl-<aPK3ttW-&+IL9(P<ziq^o20)2Y0=CXDYZO~mGg7^>U8805!i)3<T$ zID5fXdE>zxn)%+50xo}}TUQh4n&A?1?%_Z^PTdpcHrwzwo1W|vHl1Cjb;Jd|T4gkO z8GZU3jJ;1^pnY1l@YX?uASoAT61%R#XJ48_$JQo@_e6eInn(5_HT1+k0px)pJhi0^ zcNZ=ac4)q(C$cp9akwAW7d@dfnM&m6e2zTthl{4a%jlo3639a$#I1G;c-eUa-;T+n z0b88VFe(|QdE6G~C&#e;ZYlTPI0sH7#nCy(I>95V3t#s%riF@WSoACs4pvt|+9P*9 z@|z)L^+}<oIgeVm_o1<))Ol*1AN?vk0ofzkVdtC|<QKG)w@f(z3m*5xUD1hh;}64O zo8NSN=Hekbg=tZ_qAG`3t%PdbaeO0CT{iIbONxw}1brk<l7mfG@y>uc3J93Vf3nqN zagGD|#1R*K`g<F%ci2k{Ji~DKXdT!n@nT(ceR%4vGcZnKXlcE=2<8gXckAn1;ak4M zH=n+nMV&qr+Vwm9er}AFUP-V`{0BMNVch3~fU1#+81lOSm4}R_hwpDv$*r{%u5}!a z@D%w%JvTHRr7ST)6)9}SIQ}H{;Ytg42v0UgfMwtA*zx6V4vcsqw^BMLstM|6N*<-5 z?e$8BdA*(KnQ8%aa=T5H+nR*dHHSFtnj2kq><gYPMyMB;1wUh!K*WQ5c=^s3{+Soz z?;Xh$xU)YWFs&E+HjKx0@1^|eR(-as3g>{62CRC}L-_E;Nc`%VO<HD^km~k=)}QK2 z4_0Ogg-@)|Y0*T{*Ib>CZPgPSO*Q$!NEvx5n4;os8y@uF29$U;K*nk#j55xLdGAee zQMTks$a+J*Z3o27r}~r2_bb%?O*pK5JqT*s0=aX*EI1w?FY%Jqafr)Y7W8gInblRn zV7n@6ImJ<UWF%x5zNN)OGU-p?WUPuA$DZG8`NH)1kT=pw{?bVSQrCIY$KXDkno>Xt zE>n4#l;2A@tIKiQHi|1%M!@wiGIDOGMjf@|`M>Hf?~f$j{XeWf{m1;_L2_*jEoddZ zKV6_t=MlWxxHtBVtrT64CgTBRAC$$`!_*1Rlw8?Bhqf)n#>y`I?Q;}{I&G9(a^`YB zUkkpYk%EP%f;l6yRJ=LLor7xkVc*9iW!7Iz(7fX$PV4v(@_Xl!vbh_@#D)vc#|XI8 z>@u-#6*%NM^VH*mu`&m!)7eDWzbS{V>MG#4>5>O*a|>M1NF&!{g<@dMKs>odx{pVR z^!2wn+3#lA;Ge_ki|ZgXUf7Av$9#el(~m%(Fe7SCW|*bEpJG1srkD>>U;4W~U%2ke z6_N`=esimMC*&B%^zh+5%d#liC>>OS6JU4LRUCb^T3+d_BDOQy0|k=H%;0@THk!N_ zRx5rLS@kMpd>hT_^Xy=n<fPm5dMK9}Dq*y}8KjTv%8oIAX?y!#n6Y}7xUFm_?A8?_ ztoj{XxcD5NSccLbp`GYD@F+vnC((L%9Vz(yg!FmaX`<URVRBP0n7J*a4bRf)w`Q2I zR<=;$9{0eOydv7{B4umVX!G|$Q+asocS_Dv<C&eLT<oirT(Mt6w5*PY#79YRrB?xc z3tWV&<aRVC>J#N<XV9S2W_Wj=t>iDz!vW0`X{S*r4qM(vpR1JFp<p_$_f*E}av5H1 zJV0w%%J;f%!SSc(aN8ZpCv|KNn>|kxRwq0YC#kQ23))p&)V7^o&gjkyW)H=m>B`VQ zY!}>qzE*I}+>Vd7)WeY_!F(bxNSxvu0e8Dw3R@EO`H|In${ukD9B!<oE|X=LXjerh zI1?S;8e#AaTcIHA8!3#rM>8vo#i=jb<4i9A(|SezP<IrMDw#v+CW$rf<p-f-*7KIf zxiszKV{jW*0Oj4%;7Ey`#AlzzFKyn@yJjWMw(zGXuY%x0<R~cBv1N<?OR?tuJUWne zjk;%UL8J59{O4E|ENS^n6Q@)Q`or7t`lm-}YrBgCT1xm!w+Fq_ZioN8PeRtN8F=T~ zLj2w>3M#$t;fEWGxw^(3er@VS-|cewX~0N%=IwcW$-TYQM|MHg@@NV?RgeF2Q((o} z7u2w+5A@o&Tb{UJpP2q4jQd)I2syzd+<efTi(}m6yBj8QOwKxdIMIXi{3l?`uNtvX z$3-?@I&bYT8-?TkwFx(CZDqS24n&QV0#LU;3ue1|f>Caj*t`86xm806eTh(KC% zsWX5-9$X5R{#BIbwG5{ix^X|A3mqQrp~!O*&&E<>JJ-tC(Bh-;P$QcL*S0~=HKTZ7 zm$}sVb~$vK>xH*$1H|y4-ol^PXJA1>2hp+M1dN}$NA$ANp}(8DiX$?I!cp6K;B{>$ zlClV$Dklo6=hX0(zaFCL7EpP%RSa$I$`5>U=&b`$MyC>Z<u#sW#weljyf^T`?1Zp) z?;O~uaS~li2J`POPMkJiB8;EU;>usU0bKh-`xR@z<;x#v8de16g(Km6yPl|6*Z~$4 z63r;c<D-k>g{N%+qC>qqYDqc!`xj53R;)Rl96J;9?&z@nr*!yW7Z2mA2H?K~3AEus z7Aw`irF8Rkbkkq8e8I7vu*YK?tH>lz&>?;P*fUV@n|KQ5ekvh*sTLLb$eH|fRk6<} zUHtY}0g6vfLC=}0JbOhVWrZfvpQ@vR$15$?yA&ZjG%5v^McdhTdlW||xWR(x=aevZ z5`Pb}<+@9`U|?B9i+g+Ffz0zXqgEe3{@O|Nem27P_tV*YI7|7{d8`+Ff|SOT!K=nP z;h6JmZiw0{6Qo>@MTG&sS5Bn~apBUe!Uibz*oEqj@l>hYga7nBExbQt&oOP$vgG<T z*t#j1dRsiC*P1C{<@t~-$2(x!X(iF-zBzwhn-2%;UX%I#IPO~hL0p`nOdlhr(<!%& zkkA@N-<qUM=&#B0>rrMnVU_}Z);a;}cRYs^ZQV#U?WXV{CRes<>T%(GzZ6QFUjtfi zmO}XMbh*p+5b9t&3a@X9!#uVY6za=`FX2gIr<f5S{>sDPm%3b&x{SP=3d@Qv^k-$m zV9=F^;*#G#C{n!x?$ck7Zu643(+EqLwpNLK0><HBw^Xv6uY`+}FAFDaR#2*v3OjhK zaO2BtOjs+h-CPy2xX}e~3{;neNPW|Z|9ZhRssH9UN{9FSSV}8buYmPkNARwMf1(b$ zi0PY+Nm*ZAe5`R*NZW7<rp4&9?Zo!1>-CTZr*_5c)!p!gaf0-{x`cIlO~sb|PVAz$ z4ouR5p!Ypf?)mWq^tau}7y93z%SET;W1SKq^m!j%F#fdg)UyC44YI-cDyu7~C0l%S zpb^G+O+-5>w=I+QKYEwi=xLv7(4KLHO!it5eohw_Jn=*KA-15M9>*UB3miFVC5HM{ ziG6mb@U6{JH0ON0IAFpK(B3eWe4g}RJF%QLeb7RK9)0n~>ogeHD)p<&=7_WQcEu>= zOuU+7j%Bx2(bXgOh2#%+DQ4sW$f1{TNMQ-O)^){m(%Ju0;%c5`S_)nr>+qOKH<tH! zD@qJ6s5F+&68Y7z!Eieq+Sd)+=SQKgN{*oGod^#<JQ1(mKMnnGC`<Pw_RSaoE1ITN zG_D=T4Wr&W?Dy=0vV&dd$}uUM@=;0RewWhRho^+THwNR-w+qB1lNxe4qR+YZt0|~W znSaKI!Ox)HQ1bB=cuKwNP6={8b4rEhb~p}Q#)qQv=tj^`i4#qCh6CyYSiUKyd)wc^ z#W&TUrC0}-<{gHeyYivo)M-p<^c2f}|AT%(nwVzR5nEpVf??~{gS&SjEm)8)26_9F zQkMYi+ZHW|z52t6fDLH%t%JnxI6!WjCC-ER99C41!?vDwpcUT>Su>8^mzeSA{CHvU zSAFVhwV$TMZNx15M__vIG^w@q6IC}ql#L5qOO-3bXp?6roc=<uV(w=hjt&~cbMvJ+ z$6o3ftr7u>cLwu}uWMlHPz`y<#b*4)KU_Fam5m;U0@3D>8XkVRA3V+8(k!_$t|-!m zQCUaGbW(eCSJ(z6`&aVyo$b*<*@Qn&Nrx9_n`wzf7ryKki2?n}uw%$%I{4I-{U6C_ zdB_@69GOD3X-~jPN6K%v%!2B<uVA6iK(yCVK(hzuMT^<Fn4K66TIb^+wUY@$t|BYU zapz~U>vZdHDr`!V)_b?%Y(4oEsCu6izNuUx|1TFs^%xDZoKqp1?TQzU6pZ9`NrjYJ z7{Io=39=21e!Nmo4#pFgai4)7==8p!(k#YnP#SHCkzaL&*H;>{%}_h`iCV&2FMpx} ziv(#VZN0dD<W5>p(}!|rOZCSKTll8JD*E)~IavE-!<gF9m^WrL6@Oj=rrjoU>t=82 zIjBu;bVC_utKX!gjUnP+n|cbLmjyqLpJZ`BPvLvMB|h|f1Idq-;KI3{*lolO8WcT7 zI5qB_aG=`}P<-P~sz+CYTl_V#rcIL#4g>GF{z^W!d$Q!Iy~2O1%`wJ!89b<IkpKQF z!vlj(&>QCq(EPnbSbWk^44xM)7;cHgtx_iK+{`PIPi+QUuP}gPhFj4)Hv^omjK_PM zV(8nqGcc^JT%0-BSZ)xg&)SnV!JF6Gxavy=Y3Ce<(1$Zf^+`8)EELm<F5CEMLo<9x zeI%Y0YJ|S?On8axA=qlfIz07ugdwU6DLbTqrZ{BqubtX>t0_>t`uCBjqdE<HhF+o7 zQ~T4*cgJ}9#Sn1XdRac&OkXf@jNq8yG@+)vJ8y=1TImtNd82xh<_4*6e{BfI`DH_< zrzLL*G8YEh>+r8Gl7Gi?4gOw!jaOx@qhP!1(8D=YY*@b@6q{GWzCW%wcEmK<o$CfP zS8)>cEKug1`ya{M%-rdNpFWcAS6R=R*$(rPWBHrIbx^L1qPx;rpp!==-Mjh=P4D>7 zgcUWSYHlzYlxz``56&m2iK=|Hun*>KIR;$|X0pPLZDRM$kA!*82h(4v=M*@;D>XS) z(n#$w96c_9-4+^QK@TMyyZi|CP5|h?u_K!0$I_9l+I*qmB3*Z>qcbbwQ2ux{n)O$} z{^!E5ZP7R!xw;7?L^t+*-9!UVuKAxn;Yj^B?3M8o6kOH#UQ~;izcE6XHa3k@E(UPO zGhI$yq{iXitzw3qA|8|W8>>4<!t8-TWTUndW|`It2Q+Iz+Kpj@!bE6J94O7wH$&f( ztquteI(#;NGCyzKLv67uD7@zl!BVk1?ppU%Y%}nKDp@$&?42)0PVEHB8$;>Z;2^qr zRts-9R#L%}v2c3-c2=$lplx$?A$;F?;b&bEJ%5{o{W9X=im@r_eVr%BuTAC=MbWZE z?=!-@5@V{(8;0$N{iLHe!YRIs7Pl`+l5(uOq05X!ytDSbu;!;f?{d0J$5SO<m!&Z# zo)d6VMhHz(zapxd&E%U?`r(iP!*SxiYvA0qfi2GT6PwmWgN=7Elg@8AnK1+FB{uwC z3s1b<bR2?y?-R^YawJal4%+@^KL0$vTX6iUOSPAb@v+Jq`XprwWA~hZ!^7L*yTXTJ z*!@RT?V%(5=;TDISGq`S%_ZpDy%ZEXui|s`8v>j1;7j&0Y2Pe|h_{WxHCX`c+cyOF zC-moUN#XKE6<u+E%1har;gXm1;XCr&^+vdOD4RW&jbyWq0`zs=DSGEc$!~O4qkSzy zKs*12*j?cq9o-l~(}O)gyq*l-f33xuBSySX`u~3;&C7Q(vBSrwdZSUX4#&Q9=FD4H zg?ocix$iD(dGdgDVry6q?2u}2o4QWniQnhIqR%g3K~gdwb(Vovf+-~88DY>ABUWcW z=&<UPEbwM3#!0O1v00H79`FCrRP6}4V@U@(7Tqew$tJOA_?<eO?1|T`8qrBFAC~y^ z<E4icA<uahEI9NJigZ$FTjN7=-7U~u?@4s2WCD%dFXh=+MUrc7H|VvZ4d%2jr#aWu zcx%RI(pF3pyuDOmiQ0V%-1Z&{y5$Lb*L@T`tBYiY=TFds${Vz)+8JdVI<o8iLh*ju z2=JZYNe!Ql^Z9QJg<D0lu`F1hrzfe2!C95yZKTB5{e^fqV+j^`o6A3eK1^0Dg8!;N zoHzB{`+rz}_<ucr=s(QA{C|}HqoDfVBS~5cjtahu{d|`OEnl*1^^gVgSI-|}{U7sx z{&&2guK(@vhDyAhP71UBmmi58^zXku!w2~n?3AS^{a?TTyK4&n{hAWP#{U0u&Hv@{ z|F?fety7lRaz@<OI0`fqB4iIst;jiIId2O72Kr_usCUMp+-Txl4(*>phngh@iqI;% z859fMPVR=}r1wxs_MEfj9jzGH6Rju6=;ieM3Wa0iX>;8Y{xjsdcr8ksZ#Ecnc#m_? zl(LVl9_z|xOHAWlY7@Z7dJ`QhHiQx8`hstZA+MGCe6`lCaP9mDI+>}%ZViKA!oQWg z>d6(MU3LsueD)I77xm`YyrbmPJd|42ON<ajTQ={o6OXoD6OUe=1cy-#G~WIaPiVdq zJ3M3f`&y4G#_g2vc)0_Hdg|clYt;~zKNT;qm*Di}oYb%H2WhSuG~jh3)yhS2{)OIX zFfA7z&Rc_ff^)E?*bs{;jq(frL3jK9P?WQZydVA(XU|)}FT$<BV9#*2J9JQ9C?80# zOq|HYMwy)q2GAPGNB_P@C$YUjEG;>{1`7%o@r3MH@E--J((gN6iyFx4Jv#CE`y#xf z$DmvEipn=>!qGmWEOmT7T{Mxydc`q()nx<^8CWFl86~apua=?B@6m#B@1^|rOBinc zJ%jYhRrs>!KAQWzCpXu+g8F9x6(@V+`;jVW`AOn_&x?bh8IgkhBV}GVp`0EEbcScm z1@!x_W`&Ql<PLo`33JN+2uF|XhgsYm^=~NhR^u;puIeI<J)=qMHmdTYM`Ll`%K~Ah z?{S*)#GPilC-R7haOyF>3&xkY;OMGxd_MDy`0HyR-?df%|KSt3vcVSr?R0~dSPN>~ za0~J_nBf>pCHnMSVk1-@7l&nTL$5vEnU$ung;6YZ=T0;$(~6CMI$>x-Z+=&u!pjSm zQbdtCXjfU&y~N?@@yP_9UpOH|X$>n6R5v8EhAxnK;-pykQJr3EC6fEEb9_v91-8gC z*uZNK>+Fx>W8sDs7w0Zur@fV+o0u#7soH~4)qmm5>}^!KNegG>OSOW)OlrvN%_<`o zfkoP9I8-KiRN7lpshvHlPg^8-msnx&^X+n@=XaoWp?k$sKM{_qh4EIW9H<>vCR`ux zES%gahme_;km-C3%mSBC^K&))GJ6=swo9j-CL0Bl+Y`ZTs*-fxn+p4<DDwxezAWU{ zQ`HAYT6@14JcIPuzq>lt>s=SG&bF6k&(gs|a-C)<3_+j7-MoK5HwZuEfurv<z^M#( zxIa*pExYgL_QeTMpfrKj%pXPl@0-xlr6bsA!2aR2X}*-R(*&->`w2aIodBz$!ECfB zj_qQGWB>b+czN&_+LTek@$Yo`poapS*|P`i|7@XdGhPVJk6BzXL`q?_8!s-(w&Wxp z%z-i)O1nI9{^m&Wypk^+i!$WfnLW9myi8nn$OLvs8HCX*U&`Yj&t_FiP1H9~!J2uO zFgaj1WZ0;Zb5tlOUk!p@n{-L-<~qK9${%mtiGZ(9hQrJ(i5DH*9TmqfVygpTc;vjC zEtjq3Cx=IgXH|@bC#Fc=hcah;;kkp&9Xl{yb>$nuqcQ9AGvUrhANr-ZnXN1o@$({0 z41m!XRIJZ+b=l(MbtQ0fTQN<mwq>)rdxCSft@!Xt6a_zCEY-kwVK4g|@IEqG7IJVP zRq19yKYb6rH~KXD`R~O;bMH(1SvT>)zkwV%bOqR)y$S`k?h*&z7iXkwAmw946q#pF zBOi@q<xREJ(q=ARS?>x4-)hCt(ep*8%}LOfJ%E*R5-6=y#*25{6<)6I&Xaz*bDZsI z8XLb4JB6jf)gn#Q3)bfL^XF4=T`S3&4%5@NatiyBfz3l^U`<soHrbFt-M&8{&u53= zde}IW-|>N>8DH>h_wjJ9yEeLdIpU-zw<+gCE&sdzkL*g5VBkB%SE4G$-dIRi&&$xI z+!-fSsNtq-g%EpbF!yyf5UgU|@u{^x%$N<_GxV#xBBT-S7`>rIU8l%?H2jyh&l#>P z-A13~3LIdROlO0%ahc|2*fMMk#wM%a{7h5avHmHE>c7ZQ;(@n+HHA{@ouD*PnWi@y z;+?BnP`39VrCl=T4r&&(|B)lCdKW{ZH#=cNuVnfyF<LHO@sxU&GQyOTU~?&7EZbzk zV>U-(*PDJEko<}M7AWHTNs4r8N*pY_G9BC9=!MgLhQOVLi_mGS<kWkATy}1XJ<QU4 z1qD%e!M~)M?3^;mySao4Taw5!y*<uJy$Fv-DB+uBdxT373n+2$RH#3FMB)e;W4nu^ zaBH<RqcdneX)2wB>dR-uWg0K(-|85VDvfyej|FgImoX0Vu;$fy$^7lr0S<6c<gx+r zxcu;CaZmqdD0A#4&F^l6`C8w=&B&3jx=#`(|9T;+H@3q*Qby!PqzXn}3&R<m3rMBu z2h4m~B37iN(2D!@;<>jS*n5yJPF}nU@9w`rF-Q7vRPSt|z0VsMyxS0p0z2d3TkaG> zhoRr#SLF0#CHvRkk@#mt@b0>trcby6tp*3k91UoV(gC5UWj|_YFT|(_ZJvJQ0FRtL zn4J$N)8<|pU=m=<5&rsOo%<&A(=@^f-hZL>!(sSxp)*<pn_#>AURYidCEi<6hZ4>X zZdF@DuEthjMo<W@vLA#AIjQvGeLdavdIYY~j%4Py(_v8f2a35*^iioBUWzDy)5^h= zd0->teo_U;autkM>Bc%kH&R}y7lyrE3&*@Pu!F;N&foW)Y%|YR{2o#vr0Z(Y`yY+s z)wJLA{vpGifAgWW@w<@xtr%X6Qe|OXJ@s33oR-y&fvgd#IRC2>&AA<jvR=a|@`wlt zeWU4S2RA8C#zK#_8lksWPqsJ}2N6l<M4i81>0Xy;&K=i<M@&8;8@Rp_pG_*18GV?J z3wG^+!o3~X_H>4@sWqIHme+#VVgg&LC05EjZ8Q#1WS1=rL2E3qcHknIQQ3`5u9gdv zs>&p<#7L~1(vv&bnPLB~Loge*$+Yr4uuknQTzBh=js+R$A8&!?mL6Cizf$J-+LRw0 ztP^A_qG^uCU(kDR26ns3DCOD#C^^1YcptusYku#N%~;=o$Ng@D0Q){@m}SBt2G#UK zVuKiqqxk3abA%K7@>#X^@MBB>4KSL@k9I7hCh561Z0XH~n!{-Lwo$Y)qgt@Ey)WAM zUV^f1{*WH-DsD0#1ec37X~QT7?)cP~M#q(cQpC>+*G}(f{^jrD;=gIMVPh}MRxc9I zxST?rH*?wjYXr+zgi=nA$Kr$^?r_O-7A);Kh`$W&hoAJB&L%DwM_T7`wDu4RTcrqf z4ZCpU+a==d`ECLvrwf@!@6iVBLHPEEqx|c}Z!oUjg$KL1Q16`4cr(i!PIQa~qYqVL z2c>p&qt{K)^VvutMw(!EF`UZ^OeFZ#2I&;LM!J7?(bJJ>@J;)ou=v?FaiP{TS{mI3 z-Rf>pb=C;Za5fgjXUid7`x%t_%n;yuu<UI@tK@hz<}tUu#cYcsWIkg6YN>4&+g6X^ zK6|ENFQ06j|Fo25hV3U>`3+vHJ%tw|6*08yKukM)gZjTGAva7N)2jwy{iz%<eM#b# z@k_b2Nx;Q76ERzJI;}J+5uJSxkw&{E&~A>D|6AILCtgUPCwa%A-`)ZIy@MilKeU+b z%$4~nmr{?=XJQH!2stfVc;eDrPz-j5X9bt&NsuYdwbf>w`vPj4&*pXOSJS8OO+w7g zQkI|3;IO*2RPylvq3b&u@^=jDO&E(KTkn#FYod79(3g5BoaUpw(nuva7wmGcv96vW z_mpb1Z@xbfw*MN2CSn0RQppCr*(tQiVh78-PDvh_Ote}xkH0lPqvck{Xz!AbYyC%H z|JS*Yy|WIkK@L91IVR=dqp8pHli=8rM?Zfag}JE2ajmJiCUL$fGnU*jw^*F<ek?tC zyGv{wzmE>xp9i-je{EZ}GA0i7WRrEr;oYo@;Jn6HeC)ED4womv;oPxc>fuWv$@k#H zrK94&z*`P|JS!ljXQbF<Vt}{n6u3EI0V;I44_{x*#O-6261C-$@8&I#9nl$MB;MC2 z%@efK+J^O&x>NEgWv-BV{S$A0gJb^)%raE5P1S<GyoeAyN?y~ejC8WKBiWj!9<a9G zBUqif6b&vKLQtL_7w>h$>%D)|UJZ$<oO1zYu1b^j3aqACV~$HRMuxD)@gpP@`(UpR zow>2~3$35>6>J}?^CYK0_?uTw3r4iU`A?GX+5D}zy`>Tcul*~mvg^lr@g3MYtqG<X zmC;7`L$vK>5w>4ATAa{vF0cF101r*Ppz|zW;ZJrB{8k90TYFxM3B|Jox66Tep{4@- zb+@x^W;o3@>IM-p7N~1vAS9HZm&u-ckab!psJ!lrm>EZ%GbILdViX;^5HF@)YZgTq zkDX0J=s>TvH0RDOdOZHEn6&k}ctytmw@-)@yFI%nD0SH@^ndUIj@J!>fmY?Pqt%an z500kYd;vnQOXs>_OL>odM~F~T0>x>Xf~jGt_~N4>uTx&ecXzJC7b{lL-oRm$ZY(E- zFbj^3I4-;vzYDL94?s;DJFafpM9DGd<$G!*hj^?d_8!uc_i6tVJ1eC_utz-X=r@a_ zAH617?sLh}5Gg!spGR)P?J)F~1;4pH88e^jK)3r{@qnSdte{UgPkvKKin%M;tZ^Ht zWK7^Yo0rnm_B+HOTP@)GgfXmrDF%*R-a*AbM`DZdbzyj6F3WpM_?+;`Fe{pc$1cvO zwIPN-shtviGE`{H;MKyONtwcp0oP#QU_<gIMVR_J3@@4HQsam5-1ozJi1WWrZnLcA z;m^|~o?;p$ZE?exfg7Oe^8=W0=Og`d`9~kVh6>^*2X=gFhlcN(X-vyLI=o~hYj)|) z?YFJvNSh_RZ-*C`&Iy35%1+eTcnv&DS}YhX^5*Uv=b?7H5i*Ap#qi+wAF*xXC7NBJ zhV!@Gg3|_?l9zElUDoTx0pc#S$#-F;=|PyfVF^Fl-T+OTYp7|a86~cnNX9*T)8)7c zq<SI;<y0yh<2<3;z8>iRbSwATxeM-IT>)LvqABXzbMPNJj!jG-z^rb4QDadkbpGs& zz0{j&O}mR=YaA<_+*bx;wen=H4${8w-X5Oewi>QF{1p~0b0t^*QOrXd!81F9SN1qc z@`#j*uSo+rTTcyFD0rh{R~MnER4y3ryhVPMTSViAW|5Oz;GW?C-WepI#i&E1xu#9r zF}OXBJ8FUI?M-1zv^gnz48^fV!)bi#2spP$9hYmzL37|uP|9|cx5U?x)lhp}eDJIg zKdKiENVRA1VcEU3GI*Py$$~~Y+B}Lu>rb6<&8MLlvidL#88I9;EVqI64HocweF=F? z*i1*(_uxy~^LTZa92(nG1(d8dOa3sa7As0F>Jz^7O!h$tUABj(yHtav;vx8HSxqx1 zn^SV^HMp^6pBQ7`fkzpi7DJv7hkv#QDBt(6ym{AUFs?6wuFtno_{QG+s@0RS4<bd1 zdMun&<=u}@)0v0%RN-We14}fSydR6RA5Egehn>0K;C^ho?<F~Jm$(DT50M^yr#-!u zF|~u_LG`;Vd)j5Y(82aEy+c1<-qyLIbsC~wUMNKmdQIQ0+QG_@->`A2A|#J!7PswJ z#^t%Y;pz2*;<YE1xMAvI>Ns>LPI<NxvOXu`%PH5y>B|>jsm?X@v3^Xyf_JjIwk<ga zJf`HQ!|CBH70gT=gm&}NsJ+!Nd|YA0k?*g|>izoi<%x#>gS0bY$LfvxHX%gD426m^ zl$liAdtHS_m9bf9P)Qo7RR7966A6V%C_*YJ7584Nq*)~zNM%eT&GVplKhH;a-f!S= z?0fI)TI=^aPlu&TtFU?Y3~1f-jk9>U75~IIu~JVBR=HOUN#78*1y|d5C#_@AsWPm; zBAI{D<qV+s94=}$!10WkXj*v}+R7$U{b3WfXiXrqNjkzA2wc`+w~g5BP=Ys&7~Bp? zM=K2n(XQ$Xbm!V%R2n6mMX#+!!-8D);QJw3^4XAmKQo-!EJ;JJ>})2XmBivz_tS$i zCA@c276y$9#~-W1Sn)t5_^vnxe`L>L{fmqFK7UDiu}uT4|C7Z%e<niROEvL}&o11- z@u}z)Ghg&3dptR~UKQ<rcLmg{Q!(y@7877zY)R$_T6yRtTXk{)(;TsydS4ZBu~|(p z>iK>MzpxA@sU77O#Jgaqkj+;V30$4?h4^9TBz!UX1@sSE!tSW5Gvy;G_@aI^8!`3- zsA=iq*Yu4rV{VVQ;{9xR^WPy1H4s7HN;B5+XSmqK{581b-NlBZg6}RplZpI>(d#kM z^!;-xckH<wYx{i}0!$?+Tgw8+n<}whzqb%JS71sgB*E!CcU)EX7u^peK|>3V854i= zvP0H@&%6RQLts=Wv`iP?Ijc$gWfyKQlo$L_nna5{xyM~o$!N4LnG4LQ*MEH3iMgt5 zh|LQjAN7>!&bu#evX$b0oz`F%e_ewqd5WMtWf;kqmf?+~MJ({U;2_jJ2=3p$;0Bo= zc>mR3n6O$FgO}Ixlgwq<SIGbh_ejK|<=Qk_;tFU-3OQfBD53ZJ5T09wQF&k_EGt25 z_o;`J2&##W%w~GdyKDS^6YGmhg;$oN0PX8}zs(iUv%roODVOqZ6TE<RSdq>WXKMK` z8cxNkKw0}R>fd`6b$5hvGp;5HzSv`&a`G|$`VcAF?Bh(OkFD9f*b7{x+<9E^Teu54 zYOuq%Hj{00CAj{*$DNgb4Jt+oY^I+dd;Mtwt&<x8!}J>9y~{kdzDJYHCkn~^M_Cla z4W!jUL&&xG1~yI|PPxY!3t#*dcKzJOkJ5S#TkL++{62eE6c}O;HO5E5ddU@7fPxdN z=_8uHErNres`xyo0Iu%94wftI2)DcXFo~4CoZHb4cxvkpT(+Q-Mkc+1f9J~W|5oQ? zN61G`S7Hzyx}6P*mv@l4i4G^#lMI(ePiNhKPT{opD)EQD3Qj|1FSA<VM6W|fvp+({ z*0cOBJon#Df18%ji~;UgINOYQt&pG{Qwnjp!WS?<_Zp}5`(gClW}NGojN*c85ZHAB zcMFc7XhmO^GVM1WndinY-5UUN6%%MEAISO1<npawkMb|qPh!`0WwLF_&8T{55mj9N z#(i))#@m;KVyo{;cx_gV-cr|b)@Cnwu+Wn=)iD}6Qw$dTIy%3l4Z<GT3(5CLTB$dR z<#Z^J$_*Xpf7%L4HO}m-ZVZmdufYnXvF!LRDcayXmPJo7rF7|i)ZLJd_IEPDeDZFt z+bEG+Fv*F@Z@55>jaJ|(`U>XbO8KSpd6@4iIBE)aJCv?lOAlk@A$Ot$d*dPp7B{2> zm(WsLXe}o=^msJ;rVO$2H{p!J3<gJy*yD>=xF>^WQ<?o2h+JZcO-rMA-tq{>ZHT9n z^Hf;ID-$+o>OGh<y9@d?r0Hpd4Ab3WM>-Fe2w8u@v06L|It>EAw>p}ewQeFaT4RHg znis*HL)x&RXa`#8G~=|d5g1o(P2w=YndI$(udWpd=f!k(Nv428Vl>%-e|JGwt&?Sk zSWsj}CgzQhBA@nXR{dWu^o>YE-~KXM>KP0AULo|gY8e{Zb#wi>lPSb>2rG&&Ao;qD zbn3W1B@7fp;ZsIBi(0T2zt!v+UB%LlPiI{gljz*V-;m{$NBjL%G5w_oT=t)U(AsCX z=3Wy{@BRWZG6&K4>_SRvT*dZ`@&xaec#z2*1)>S{Z0xsrY>cKV+o55`-W>^H_j<l# zY+Waxpd8P?)A6Rc(;pxxZiH+5$1<6oLJIO4No#H_A)^;1Oq^bdF4Lwk>lv1O{p5-8 z-;N3{+N&3b2<HL4!?RfC;_G;GTsA$MF&=#9PlMTBK1{b^0ke9u2R<o8;Gf0Y;L?XN ztmLQ;Yx^v4;WPF4)ajeh^TjT<bdoD&d6wdn<3VEod>Q^qs}HU3nuAI4fB8DgBCP(S z%BBkM$HiZ-@VyF0C~w$NRMP9>|8mZ3lusqCns^u<uA4=Z?`m<DE$2WVW|4>NRD3u3 z7EUwlMe~y#xbo}+=<b;g_rkj2W`{gmF&N?V<ukZY$`@mgT;{E&NVAip29xsAl{D+z z05ZH}3g7PSL8okihdLmT>yrM0nm^uvqt0$Tw?K+n3ohhoO_Ql)^J%<lK8>9Z(MEq^ zHZ#hm9Aod*VR+n1Heh`!*t}dx7Pls_DdYB`g2^P>;Fc|TwdI8GwH}7-R$xyc7>ECh z0E%^ScyYd!Hy)_OQrdICx9S$A*UZ2Hzw6-1{ajui_rjdm<)Ze;llc0`WQu10@XpVt z5chr$SAE2kJ(B+?9-)2>&~+zj|B%8#wWpYcRX3zB8p*6~rl8J0ZAy;)&fQu506f#$ zz{olW>_5GRsN7g)ztj}8H6_T!Z8*;A5RpYoGH!Yok5khYQtNJ>^V_aMqh#-MAAN^# zeUIhgmX0@j`_7jA?wL(Fw!_)@)&z1}nZxz29m*z+=*A((yKoAmQclQ5uH#<-ZOR#l z?SWya*8G519&`sjEPKewzHa2|&+NbtI<{C|J^*w~O35oZ3CB7XLqN<f2(}Bry$YIW zyDS<$ZVMMx<)+|C=e=xniUP&xRABI*t;{as88{5gMfrnbFq4cVl{Ye^{U(T=e72sx zwAewU<zbu^RtX7n@4&t3@p$`oHKsNguv%F|ur!UulMQ<OC#iX?-`fJSwidEIy+E0J zFJi{9PTt3GB+N>lhjZHE@X6I@aC^=wTqp40a{7}+K6h8p#htU*-uFk~(H8?e{XpOe zw+&^U`$yBzyx~~etAq~Qcz%1WA#L3r#WEI&=zW?CR?Y7Kc3=WLDAwa{cRMqme0e<N zHiC^_l#B_&Jnix6^V}_k1_l$GVfq10VeYhqE1xhJ0|OE;|A@e*QSv04&r$fN9q4CU zFYZ2E0rgDiLb}}nJH!8BP=Dk^%ZCVg^bFp<doviF%fziut?1CEB7SIcE%YAWf{|g{ zp`>^el}*nEJ5?o-^zS+f`S*pp=wE>IRu6$sovN%NMpl^fUE~LZXOL6ST6p~O8r02n zsLA*`iMWd@_{FG{OyxA$G08|Ad8h$JXZGTNzq?tI{#Mu%t4-wv+u5Q)^H_7guXsUC z7ruI`OGbNi(80!)zCZoK?R)qY^3<MltB))Msaf~wy5McA8rF(|D&er~i8m`<QU^na ztY`ibYS24vEPKYTV{?%jX<dqUNb7%x(|6p0BKeKrFYgInpS!r<50}vL6CQN>YXtjQ zp8(&*C$ToxnM|Pq+>g0&Jx=#TlR`a6vCWeHxwulR?IUO@?ci4ZokV?GLdC}pk72t^ z-;rP6NeKD+w&v;RW+68;g>ELQ^DA$Tp_QMrFlMYKm{0qR+f81>%oD~e`}!J^Tzv!W zDve-OPz)639s{oxX?)X1J@WhBh<UNaqWpwG;2UEEJD(1sg~D(B;Smp3lyQIxZ@ZDg zFUH;!d$FH~=FpvKr$N%vm<21%q3aiKa`Q**z(qyd@VLDe`W)QB9kLCD$eA-)d%vfB zdA=jl%on)Hdv8GSsp+(0!+ErhyN;=Mo`OoJFH3rTnhVbj1<8{G1b%1~|61FWrr(Wc z8~xhRTeFZZq(8!nlU8hDs~&y0G#t#I&!oF6@*rZ%KG3_F1s-#+p`XoMZro)dyPw<7 zO?X|5779}Q-39Vw8gUX!{9M8OYbO3v7P>m`E5PHR5%~2F!;SL{De3!oc;lN1<L<`O zWOe}ObOgYt<r%_rmB?8>wTA2(J*Fiu!)C`7K)rn$KkMOG65B3>{lg>JN85{B^L#@# z^jZ<W>q<TEC2;M$=Z?U(|Bln@vwJaS`Y0|{K0>Upuz;n>_wnrHN>TUkeRLr9i^%`p zG}yiV45~f+1ok%`g3ai~;FIIX%O&W*Db<m5c8DERm4~Bcfiu;OUJpa)F7DSlk6N07 zBmPA^Xe?J@op)qtxM?!AUpo#XUo`T~2P%>ArD%RJ4yNw^gIOyt2z~Ed=GtNp3w*qx zs;32#=XsFynCJF6kFO%<yAi^QCP0GHO|Tnm#X7n)nW1krbcF`-3ltUDk(<iw+|LA> zJZ1<CwXvhYy?bG#Kck}qK60{sF;G3pl1y%vJ8Tc|WIbIxpZK>37ay)eC%l650=@_? zOj%sGYdNMY{>FWqN$g^zEU7p}@%tQ;*^C+T|I3eg9Jy`K|IOcJ`~S_aZQtWd3;UCy z(*LSRuVE(>UP1giv4^;!Y6PtApMqvy<0!=a1+KZIPgY+wdGT0dQca7+#_~k|e47Q6 z>nufUwUzj3ff9wNttPoAUA%15Kq}C8r1pdsDEz&QDO>!2xnI8E%+-}(c<d3s@5d5m z+|w&I6?RiP+EpSc2_;r^-50E_KSS&eSN1-~n5J}fa?SqnSl6}E;kQyOK6<=~V!t_2 z_}J0dZtcUC=v9hCZ)Wg0f*<d>;D@t5(S!Pa`!VOpbP5#jpt0Y7a*HV(`Yr2$8`{A= z5@tsgkum7D)D_-(uELfT=P+`64b<Qlbn4fG@}m=2dG>y;`iTW*81lUI8FTy;8H_0b zfM!Bp(rKF<O`PP;t}N>3#F4{E`>-#*-XxE|V#^^edI}#?@f-_;7re!r9$3Ed85;e! zo*fe3gnMEo7JW&T6&uw%Xj$E6N-Gmt&7L;Vo&|RWrV__K|B$D|z1qx7YZ{BHAIKUE zJn5D7ZT{5H5$s<>F{hL$ML`y$AjwdQ>hyQgI%dQiox5Svwi>K%mms(PYfQuIGOlP& zMyKZ=`4>h)ht6AuB?soRxnCX$9_@iFVS_6#^T7<YpAN=5$|}q&;t|}fkA)6%W%2Uj zDZD{@HDA9Ygnkq?VvcAfXQKH=^j)(JPPLDu%SQ&$b}2O${A@W|X|3mt|Bk@X-zqri zDIYP(Vi3!?Hi{XX;<=#ePw3r~h%G4<{Fo+hey-s_s#H9Ob)_SzS;rriZ`Xrrp%*n` z%@feoN`}NGL&!VeJ}cFn10Sl!Qre+JnkulBTF;!IqNH<Z@1IIj=H-IAS0pM-&BaHp zDYSY>J{0;dWzl*|L_TSH%%^b(B~9$XXQ!t_xX&S!+vUN6Po=ZhkMbepWfDkRw&1#h zv+>rAHniTm-2S7LE!w$A(cR6KtkpOYj|V!kta}x_<b7%Sm$H=iXfj9eD`extE<&%h zz)V~#xaM~cs5v2jnzqQ_gTMp!%wzpnmQWjr%Qcf(vG!k#4{)IR3Ip;}&SbkhT_}0* zFVxtc53`yUQ{Kw4e8%6U+}zqOhvR8x?176E+n%wI?)DC#wWqfWeeq^)S7#cmvbKfU zCC*@6bOC%4BH@OK6EkzP5(iG#Wa1kq@nhRT(C~eUhMGIUe77tdW#{-Uo>g%9V<tu$ z<x`lAwzxml6nF1!#Lr1%7*1p4+U<z>e~PhRZ3tZx7>V<RUi&Z^OLkA-MMNxd1#yin z1!+BDpPNKX?nwf^TQ?qdJrcu|$v<Jlgk@y4ax)p<38jowFL<Sp$k{}W7aYijBp+}S zxbT_wen}&sSw)FGix7Mzcg8b!HEnowPzP^GHNuT=hp^?$OMI#lB6h!V8J-u)()%%0 z++%d03*2CS{*6_vE%Fm8#7t(5&vx?7x9{Ml?L166T;*`!UMhGerh>-ZU2x!6JGNYZ zhUd4e<<565VH$;T)c#kFIqw=o>jp4bnCe4y!}nmpR2eqodOAxQwuF6o8U|W>22)(Z zXgZoviT(B*ye^fXamzE9TA?*_$`{Uuha|a6L#{yRcf`&g+hFUfcFuTb1thlZ$2BI- zP<h1@AKqEZelC}WW8FotL*X90FweprKit_Dtvf=8Zyvw)eKEMX$J3A633TwiCDcV# zV`XMET%C+)vgs|@sc(jo#AzbeR!6S+R4M%UZbFhPA7EmN5||#}1_@4a+)7;?Y_qz9 z?~cuddfRTmAG7FY_cJV=?N8lqLeFaIN3?ZPV|P97JIqu1#-FQQ2FsF^SY2@}q?_{0 z=&J#~lHSXL6h^Z<>VwF;^%<xy{EuuqFHqLUDDmA<*Fekc9$zp}iJAouaE;4nzST2| zv&tx?<)+JU>Cj_*fxII4e7i4RHgFJp(&&Z#t>$PK=EKq=Yhk6*6U_J#jr!vQXq%xQ zn~_|>YaSK;zWr|8BNJ2ZP^B*|x;2sfUG1@CLNMJ}<Vy{114(DcA4nOS#aB)mK&`?Z z<d^m_`tCoTyml+muzgk4#kcAqH1ZrAEPMrzZZvZmUlK6r6~m4bVc4Mb79>9&B>DLg zv}>CiybhVghWqN%nKNhLHjmKoHWv;y+=HS;A@tcjhgWNprrG|HIMsw>$^GZ>QjRTq zmuAK*PUX1`Z}r*p4mG&+<R1LD4B*u!9d_OVKvzKvQ||6(`XNs6IC39Oj0?luA4l+f z)i_YuRSo^x8O*M08+$!zGPAjG2z-XkXM+}m;_c(!DAQj6gNB6CxbU~!c#jRR#PA7s zK+=<vZ<tcmG&M@JRY6aO-7wDlmUxTjWo&8-;cjRS!RmFYFn_5ci&aX*ljTvcN8X>F z*~YQWKPND^dTC}j+K)D$ECXYV;vV!Gv#6&jD4$r(MK;c%9jolI?7QH7t4QbM*BXM! zh6YMG^B6Du97B>%?n6PWE}VHI1`n|YO>Rgb>7Y&EaSrh6i~@G^Z7kaO28ol;_u|nc zWBPJaM&t~;fUmy+t{sJ(;Zu%|IF4aYe=H^qGbyTVUIJ060&i;O75sQunk`@H#Ec7P zGR`v(M@R04CEdw<x9&k2+GS32HmcG7#SK{3tVoX6)ai55Yc6xj4bh&(VW3_eMdKwG zv(PDZTu@~Y`}(~c)28(C-IXzPIU^9$+%9sX{e?MIY%}QTO0k_mnHZdMh<;SaP(rCA z4e>sSy&Y%ZjrkmUT>cf4w1xcB)!C%}V<y&g>?dDl3o|@ki4^wrLw#N}C*c;&$KUe8 z=Qh^NHbaNGj4Or>6ZSI2FJ^3uRu(@)u?V$YEMRw{9Q9ouL%FXP;IO^U%&`3ze!6u4 zUNxKGU{Nnzy-<q|EeBw$&3cx+&Xzu^yNe})g=eWMhowi3q#JcJa9orO-RV{nKqJM> zZT)m+{zgvVe8r=4{8Z8E%b&y(2QjwqrWX78Yb|RI7%5~y_Jhs>0@Z8BaHq2~RXp~` z+Q4OWxu=G1heg6xCmA>$mCD84kB3XIE@6i-6P<JLAnY;mW>WJ9VdLN<SmYdz_Pr-@ zl<Q*b>)*`AEjcXk5dNbY-`QBgMPT#D`=IzL14d+<a8q}TB%39_cs<v7w8lMwvhI68 zUUoMYv_`VnnMQQmy^IT8aud#O>_ndqOPsZO1AP))G+&};a^bb(*<91b^rR@A9Y+H? zeXNI@r07PiAJy^xjz_q_N=7{BgA|Rrr2?8--gITS3Ln4eFMg@y$X<2}E6sTXx*Gf7 z)Uzu3>o<r#9#tUAHdRsydIzaiLXXcw8U5>~^OXWecJ`AY>~PE~>XXaozq`$#$#F$= zzetXnwEuD{$JemEjzej$+<LC)_)k9T=`>iVv`;k8y@|^c&J?p!{jq?oFna1}nxUT! zBUD#XnMRj`NX-i52mD7>#loDoI|Y6l3I3FvR!(+f1J?KNq)Pip$X>k>@;?2;a)U6u z)*DTUozqd`<q!&=Sc{64<t!lkHpVt=;|@lS#d)vnk?B^Vt$!|PZdifxeY+sQTA4lF zQiBpF)u}Z#UA$%RKdAbag)UDzMN=J$880=L3|IE!hu1rBlH+4GYWr@MeJX$w`r6?A zslOO<eH^VA+bo_97xBs1LTvpe&vbov;C$Iqj!VtO><@;tZ>$!-Gog$s?KaS$z6}DW zRvMbZABd`g&Qoo58KukqLg_W;uzPx2^+)gLqTn=VbSwV_+P2!n(q(DoYkMy7$so|b z%CWYNOYm5I2prkf#D9FR$L!~CfJ7rzcIrzV?Emmr6fq+aG8>;G?NOrdJ&Ewo=&ESL zqC5QQweG;L%H<C}41s4`_Tr`O$>MSCj$rk5B#Y`PWCdzYkp1c$uP>_sVGgr6e{Lkp z_PQ&|9ryrd9czP9_z1=$Yy|hZA9}wOv)!@+Z`(K>G>tcdf;gWx@9Tmrn=B0aFBP*| z&Vacc2NfOv@bmqfIQQ2WOd9?f9%`6Cp?)}b$bTwZx<QX(({gFZ#f^|N%aoZ`dr=j$ z$0>oc2r{o>)X)Kp?e(G4&pR-r=?PwZ8_5lwa*kU#;SS{0M$^;CV3GS$1*mT@rQpTd z^hEV7Ob+PBEQfFwcQg=ZarvaH{RSX<J4tFwP^FM{vyoEfu1qn66W1qEafg_;Y?uKM z+QK=WD#L_zPf$_mAv<<gh1HkE!`8G<*wN&L5nlsY@x+^KbU+O2&>zpnT^tOPTcp|Z zm>kFu`@vw%aUi2@%48?bV#!AR5FqS!=5Bk8io0HOFH4<Bb_T-hphmFdGQ_uFGE7~Y z3vP*Z<gT`ld)s8m>Tc|Z8#|xVt<3eX`gS|6{Ok<ZOB(Q-+-dfHqKNdwuBbCohi-50 zfP14l9Mtz*;#vK8rgjHm;ETh!ecB>gYa2}y1K;3?sx0=ib_`W7+DuWwE76LdKtmSh zLs@h%iyv9Q%b)s&>xS*&wMG@<W04vE&{mOO5V4t6-@StcV?Oa8H;(34x$i;!M{CID zSSBs?7{okE6|r@XJLhHh67_YCVD+pvhcjz=JT-bBdTGj1-HB4d_ao>-)L8U5rwScw zo^y>_TC9a@;M{`dLYki<TYJlqZB*d6Jd**`sB;kZnk&HYeI_u;`UFVs_u&&)`jE=U zz8an7_h8zdXs#(}A21=G`Dtk{XZ=J34o-$3-CY7(+l=@Ic7;1K{sV5gmqBkwhcL=< z1cQr`%+pGnytIgaRx%n(b_TH>izINMu^jbYmZ9r+=X0uGIl6Vs(!u9N0^1UiO#foz z_{Q@IkkKAYGdC!(niWF6Q23qlo0K3^;WBu%ufg>f5<rp-BaMzqXtPnJe7+WP^u$mc zOnkv|E$U8EM~U>OocpHB_%p3XjBO9ltz*5z$bxag_f+5#9aE&Q$22gdfPsFoGW+^U zjxCPY!=X~|QTuQUhNo}hRuxI&R#7}W6K`ba#t7%Wbx~NnUlS*eKh1}w$Kr)QgGkjn zoMu2Iv+j<hMIl?^aLO3k_Ui&)H~KyXb;hI3l({f;jw>p6Z)3SrhY;SXsxCbh!7ugf z!qz5B+-%XpMW#())!BkmusI4NFJy2>HD<C(SzXF83?R*CVs6HN(ReT8BxG%y41NV3 zR4rbLDXI3fe!T+A_Wl716HA~XEEFH=OeQH~6X6k@kNX1Vl6!*CZ8mejgs*CFwemVv zRxP9kZ%Ot!;10;SEyf#1!kNK;gV~H>!(md$6!!994mSI>pjM6?+%pjJalQoaggu{o zVjJk+_hSz?s4?e@54n71NZ%5SK}~KnHEceBYfo#lJXH}L<EC+nCnnHBbAg9;Xgukw zYp|v6S!kIn&+O02v9ASbf*Vl=q#Ny+ZdehWPtAmZF3}V*)D_CE#B<#3(_kccl<K<$ z^plM()3OZU_b3pPTW(E~3mw4ttvVae*YmY5pFpMzkhk3noyPk_0sb`@s;$5jHk#p% zv2|$V97k`SB=E|nGcZVlL65ru`)qK8Z3A=IQ*s`A-pjN9<d;(4KRYHGpv2;S2>a|y zVoqs=4^2E?#IHHPL+&4asCm5}{tJCgdHV)ob+<B29Noe;<Q%~1W<gN>)`luOlksW( zYzkYO#R5bs%vJRrbmwotgV*FJComanmTIubMeAv#Fl%leoX2V&jG<bGJ#6{cgVc2B zD5~Gd=GJ94<7-(1c6R$_>?zNK=7@uI!>SRYoi@_SN55eHfHC~!>HF|$^G5tG6Ae3q z`nityH{3Fj1uk|NNg2xp_knU4fNTku>~;oU#;cN!#AV*LTpDTx_GIRwV9*!7y~;t8 zS@^SfP#P=G3==}w3%4!&n7}1${D^6IXV_%cw(1L77fxgzV<+N?&KO!L+5kUHTkzxF z64u^v3l_*(v*DkG{@ms{I5*Os-CmOfpsIkUBJ2guL_F-=ULvr&t*~Z6C`Cmch7&24 zY|||-=BM-y?*8`0Ao>U><h7%ZuSFxFpL;1dhRp7_!+S@;$zt>ylmDo(#LKmUBRT`` zXTHax>62JyogK)x3j3_*3z(nhO`H*A%Oq~uvC9v}W0=iA`k!~!_2DIO^xtIq+u{nM z5m)&gFITgRrGxo1yVTj<lQ(hpa&2yPb`&-JJqKTJ&0}H;Myql$^mkSBK6_$0(>u$V zNz)3!apZ-9FqSQ}yA5Lf!DK1yGX9>eMBJ2!>YroShx1LiDsmjHw>E-Pe)nNsS1lvs zOmW-BhhTo;7ngiA8(wH@h-wx`bD(<_O}>r=<CP;>;;eXB*z!)82^+ERCvDMSc?utH zcL<F&{y=?_CzA_Y&Lp-@V^v9i;h;njd~E!TIntN87rhTqDj|inEPTT$oEppW%dD{M z`4`Yp%0kod2l(fb4*SyX#LU%6e7k)xt^9TaYRkeH+t~=e%_N~GXE~fw98NneHTctB z5xlCqA{Fl)&x$78z>T|9Kv!L#25%mP={MGJk5!iPA9^mr8PjE;JfM>1-xVC6C*$$X zDg(MNE)jB6Hr(l*iTt5ZGiW*SA7}ll4(@(;WziIc@s`$@tseqmSGPgCn<51V9Af8o z%%lnF_N-gj@kh?e<2lbOIO$y+Rac~m1Njb2y*`totV&o;+!W#lo))bT*s<L|l|g5J zDyE+}%A^<0<kH`TvKemO;PJ!{YLrG%^w1>U?oWrXhaAau{=R@?PQApIJ_-ESwSxJb zIwc<QIsy`RjbV}@_DoIqr+htD&(w}tLSvXQ9qxTE&Tf)~!S5Ht`;*zc?5jpJcp;ow z{TX+|Vm2-@bD<Zlb13fiZ_K#r%JhGD!}|@u+Ct{R>am9$-fvc96CQe!kx~{7?h7Kl zjAoJaSwpfi)#8^ZPh{r?RP*1|hOpBUsvx$1CJjiHrZ|}?peST_7c2@z=aD(wub&5D zkbwr=keiJTkC#yC^*nCYF)6aTS`0pqF9>YX-?+e{1~2xlBBju;IKE*UMOetPNva1$ z>q0Mb<uOrsaYHEktu+U`QV#Me7w1r|u?LOataveD>LIH8Jq(9Oi^$Obqxjj?Pp~gp zjz`;T_*Z5iu1Dcrt-b++V(q{~#}Q`b&8Ce>7li!-Va2pq)_>6(i%;dkhxkRbzHb3r z|4xEDo6A7vj4fNJVgp;sgl8thh$%fC2%Y2CLeoBX9Qbz-ujWzDwYWUSU58SLKRgdy z*4x6xIw3P^Y|g&BJF$1_8(?qpD|~dO9GkU9Q<wHRIBcr`@)CjY(n^OF4m~0|RXhbM zcRR88vGego%OrO0g)3WLxF7v(i=e!E9yjxi9ZtEf%p|tgLJ2(M0%Oy#XuZJlvUv1A zUKCcEU4d@9JyEvt7MG#Ciyoc4C^8@K%I*K1$|TD^gY({vaQI&u{_ykyM;}Fsy(Leo z(-OE>SE^z7iw^u_P=aCyb@q8s6#Lj3&IbN2fUy%ixqWwISU{;07`N$@?-3ty&AZL) zPlIIj;u&*bV+}B$dP(Ns><4c&h3}sCOHR?T1BY8CQtdxo_N#3bXbjyewr-Dx0>`yX z@DVae15JALM1_r<rog-POOj)+4%73$&wq%iqiMPsFd@@`Ia&m=!7@YH`#gDe>7fVi znEn*11#W=npk<VABuP!L4uX`#1CS8r*dLsvnUeoIw9j#*rGFjpgncJ0`n`mi43Ngx zy#*+7_!pY4@h8)jJ6VwLe&kndg~Ht%(bVlVpII0Or8No;%4r2SEhdSHTAp!r;pW^3 zAv?Rv9r0~-F~6(-qIkQG8@p^3jdns#!$~xdT-r_OSp75rYiTymXFUB|w~4>KWD-f7 zEx_}?U!wQvMPxhp5DXN$butD+#H;*G1U|z|dTABJf-DEnj@nb`c}2t=&dIPhodWy# zObjzH+6S9yH*OJ2uo7X`ytQT`ck;t(VgH)Ni8wtv>adX4zbZ=-pWi{hcOi8i-bJ@3 zhLOgzT>LeA3a#*44f7KBQ}LM%_#Y?7=H?ngTZsnTQ~isrU1#|H=6&3PQIEmc{5Pz! z3uh)v$}#g=96l?4gp<C^N7D;)95m++WV`ySxLJGFLvW8ee@Hola}C@>sl`%47IQ6o zksOU`Ux$O<<0bfS-FL9IP#{s$O)&cNo>z7`fCjN6p=-S!MU;pr^hTQK?wrqj(a3Br zruIBE{e30AlpG*%l)j=;ivmqfUd;Y!HNZxriFi`a1|x?b0gI>Wc|%<XKB99zt+|sZ z+G#(ChK=tfpI8~Tvv(YGkn4lIA>YONX3My=K?k6Fj5#O%=qt>_87my+#OIG(1bcoi zV^dY^_?IRncxALOkKP+YFQfID^sXM!I-Lu=OkWkwk(Q%3TiQV~=7fXQfN&iBWiDK3 zxe6|2`uMGKAjFo8<DT05M-NK`Zso{>F!R|SXt|mq9_)Ccrrqcoe`dsD`s%xsLLC$+ z<3SCZpfm?l-==}L(O`P`C4z@Xr64=;DW0|5z#7}D3FPGH{qKoXq8&~8jp<<e>jgBM zAH|(#pTm%vHK<}>409*f<95kRlDs;TU2afe7pCUnWsA>XTDyP>x{jl3{TmA1E${;- z>0t9GUD)%`g(4RJ#46(;Zi|Z=>;9X9h4zA*_VW#x?a;y*zjQ`rgJ@x{XT&<I_rcL^ z*TGrW7x$9C&>s?fZ5ySS--Zbc{;hyr%CQbxTTbxr3V#UaX9dVFcEv-D)xx_&h6Q{0 z(4#qOY}>&~KB;OqdNl0jmuK3*36m;*oQ5G=SCY#*Ga0LC?By*#mZ7y_EEK#sArdc) z#}zID*HdF69qL>N+c&0Q>xnh+T}lQ;pI5M2^*z8M^jYP#-wsos{^JT=q}lniy%4TY zVQ=<YhH}c+F|+I|yzf10QZOrn?CX9k^ms62IVynV(o+zrr_R0|S;drd71{RZKFr>F z7{8<96E4?}Vu};BnEYCSr?Ubma=kH5ls4kUl8!X$m~ft~Tn=R~mhdAleZx`u>0nz{ z!SUx(`NiS-Fveeov^E_C%bR)P&e~brq~E4^O_*)q?ZcQpCJ8p}zlADE^KkIA4a~t; zk{Qhi#_s**{59hoIOF(I_=YH=<?M0{@w);FLFKGoUK+PAImYtMe30X1;Fr*w($k;H zszWP4(xZU#?dRj;d~G(m>>M{!yPmxNCE<6kmqJdsoR`tu#Z5aEL-v<-*)(ZYOtY%P zqEid7ar`;7T%T91<l-xqv9uwrnr-}+F;Pr)!9LumSB2f_-|$gS3L9h)2T%U&My=%K zjB`k=nPZ(Ue1G&<$W4y*Un_x$-+M89%WU>gm>YJT&}IG~exgJ8du&{s1pO_=*nXiE zn9&)|$H9<#44$In>z9HrvRbVAz6?KiHDSpEBP?9zM<4PQu#Mk^`&rIg?oq5AAN%Z! zgT}=z)bQFCbNnaMDS^+~94;_+M&5;`NA(3~K|T%_J%>=$vA7^l0WW3igZ7zytmyO; ze$BU0?BwH(>}vBFdevVll8T)VBZT?J0_6_g#Nr%W`y=E^-n`=5j;F)fkNqI`Jc+s9 zA3#ROBI)$wK#FpXVruV~<Ni&*MBh7JLCa55)?nU?L6;S1-1MiIZs81b2FzkXN|*7S z>;*1Z;FQdoK2C7>yx=*0Dc90&!ixVq;3V!DbG?$0{M*bDAr~~9mFFjdL!KGaiO^wl zR-3>FodD{c=Fc|84j|>?9C4t1Dmn=MB>Cg9wD{v)EEZ;)m(@+!%|olWm0k|aZ+J8= zzh!{y{uJ_0<kVTUy(ay4b`*WrJt<!A{EzoAvA|JBRmjO@HLP4`#Lo!(#YHrH=5}*4 z=+iMnX54?I=GNb0R;g3~*XKLX&i*|l-mTAmUoK|0l^f~L2!`j=l$h8nk)8(w$+)d{ zP%<>3&C^oYmBB5X-0uQrKb{va%X|!{{|sYx7T%=SF^COW^Z~s#X_Iwt0E@WMC%DBU zv2vX@lZzNfCojz;r;kni<I*BLz4SM_*3IGdpO~<xy1VI8wgK$ga}eLma-+LPMzgD* z^J<nT%%(Zh8C$#II&ATZN83JM@*ZkVPsZzE^UMQWS!6rj{p9e!`Ge=dO#}Xq`Gfy& zf6W4oG-jY-M{|E9vhe9~u)(&Fl6|((ghPeYc)m;M*VwUg<#zbV%@thQVPf3_m*M5w zYLt0HuzlVu?$nE7d>&B3cPBp=FB~@l7Uv9OGtN%LPii)JEV>WU`_1Uyr9gVOMTrde z1mU-PJMgtlC_54|8S@^Tr3>i6eTyH@|B-plwO)V0XKP=@O}sIlYx__1XrMQhXkHRI zZU|>b57&q^*OkJNH5m?5qA~?1=V8G)AV;~~o;0>8n5wcKfpnlEOkBMdQ~zs%h9)OI zrRg{}R<=Xxxfzt{eiP5R9mE*nEGKm{6%BHq@p~4X#fYmJI7O^RY07hHx{VYKGn!9f z<*(5sBooh%@uVxyuJM&ae{zrg=Ri}IBuvmc48ftZh4ZMo;2vGWevcX^K6c}+$gNBj z0}T(+p?iDreV-n_FjIs`dp-DX&>=Qwgd{Urag0Cp;tXYowzBBn6j)cD%}SmP#@5AU zpjc)`?W#A?_|picez*q9K0U_?$8|C1dp8<b_%W+yKiKw5m8MTW1U)t_*r}Jq9dN5) zTTTE*N`K}38-QQF=qDOmFQ#s%Eu3y-9G|5q%!`u-;FlHS*w&D1{Phhp$SPz6v^HM_ z!!2G^Q<RIn&(hhiQ>x5rl_#6zX(-Yg(FfZs9oWH5!$s5fo8rTgB&?H~0^*CMxH5GO zZXKgelO1Bw^S?q)>(T_)aN@y)Z{!GZW_O_Puq0iWv|q@r>cYPRCT#76a`ZK6z@b&% zl+v$7d)HQjxziUmJ>nj&`>9W7&w9{O-!XVIdL{Q(a4;79Eri0TADqdfICRx@WEZ{P zW9;ctXg;`})ss5&g!`y<Cj|!T*P?WbkV9`f&6mmu88C4G^sH{?ljP6Bg&i-^QD>yk zZ8!r@<$j4DJbwt6Yk&WbRu3bD`}Rp0!N;6mg;Osp;rgak>_&AZxvd+=u6GWil(CZB zr(XfIQgEL?Kk`q+k1PS>Aa#`O6uKmyI+%aQ5WONE@+a30XP4*2fvn12DpnprV_nW- zm3SnpdvzA2=e+^_ZL`TQTb)S-UBh<mQhHFZlXbq+pe>GJZ2HS%SR*k0+GSK}hO#01 z^7Ax?3@%`8fu5lIM4Ek@ZNN!fdxR22FCe*~4rHwcP`NM<eDPP4N_Ec){e^e1q$-BF z4P3;O^zVbNEC>J8l;~YXuXsYb1RG>zMCQw%K#X}bZ0E+HjL;=IH@XZ9om&N_pBzLx zq=Al69nA7+=dXP@#8zjw;+%6S*a`38WS<4uzP~Rz@3R4{jDMkCbQ!y8SI+%@JBqHU z#E1jS2Q%xBF_5rY@B{Cth7iY9G-}*r==Z(ACLff=tq%k~_BkV#obm`h95=&c!+dVz z(i1p-ks336zYdnG1<{^_P?&P31YK5f=>E6`)q*5p<`O%Om;cNCSrox$&ak8I>Ak#n zlKjQVm1fL-wKI49NC(>OT*Z>zWNBW|O}v~DM4qGe&_mA()PK{8;ROQ2`M3i9nbZOr z`I}gq{%*<&vqDI|&YfUSINefvdj9AUv_BZZI#HHhte(VqW_}0z;h9uzDvuLa+f)7A z9C)m~4d?dPp>kOR|KR*)Zt(7WToh8qrPZGo?b=t(50$x!*K4FuLh!~_9T)%`3*+JL z#1#IZ-Xd(d<HVh}i*)D=vO=fVGVG|=c($QCA6BNPz%;k3{H!18T+hn^O#N*<Um?uA z{ym+=#^4ZIAlz$zxa;7{@lvGZGmpj|b|AmB?aXK4N7y<;lPwX7<<`SB@nMcKTl5r2 zbV(IwcIUwq4;fneyPx}IJeC5k#KWyxRY-L5;|8WVkYbvU=gmHj&#uJdxEu1Ur6ZEt zk*>sCD%a2z^B0(&C-4&XT>$yQ>GbBA1KC(<(fEcPl<`f5DNl;VZSAMUzh}Pye)SN# z{Ou^XDeR!=#<N^vj~=;1iFwa2VWd0sI;SLLTJqN=i59Fef}82UT~i*!HCTM*`|4Z- z&r=+FIL<?@T^@8WaX7asAwbNXIRs8|uVCbbQ~bKa7VL%f4{Vte5A9tQT<(y2=&<$& zybY9P3Ik@aRjRTquecn`?<PRu#dT~-a~ni(FCF~PNiw(iOHg7YxH}i9lXXED;|9oa zJ!ZAs!rSuvS3Ngc;p&9jmOntbOUr0pz(A^Lso?%8zu~7o+l=Z1okX788`ybh5N(*5 z%x4`;CI78!V8!^)*m`vUeRxpCKX5fc%l0|UL;9cS#+wPOdCGeJ5)9_hr4*jbljH_# zC&AY4CVtd3V+hr^WT_eNu&QDRIo=fV*7m*_et9hn?a8Br-6!xYE5I#_H*v~ur@--T z8So%qf-bAPM_u_7+>V!r;J^0<BwUur01~jV$dxWQjigV_YiRL*)o}g904A9ckKSgt z@l&G$TchH|*$7PJNt!RXuVaK<hDe8P-hCCet`_6BRw=yjG8KPrl@+g>(adw&>zPrC z8aSvmab?2~a<zkpb8a(6i9dXOhl*z^Agj1b+-0<shS^GsT6kcyE8;-fMV}qNugOVf zsncjJ6$+d!4-L;VA?LUxY9}uzw=cIj*BEyybKikVLC#EbM;7``9ZQ=gIMd75X`IUH zi(KQETvUwQ&$Nnn3A@f*?AdC-W`t<7Pe03nSJ{W1(Pkt+^e3dnoWOX9W${J+urt|& zS#G}pH{A-U^t}}qZFLJ$-Y#VMr&^$>%NtkdheKboz&kxLkO>wnFam4JKBf<Mhj`OQ zi><I@n=EO}%M*8e7{Zo*^M)VGPx8C`yeWNYD9WV&!81bQ|IDs`+-<AHu+B9W?+?tV zu~AqJYCgwk&AU<Do4wDmwm$^wcQ)X;L(ll-<G%5KI>xboYX#rY*Abk)(8qEwsOI8* zl5yYeP@3zqlopI1#{8zP#51Ay_*E@Z6mBX>iGcxh?@TCMj-5a&j5l$$aw&Mz<r2;- znuG%-s>s{qIyC=T1(VzJ&}`8W%J}*e7F~3tWs!26<f}($<a8E~E?bN#QBiFB*VFvu z?9-6+#h$IbZ^kY!$`TLV)F8TazZ&lx-w)@KY}vypUW6|ap)%kd?Bo})Yhi&7rOTGl z+09caZ&4>US8{CF!S$4^r_C0K|KYO#R<IN8H)_HHPQi%A4w04W`<m^wv&E4?Cmf#N zih<yfDpar`kJqug2H%D5>>Z^D7%*{>;8i)m@TU>l)rw%sn-p&Cx#PTgh8D~9I*7NQ z$dlRp7#dVH4=3FGj49BA*6N{P_GvoY9x{yfxF2W7x}+#=eIMG*X@b5F7U29YALoW< za|4{ixY=GQOhf%7xYP#I+`M`8Z0#j1jva>YPphzBcT><i|0^dmW-)B}If9)N`l4f% zkh{O%msvJ0Vvlwwfu+xC*5ld0S!=tp2XQ~quI~cBDl7+oM>&H0mR|gkY{<&KAA#$x zyZH&(V~KtE58tJU(L$pFCRa*u4x2OK@4>SeSFi;OhnMlTE|&bRBcn(?cOuSJlV+`( z#b8z8h5Vl-WStQQ<F1vl+M40amosB&N4nsW<7s$u>lV(PwMMKu*Mr^;ZiGedLWb;d z5ts-$zTl9BMCFBOd_93BXIs(Uq!n;+Y%!_6NQKqse_%EpU~L8?SoBN<^lGT%<*JV` z-*R1IyHe?R?-2_4%2QmwHSQIBz9Elfab#Q?tbaJ2Ivk`iuvLjksOiz*{Sq`NpoTR+ z&c$Z4J>2AAIa(bda9odH=I(CRWJ7n41^2Cma7$gDd<G1F;kBwl7wI5VzIy?mt}%w# zP$z1YIRRP+LWMim71Wm*%j$QzGG8S{_C(r+zTX;*BAe~((+6LgbAJ}b_Fu!KeNnh^ z={`DKTY)12?eKzmF8h+L2D+iy{04(tuv0h-7L~Z*XjK=`dRUAyf@dcrb1Br18Nr^Z zgoAo(AX^Zh$L?wD#=g`EY<0IH+!>S!(s)>49#3Leat?l8TgRO#D8XTek8s<Tw?MyF z8Q1gIiliSl@@MyZ;=oWB)>IrT3VvCGoeix(-)E5B)O5DMN7(0!Y{}DdE2Iy~#>G2E zLqtpz%r~Bm6SfYa-6Jh&@&!lu<t#1y?-#Ik)8VvIO>nR}yyC`%1`Es>3m6clORFO@ zS&xmt@V_?;9$245_ir80B=lnbZTL}B=suahYMTVJlZUdbW3%{41!4%TXyEm3<iq`% zRUl#K#5Mlx$KjIWIB~#q)LJ(i|1I{Smsy3dX{!-k?AgNE<nINSr@r*hejn@W>qiOW z_o#AqD|jfHGx^j+a<%Tjf7bF$&xQxD&OuD$AEUh7O{6H*B=jEcp?CHw_%mCdZjTRR z1rKlF^<9?)M$Qs!yb#E0PaP0Ht*gQN_Ey*tIh2Zq?Zf1E`cyqffvz3a#=??HVUCu- z$xl+Gseg@_&-*Njc|8eSy;X(Gv<TC*6lnN(9$spi<MxxO=-MaDH%dLZ6VKzhNmFz1 z>{DeDKXPZoywBq7xjSg~LsJ_2u@A=m=*3I*Md09Z8!tI`^TXT*u@7?Bu}|RXW#eU> z6S@Lay<Av-XPNlTb6J6X{exe4B^YC!^*D<J6DE0m75m)Tg^$k%lId?>82*DGB|DBe z$D3l~uWNX|^Bk&d4?}b93QqNy7Oe@)z@@$;*ouw%aJIJu&zB9S+x7OSQuv-rh?^<+ z;hdRP@MA9Qh$MS#r4NP!joA>bMsCL11uU$mn{$3w#FR9ialyHroZ~wMCUf2b&9b-9 z4a<CZRk)d7r!*5JgVRMtO%i--XCl4RnaX;<Mp0$6F)h6#$v&j0Fs6SBtRKa3-m=zg zO1LVT3GAJb^T)B^!YGKEehkX5%X2Gjm-5|aH<-mu6Pnr;?~oua=bE<<!^w(i^zZLb z)~hg*y>*Y~ZtihrrCF9F*|m^zpMgl)K%HXO4kWWmQx<2f2;630G+QT6X{!ubuyPs~ zxUvY>uQH+%2_vRf8H$QYnzVU~3vuJz1x{fa=n40g%&Jom^dgRxWeFUAV^iiTEz7)1 z++ggOdH7R)G2Lq(&w@8TL90(YsA8xtzgL_Iqs@IVXWJQ2+CP!S=(XX}Vo$nM;m1@K z6@#gMKfHSDj$6!jL0*$I9Wp!y;g2nu_LT?1^VR?-4v&LGrUUgx<5<z7SpHJ)U^Z9R z9UXKE_^0h5u*BVi!g3kAt2mMzb>dk=axFK1b~Zd7wuZ`8BcN4N0j`U+LF=jmlM;Af zN)uk;(W~>=OrhsFb&w4<B`>3uQA_dk{7A|(djfA&Hem9Lzo0i?*n>?@<@B4zQ-`qu z+xXszV%!#rXR58_P32_C$F7HeExCm2-y+K<KU&8e^oL^m<;4(Q`jlIoTqOR!<ExNC z)WggP<?P6R@~rN6GT$DmPvY${{Lie}?6t%$P_0_e9gFMXYaNe?-5<DPgsUoB-MN^2 zhg;E*rD_yzKaPb9nWUZd=eeF8V%#(IB*4kR<Z+~jf3DPnRoikgX>%Vx%kB+U_W+#V ze+>>Q1F5gRNvSh6g!4i)L>=7((!$x&&VL`XyAUPPIi*aa3m=2nID+P@8?nAAvUK5R z4_DT*gjWkoV<|Ri*gJXxwWv4Z0*xH5V;6GKgFkW`M|<)1mxG}vyAV1pQbg*d<=Eae zixNA|VekM+TJLGWxS+Ri?rIu7c{36p3Vn-^%|qBHc+Agh`ps=o9mDmDW}xPQMqI1+ zmh-5e1N)<aRGKcp#LEWM_hTo?KH1M&QpX9o>3%+cwHzF73nr<VYv==c;vv@prm`iM z7F``pHxm|8-SdrjCCdb-1gygJu*D7s43*dfT_@&#D4PojF2=5M3(>~*R({(eN%mfD z1+HrIAdk?0F#S{n)h}0}oz}73mLXm&%hixH*3Y9Hy9}^%ccul?zrxNj6X3%(Pwa6z z4|zfle%?A0_Wrv8%$eVf7iuCvV|5=&k&t7^$t2Z>^Rer`Jd;c|pplcZ(0)n;Hpu<r z9fU5eszwX^(EkmCCFH5rU5j?dABWpjpCH6)Cgd-h!4xtzSonuE)LvA<KNg(kNq^Jm zz`fOYdh>T~`=LT!O>R1=2MGSCVPP~;a7%nIItn%#HT?FTsqpHRJC5776Ae-`(7s2C z#(JL+zdAeyR*W7&p}ZX(k*db?y|-B5&ls2(AxGgtz~0dL5p>nP<*Rd5m|TuMypH*a z$%BolWx6BhCV3r<Mhpd=X$8F6Cs|6?|0-@XUdf&tOomt^Jvwr)1yrX-v9#U(Xf<=a z;M2OwKUDjJ@-uZgJI%Z3eKVVN=XK%Vr-3AWz69@Gd<`ZJL6A~t%KH@E;S+VTpw{u8 zNNH3k@Bzes$v?owy>(#c=MN^oa3}nne-f@A+{7jn-UaiUw?rGiO$ViUZO~t`1I|AQ zV>V?nX!ifgJM*}jw!e=zNHk9(8ffm~YF3^7T~UaHQc;prl4u~h(m)AKQkn=2rbGiJ zT-x7-LdL6%880%GS&<0O&hNUv>KylZ?(hEbyq?!{dacf>bKYy(-@VUTYk$}GGuIIA zRrdlPE*=eYb12WO-%sTH5pK>DUjfm5J?RwJ-`V}O+i>ER&3L4|7PikBKweqLfS)_i z%NDlKMjQ#FCIbdyk~IIzI*6Vf4@wgZNz=lQto7QbRf-}h>=_&kr|T5S$DBl1L~lar z!W<MyJ&b!|OPEG(eKB}m3%IA-@^rJL@ayrRD5AupYmVu|x#{I_r$v+VnYIAW)H@hj z=Snxey@F~f>sjq9Lx`OmcR#syDiw3oql2Z>tFBfibH1{J@&1InOytt5=rpH4J@R2H z70r^Xy7D%GZp=FclC92UCs8Lg17@;!Z+?c0kx!tFTZ7SpBC2FQ7oMx}==^}8C}UMa z_V!<c?R8%;Ok9S(5KUrtd$+P9`@Z9(h;8XzRRj-Zd+?oaPAbReQ-@6F!{JXB;u#q| zCStrlE}fDOyK^ncdyz48;mK08l$=Lqyx#_T^X=%<rW9zqpv%Sl8;sL6<Y2v<9QL*J zCSExqWa6DvX73amnzwNs@dzx1`cVkZUaN?+wi{{<ZU@i)i%H4l5HyAx%t=nGY2=!t zY|)fKH0;88MsnbN)blc<QDVF3rDP@2_%aJ7<VlmYO|}rURhh-Sa##?v3dJ5P61Q_d zW1wjRO0W$~vBp-Wp4&$_gPlOMZdhXa`~J{4dMvel>`nW+<q>bWI3g~zld(zqm1NvQ zB9T0uce%WR=$_?bbzJa4&51vgrmG@^J@x^e#e}J5!45RwbZXpJq>{E%zo2D*e^ftv z9e*vq$iCWN&y4n44N=<WIK8x<xBSHbNEveg3e{)hp-e~W^12M{T>Oa2Wo;6;A{OhN z+R@|18YXh_9?1F-Ogqx0Sha)Pe$x3Yb7^l7xcY2mN{ZdthPLT+t!h=(sRDo6kX*pc z%Q-(?y?Ao)XC>nBt_6Cfq>%GE+p*f!5+(;3)1`az;6b0GOkw^Y`lQ!qMp`r*-Som( zky)wq)@yyby10<oF6%{R584HFC-We{emt(d_8zo;Q6_gBQ_xfYJ=Fd*j8yQ>LSZxo zGhYiTaqAd3P1mFf7i8%2M+vYqWgpI7uz_ypbhYQc8p7!($HBt`qsWd&^T?DA7aBWn zHGQ}FHnZ1tEt!6x5RS)hv1~JNz;uTUwzxG86$3o6s<H#dt=IvRBFeCJX)ujDGmLzC zyPp|RnGGU7$9dlL7`t7uoH@2ImZ-K`qw+&;-9W~P_$(G928N5th9y-;J`BDNBj1TL zX${rTvVe>8pA-j*gSJEJtWiv+>L(_{RE&-Y*@K7Vuc6tw!FcO>t)<4Zqv&)jfelDE zU~EO>xj2x;xZ+L?yj@g+rc*v)ro0ll00)>j@g%k}`Vs^jILG-TT*Qa%Ze-n~C(Izx z4Ln`sG?^sl(OaL_v*F_of^Y6doOq_T%C{nsxGq_TvnFf-pCO{SFD93ce@1Da+w;kW zj(bGtaylLeYpIfZ^#qbe-(Z)9@3JyE9nV@D7FcL{-{l?6I00Lo`hjCYEch!lKwDTF z^O?S8^)8>ndwCu>%hd{Qv<Ff{!$>lAW)fQYg`@q0d(0d=dHUsX0%jX!;OV|gA=g6( z#pdNOpH|4x$JL&6L(T?xR5p{m&e#X%cO1vd1JgMl?xWzEu$pO#ea!9I(nW<)Hq3`| z9qM=E6JDKFj0M4C=<iBNc<A_Twm49i<=pVdtNM{d<#Rl=&m~0D{~qh$cM|sBjbd{H zZn37hy5xGG47%__!RPTrDtn_Blj=7VBM%1B0*^2>8yrv6t|-tWSDKlP5B1nN3oBXK zTb0~A$plu6PJsMT6PW73BD5m-5cAYhljgJpkf-MA<n&%A_+=M&-}j{jMx-8sY<)-6 z)s`X^_Q_Ov)&zLRX~QeD_z<^AFYt=77*##p4$5_5*kU0{ggmogY~xAP`lJZkZ*IUh z5!F<B)Jj@u90ci6A>`aSXLxlqlv;k)#wVo%NvOIlT`0beDRJgBRHN!(Nv~Vzv_qX3 z+6*9=9Z%(-8S$2Vy~1-VU4(6{K6}h|6&tnNg+9s?qt5<nz-*0Xi<>U8vx4NfwKrQj z#o-}SnsbiXG&c=*RR4x=v}&P7C5U-183%F=Q|Z|`39z;DHvRpmF7?^DoKcQagvvfq zRC8JYRFBZ7!Q39qw5{=|+abn`ew2ap1}>pH{9jZRy6e%%6>HedVwD)LQiRb-lSy&k zK$@aI6K+WcLDn8_f7$2@c<{j{4AM&=iaQNSiP1UU;qCpYlRrb^KP02tStDY<VjDc@ z>p|RJABWadS$cDy3wai22M?dG#pOw2)c5)c+;=OUT=mUlqkmmb#R3e;z6MjOA$t-B zmZd?Brz3_W{YoXbF2h7vj0KvSBvVm}SUVWunSm?F@sR=0KWhQiIPC~3wV6;e$P47d zvQhqNAyy}E!eP=pc$RBTv^5f0Ly17FTc|<AbbG@}VGnvRj<OlDL2%r+7mgiv6^-vt zhOb((xO+Gkd$zcpk>z~Zn=MqqWcGS`RA(j?AF-YhYkLldw)UerLw_clDn8%~`<skv zX(2Y}&!#JOzkz<!JFxxJ8rXI1F<PeTK$UVlEBCbu%?}SD`#=7I_axG3u4MuVag2c{ zv2*F<<~QI}(uY>*FC;BDfO=l-M@Fu8z?{0FRpZWy(>mTp=Izr8EVvW{MaCJ}VbYJB z4G&|^Xf(4{Z#TkcqXaNdd4(;-TJ(<CQ?yv{76vSfIh@;(K%Z$$z^4UixHtR&D3Fct zwI-XG2G7G$UO6QG{*$U4hh;cfYaVY`?qF~msg2@!X<$(10!xdyeT2NBBx%1LaWlxk zYi|}2_v_a%P<{}WwkFaSk`H;GQ>E$KRu8&q#CZTAW%{`~5yQh|>E;EI&~acI(#vyT z$NQro<zE6XD<$YD9TlQ*&5*t+Sx>eE-Nlc4RH*S=Q+l48M?MghCZ<u#=*DMD=&<|< zdT6*lebG;rj`1G>t*`f?$})M97S_SVDPD#Sb_|})u_H5M2@Jf`mke>0U{35*BtmtW zIMZ_*q-Zz8n22cvbnkO<mX5IF3{vQ+)7y#k%vHFoViOEDmm>Z)?lfi9UR)=AsH((j zB@x~dgBB~L>E}sDasKrjygY`-I(|vOj_t9`jwSUF8`YPpJ1oL&vmT(PVglSe&-rX^ zj)yAEtIUepe9S6VW|H++!91%(%)R{_4W19jlHxknD@uwS+37<4->kwAn|hSmegq;d zZsPOlwQzf62|HuVVU!8YWrqFQiYK@jAT_@T(PAkNcxczga)O5RqGvHqdA$;QN6HZ4 zR3}DBGLoIq8bj!F4Pwt2F>f~XBJGFz6Jgicu)|W9G^t+#P4{qA9vg@;syFfYw!!GK z{9)B3uQSZW!DDgddKYZTF~fmRJxG(jDEnbHi}@P|u=)*Kp>9C{8S`cpJy@VgHcM5) zwHPk0mDODO?wJvN;>N{Rj$6%`c%;yz_(F1R>RvFnnGg1ndgS?DC9)}4nmQaZXX?jZ zh79Wn>N+x=b-i;IPkGl={g!+f&gY1cIGg>bnB9z5>@u0D>QAliHw}fwPm<{`rh|?o z7x$z3?UMBEuWK1%p9G_>uuwClHyJuog1m4K=J8Ip^29xp;rJvW6hGgaWNDwog4|Iw z>g^XM<=O74a;tE>8Fd~L5*M*8c9V$x@N`fKa;CM9&wkSn@x7Gt_CNGLFdb*Z{Kwi0 zo#G7qIu-xF{?n;_;kG(kT1c2{H<$iO`+{G;!^G_WdjJ13{T7pdte?y2k68b2rq1@C z${sj4hCBS7>2DtYf2`l*=f>?dmOKLoV;Gow|32}t)rMe@pp64Ik+h>jL1S7nIk59I zjxJhT^<}phz0^B^=Q4U9Ex&UYx4s@uMyM8GaGz?tyd(x>oQv7A`cdrufh`~%b_*Po zKEsS{33zIx3iEtdIa4@rB^FC1qE6o}So?A={Is15X1dXx+kb5Y`w#1o^46z#c3v=v za*-o-2j9cRQO#hq>L#x!Sd!Zl*N%HWiDIJJ379Z{3{5<?8uq0Yp}SWV6?!cT*&Ti) zQTQsa>i35zE>(aZ`^eGTB}pJHV!~@yjbx7&?V>hukJz8Y0<llq@8CzZiHmy@DKOYX z+KXh-dc6XjRQHwnq+vr=M=*3!cr1x+w1S;aF2eSYedxneitJ%IZ5lR8jq06w3F0>| zGS@e8>y}2tF!xM440p1oYW+gki)$ajLAe8XvD_Tyzgx#t#}B|MWrtDa?i#YQc>@k_ zNoN+Q8Gs9SfAvORPE&BkT<Xz&6r7XQDQYy~?B_RG+r~07Xsk0EHLMct-e15~r-kXM zC|!2z<_0t=EP%!POyP-T4Xfe%nB8Q_!*g2~Qf+NB>exDh?s_wm<QhdYzSTT($vc%c z?|p<7a$dM1LxoBtEC=Q0Vp#9p0Y<|sF~G0}LZ$jaYuPa@yHvvT?zadEvtv<VPB{6r zwGGZKRi<r;H>pi#2I`dWBFAod6EUTAWVd}9QFL7j%+v|6huhCPPE?(qP1pmCbw&8{ zo*11Qa}KQy^MEI}lQ*TD)7SN}BCnR&@nWqKU|w)KN+^ZWD*H6Z+KDhYT9rmWmM0xD z5#amEoNh)(YJ1lL1H5NimdYE^#NoaercjF3J1&BI;4!xF{w8Kz^$XV5-PbDeSqhdb zJO<hrz}~o!%UrW7<uxWPfDZ@7Af+mfs(SUOBa*D>i08@df{~^~*LyJ&E#?KIhsx5S zX~T%FYARV&8qWFsr{PyT4U$0UwJ>gMv>}W=CZkJ)M0$~9WzU!f#R#mQ5((bv%Q<br zCTQy4kNspJMHU;?Fgm|^kiE+;^1hBqr0->~fzk&_EPJZP#r<Col687yj1NHzSx2zB zn~J+X_a%KsAEGNfR<f72Ig<?gWRN?TkNTx`*ifuSHpY(!*X3jAC_PR)a-SBRpud3E z&)bGJm%YcCtM1|LS@QIFzYjRTS{Ed*a&g*PKS6};T<Svh@U8?Or<1urY_4fZXyCpE zt}Q(Rd9~}ROmfGQ0r4fM<=DhJ`Y5n(x%;E1MraTsT?f3d0=Qr!gsNuKU_j&xG(4UK z^)oDJi>MV?`n-XfpF)ZFfl?G!8b{|@t;KszPnb`l<EV~GB9m-fYjriJkgeQ&0(4q^ z*e|<EX~7K%7&@t(z34NXsAcHThsMX5MV9^PbM*sYy*-hP@BJ7z9Err0OVsJii%G<M z<{eDfy^>xriU$33b+{9#O@=sTb8+7N@kM?wW`Be+J@n=U7xU;hxTE*vaQMA!^jI*7 z<TDy<U&T3))fz=lMP$Io$;njeXd%&hS%tDvs`%jcD@HPl^QoCVne+_^A#JmRap74R zQk0`jLr<*2J*Opk8hY|1OYJt)n{vL)W=FB^kq()#=m8i;dSd&~GOGw_9SqtwlN!dK zW_lHMRHa0@(>YsqF}@L8Eag{7SLvK#z695xe|8FK6<USXL1XDdtv*1U#fY@O4qdiW zk0jk=@WlJ4Rr=FrqvF6=qEM29GC`AQnc-^s(NL6VKYqnJNAHF4*IXH~f<w^SA%$;h zR<er=SSEP%LmEwP!h6H@P}AoO)b8F--nbv*^gi<NHv<uRO41tp=&XZLzvkg(PfarH z{3a%B4MW?O=dxo|Z=n5Najc8=A!lX8ah8rDsmPRv^Sy0AyJ9zf%)AdXAAE+AhKCS4 zP=`F*)xh(8*NSlw#h~dmgi+qP0g82Yuocaj%r22OD6dH-b^b58K2jlyBKg>mwE{Pn z&cOC9n$%*f3S5wsqAQ0e(&Ao<RK8&lfm^e{*}#C@E$j#Fw{}9NgE^H6dxj26q~Oui zbxih(GT6K}2~wJ0RSAz9jk@Eb$z6-xtn$ygxQ^WoKbfs#Q`vS#eS;e`H3viQF?wXj z^$Bzg=l2koO33D>Ex7pY3q1ZlgidQ;M6y>-Ba&-3A~D~J%JWj8PIee^@1;Rc4q+kt zycu~C6bt+Fe&Y5r&7jWY0DhdTiDswWsK>I`?AOcL@ToNcIu>dk(ck5Z?=rc4&lkLC zyqyFMZ8{4TPQuhJAe;^`C_ty&RJ0vl50c#d?ePi1^o5QIZG6_7#%G4%2MKqwQ2Po7 ztU3+3Uk{M{)^)gjRvDQ(U7p-J!^Oj0Z-(tdLQx7Wup-O_6)N>{eSr~qDjEwe!E!`X z*q+(etWTyCaq9!mMaY4`JZwCZNK^Ag;CwjexX<ake3Hz<{yR1ibSq+Gn+fCjn>pB& z%8~u``|vuq|L6|)-BewuOZQKjMo(8%g1L(ttvy?Vlkhq^PhJaiWdj+l&25~Ir!HQK zmL)IETfU9|8<XBhd{d0$RO9~U_~U<FKd*EAWbT_1Hi{cNb*BGu{eSTbT^tY+{7=Wv zCK8=BsL%6v72!(nOh2x{5TOxW)$4x#Cc(GZFEqeAFd)Kv30E7hfH~uAhW?!im64%| zskxb<xw)x{siBdjskyParJ<3r8TYrLsim1Q_iQoJ$i%X<mdp1yCg^iD>`aamg*z)N z)aMVSz>oPsu}=pG33VpL&NsR}dguoJyl_Ea&kPQ|x*hk&97bqhH`qc&s9Uq=hJ*%r zFY)^00QAhqdg-F3?_H_ZEuZy0@>#qjB-GpA&x`9l|KK3M;3Yk?S`CdVy}2wplTx>= z6u268X_L>gC4M2HbNoVk=Ah1#S@ZqOknfg5cJ~~<9ZJKVxK^>P#y3SnPSvnGE~nM4 z0^e|>o_N=JL3cb&j~_Sw8#reHU7G)`bd#QV$i@wY-&I9>ly1O}oA$(qTzM=k7?-o+ z$IW`;xgU$FW4IpgOg&1s=f}<e20oD=AK4RMWDn(nE1%}Tk6ZM_8@=sp1><s_{5W@R zc0U)pwq1d1|Gyt;LHxK8pV|EdA1r|X)9wVThW@*BA)&Mo0X(1Kb=C6hUVov50(d^l z6DdrZ{e3;EJMSWh=QF)U2Gx;*`MNX#<@4E|trNov<{irg@O;MCbv>th{p;!l@O;+y z<KyReSpd&xeqFB{-OG2pB7o<!zrWzG^7F=g_NO^Aa_)EY?@|6W0sNo#_x-yl=GMe2 z!F=6g0X(1m{YC$^3E=tc?=Scd0(d_Ad$}k5s^I!NwhQ3-?60eybNBwM>nqH6;Wp;8 zzrU#eKmq*UvA;nA_&@FMzrTJsstVxw?C;0dk7ELO6F&Rv8qc?T|0${Qmv6#ne_i+E zb?4JS0MBQCUDwmQ^K!EU@O<|7<LgJ-asfP_{r&j(%f$%b`Rwn<=f7O60G`kOy3Qwh zw11@lp3nZej_14cX-NY3Kke`P^`q-(ygToh+>QUi5VY$!c}Ul<KcP$AdBG!i$BDgr z5BmdC7ajg*2H#(OfA*04*IVIhg}=SvIPuOGmpA<D@6O+j6U915uR_DQpU%(!>$Cp_ D4I?G2 literal 0 HcmV?d00001 diff --git a/tests/fixtures/tl_checkpoints/gated_rms/meta.json b/tests/fixtures/tl_checkpoints/gated_rms/meta.json new file mode 100644 index 0000000000..1aea27386a --- /dev/null +++ b/tests/fixtures/tl_checkpoints/gated_rms/meta.json @@ -0,0 +1,13 @@ +{ + "d_model": 32, + "d_head": 16, + "n_heads": 2, + "n_layers": 2, + "n_ctx": 8, + "d_vocab": 16, + "d_mlp": 64, + "act_fn": "silu", + "normalization_type": "RMS", + "seed": 0, + "gated_mlp": true +} diff --git a/tests/fixtures/tl_checkpoints/gated_rms/reference.pt b/tests/fixtures/tl_checkpoints/gated_rms/reference.pt new file mode 100644 index 0000000000000000000000000000000000000000..664ea035e4daf7a8f597cd29166bfd368e9392c0 GIT binary patch literal 2165 zcmWIWW@cev;NW1u0747`3`MDFsYR)I$*KA&i6x181=%@nPCN`zNT7kKwva(1f(@v> zBtJVfuegvgxg@_RIYTeLv?Md9m@B?0HK{Z+CndfFD4AarUuMKr$fThW!4A|>3{;tz zo?6HZ(Zc1EpO+pC5#uUkiC_dOF(_n>U<T631*L^--Yg}B?1A14-i+QXZG{}(EZ&T5 zg`Azq$@w`ssmUdo`FX`${zWN3C#Se%CYNv(a%lz?a+ehH1QqgvotTrKo>@{{$d>_h zK?clLx19XM61cVeU~3Hv1v1#Y9ZCuXq1Jf|v=<6_J9rCtGqx8BcV>W07AYwd4Js5X z)eG=u=a|@2q`(SvF$f3X3{*)56Qn@ZOUp0HO)QBoOD!q}I^NC603~dZf*B{%5M=lE z_jj5>fd#_2?UrFMMYbCn`0<H3ndy1Csd*)CPKG2~A7y3&a)5n#;Y%i<<sgjP0m2Nn zNDfG<EJ@7=MV6bBCF%CU0wgE1q$DR5<c3z;$C99A2*SACAi!XQ<OWcx);Dl-vLM?5 z+(6qvfE7Y9!1z2+KFDQf)`VSC0P;Z?pUafcU1sR!BuBtycA+WzESwM8SXsN-A26xk zpH-x`|8jwfeI>8MzVt^H_Lp<F?W<wZv|pH=Yrl1CpPg9s1-l=+a_n#CWY}HJzH67d z`>lPD_J(~;r)2H_9N)Fy=z*Bs262nMueBfSOOY+J{~ED)ze~bnyM?;T>>UbI?R;!x z_a7*X-si?rWT$re#=h{wzwJ&m&9YTk|K7%H-h%yK?zG!ARJiR^5h>VL{<qG4&DOyE zi`bLwq95+H&sZ2>w`e}I{XUO{b|o_y?PssKvj2nEpM4+Av-d5yr)VQ!>uk&OxoKa4 zX1m>+a8tXfr(5hhSc`2Pmf6{dUlrWnKI65WL2lB1^W81>iZ7n+U2b}EA1HkJmXtjM zr+vIxPmsY8DSY8s&Xzy`!x9<}+L#OEf{XLgLkXvU1qK5o7wCZ#UvX+tCNKw9C4x$? z_{<bHCmoU;WME)mZfRm<Xkut;WMpn+VPa%yU|?cuX<}k*Xl7_`YHDO)W@%t<0dnOI zga7p!z%&TL0p5%t2g6H#<l;*RB!L1j>j8BA$VHGOir!FQtRVEG6?y2!AQvPuD8^hw zH3p;fLN^JyI1olLDG2CsgoEJ45V`@#xmEzhfNLlQK*|tw-N<PYRd)ffIzw0w)(tGg zQDXx+Jqe=NeF0coBXmPkF}il-h*m(+9t13>5ZbXP!2oYIHXW!gIc8nBepnj;MoWVe j5Cg*xXxRtU2ddN=grN$65e7_Vtf2CkfddFZ>LF?Y%flN~ literal 0 HcmV?d00001 diff --git a/tests/fixtures/tl_checkpoints/gqa/checkpoint.pt b/tests/fixtures/tl_checkpoints/gqa/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..3207329d18ef054da72dfd5699e3d15644b3b47a GIT binary patch literal 81077 zcmd?Q2UHZxw#N+!5>=v#A|NPPB&X?JC@LbDzyuhOq@o0S7|a2Qf(eWSMFB-bQB0^z z?=lCBn7{-m<{S_)`}GW-bK#!z?swn2Z>_h!^}(8&ny!ESYFAbF?yBnQkv{F@Wn`3; zWd8D}CDTbJbXs`mjF~YJ(W1U#!J=U6nKL3iJyc}^|M3?u7aAH985tfbiinAhRT&`+ z3m1lm4U7mCsjxEUeypsSzcP;wj|vG7vmO^Th?NT!#Rx;ESqH_5A|hi|f`s8AaS@SW zL89>J*cf5ZEISppotc@xBA+RiR|ZcFXXRV6sCY)k1dB$s98_Ty{AGEK4cp$o9seCV zGmcdpCL<?em3)0<eAImmQ&?pmLm$}`RwZqk#1d5ztLDdc@bAQ{XU4<^{b3RxXFE>g z%bX@Zt%YgJcACbsWJRpHgry{5X|%93TUc5mR@;x&@$bm9LLy^AXT)0DSVu<NTF+)X zw?WylU8X7e7>d}g5|UvutLr11%yw&`bQiIDer%7wQbJh$HWYi-poL=io6=K4F>0Y0 zi`ZU%Z0~=j*zvZQw4pe#rY#h+-xPBR#iE5`DPpbs*gk)ygs|3aD2{C37K+VpimilV z*Fv!uu?~K$qrXP$b_o`VqWP}(Wu4l9oLFZ`E6a;m7YRw8Z(yZV)|GE}zQtuzS+^FX zK*TbBEc`8U1WVc=omuza$bKzIzUxxi{t_gACZw_+Eyw{PcAy_S=x>lAyp^79kS^@t zmg;)_CZ@1MB=)7S-Yqa65j)h69rm}d5$y0bFjsa&3vA@?w)2&=onH%Rl!zVe$ByyW z{7c&fjSKQ&$Nou@;{~=h>^MnP`Saah!jfxgy?{SkFR%qVUc^rDV<-MSbSyimE!36` z`VF1j3L5+;G^7O@Dq_R@*zmuDhVYh7X$!Sur}C9n7qQb?m`W*ZL`w@!Z^6wFv5|gk z)ZgRAve9jE_H0ZGZswmjwgo3_!NrPLksllPH@K+a*ct4s7Tj-_;t>-Q`KL#j?JsRw z2X+o$f{cirD=C5hPhscz$V$3=eoF}pMC?L8c9FmKUwUfD;Nc_u2C<9Vn(WBN^FZF6 zEa8FX<}Hu}5u518Cbd0pq9SMVS4u=Qo7|S*#HL6(MoGk`N^COp>6gMT_30<EDXj&U zE@GGYvCIF83z6Vfw8c5ID|v(ZiP%*VTEApALvkKwwvbkf*foCa+P3Z2VpvR^h+Wqf z<-)H2ZP^AXOX4fDBv)Gso87|85wW>`?8d+HLV_f`O>KFuY#whK?=Uw@E1Jx1k(id> zf+`TPg?{YT)=tXyx7*#;24iQ#7PVlu|Hc$cFgsc>B_ejGAG_<Xm=LzKEykAJ-GbTk z8&f91>}|p96S4dK*aQEJkvQIiZ83Iic?;&yZ_Hr{=12>sLd1&wSnjWw5Ek2F>{;4^ zIr<xOOoBPyf;k~#Px`T^TAu*6)_=O*(``u(>={X~@xJ%01ohkZp8NAcuWVtS7qJ)o z*o%M59Kl{{%XDNf|7KokVM?8E)t}DyY76t4h`sK|-uN4)#QENA%XDI^TWVbM8=1o1 zYPr~Nx6tm0*t>r0y}zZ6U~AjboY}e-TK(_#yDw?K2Q8!q5&O`Oebo9i{oQ_2_xrdl z%7uL*sjQrceJWx7cE8X5Y{2I&&=(^1r62q1@1bMa*KMJ$?3>@vx2>S>{)E17fqoFN zjehLMzk^ELZ&Mqny$$<`uQcy|n_HN?`~BR~gkM^4Uq$RUKlc0I<HoW-+Tv{4pDnmw zf8c~N{5VWX6w2~3DO4!O$3((*eCQ{X=fl6DOT4jAflvEgK#TXaw-dIP9FsU;p&}oX z6eID$LM8r()C~)j`B+OaDtt^NROLfIp&B2ycFDhcuf-(`JMgJ(Yho|#C^;r^%)(B5 zj4$Z7Zx*WaKSe?fKJ*i6@?qQ4>klU_)Z&xcG9854l4DX=E!5$o66+=2TG*NYA$8co zE_|%T`mTITB-G_YKVdgM{A)21=Pm5cC$%lcQK%<5CUN1y9(+tvO21^GJ|E@JXrTfB z-BN-f9}@|C@}Zy5h!5L#!=FxFXw0XzEy79IOLDBGCcXKnv<%+63r+YRQU@<I<zp@7 znDH@@(3}tbgcf}G*K(xJUTDduwJpb4XeBu&arweNd`#M4$wF&BDyd3e{=20J8$Koy z+VY{F(2fuNRrtGaWOUG!i0I%*>)Aqk{#aYIi_k%GOp12oW53Z(d{lyV=D%CeE__TR zbmc=op&K8zMTZCl{IRxZS0N)gCPf1u`;8`iRDyQrzgy7#_?Sr8pAY?n9(?Gp%3B>5 z{m0y6jBo&dunpV6MmSJ%OzK^PgZQWv?xUU}^yGi=GaK0y;b1=2Qh*m96A6d#p`Xy3 z58D(F5;R8W!yjy0fUR(-<e0=M2#4`8312-~IGm44j32>&x3EX@F_F-h5B-FGd>FS- zIEw$0PUwQ7!bQQ7IiYYge@HUH5{}^`X8wHrWQ1e+=tAK*iC8j`n-wmM<)?>2$&AfM zHr8K(KPGJgVSrQ^$cNMaNR;&Jw3wI~EfX~1c&TDSYlUP&`R5EyI8iE?)LI~!mPrx+ zPz6a<lUu9W^QxA~n=n|K8qzvdG9&(DY9|bpriZmoS8J6n8G{JJrJ5<NHC_HmV=Xl* zHZ(XgTsT!KpVnHg_qX!s7-3X!WW>DiFkyr=&-B)L{>cnG;S8xJvbE-)%&=>1MwC<@ z-CF)nX8e}7G9yNsXJ+d>9a@_%8FdI*sYuvb^w;sSv^{1BW2IWrzoBJ?aZ>H9e?vP< zI9sZn^KWS93Fk_+^IB{5|G82v4JWByP;6+7Fg#W`Uz%${>s;ObKG#e>TevV<xKNs7 zQR^JKf1iWr$3=>Si=`RjTW4s~>AV*gE|F>yT5EJ#LnWTR<$@4~3lpW{q}Jlje=m-S z6$z82>Xg>%HfSkU+9dp~Dpe|8`me<-e^Z+oClaPf#p(ZAJma@`nN+;|UyEn`7O#+s zSGE?nIrpSC^KBxzyRMRIGg@ogw8lT`$In)TnNs!Y*6P2uhJ-8;uaSz^widU!6s2gs zmm|W0X7c^APAXsj@5rMf!h{>7@~qbKuC2SVHC@t?*;0K@YkeypXDiH=rf+PW-s(J) z&a;G@q?)|enpQPxnTtuwZSirNrSdJU<*l4cD|svDk}u6u&^pgQnPDd^lxnuN*8Gzh zZG7A|sl2GQ{GZJDtB>0*%~RYuPb(kSGPV@%kcvuLi&~v$zXzbw_Lw2uDb?=!H?*v< zRI1(mZ)j%;_eiy6|Auy+aIaLmueG+7k85pjYah2?n(IL8T&;Xu>s){MxP#Ih<*jqH z@^P(m{PuB&q!|vk&d{dQB|h$mR8!Gf)9TW*wQl3%#8NTWTHNZYZY`Ev8YorM*6KEB zX&3(C<Bm$j$NshW4<C12Dn9YA#eev?lTz`ie=YvQ$DNjn&$Je|IrpSC|MqcbrP_0? zwQXACAN7;CtxBo-d~5YzTjLKOcR?z?*jn7?Qk0_q;o~kz<(K~*xx~j^k;<!D%Uk(4 z+m?IUZ@Q!-uS)gTTI(e~?vH7L@VYepM(cE~R_Uz=MmMF>>ef<8(SM*?q^<l`jZ}ZD zwO%sx{XNzf-j*icX`L*25b?)UK-x_F-RQ1VcCWQe{hwu$KB|@K>RRdIVneJ)`Y0+r z9n)yAg1_1%pOOFN!<}w2maQM|SWl4*GDShZ2kM?4rmdgx{NriQUt_kCG7r7ICmB@8 zNIw5{nLT72+LZbG@lFswxSbju6&@|}^sxDNi~Z9OCkKfE?z7qFlC@jO=f5^UTZU<E zK*(HCc#PzEm8XZxf4kg242X;ni6X-#HYCGCUCER}^7*fA(2x<dwm~w3>TBca;ru@} z;HR=N?eoBdt}@ZUfYI|&tEve?qs~EX2WLz&%ccu<yTOP?CmJO7hWLecaH~3<nIf<! z@)bRaI&{K~4Fl-CtT>vjzZG<^KZb#9DeSZlrls5<RJ2MVHiK`&&ow(Rw&5H%FyjWZ z)qV)McqE)WR#qnW0zcx+^1V1|>T1Eoy(@9g<4*8t-y-NONMIgrI07ll7DHsmD6UV^ zW6--`iUy+^1t(%Qz{-6^<l~{y^zE5&n)~4dre6FG3OBow5ar8ES)>B)d>w*yft$eZ zNf^q(61=>y2A+FNAd^3QA_AO9FC4a`1}76q2yRB(`mK21Re>nf8j`*j+=#6DSnBxj z2sxyfhL%Adu&HqdRdU^o8DG-T?nnkcPV7b>#q}ilp9?s}pyzPi+n?M2;|z?s^a0Pi z%e(HbZiHn=reZ~EXYyp$2*AZ1(eH2(Iq1HWuATWrOh$L1`bjGEVrZ^_?zjZi{?Wwc z%Ru^h!(gb*To21!k;;_DQF|wOyk=npUz9pQo{1g9To?d%a}%M5e-eZie#Hge4Y19r zJvB}F!435-qeDKJaf23b5<7*ek`xbb=*22fv4b2pJn#U%8lHlWOqP+yeIn3f*(!#X zni09f3Z(9d7nyM97xvg8#G&1uW60G)xO}z@M!BYfAz49WKI+ms3w!Rw?G)yy@-D11 zcA$G#L~@IE>Ck1mj(9$(BP2}ej7443nCg)o;8T$qaW*%F@9%=3>FqO4Y4cL<N=Y5_ zwXhyjv(xDMunmmI_X{{|xD&h>SI*S92EoaWYQ#FF6Ae3P4NHsXaA_OV$s9HVn`hnw zqP-tByl{kLCe6%i?Pc7@iz8wCGe<0Q(xa-2R6*9xo38iJ<!-MXfzymbXjJqt3|Og0 zRQ$4-kor)va8v`_bkL)4u?oli98DZb4B)2zaX2lbPL;rln%nlF8l4Mpc|s0s+dBh_ zGQ&V;{~6|D#Sd6C)C3h8waApoR@_lbeJa~I6k=_6L2=bl_*7<1-@e%cZgcZ6af3M- z{xq2Rkf2H&LK=Zx?nvpgjwEj2Dr~&uO3O#9kkb!bN%5UyXtcW!9&R^*b|pDt<FmT- zct#SJ|G*P_E(;~ge$+BQbSA*u1KYX8U5DY)J0t9~z8}4ryp>Bnkb_R^H{hP6FJL8< zp`mX-fLxOaIi|Fj%E^qSYuy)7_4zhTm7N|Ht{X`_4CA?6t?kh6=SglrfHpTgxHA#? zwIlXN((s(^cQh^A&S;)XBlf*Qph{FOUf%N=hNY~+p%K2AI(r99oMuYnx)gD%H$LEc z_4nise;7jMKG+4Sx0j)b#%NsBryt!HA4eXpTt{vsE8>EC7vb}-?Qn5=ckt9%N8NVj zfU!m{zU%cB6^{jol}pXZ(f3new(}G2Rk$hjj&Vkzr4l*O)r6k9l@Ecwk0G`3Cb#|l zb-Z5h#;tnYfwJam%%x2$aOv)s_+im&_!4pmjp7Bwpk@>a-nSopjXDt{|9SW({s1mc z`O5W5TY*lmA40SDT&B>XJ1sO$<z5WZq7xSMq4KNK;cf>zsyfw^j!z9D(;BkH%N7Z7 zS4VBkJ+zJ+@$0+TB0Ulo_b?`(^p=yIuQf=`0u9pDU_PdPu&37)+=#nl2M}${hL~OL z#Rd)ev~t*JawpA(o?c%8?Y%m>w=>D$Qnf9?ws;~0J8#DAjh$%ra}QGVU^y9I>qaZ} zOQ`dub@bT2cI2~lCAvRYK|b6zphUb8VrnOmYR?{|bg=_*@J(iH<ENAJLsQ`HM1~l> z_zpLobY>=gtcUW2YP4Xv3|$@Sg53`GN8L4vV3e2v>wJEK-iBbvxG;*iK1c`Wg*kYC z%@*;NAPQ#(JD}Q`0<2P6jukF0^mtAsH)P=!*c*F|8-7g@V`o^vtJhDUNz0iGn5#&N zzik5d!b$YbHwWI=RKwnOek5|Cw)>&CJ;~&!A@m7*9vvnWL8P}m>6?56VR|htNl=By zmya<gvh(S~K0`?P!c4qJV#z5lZCc-3L>$N2(A<@^uxLUb@{zraKi+lWtco|2jUGz~ zoqxBYxvm458{Cav7#2?|PxeIBDTX-b`7Q99Sb+gIl&Pgx56Db70bd%E$(7kw#HpqP zYxk>?Yik0j&X_`S`-3exnzI}|CMCl8g9FH{@$+Ha_O&QTs^JPt4Ism85i{9-Bm@q> zgHIpG!-Y^SoP8yXsIBOL-bH=rZ1rey;*td7A=0It!#<HQ2ex9!p){<ISwJ>idBsHz zamMX|0+LhZLEOvkGoy1tiT#0zn5F1Vx}<wTbKo@Esapx^KPrWBISXO-!XWPX2zNM2 zZgXxshSPzYcS7J1Q)+*Bn_zbTuT0(!dHU;NFewatL?N*N9sAG45$*e<%ljZ|IYmkA zHQko1JvW8u4UD9dE8B75lgi<}Ru5Rd`wZUbu1_CBEV1kH3#)%_2c_?R@Y!)FQ{w^D zZ2d)C?M^wlJ!+Kk(xLa}h(YP*J#JRU74W!`LC4J7O`kgaWJC(H=;zHqD%Z}Wl97kL zpks=RWSq-?{?*8%|6fKPww@mFKOK3rjDYT%<zanNInJs&giCsNAZgtj8Jom(fo95V za@_tJ`l{T=ExUKYJZ(oh;z@7v$fz0;=R1-qN8@q1+G}85y~3@BMuPUW2x`@3KP_Co zT~H_R#AUh@ap?JW+^n%5LFL?Os5;w3oJv^wplJ=r@92%TIv&tiZcc9H3dnvrS)y`( z1r}X3r~W_A;I&RW=*GtZ%*P%txb-iUxq-VfiEWnw*nFLV8*;g@?ZHZ-I_(`7wr7QS z*wxj{#4W?zW~n~`=JpK8=;({5C=kozsx-L6FX%ZliW{hN3Vk0$p<mc^+N1j#u&^&C zTXYp5Y-%>1d;1;Jzx8k%Ywkpy)q2nlo$rA9JOQK(Gr&SkAQyd`n6+s=pwOi~ef{t> z{Wzr%_a4&68SnC$ET1(Hk!T17x*h4SLo(!tQz&U)4BWFYZ`4V8!4#^Q;`GZqnG~Zv z<eX1GqIat+DjAG|yAFrR5AFmtt}P?Oh97gw8m><F@9#jC9$buJPF*lU|0%o}ZAfE| z6_bM1wOE#shF#cIXm$B9cPD5InKC|)YcJHIxAt6SIv-n$_vaQO`%|8t>3SP)ej3l2 zL>iDOXMn^Mn4{h3NBBAH6sJCK6YZH>1$qI!p~p^n7*p*^-c<~yyYn_eO@C!N*yaUu z*~$m^-`)i8ZGuRL+oiDX;sAU!bvhg!D@!cb_|cMPOR_D#4@M6vB29DLVf36$<VCat z?Q`Y;ao{fE@Ej9z+HEt+*}sC?I~io?;zhX8dLAzOvJ^}58MygVjhc0R3OiJba6|bU zW?Wt~qwAzXM)}@=BF>NQ>A9B9=zWQg9e`D9MfBQ*S8%B`7+1-=l2ID#pk2cdxHz;k zwZ5rB2R6$S<JBwR{eq4pvA;SAyJkj<(sy9=!5VIgy*&*QS`zb_4cysP{pm!PuEgt* zHvP8L1i8z}<XS{2eP<pC23JpUW_CMZl73h6$<mG7>pGq4AF9KLXhR!*ZiA5t=fO8h z1D^QwghzwMb2iQqWLx8MZo;D>^i%tG)c=MTHdtPSr^{9d8Y7zVbH|O)upkSgpY)@t zCxS@k{z%w*dM!=y&4LY^OQA^+iQ60#;m(H;9Q!qg=sbGEt^OR!x#TVfr?(ejWae2x z`SL~BaP1@xzdE0U$0^ateY(+WofL552S=(fIv<W^motMoZE{!Mj#vjf9*Jt7NQLXq zLFeQ?WNaxR2B(MM;#zBxn=i(=#aSRKQ>FvP=hEb|iBK63g*{*0#KMCygdAEA_i1~S z_#fk)0}N#NU4)X)f6xEe{m1@CMke0<|MDZL|C#mO4CDFgOFsX-`rH36)SoZ(|L{*z z|4m<uPsH;!NIw6)`aAqD)c^m>{qOj{P=CJB{}2EEv;KGbkL&NEkpVN8b%)ROW60a2 zVG!drgKVka38!o4(p{zpP`rC;#mquk;#0A*B53qg!Nm*P!QU#Jnv)W=>5#;U%r3y; zms@ZTKi*JMx+JbxupZX+x(BB_=+IAEjx^y=B6q)fFYfQy3_*MLGwV$aN$4VdVr#h# zwC~-AH46vQ{N1(4nt$L%Zq&uO-Vd<Dc^{&mTSK2KZ|C}3e8S3PE6lOVAro>lp?;PO zt>2PC;v3_rQNale*>n##Z%L<zg+{ddmEHKxWDgyCNuFwYUI6zWcd$Vw5UTu*Nz*%T z0Uha2_Mgb3rsWH`+m501@V0z->eG$xu#13loQ^-5igDL59ZU(`gVS~HaD$pv$kxVF z;B~$Z-_@IO*X4a_*aav0HLwKs-v5D?uXJeMx4twf=@hqUTQLOQSW4F1)Pv?Z*C8v| zm0p-P0w2BWz|DKnpN#g`rK8@=gb5!W;D*2u@sqPkbeE3}!zDi;7q7iQ{SJoo{lcqw z+2;|Q$r(gUS%0{5U!BShoa)*bk;Pfb#4x6gW3fc{GL}zT#w{6TPa=LkW~v80#muJ$ zFeo9EkvX!T48Pxp9=@$brkhR1u~qH3@!|T|II9CEtCs`nuVv|%#SD$Q>`Bw=MMR}8 z8Mohk$y8<PP}Oo{;@xK(RvKhc`MO|G4)$hZeLthVmI2yMiig{eR7l^4{Ydtgde9wR zOKOb{(EIsuRNdy4IH+U@F_=(9)*Z|O!{9wk!oc-#{B#L^d#+BZ&JpUlX@uL4xm%$! zc^JL;&<!sp9AS>vjHOHFz5$`D7D*WAM(U2OC(qqenVuCpx#m%_WLnWZG^&|ICp#~p z6Jsjj^R2zK?z+1`qi-g+!d65CSGSTiD^=*Rx0hl4+K-^DV#G9RPIY^j?n9@BrZatO z-SKFABi>$-3{F$`!J!jBxH;2QQBCb2Rt3KT=PBDTGc=v6TrnG0-gybD7Y?O;t*3Hx z^7}AHzLY>f-4tN04r9QVq4>bWfZ82fOVyvaabD;04sW^Nn=a^|NhbBU#QpFePcwIR zClB0T3S`d4(7`*Bu`aeuU_jmJ!@XPJrQcN?SDa0v(w_>L#soO@>Lv6{&gTSQM`P8j zBut+pp!GTzz{D$^p3Ch@R+e<3SySX-^^;Rz-X|MI*bIYaqhfd?yHl`QcmTXz8o@dC z7d|?sO4PP~1C=Zp;`d5{7H|R}cCXRvtuGYqkb&p<Js~O633Y}~LzNd_m}R}($hxVW z>HN)+v~qhch|cPfv-@Y$H$U{~)ZXj3n;xfe)?T3Pn4_S@x5c9BBy!R)3C4RHKwaGv z#{Sv@Ce!RAZt>g$y{;0{UAH@#_{xT+7?*?T2~9FF{VBd%pUFKubrQB*{sMa463B&d zfpmL@8ME-`HX56m3*|AV!ELWU)g0WB?$M}%MZXq70WH9{Lyv*d&@6h%=_h;(Q%3%( zCSC`7Q{P+1sO-&bQu)S<>`{FQ?~ihfX4VN@WfDi1lys*X*k)KTY$tB2kAoxY?Wx)^ zW6ou!CTA+z2{Y7|U~`HMjm@7-&95$nqJT40?|zJ6g8xHKC82@$3%PWIUw>%7#fWyi zvmcJ!_<<W@<0(D(0)OBb`qbt)R2?`Dm2TGD9e>K)yuV2Bc49GT=_|o`+wP?I%ncax ztO_qhz2WAP7nr{`1(Z*RK!dRd4V1qJd$q5FT7EY!L^dC0-14D17V%7}dZA!JVj%_= z$df>uK45NfiFxQU822BPhhxtcK))q=(Dl<(Zqs}%GC7gOg<DpNf353CljB}6hmOw# z6<=%mBPxg6&}5F+x23><Cq>|VUxB-{-JINhGM=vMcTXU;J6bXL-Ut$KLj%e(sv%C% zfF2*UpK)00O^<d)!R@O(Y3Hm9FwO5c8hd0i&!eY<cNrqP{Jo&hw}Ei4^CzxwObB^A zsU5Al975y1+!r*}cOvQIvq{OW>s;Qi`<SvNp46Po#H!Bi7@&FB;C~L2z9DWLR0$(I zQ_*(q32tiXU8q*GA`i36am?~A+(VcBpx=~1j1?@w=xH{%g>AqGi>5H%5vg3*r*ZJS zPK#Fh>5;FUO-SI$bKKs|TFl6&&Q$UFeoT9&jK=$hGf|-l#655weUs+~0~&kLt2gY) zu~*}%>7730_^WuVcX}dl?@`G`_Ns#wzxF}B+<CHf**rMrkwe}T9wyWL`@k>t9q5+M zK+fbG#%gy4<6r1c=Y2L4jJEZH(v2Aczc*9i?v`0F)1(o)&#)(>a0ARRn@QJJ<-z8~ z-gI^MRrLCNJJQrH536q)ayzwefRn{;uI%zfyg4Nmf}?uE(5U`o^Keu0ftBMX&YTNJ zIw{gIKLhB`&=^KPU;_DeXcP>8Gn|Z59R>ZWW69G%O|oDA0BA;UBbu6FB)HBVb94O3 zfngeCy5&fkbLgy~acDFR>o%H3U-JgP&^T_woNBBrG9}Flz9j$lV@5;GkXFR6p<9h} z$fmki+=AVyP(93={LFKNinZZHHRmS`HaJUO?OTtp>bB8d7gecm&opX0*$ZXz_k+UX zb6|a{2C8<or>6tY;LSPZu;=P>(*5&fh}V@N8pekF`e7eBue}cuDQ7_Ct5dLfsUhjO z{yDTextH^5Z-8S$dqdMS4U(;C$=}O_q~Punip2uzdGtBn5oVLFYes;_o57%?P{Y~v zIL!5ou^~owM%3<c5|$7F4Y1TBovx)Y#?{>+WWZ7y-XRayACZ9^S5xe8`#4;S^e5Vr zg9QhzGm!4P30l`($R?-#+>`2F=rge#t-p1`qH)h*ccuY-lTnQuOW#1sW@Vc7iNeIw zXE64I87>V^C3%{L^qA6iZr^Gzbg_8@`T`|l)UcU)qY$pjs$=>U54yCx4^D2#qjK^# zm=^kh8(n%Dx-_rjcHZhs=6W0;QMEN#v|tYHDC0@Kgl;A}=7o%(%2hZqdo*dv@T2u> zBWe3{)!5{wOEeXx(q4BCg4?(tT7Tjv^HBH`Dy+VXqs}kEv_Mm`axWn!5qfm{C@~Is zXHE(%L!e3CkT$*j2)f0os5$fq+P@sjOzmM$oEz?t_9e^Fvs4Y@dmiNI77@k1{Yg`B zCeePRh}+h6r_J4-VdNbd@+ig_1?*BBn!TLJALQ?$fkl|nPnC9kDT}7oLn-6ljTHL| zK|AX*e8W+YrSkz7bXq}^e<qQK<Atb`mdPm&@}=$;t6cY-0CICs7Dn3Si5-89CP9ux zWVKOW(%t2=fY~$_`=;u`hahBhcHhRG?!Do2Vk~&Bw<5JJuH5DTYvQ*sjgZjEMEBYU z#{bSWLHi_*Ihw4<{YrR%iw<a$3ug|%xXWhn#xEZ<0&d}}6UvNbCnKWq{FdA9Y3&8e zCtgLX=g&A<wQo4$h6A~i@&Q9E<RE&=M^s)aOVX|6VfmL+jBs=VycwN?waV>j<&c-$ zi)G8;a#jl11U6$qN(g1G%DJ98O60}aUEJ`1BJSP2_U<=c&LeK$tMSL0?aY^p&JYq& z4gvj_qHC`&3@eC1@3l#E+?kEyQ1#KUBR)Vpu`mJUhi;}0Cr1;@z*sursR`NClt7eT zhS0Z5y~+DY2C$pykFvu~;VrQnk)M|VpFFbBR&_RW>8KiYSs(`smK;P(oJr!`pE8dI ztC6!KvoTmXmV~E&LZ)af9eH8^l~qg;t3*A6fr74(c>gTC>@)=iyBC1tED@J3HiQkw zx{#NlDRfuw8#r0`9ptay#r~gXkbz!1Xs(wb&R?tpW_){&^#~^`AI`;i)d}Fe_#>Y2 zEEewx2xWezsgie!^{D#VoZL5>N|)=ufcXPiD5;mjs6aQIYqEq6&Q1kpu?)>EbLV<p ze+l&ENwU-=2i05xNZPnq>eTB1t(;>+I}~YS(>QN(|K&)AJM76^)>(pc0!F}jVJh9Z zw;8)0=|=9anZ$TmpTTggsc>h!4isfPVc4WYoPk<3Xq#)`kpqvoluN7V?8tH~eq08I zUkkZ0*G}WRA}^9?2i(;@$FbXzWJtd|4V5ekp`ec<-5I?U4{qv0SInIStIH<SZ%hDr zHCu!J?DMAL0n4ww)%%j_Ded93W=O>wpE2T}FMATNeTtCk7D;0VGo*W_3QX`#rbC5s zbmBuzdR%h}E?sp8m-p*U3NtR_Htq9peQsY`b|9a-EB}E@-`b0`8~hNT?QDP{CLQSO zENe1W`6lyvnjsuao{6%T6kxc<4d%mF7IR;h(BrbUR9M%I-Y=VqeY6ayYmh0GTMyXI zBM_zzRwrs34N332y~ttfa4?;`93D*`PLe{m&>sCeqvyy9y!&7VT|cuAEbE>Q3mzyi zdxF*JymuOmaOQe4Y=$YduK0l8N0_=j6uNT(k-h1AleOUeULQDt6HSP}4PR@2aO$1Q z;X&U?wDwUYzM1in@t%|iPNh0@)8lk%FhUEqI7ZV+4fAnI|16ptvx2DZT#C=11d{6> z3EU*X4{)om5~$ZL!;pOb96zRsW$zrx>DnxKA@C=~{py%UJ05cRld|E3+E%1a_qh2F zmN3`PZG{u{Ecwu@mYLjcGu64g8>2sKxdk<Lqr;Ey0QdPz@rdSbuE~D~n*Ury{7e1C zr9Cdgt=GCVY=t(Lvekkt{lKC2Oeez33nU|-ICE-aUxOO+0ajo1#oR8Yv~g(xhALeG z_d7A%+rDNrTib&!%1x!y&f3#F&o#uLZWGhnC=+Mtz5uOX0^+$LLLA`XNQ%WWwEE&z zoV#T%-1^yx{tQdu61>gHw3iNUoAx`C%fTDSk$!XNn>u}dy}27*y>=1o>$)2^=WM1u z?+hcSx++tr;jYY*?fdxg=n>G>wSbS2eQ9pLG!#xx6&O_<LiL5I_;X_*Y%;ij8#U$0 z>mDV{%87eGFiRIqQ$pbULs>%aWzr7MeernJPC>VK-<a{Dnb>f_oR+lP&kcR~1*325 zA>I3&z{WZt+uI$7KK8F6ge)e_F}Km;)>Sw-m!XR^ikVki+Cg@KHr?EuLN>iQLk1?O zK`CcR^}2|`{^og<KQ2QK_3B9)G<C^JHxH;=B%me@+PHpxx^(?9CTl~HjAWY(|M~B& zKRN%$>kq=lG#b1hj>_|6pn$R~=)sTwl<wJZR#W4_MYa_CKT@UlWXFq*GA+pctRtvD z!HH~HA4RRN4kyp5I+EIt`@ny|JQ`cu0}Cw}VzteXzI0ehZv->cp|lp8CZuC`7dwhs ztFY{-F4^7Jfjn8d852&d$K$R+<a1?_;LDkxc>YZXI?qZ(Hk&Df>E;m9F-R6|yI)`? z8V+Vs=T3%z3l{X)$}qCl@;w)u%-?_4o)@S4r*JzvJ%y_IYIxv58d}$l$CvbaMOj$_ zKK!gpVhhwEyYjo(L--O#?o^>)FZ3n1u57~vSDcwOc}CPoei&8wDUY)g%bAP=mNdW7 zjsCnQ<MzX^9*bg*lReLu(|(h-kmz_JnKn)ZXWmJtW)DxH&#%LD-z^V1GHeCeng0Qz zVuz9~_ol*y)YEXtNrsMlFdo(iuEb+=R)DFsCwAGC1iq=BWM2FYLAi_^%@laRtk;im z=+#QttE_-M29{v4Wg6DbjuyNuOTk-KmDsoGq<GR^IU2|9fHmPM5S?Q~uZ=iFzj%ph zwas_D`!<ic6tzrHc}|Or?K*<FGkgWk8QGoauUEtjJ4^a%qZg()>%sn%hqx@X2DFVj z(S)ca;x~19;<v&8YTT(abt~wMtNFDWw){Ny(l;PF<;#hcPG3sftr9;Tup7MZjigFy z5nTU56;AHk38wzY3rMjN;Hr@2^w44r8fpEQTQkvz#(v61gEAw0%MOO8TnbG2nE`Kd z4&%%rk8w%fWf*NSmU<g%;Idd3Ze#dPZh46waj1`Q8}&Q^Pw(4G$~wu@ZMW9IgXV0o zd!j;Jz9w@UCai-6fl1uW%3}20yPPRfIt>bIys;+}Py9m_$?%j)2>g&n>U~1RVNT`X z|G)tAz8rV^qHIPg12;4K?_Wjtc*JX~)6rAL7>~?c!OXYTB*tYrB*<k2ys&IfkIpN? z&(YW6?t=Do+_Or?d`TiR;E5LveqBXJS}1XklCmKsU@|SazY28v3CXG?Cv-n+iK+*O zV{DIp%tnSE%P;ca%shY$3af&*Ute-sD@@5Oe(rlN;}E=BHkQsVpGQPRdU$ZNKKT(K zq+Ph7<iNd)c*6Y`PB>bEdmTPt){`VURxTY3Kdj`+jkWRKd1p8s+lT`L9)n}^1x)m8 z;IwiL@l{PR^u2EheP(XMgLj)DuhVR<=fj=2fq(W>&{zmc?PidhJ9W4f4hK+i)Hl#c z?TtSb_P~LumW)!>Q~WtCn9F=Mk@mSc0nK)>bobOAWbWl-;G{1@`g^B>#{CNP3|ov! z58N>OK__~7%WEth<ceEM?9ugXG&pDY(%v^L=ybRDcpx|t7MysEoyKdDE!+E(wLzDe zr9W)NA#@GCb6YR4eBG7qy!(bzI;KW8Mj;*jY&uuH`U}io;DG*K_3-UjBhKiQK>X|K zn7fZJadKvEAg{fYj4aY1S<c@f_u*L#ud<@&tKLDqZxL|ey6~n@o5cE!Mf)Ll`8_$V z(52lZa_#3U*xJ{TxK2*OThN0VZaXWu7`Y0J=PObN=L*ce=|bDp9z*Bi$J`>PJYu6< zC0^oUM&~WuiTYn_;QS*?dadglG##)9nxi{W_N*qZ7B&fjUd9kRnFN^gO#|i+-AXeb z_9Tta%yheJ2j`z(#f|*E$+4*`Ep8AJ4;3reX<iSjhUaqh>^<h)OCMsKSpdVwy@oTD zy{YA=?M#jFTz+okLKnaAg7i!GxXrUQz^VH{^6;XsYu{uodM;3zUKn=)G!N-<zVr0R z!<sF)#BC_uJ?<VZJD!0PRh8%t*awMAe{y@D=m@@qtRgS`Cz80WX|#Ds7|0tJ!^bKj znBO%PM%D0Z1d$GO#A!`tP&x;r?&{H-9(i!BxF6Z;e@dJ)umHz8R)U|U5fy&=DDJoQ z9@I>~hrNs&u}i>xyfOC?CNIguHCFoAtz7~cVsR0LNBUt{|9x0w&Cr7Et8jf+EGfM; z7(euKW-h8m!jM^=Nk;5K)Lqtz1P+M;{~le*zV>JNeKIPv?}lp-wxkaG#=BDkt7VWq zaW2|;>=1-@Si_iK*afFE6tKSUA_%IO0QzZjNaQ4qit$g@(!jJS^lgWZI6AB^yjWyO zmV};wkz+LI)1zrHVB-h4l=1<Rl%8Pvd0+f6Ya{74#tFK5-Y2r9_ucFZpTVetN(?jU zMyF}1K>Kw~ZYt4nSei7JyHxQBpDy?&u8(<!%Ptwx2Fovum(xg`w_-b>W&=z)A|PQr z*K=`iTJQ5n5WhPL%73j8yxy$AJ?gX#J1(}S_1;==c8?YpHsvn!OX)3W8DGFE%SXWM zFrcHg*Dy!BB@y3@?Oc$|9J)2#7v?u5WBHR@8lIdBuj7iL`GGrK9zy7dUmNiBJ%+9e z)2Ea7%%avWPT=c+)nM7+PriTIggS%!Q_rY0_~^YHR^-ozRWUbkuze!fDluR*4rqPa z5qvix0k_oD!^ADF<Vn&pJQH9`bVLi$blx~9TH}htBQ)vo(mr%=k7h<uHi6h_MS<$5 zenhTcXBw{}2L@BVGZmfMQ{@AL;ZC40?ZIC=$Cu1Th3rgvbV!q+O1lI4mIcs`2aDkd z=S8<g7?Hul`;*01mqGQ|1C01Mf?6D&OeYEDq3}&6vqH5SS+m2A_DtwTD=nS5>rE*% zb-`>pS+N`0ci9=V^2fj>%@u+>Z$oH^bx+!$aDu3owWlwY)$nRB8S>uxHS<zkm-gew ztzWxDFx{^iQTw4Ykt^8>sdGD0tw{>Rcj01g^rlqWbF?jGrx;+`hEAk;(IeQsr7LNU z^QAhb85m!`n4X>7nds}vF^Q`#;rG<dFpo|m&GG|L{_AKGZ|;Vc;>~pXF%9xVYalq7 zlrw!EZ6}RU_qp-R6s!w=kE@@{(dfaU5HsQ-XOY{0??!3T9|JnjYJS{X@Wqqpq>m@t zyTroG^F8Q;B^Q~hb9cc+Q=4-v8Vv0t*TREg74WH9fm%Fz0z<g{ZZ3WUxOu%M(1%C# zNSCF>psGHUFdqzQ`K`Hde*YqV-<1Z<ua3t%t5wizy+3?ZJU|LYO@*88Yrrzdh5Kom zC%&y{N<Ljbk5k<9nJwko+|x^TR9A5<Xx4>6_a7FtZ+Q&fZ&Ky#-5uz=rW(|{AxrK( zDqy;NiXb8Vqp)kQd*T3J4X)sLJzmZ}g$orjv3gG>opi1vS@ScV$jHl*RZn-KqG2aG zdTKjrr$2*EmwN^_J2J?w3-d_x10k2X_BhBzFM;8uNwDJV37i%(ki=cFq5Zzp;kR+m zad+`cus_OG4E>f0^>;=<JH_|hU`2JhuVOfEo6`^02d$*vZq|UTSt>CRtI&Df%c1f@ zhWN(ELomM7Sg^@|I*EQck{fa}3hqzPAl_kyw7yb6#Fo1_^Mcjn!RytaUAPnn=r=Lj z8avYx1sU?R_bt4xlPr!eGlj4McQQ6ngD&+dhaU&}Gkp})!PfB&4qEAlto35C@~H3N z*dqd_Xn0}N23eBqnuu#Ey~!olTA&MSnSqNo;fxGjD!(t53|U`7V(S=G9&jB)Uk)Pg z<Cf6nUtTfW_Rpk;@6DpUpYBDnZZPydHx<55tl?JWM?k{zo1kHw2hPuJ$*pWx+IOZJ zydUxbrhV;CZy%EpU<U&_V|F??Y(HAD?crN|fRpG1{ej?oq!+n2dmP!dZ~*C@lg9N= zF(c_s-D%RO15B~g7Knem4~+csVBComqI@F*UZfK6YVg5LJ~m{7mLXYO)c|u|8qvy^ zW;A8PTYU6vDSYkPB%XIv2QwXp(1=`bf!E<880eoT9)H`LT*#<mZhhX(4AwHGWtZjn zIq7+(G2$|0=`BXPD-N{RtsVFvaSMDr;7^r>!OX%ELu#JyNT0_p$HK#DobonRyz0b( z?_z#!`20YK%ep4+6z)nkChihHdhSMV$XLMAqds*0`~vRhq35{cg)vikYayLr*pWWj z$nOWyPk{Jz6TyO~FU8d(5=f`;TG}&W9#x|%wCd7R*gC68Ja)`$aCtO>NpvcO#z`HC zg2#AFG3iMS&3ln|$3LTr<}4WT-VegFdyt;|6N9t;jA*^~Q8%}K4mh9&!6>jJGjoRq z8ofS;M`ZRhlWV_XgHbO$RkI8y#HkV2CMEnBnMxZk09<A_pw+SV{MxZB^;o`zqi*~j z@HI!pUFs@9{ct|FX8TW2a(|9v&2qV%Q(qwC^iXK;^ATlqin$frT$yU~VPxUjY$9WH z5U*DF)0ac#>FkJ8IDC->9bw#=YPy+X#@k(RGtQLuUi6u{b5NTmKRE*4vzO4Z_u}d2 zjk8fHQJ$8vgQ)x)plUW{;;SKBu{8cT2jTJL{PzeZM8S_Pt{p_{<>g>$?f|lS*nRF= zYIknH5*Ko7Mgp<opNW>`>ys+Gb@b_nLa<oxL%eDhL+HEhxPDs=bM;^d*36m5%wE*Y zxY&4-TMbiSozpQ?Sf>ul7i3iEz3`zj`@7Pw<?^IB_BhV&CP&&mzet0XOz8lt+bCZ+ z({-z*JKf=X6O?jNL8*BucB|7O8#b;c!~1R&)Qu96;Hb{X?!Ciwk{`=VzoA9*ngF5# zF1ZbxltEBUj$Gb)8`Me{!mVj#Fhq7TIiIi{OKi4~H49CN%yxdyPxmy4nK2*y`R4~a zjSJ`pwh?AMsALZ6+hY0zb?#9A=OCl*1z}2cTm+|rmN%EtYg@b1a={Aj(6twWA1A7D z<`H>p(D229pSwbQ?j&MlIgO0GX-_lcgRv;P3<drZn3P@X@!1U{@^g<KcYnlYlAlzK zC7L7YQ;iCypu~a%v8w!h))L-zTaPnWW{@SlhTz!wI>eyx4Ca-xTzs%LX~>ucAMqZ} zTzZaMb-V_yy%X@y=JHWx<s)3@tw2jw>CxpACy58>kEi*1&CK!BKhVT&7o0O&K-XSq zM|O0PBZ}WPi7yu=kQldkNO(mVU#<%Y5AnctQ&)kJsg|IR`9rvGlz=^*w?f!pEm|{` zp~dMo#PH1{+))#OWfy@uM}Ed%(Iaraz6UAo?oVsKHKP5&bKC}3e-sD%aPpfE;C0T7 z^nK5de}-ki{pb-O+5gb<#uM59+4_Uaf4u(if4KESTJ(Rm|HJkFNA=f~>G#+DxBRO= z%zv~2ciNENp!Y+cv$9DOJ8LzeqStja86xBQecLl^p65>^ckkogjob(WhFX&htNq~R z{e;=z8AnF0zXNjpH!-quck#%c9CCkd0?Fl^>B<`%?&>!|tX0sH-_yPcE_N73$65^| zw-e-vY>|%m&^CF}yv>KmCv=3IZVI&5fKkY=dV$We%iP5unq*qP9Q?6qhv5E}`M6NK zFCF<^kILFqK-R$5cyeq<YOA`CQ<-K*d)(&V^U(W_4a0{K_j9N4t?^j8Zth#=YvxzH zJ*)&v8m@C+yR3tP^v<v+ZU#;Y&?U3WmM{xW_jVnbIuKqz5Mx+63ze6fVW5%=Da`9l z&zR^yaH=D9R67Z-U1g}Mkq=2&HwDM9Uk0&ZJ!l_!8#;W~aY%F2KKiL<JZJ5~4^<ZF z(8`8XOq>@*yNqZA1BI;=*4Cizd6w7v5sP!X@#e%>IHl`A*Ctz#EkPY{PO2r*-|2zv zJN6{U*2OTpAC_^epN}I}&R&=o-3`@O^d^&sD|7ckG>QGy1ll)82`sl~WBK)DB5Rh; z+<vx+2(PS%iJEp?=Qs0+-o0s@P#6Xo=L2x?{T}4UJ`1d?x{s$$$>WO}JzA?}Pqh|V z(z55PxonU5q|8H;K8?0O;n4^@>UEQ;IDQZuRI^Ct#5xG?+lb?D48@^gPqF*kT$uY= znW_|}!Zt-Qx_oe^vpW_a`8e2`^qjW~;f5-SOxXabGkoxz!dY0*6e$=tZ9FMCnm||P zcA%RahH{OEmr&Dj71$xgl$LDF;B+4xgq1h<<Fj2$2)a6uv{%bW2XQe@lq2+$LNzSh z7l5ZOMUojiHGtGv<Mi1bz-R9-?03FFU_CFEbnTW(5074fW1gzhg>psA=8uXruXHzB zEquy-JZ%j-E8^j`Rt*G1KS!&rFSwvyvc$H>U1n<LbTY1B67JJmN>7b6fiAQQoKG5o zi>oe?hOi@;XIO$qi;YR4V<w8^%W!<RO`Ph@avWBFyux}yH5^_therQ8R57T!67AGx z(?(}oyj77)YP7#I-+Y!~{+xaA)7b(S?9qca%O1mk6~#Epg`Yp952i_#d2lRwGfAka z#ghsHNpL|1ynP=`qwg4#gq-K%<TH6tch8Q>TuZ}?H9v4~nKl?!&n8D6{XnzMU%`0) zQ~0`jBzNraZS1~Jff2eXQcFWc8uHDAKW7ce&NTjhb+ihqcb{NB&3=d8vGHhZ8OuNW z$z?*#I+Ay%`1gZ0W#Q>X?j-4623=iw81)OxX=iLi|G;eed6@$@#p5y8?%Nh15Kg}e z{Fy_&F5}(N3Ap3J1nPHZEmbP(g9}bqAiqcn&*z-Q)V?NEA@2;-eoGavdOnA0%(B49 z-IGYn_JyQA?=Fsu3cz;@tf)-;Lg-(fL{G*1f_otYDD!j+H1TU^y}!TU<m|^_0)x2B zQI=Y!%A(@kg)r^iAbQdwf(DJ|!0)382L0GT`dxg%rEQMj_f97f8~*v6TDcl2$-BXI z9=#8qYF;IallVO_!)u|-+{M^1ER`lVY=_gH&NS=o7s$fZV)H5^ylv_M=Yw?7Cj2)4 zY*>XEcuIlZvd-krjV!=%XhK5Ol_66wlnmVY07lf;6PfZty6f;ye7o75bcmM)gNP2K zDsv^LZTp5%`Mek=-&+qpMxL~yXPKa$%46J41^o7UC5|2wPC1WI?rFXb=4p20H0H{4 z?vu7LzaE&9oK?3`x!Q=-B#)r=s=7GR$pQH7{@5vhI-T}G53gR<BdZHt=$ejm#l~}Q zK<9T6xMsHtDm4`1jJz^bF4SZu@b6ox*qp(|6N5>X%M^Sx;4>(`A4#m0Rgm_353coz z_%-te7Qrw&DtHbPYV#8Nv|mI`FRRfxp^aFICqez(VGJE7kFL)SFkH$BY>HXQY)WYa z*Sl6Ac<l~P-tEBT)B&95U>!6|%)tvfKhVGD3S50b%$Zpv!}M|O$=RGP<kk1%T*yp@ zd^?p6fw8@4i0fm7rI+AG!&kvVbADf*wj1^CSO|&h+(@3@AWHc)j98<++>Wu!=(ntS z)Wl#5b6T8Exjnm?J=3eWDz6Ag=lAc=92ZZL9%&Nof=n2&*M_m%_yv4MPp0M0$I)WG z4s9NDmFW`TMEW{6qql53+Hf&}9C7^xQDYS7)qdl+`P-LsPt{Jrg*|G-@lYOUjz7k{ zu}mZ%hn6$@T~6YC{KVXK98CIc@+4KUIaFc1H+g9v3sFZ_K}f|^&aA|QdQEiX*0eXL zL-_UNgNEC|x}liqlOlpH){~&dVJ{|XeSj++vzU)r=LHG2LXdGk2tzhdGDL9&x@R+} zTQZWwsy||MZ*`{whwLZgU#!Hzp<S^?VG4Ktjt-u;alob~#HR)Qu)1SMGPLhW*Hb-L z(abv?$oO(YB6^vFPeU()^RyzaMt(QiesY5AMtiWmnKho85P;_w1;RRyLN55LDe3CG zlZel{!oh?o(DUmltZHXX->`b1WLP3T-JnKi#6N~LwfsAM>${21Z#;&%i(i1x+gNHm z__07T{@Qutbj|<!`#*m>{@`B#`p?E6apSr&&7XUMSD+q^OtS{Z!J6dmgEVf+3nBLy z<%xFm23)p$IcVLqp<_F)C6i=M;F!Z}nG<u}7`X@IK-jxR++cSDyA2G-$?S8t#O0^C zD+ey3^35-JV}S*AFBO1F2X%VK!ik3Y&7n&;XBafvm=3@)XkOfpJ{gaAet{OE{6R)^ zcN3KUB-p+ERuJ*;&w8FbgO(o+#2=3bgVKmd6i*%iuD|ZU*jx+p;avb`ZA#-_y^O^p z!r64~k%3TmyenND@|BU(TuO#5QsAElbfkUvbRmOfuF!yIkFZydUJR2tguczOrVjOM z$bj{)Ak)kQg*ceZ&5b2HWVOhPu0VC&ZZYjjb!l3k4AP$~<SvI=GPgzA<W%BPLG!m$ z+{ssd^nyz&oHs|*`x*@nw^o5DZ8!0*m8I27X5kIRqXN76n=mc16Is$viN2O|CeL=J zfac7hbnLlq#G&*exo#bb8|RpkN$rNx7Z;U@{(Ko4=XulZ$b?VeFj|q+D+a?M<_xht zl0o9WY(b;a_soH}`SeN`6ROr(jdbsE62N&9QyuaKCdjWRhbM)AjcP4C*&2=WA9>Nl zb9ty%=8dnfBoOUgnpDlV0kmHW7#n*h(kWDfW?H(@$GmMG3$8GS*>14;^;u5!Lr;*u z<c05t?Z7MidaC{4+uRqcW6){HEBJ2n2#zFH!Q+XI%;zLqD!P#cXUpFT7L9xkUv-vq z<DVCxIr9<{rs<NBMS1Z3p)+yXRZ4!0Uxv3n--N2VSIA@+GMz~gzHAJ1GY?w~?L|*v zo`n*8tFaP9h20idZS6^S4tgPW>HQWz#_Yq#o&(@eWfpUCsen2qs*oJ>caU}7i2Czu zYv*nq0L5NAV2OqW?(OFbk9t?5akoUeOZW>!`C8OW^*R&xT8)0R(ggR~siZEMLin{l z*ze^Rc>8k){u-r&SLapYLY)daGX5ycuB_vdcHRKn(T7my!5(qaTNCP(%8#pUI?^k; z<A`SGc@V5H2i%Klz-5~+-PFB6Jb$hu-0^8(`WjR~T&+D${x}|2_g{%JP4npC0n=d4 zlsD))(FJN5Pipn%EOT%|S2Qh7!h^-xboHiB*zN8{ZndEU4FUtQb&Zu^S+)x4w&5Cz za~1d(-STi$cSok-oeDZI?ZENia+t776(i?Uid*xMe?1k2^@jy>20a4P3p3!lbuozA zccS6<+L88pXUS;oVW@s9j&s<YAr^e8X8M>|;QWO>>DBzs(7)b-8a2)Ux;OxGY~*mR z=~UD*dxy_v?Gq1?`3_JZOaCA8-aIO&`1|{&OqDcfA|Wc3WGGbE*;`7XQc{MHDI!G3 zOqvv$R7%pIB1BOrb)CJDu`-5C8Om6OOqu0A-``))y`KBI*Lv>XKhIjvwbr%z@48Os zI_G@$XYcp>_0DW4g&tuZptE!>9nsE~X<X~gA)31cZ$kmwTxL+VmJ>D^Dq_2p&jk0m zzv-Go8cn`%1x{62vBTY`bnnPf@mNhD>(4kK`?{(>Sb3U*Vn70>_n*Ut-3DO7piI2) zuOe8tO%x+ghP4R|)L~@^^**A7T9dpvzn?ls`7Wo@;6lkeAC3m;E3nn|ES9|=A*hc$ zE2ekIqd%AZgyK<KIr*<P_xm;;#)=Pw?_Pxvc<umLg{O&&j`R_4nZ5<z$K~+vkP1!^ zuTX`v6~%vPhZe5qDE#6vSTdy>UOp&ttLImtblNiEWbPj_-rR@3E^~y2)YDLOcpEhi z-6tN^wBe(kk3}#0qxjLmQtE|VfUzNy=(g@1y0OhtbV!mu=d3sPPZ=SOQQjvHhGl4S zs1w_0?vV{C+Q#kPs^b?;eH>z%OTpPDTySq1)jc{3sqq8YYsD=JcGTovvnrujHI(H` z6}h~NGj{h6#y#dwXnmLl@#ikEY_|(&{ZQqK^hXfecpv8IO{Q(p?J=s~KB~NJCiLyA z2;0w_Vt2J1SRHVlvNm3XLH;MGj8nn5%VBY$&m}RoUQMb64$y_eTcOMSN#ea3L3q&l zny{%|Liw-h;qdm@OE@LfzB>*KqUAdu2`y1-q_E&qMYOR4D^2r2kJVG*YpW4<%pC;d z*0kqa1HC!dZ8{zQ<%}xb!#GRNO+Lf+9;HrNL91U)rOl_)WV>Ez@bHyMyus24yWW3J z>)ISCrp16eM*wG(T%uk5ZVHYao2f}9fugIsb5i+Ha#EVh9_#0D$hNZ(u<SYQdo>0> zd(R{99%EQ%?@O^{t`Vn>pCdfcdkV34#$)cV37{@LKZ<hKVwZP^;Am(D1lt<(*Yb24 zfI4{ir4O#w(!oNDuQ2CFI6NHrfed?{7rkp8z%TQ(ARlAIQ>v5U;|>E@u}#3(D_ZFE z!UE^I4*~1*TKFm8zF_Qo6AE;#sh0ckfI-8lDB~i2OdrSJKklOt<q<HUi!uge{DOdu zS)ez-wPM$+fADCb3n~Ohv0~<PDA)N4Hm>8bDPTKyRMLS%Kfm$;KRasIWd`n^(T%$- zmG<ZdpOcG<5zJpw1jSdYh18F3SXbZxCD|EbW4AG~%h#i1O^$MUZ*v0IyfkFBdO4J@ ziiF=c=Ls55EwOe<i4c4ykY~M8hlZ<Txa?^;)CbjzHUF$_jYb+{7Hy{k4{F3z6<fTp zL=XQ{f5_wB4EaB-KO_U-f3N=d@Akj`bNnOx;#S*BAM~I1|FeGRe_20C2EP9<9{+#+ z8q$CBe|`Ngk6$w2{r8Ule|Nsr|L^@S$?(?of9iKhuEBr*`3)W(x;Rp@kGA{I`+xVD z{}aLIzvwwbB@@~IIlssErUTXL-w=h>>g>=Q!AnAJQQm!Zj99GzQ~n)b?S1l!p}lSC zmEvo8#I>%xIDI&ezT2MN$~tlBKvgcd8Ajh69>YSPv!rHtfR-E&#K;Z_l8-A8yi0z{ zcIr>%7{4^?yC$4Jwg7jF8p`R#9kI*#Qt?Q<BQG%@%kjk_Tw?u1R{Kz!<x9$O)FC6` zPUo+}{M3C^bV0y?ns&U^eiN-r9gR;5)<Z(~JW?<+;BEO6V1rXUWZ8AAcpP+uT>XcM ziqTehsjMAuN;wJYzn?&aV=n%xNaaF@*HHJ~5@)r=(wc>F)L?O+E?WH)z5Y528@f)U zh+8K)z#<2X-qi~~G_FWB+b6j2cP;pQ)MHE45%_w?NlO2INsLQU#BL}0VA6;Cux6tp z_iL6s>I$ZyUb7lcZRig2^yTo;HjRc4K~7PA1O`DKGV62K=~c~dTCQ{gj$TXW_SuJN zkk2sT&^BEx+31EYPsZ`}Yo$W9yFPU}X(Ovy{h99N+6#LoT&8y?tti~r197?wq|CJz zXWOO-$7c3Hn}wYj(vH(M{Y3C;j-snwm$HuaesTOGOIoIBh%Wt->E_r-IHPF^eHttI zWW5^8q9lKq`8@pEuMdCE{wXTeJHYRz0x&c9M4vL9IeddIRyJGEpPmQcMOzj=cHUxl z-Q@|Le>(~P9k?w2-C>+SCUI~%dMMRI+=jUGS7=F^5#=3T29X!EVavo2nEv*MFst2O zT3u(wN51TYUPDZ&hhW8<H(wBg=bjOl7%m{I(-JqxWeO}#cqBd#w8M%u${1T0&qk-D zI=D0e>QsBub&~{gUE~haLDI-Pl*=C#?jxsrV_=Xp%Zd52jtkDe6P8TZCQ7WP<<U*_ zE-MfV{_PV#jIXEH-q+w?nH#TMTTk~#Z{Uy$W4?CMAD&)q6+85JB~1IJhgV&S#0^b# zFg3dt*174@k3%o*3_>ete?S$5TzBNlv>)Cq&!e-CEd=#$MWV85E>8VkC_1Y4=5^Or zpe#v|tuj<0HD$PbU+-L+u_gf~Emk7s_Br&u$bx40#ejXzYvJ7Guh1}01$@e!aK-jx z)WzN!BfD#}lX^7Y*<44vdUeFO6L|!eZ%Xs}Q!>4z&b<4@7*c+*flf$zr5VlP)PCy{ z?A?|k+;F`s(|CD+VwzXemC1@w^-!7P;vB&0Z65f%N}>D%IT-KR5qsB`Qsm|+P6~F% z4A~IKIHG`7lIF2;b|cs*rE<>oM{MG|ft{xpfm^3I9QQ#317-z?lOjumH44LV<&ZpK zxoj-v+f~TaeM4w`@-kY~;R7x05X`m{r^#0ScnNQAlkjR)5nTQ7R@OMQ8x9XLz%fN3 zc+Kz$&9fTERc+GR*r>pRHCK^#KnQ=RABB}AJ!xQ-1{gi}<?g16vejRZ#L9bc&m)<x z2}kMlHB(mqdV$os9H$O-v#D*vFOXW*!m1(jDRuWjs!7cdTgQm>E{%jUZn13hZUELi zXriyZgZSRT!<4%>T=HC1larGHB`0i#q%+w(w*G{$Tk0KGmh_{5Ab)PyTY?W>M~j1v zFOk{uLP+%0rYGxrvd8yFwA&;L%*EaqG|5^}nwLQ@E*Fw{TDstRP!~-ijzYHZ1)grR z<5NrR`M2|I>T@xL42GLw*tSfv`4%HoB+dby_Xi-Zn<`kzG-30!kE9%#3UhwP;f$Nk zTxdTW({$I8=M^uU)p%acr!4u(uV$!h3g<)dhh=9kl+h}y{bIqO&$K#W51HpHlj}tf zQ4Pl1r5<;Jq;8Mp?gvgmw&W=rEqNHb?HUN+qKV3{-C4h+7`8rJAeejS3gb-*;Cap! zIGM3aw&HaaEZn5e9nZRn-`{lOY8PD`;!`eu2OUiLYl)egoY7|6JGrUHIsDxxk%DYX zA<HyTY_dN>OTNAYRoC|<OH1bgwFKvf=i%X-4luvxSf1dm4l2p*v7?~K+M`FK&xDaQ z*FPWLNOR-+Yd!H;*lfBo-<-oI28wU}%ILGz5RPA1fwxK)LdflQuuS{{Z?`9rcA7Eu zk*jd-y;&%qyONg%dC<6iYiLTS121n1rapfd(_f@er%8(i&HNr>$e{7GWPEeQox0m_ zbF?4()YQSOodKM6sW)V8yhPrIcF9+BH+nMAL-1;PMjy5u0W==QaV5GiODkNwShbwg zVxI~x+><z>|6%&r91gRj{?+%LQ>f*aB5Qi}V7p#D(9NuouDH2y%Z;u=!p2U#{#67Q z@>DwGc#*F6x(lPTRnch26X8y75B%zVolX=LiPPJ9lg{HP-j)&}uI+IbF3w+$_iwdF zgD%>_?~NnabVM8dcyW_PclD;=P;G2JtjtAy%sJw+Id*Yy1E-ivkomVWCUrl@T9?95 z-{XZ~X|G7V(k9|QJA1x7`U{=hXoCjTanK`B<mvk-P*|VQU=*Q>?n8eFok}Lbs>hDJ z^P4K?BuyZav|PFa{9wA~Qp|a%!8iII7ksqxWk-(6&}?4_m%eu*|FNZ{=bs1%T5_@F zaWB>>JqS1ND6-8iJ!;&&g$|Yk!N%poaKJ!wwi)t>+72d=d5@QHJ0f1VkP${HKYLM9 zG{HQ#uCfm26QuK(OYc3qi1pu71?_7=;Qz86&W=1zxg)a0x(*4Dty9FRUu`9>O?%eA zZ4S08qDVD2R-9X!2P!}B(ZixJp(>+2e32+Y!DIdk;ilv8-O3lDn~D<kTOSJPcMNc+ zuY(Y1sDORd1nfFMVtU0(zSI*+F!anYmbIi~PE-*FPm71E<M+{?5&y(kon-KCUdxN? z8sX>JLAclWFeY5C1&1wil0R+_+t=^LS4k7#(;`>6>arhBom@wwb1%RzNe@^lX&}a~ zng^e&+u&bz0gaH@Q8xxHqJQ@;(%4Rla^<}%NOM##Q7LC0r}!E1{>BdUrs@Z|Y<9vJ z_aH8dTFl?-m3Y7HE6OanCHx)Ho^LE@rf8RY!q>`<{Gs?hZF|xrzMAZgs=EG=*{hh0 zRAR;bzgLjwsxV$TZZN$muAx1P4#H$*fy=zjxn|{4YCAWN4}`{I#r=-lWz<1d@wCBt z_oRK3!4!VD_BWf(7|wH2^d--J1qBXXgYE3+aGQ^T&}==6XHVKC1e%T{N4rI|K(PZA z6@-IQn4~{n@Pf2#?tpc#z8GJs!ir{5lBUs&i>H>#s`u`r!G|nijIj=TC-meK?;U8V z{&lKJ*$T%dZpY|Op_pT~l-9QN6!U$4(Twiu_^EUjo(_`cHk)#wI_ZGu8m+?Z-p}HZ zjkV-w7sUp(#^lvSVg^}y(CJslaoJQy*qqP-;y2ZUUwX1|bDb7fwfim1QSJsEIuY&L z+l4=y)zQ-{%h>(<2E4WX1$FI+#I;PW*}vrn%bM*n9jwIZ85J<&)d=nz)e*a`xgi?t z2}GNvLj~W!(X!Hd3t>+GHCPz5QjGX{hpu{`6c40?Qi#HCTD716f(GhgVZ8-~Ey&{8 zD@tHV!UAx6?~lqBPv!G>6~KuJt&qJ~h7CWiioHf@pw5JO6#X(17wP(N;Du@`{_QDT zOut@nZB-0(SU#Fxgych=)-9;lO2q+rN2K*^8f8}hfFmQPaG~)@TDwh@rP$mOXa2gu zTg=;|_$L%Md`c@18FWB!{kW9`Lmdq7(I2vpI^mXpUGyh86n;40qfy7L#63QR6(%vJ zG-m2=IM(VUZkV=^|E{QlGw(iw#_nAZmUdS7^EICqcK8hEi^rkmk`~$Al;ae*Xb5Tg z9f9?G%te<yQ!sE~U+!A1g;`mhDBNoXR$a`dr{!L<Q0GJ7y)_0;o{qxEfE%Q!uvgNY zid5|RtwPAZ1Bu^sc<~=4tW(<z_xcT@wH-Y$X+bRA-trmNQVvh*@5TEEzM{XqhjV`1 z6R>tKgG)2MiC!n-`RV+7keN20sy=tehyAAG>Hcr%@>qdSU6GS%iYnUH{UZH?Vc<=h z`0wJi^kY#pf4mVTzG&!;T^5~`9sar>&W$<<H^Lglvrp7mrP_c#R(9a-onmOMwinZJ zf3f88RWeJAhmn;HVy3Su*T$=3%wjECKG*^OCK<!-40})<rpe)FCkun#`OsU*b5yf* z1pbujs;OChI6YYd)f*JJgZ%;FjnhUjlzjJ@N*cJzuL3l8#e#P7d#Qf2<Shl7xMoDA zkTCrgoQ<mh4XJPac&Y}<23!$+WAb2diz>YSVvL91%}1j>XDPIk1{}J&RrCp2LHkc! z60iJtK-+bd`P8R9;*jB);2C9yN1X$OMn^*)`f)#P`I3qX;r%G^tq-hF>wuGGx;*<; znlO3*%Pcb|^H;Q{qq*(i{I5A=wed1qd|ZehBh{I<dqJ^wzNopQj3ja|hJBC=t39{N zR1W#tnmA4X(f)(zZ~g~tkA9+{Pl`0zrd0?!C#@4!t4Z(4Jn+0J?J0WRfe!Bmaa7J* zI5X@YEY(^9d3#Pm*N&kW9r%t^UDolH2R~qHSQmT|{1Z$QW{5N2tj0%2|AGFeR(LRX zs<iJZBG36n<dNPUUz~UgcNLZjeVhw9_<A)NN_q>wr=2k~K){L#&gAUkPKuiSc`3NV z!@eshuk8>`yxE`cd+j8v$!Ep>nbSEqXfSS{Wy~x455md$t+F0F7I9#wF+64G5MFv} z9FB7w%1@J%;e^>m!8l?Kw5j)o#SgB~rj4aAa=|0em2?KbkGk-w?Q<}({VmkBDu8{1 z--C0`bjoj84(s+uiA5XD(8Z@gzTluVudA`fr*V!rA?pb^7wr=}5BLdRR`;U|%U;o% zzg2X8lMNh;FO~Te*HU`bW^gHsL(7bD9DUo0YQmmF<@x1o`*105nN%mPkb1uJaSwhx z-U|(n)KK2CUI_Q*Lr7Ex-SqD~>PWMPu%!Do3ND+4C%*e~gF~vY=~$c)8`liySLMi) zY*#>WZHoB#hdO6pe1?6Xk&X{b6I{~zvV#9%T>PMhl)R*+|DEKMjA^4&LOi(4I81}5 z#lj(_0eJE5c#OoYJgc{x_<K(toFDL7aMvyurUkkR5t&;2>FrWsMyJ=XNB<nSl~}QJ z=uq1CGKmHlN7D%_Gl}(*$VI`H7_oOW;#wEJo-eW6Vr0-|e+rD4H55+8Y(+=&9WbYT z&(njZ&Vg||enYJ^=Ub6eDy+IV9vam3v9^B%@BMfOwrURN)Pg}6^ve<+?$3lx$2EnA zrB=Lc?-%NNI0S4xt+1x34c25`1=Gq#h`92Z^kX!*)v>2w{b2&;RvwUz2<$;;kL`r} zCTnC1I$sk%oN$$;bc_*A?IwwFk9^Qb>njCnMA*&0Y9Q9A?G%6Hlu+p0XyHXKFaEi% z8!VDMhf54pcuwze{3%q0=F9&GSxO~vW2q<Bsy%}D((|q~cRLoDKZKSGDzdE39dTEE z8%#X$kDi7G(TaFoj<H{aJ+i9FZpjS6Oq#1#x@*Z2YrSyPkB{;|&lm`gSj9b><51bY z4DQW42W_o~fESz-?bi;%8D-z8*Ct2S>LzJD`t5|n7h^F0Xe%fsO{9{u=V|WqKp6D= z4;`2tK^NC-f~0DR_v^4nXn7zfm*y6TQfPsUDTG~4W{WNRc0ixui@AH@LA-Rnq9S8z zH#T+NBKOuiFGMJ%i@!P><8W!-?C*L5_Bt)1Jl78}dtnf!ZQ28C=S-wIzuU3a(XODe z<^#=sZ76mRUkm3GE2(0L4r=sP#}(`S;bHGHLgWgNMR;1Gfwl@fIctg=tGDt8o5+ek zvU9M%emC)&d89Kak$Yh->DL;<>vJ=>#odJ+Jks#rBwdyq|1d>-12==Bq491HVeIQ@ z`GNEjX$?yjADM3Ajuj7K--WXk<A24#mIdlKD|-uB&KQpAdzG-@tRc6XZ-5FtHpAUp zhr|VG!$DWQNo=r=$BvI1AfS08UyX8sQ@f|3sJab?e>wv_#<YRaxn^PdFe6s+8-&Z; zon?s+Zqvqhoq5iPxxC@g3mRJ4i8k#LIp9hdJG@ZBlq2aB6xs+m<L-+#`)ko_dNKbz zrbIoG0=cC)4xfzyyz{P+cEt8Z*TX|OW>tdd+BY6_P1Lz;i~$y8tiUHp^Wm!hDYDX% z^dgo{bSac!yhQ<>+wLrU>-!qw>OaGja5t`tOQdC~+FUX%1qy#Au;S)CJi77?>DW%? zO67KvB*6&REx$#sGn9GpCtd8{HwkAw8pDTv`eJjyGtzF1<j!9m#D#4qVNKaN`dn(y zd%As=DJ|JeT@x+JTPt=7TV6{(qOpS^v394t?Q}lh9vsaPLD$7^*=}GKmnJGL3}kDI zVm4Xc24{@=lh4B#NEvQJTMnHE@0Z8Kt_n#oxq1&opEW1@$ScCb@-e*k;5M*atwxhv zeuJ&)NccvRk$!!E5hd$r%c&gxQij+?O#Ba@1BVSM<gmqBFn|0An)kq$pB6-ueY!rF zRhsYvqpiI7ehwR(+@@aNCu5|Y0=iDgCC!bW;LM?HepUWJIB{|aKQVLS$!VvAhw9F- z?7|69cr36>=3>6AZ7I6F9tkhD-Vw$ObmS-J9*TK?JD`euE13)$N#~m12o?!{Kr29> zpN~HVGrniCY0G0eoK+x>`C*G;r^+FFtR9b;t%;tJhwDMhaX!!^3GUYP=K;UW`BTSm zxHfAX4_SW>#&!*0@u3}STpJ)bEiRF`xEJX7rUo9TbdOgg{-HC=FG{TA$uQ~T5$dns zB;Lw9OFOGVafOY<R^Y+d&m|E9EWG*aL*l^HUU<9I2|~t2L&4%YF>8jQO~pn-n)`Gf z`^}2v7x9PX4(o>sH`Xk`#`r(tMm=Nv)SwN$o-agCWeYa+S`W!K_i1WBQ!w1qlOIX* zao10Ba7&mCv&uZ7G@i=WeR1c3Ion|Wr9QNF+9vLH$&BLTlCizoaxBkR#M+*7#LQJP zUV3$z;D6>iJ@{ioMUNlB$|567Y_SKeRVjjfMgX|3TLwE;l+pB_b>ui+1wY@C>Y@v4 zY4nV%uv$fiEO`br8(&eU){$`Zo)ceAzD5HyM#1mMB8c*TBTm{|LI$=qa8F34ob(NR zDRUa<chsPUGpf91QzrLT3V`E=a%w(5ifcdIfQcHLu_m$$9xqIztUE#Y_tQd}v^GO{ zqoE}>D88X@3yZ~^&%4AO$<g8x?bBeT)<HO<(_VHX&OoRd+eVLr^;ob@f&5|9#9$wD zny;(QD|c&P#`G<yHPVq)lgB~r&R+O$dWsNYYXWT}nq^y+oyqUH5@$WV!`D8T!{YDu z9JVG=`1^h&FOgUU9bO(GMFS-q8eSpjtjR`8Eo1!MSsQQt(Z#DaMe=q_?f8?H4B~bq z)2hwD{}Ou$jXMj)q5bp3wHBQP$C1xK+rk`94_qV$JGIj0qV*8!whD5s<H0YqJ@<a! zon9ZW5mUw<mLC!<dGm;K`0=p|wqDZapM!dk1Zl?9%_mO#1m}^l<ne!7s=*eEyJJf5 zWBGZ9P!5k&h5<$EaPNY(?3zCqu{B%fysWdZwo?)Ps9r1GbBbIuw-U-TU%-UV+0<LM zf$GfS_*+^d%yzF4Hf>K8Lf@Pv>%u``s69sf`fw7xtLVv})#Lf~0cBnilTW36oyAw? zUTkap6>b&u!}-&?bIl_;M6W#~ZancE43pgOb(1>3=&i*aJZ`~je`VM<p*@`5R)k+C zRMD9PLug;MpT2j>rIkPD2}wyEuxOg3S>0-ZVF_BOl6p+sx89iK{q6YQ4Hb4yRKsKO z+F0?!U;6uta@wV_?r?MbKI{`jCFrxg|3bmIn-VJ2jex~h`s0lvJ-**gk;@j35{kcc z5qsvl(6hO9;B;mYKb=-gms*BXskj);J}<xtWDiyH3b<Hske*U8@4MIxe_F~QTQyPa z?z9YNbadonZm&SJD1?0$2l)J(%|g_E4{Um;LMf+)(B7U~aqf>aadpr=VfnO7*|l93 ze6Kl@^5=Ergn_xxdiXbubvNakM@Gon|9L6g+nI=ud*|SQ=%GTtywg<i_ZkJ2MGNX5 z3%GP}7++Po1s;<FadBQhTs$F`XU(b>Mn?>Q`+d~0;lW@O18gwxL#r@jur_+W*TNf} zCyHk_Oa4lCRsJ`DAX9ix)mLu9<#K)4AlJdpp8EJ|WG)rg>tXGdWAb}8FU1<x#+vu{ z<xUbi?%K(Zke4=^w1XXZ=$clcI7=7Xt!^O4zEv<lp#ye%nZjd^7ecXV7cA2~LCt!L z(0cYmyQ>*RFlLeh_;5JB-qV5aja$p!8_x)@YVO);Nc@zqT@S&0iQkg1VN89yw?m(@ z8T{_RcH!K!&Yb_YS-4o@#z$XIXPYZ-aPQv@=%xD^TBk|t^5Zamsw~3kE0VY8-*)U% ze~hlU+VS+gX}G{d(inO^5T;)EEiCbnj`L})#FQF{MXPUvkGZjMw*DG@TCa~P1#9Hl zAx8Z7c|X?Hv1IMNw_yW)0geh63w}!d@5FH4TwOpDLXOI<Cv}5vC&!_^=_w91*T)vU z_Ym>wCcF;1Pe-1;6E}R*<Kq*;(8K<+to+?@em~KZo7MZ!moR5uw=0K?LkDuF5a8$4 z^YH3STb#~?u)i^ef^LUHz-m7n>oEk9rq5t&6_FH*&(V(+1l9YC=<>*)H1A>*3@j08 z;hjmaP_rF$Jogq3?5}}_h7{pvuP5U96jR!)qmFt7>GUaeCp@W26>e80LhOPMu*$(5 z-OfyBNt1#Vx3tOYMyimSuLF(eY;bKrC`?m6LPdqj7}Reb1TEf1+PnrAPCiI8^H<Q1 zg-W0(cjF1?=JKthkz(AhBz}8+7=QH4A;00zXh>KkjBwPz^{GMfs@s9k^w0<{BxFIx zqTV?Dh9O2z%C?*N_^7xe$&{C?#8F*cq%iVkwBV&GaO?Gsm_Fo{<ZIKxySE3j!+C8y z^1z2oZa9nM{O6*#e<a8|3YafNIQS>oQx}&+iv95qvUau#In#z>Qqn8fm{eEsy`2lb ztoIitbUX;dHkyjs!IQ-5mX|QLM~iS;sS=j_G)G7phIgfY(2ajZG;)O%u8;J>=GB8( z;Y<ts?R`a5UpxWJ@>jt7{(EE=7TH|hZ--p((i@>gQc!JgapbIbbHGkw6;3Z3UOvu3 zhAaN6b5ms<ID~$pkdBvx1dm+0_+ksc_m9AVvy{<#$p;EEi9$u2Fv0Zt3{Y#@Oiy~K zW2ACdejm0N^G44Cqi!e2;%yknUw#!n%oqnj2EL4DKj`q7-a_Hj1bCt^2NjjYppm?f zXL>ag8|2yDY8=RcQ*E(U{X8{(sgqsrVMe><uf-~rPvUj`2v&=T1CQ!L8r5?Y-<0%q zmH3(#%{)ORw=FQZy&*2i=*4fl7~%ZK0eE7Zlqv8h0DCWA#N+K=ijQ}NaOC)Xg6aEi zTv08tRnE*8zNdG_tW$+B#B~JKYO3HAm33ml!&TD#IgpwrsR$SSB@K&HDa0N;E>>KW zm?}YXp+o9;^1XJ7z77uHA6<4}?8tK3UJ!;e-etn0x_q+kU`C_vo~CtIXTeRc8uHF? z5{-2H@z%U99RBzq1Wm3Lm1VZ9^4knQ&u@>9EDPXlWg$&g@5Hfs3+d+Nw{*Lqk<=vp z+a77H>bBHZ_M@y4^+qk=6`{7|@i!C;Y_HJX4xMrE7-O__?Zx?=E>zhk3VFV9)b>gP zC(Z0c=`lLo;lKp=yYo4PV^=o(@LG25+f6W^<G@K!M9p_w==rkW6oYBpwexc5CEE)! z_fO)5s-?VV@h*HL&8^=okZOx&Rlc%dI(*wMF>@?y?SiXM2zNcU!or#wN|=*d(b~|S zFJx-7hFKD8NjvcT+0SL2JiF4yi@W*5GD(kKpo(9NKJlF6$l6LK9PzC$Yt~+;O$m0m zP5Tm+{E$;q_c(NRjpy+@GQd7V%1O}l!T}okl-t7{)t>pW<C~k%_xu<<e(g5RHvK1t zKUT+aVmMwi{Q{vQ!YRV?3=NsS8$FvxvvOn?9^3t*=n$dDd!|*>+m~ISMqVIxe;&$5 z;=({h@dDhHy%mlphJk1%)#vB#z^iYk=*Y2A!hf^}u8tkfwCgJg52W}1PJ8gb@b5?f zng5=CLi*PKU%&oy{-<qCRsFB*C@DYc|K<6Y40Qj!^Z%vQ5^T6ZSY^-v!ts34sT{}K zGl>feeh4R|x#!`hX=3o%+mLmj9EyjQQ_ASKvS*>|1@HTbq;o47#$-gn`7@i)a&Q9W zOxD3ES2DoLUb+Um?SL@#N;s4k1O4xPfisHz;NlA<xS8gQn(fYr5WiA19J@{av8pSL zz2Bbo{D$G$iz&iXgJdyrbw8Y7!jv^|8uxA8$NW+OwHzI}RhqfqXefru`-brdseiZk zMY$L^!HCCf&EeLzfvjfK4NuHXKm)hVV0m!?(zLJQCXb0=UwKgKv%V7-1{-17D`PtS zWe~QxZ|6?W%fVfxl}27YM^%4|sntDGej>FiPwYPf{O>69uBG?On_6V7QPT}NRi(qC zLvH*?V+Eek)y0K115q~nI`ukf03Iz8du&-WPYzf{5x+xm>9_GXcl`r$l9kf=hRZnJ zHH^EyONZ}ElOW6{5T|snri{2gygsWBP3a#6R|^jaIajAaqE#|2Ptm2}msBx$>vZbv zvYV<`nsI}=q|bKBrp2?g@W=ck*l0ct{%)`reMbgE|IQb%dGkwYPhCnqRp#Qj45?m) zd9Z0~hf#yfBe7R#xGdT<R{Wu-A=-~zFAh-h1g$$-oMH4y$lp3wNVN2albaukO*i#$ zPfarQ@CqbPO?9&E)(a<Iy&%+1EhB5+$>Q+3c{Eqzi-nB(P7Ma3RQXty5~nSN$0mz} z9N)#L7So9hK90p(Xv1Y472x(e1@3ilE$-|ql6QrGo9ZpNUqc#356hz7A=9|i*!g(= z+(Yu3YDfW6kEWeYU%GgzLAYinac_onMVsXoIMp<dTqhOc`Xe?RJ!%_GxnRyM=XYXa z-;UVSF^M<6>ch`$BlyX*g>ZDb6MaklL1#7MDezbjbwPK&p|%t6Jo-*=S8W%J*WZAy zpC40nhaAvq7seeM7hrUgC7aB?L9Uw;S%1z~dfB*_G8(>7-0qW<Sl$TT6Z&wo-9BFZ zT+UVcLs0dnI-Q&p242gj({l?~j5werx{l3*@^(x4NRtDI4-+u@`B_*ud>4Ovk`42k zE7|XC0=w330NucS6zI{Nn`dXj*=ia0H2g|Vn-x(SV(`X>?Wh+YMdP+M(#D2UVzWm$ zD`#JXdPOE>Pg@?A)ea*ynq@AXXW`9DDl~cF7_j*KM%XyD0=By;^T<o%aM=7}nldm8 zE}ra1^Y<Qvw1zieqdZ#naIXU9256z5fE4FEmF8G2;2o-?!P~-s65B=DU3UKtzn^`A zzLF-keYP(@*cnDMvinKif@CpS%>pkNy9&oA`XXKMWbd1Az}NQ=J$@+hiRbkdRg`?m z{jI={Bki%h*(iE%m_^t2eF6JPyTsKFE|k39OdOJBMqUjL!m@k0bZX)K@=FKiqWIbp z4*l(kc2~n;K*Ju$TT=@S!%mRGiXp)1y~$=&1}z-6mUjQPkeFw0VM$XzyxM&j{Bd{2 zu;5YP6DQ&)ofL>p)y7xn({NzP0FJ(K2ga`r!Q%2_a{H)B1|3}aY{GRIawU~r`vS`2 zW1&8-3&%R|rs;p&sc%LIczsHTi`gz15qyPiS|yREqB*|uTO+r1D}$Z;v#HOzj^yGQ z0c(7e(f_o>CWseU9;8W4%lC`@p1-B~$YtXEg2}Kf0Ld=Zj+z%J(Sj#!VuaNyFiWn1 zg~9%KbA%F}-28~9XetUzOm2$P!sg+h!#il8S_HJSas*-Gb8%tSA}~DJ4X0i6<H^>_ zOrGle)_y<LnfAbLVY^YGPY9(&5I?myr>Dy-dEbhKu-jn>s(E$7?(2O?rKm;R@X8(P z(-+`wp%=Soei27(UMc04Wb@VFC-8oOLix9uU&(LGQ40K9$=RtbqM0mNR^?qS#zeN~ zOKBbHe*3ZVGZlGY*Y1#*>fQ^v>K+u$4P{?<8xG#&AU+AWF8&ZBIMlDB{L~gemAzYe zc95K|b{dQt5u@R$s|s?L6BP2MC-$(8<|nfTaCKS|9z9~ui#JO;Fy%OL<>pOd_bY`o z#7p2x#mmq;Plp$ch!ndgnxJy;2vX1+f+>Yf@UvSQ4!zMIBi}ost3wK1`-60>PMHh4 zKCyE*kAZtOlfm_xp`f<p1vGUtK$G1uI4Eu(oX+_vYYZ)?^0{$Xyj}rUCo6Dk)EfHv zE*dwSGJ@<nKdiUdP8HerXl&C`{=C(Zx8K!b-(ez!40ypmN)_!Es~m!Pm7!R_{Wh8I zY=$X!6ofec?(nF=iRqpPT)Q$#ezPH>!l@u0H4hHp`TZjyU79bwyE%v-H>&dXJ$?90 zaYydZ)D7E(ET-o@f76S;+bC((e)=`0FT9rgq8id(;!AO&crl_QRF5^Fqt{M}^P`%{ zSKXTK_n9b4KxJ{3kjJH;LgCBqq4;g>aiQ4HgonR*M5BCEQT=>1T{4pPY%Nh7k|ENR zAQSP_-bbW=yDJ`3VAy0FkGGd9W5mEt;JJP{tUYo9f?7SW;(QUr9g3mwDGgLEaVFoG z<dAcp3Angc32s|-=5ReD%r~7V`g?8V&z=gXCh5ikZ}+4l?oH5U=r6CTyA1m1#F?E( zbMJZ)^Ui;S<<@E(7WkZMUvHpf1zk`Ib)%)*ck@l%(~xdAoqx3a5c_`dfYygMNNK1R z>FTA?<ZE@bcj8()DCsD+d-lTthtjBYU^E@ny-AOL-=>fEdSj-n0%DB5!!g^JVDqIG zw`=#OUL1w%ZfViQMiV-)L=|>@TZwo3I#TQ8cwTUJ5zNh7A{-v}TBu!;gnIiw04N-w z@>uCPR27RC?MJh0`(Ajvb{N^&pR7nRF61Sr#z=D(JsxdTMn1bo%9`X4s8{W9!Rzi; z-mUH=+Gb3}WZnC)A%7B{ICYs6=PJQr&qQHRot#FxJcseSLZrFNP&zfYJ!>6*3C@KD zV06?6Q<Ga@*QF@vJyRK%JLE$8twFf+r3$AkEu<~}(t3G)0R=Uei%Zo^@bZh_vV(&* zL#|ngT+#a<EzQt@Q{NqVRrO~uHINw4dzV%G4Sy@PTOnmG>fVy(dM;Q$WF%d9tcS6a zV?{Zw6Ra+c#c593xciq9k>6Z#X?aJ?o_|@4ovtP9Uuy>ib0c{6>|!jtbq8b<Y@j)L zpwu&urbU6h$zyd5U3q2?vu%6ukx_Fotn?lkX1a<!mPW{Lc-ZjH2X*AIQkq#P_QE@M z3i7J9RGz&+3HtAr>Y1ZsxpR<|`x&(qHjS?ZrFZY-F=I-^E?JRyeZd*f97rJS>w$ln z;NG)Pxq<prG;UjlA^(!NTWA8E48JaDUC}}Jlb)zuc1a9WUrsX<7tlK^6*^ge4JN*P z3|eJU59PF$cx9--GsIk2zxTJOyCoiElLho0cVAc~>7;r|*Lj!U5oFiUnRou$0-1sH zpeooLfB7vHs+Oh0zE39wH|Ngq?r;*m?+`_fp3M|=a*_C~<&Ny?<r}g`@!MD_%?XFJ zWXNvqX_4|M?$hDjbJ_BbD;YUAQ0xdNIG$7tNB)#h_YYCHvykMoZtlYado94^?JXD} z`Oeqiap;;dA9Gea@uwmESynh1?_T*P8#((l=-RlUsZS?7jJ@$%uLrbgwhV@=#X|Uv z_PnZ08$WN~!-|K@q<%^)jWu%N;Zc(*Doho%2d=^oV|>_b>Ic$lwuY728az3u58ruk zQub`;ebRjJk=As}7c%}9atHZO7=KI+`go-Zc<4U#YZLhB4<*zuzX(Q^QDD_&HCahL zfD;poaEx+CS`plpcb?lu?-LoXtWuEXu_<`|M+SXYx0QNX8a#TQ4x6Q%qreqcVVkQa z&AXdL&bJl#!C-6XYI%<4NjV}`<?YyQfe|%~v4+`;6(wDV9gEr%aO9>Ym=mjwn+JQq z%sZ{JKAv_s;!!p|*nCxVH0+C$7lrbKFWbb3wj<mnS{wJR8Z2+xvmX;Q4oZCfURV(@ zPOkQRs$d=XK+wK_N>tSiq!e2jObiI41#j!%*=lQeye5L3>>coG>x~MVAnBPVd8y}4 zKMTh~6nRVg5oqAP732?1**3xrOS8-{C0U<cry0V>$n|(=gfoo$<B19<cGH{GZv0VO ziM56=!5ssQv2-Wl*uK4ymVSn*e{^u8uZ>_b(3ri)*x+%u`?O}w4f=a<BfQSPfbGkg z=vlx&vV2or(euMSn*RPH+4wvFa$AVQ42Y%_y_Ma(X}}kM{RNYGM}&^@J<!GV3f+F4 zz}veo6qafW*sIGf!887yEO%!zx<u^Z&GyKnbM>h1R|{C5z5{FjULei8@5w*5Ki~Kn zPi0AVw14HSiU-kySya4E2IhOgH|8S+DoEG<o)Xp=TM3t(j3_Iv7dH3|<fNXnal_Mm zI8o&+WQEtl>|k5?_gu!iCHCq_T}zx({zuf4vOD&SP?VT~Qq6JtfhY{s=h<zY#B<AD ziJ|i+@<8vYWN4p2d$+EE#Sa@P>|``G{kTrYW{$&p-NWpDpfkD^+45P_`*gag30`<? z#7=!x;Js$3z>}?TzcdH%+d2wAnQh~I*CV9e+8-Bw>V#{}f_T;I!*Esm7wBF{lvsRb z_;QyBP+^VVEf0yddq#2JdMTG7_!rp>?@fMbe(?K85S>|b5T@Lp#lP;W!M}ql(7DGg z7`8#`O-5c55`T8anMYzNoUhQ4()YZjr5A?i=|h`OIF{N1n%EtOH}N{y*D4x2G@1)B z2@|;As9=^mMuAq0gLrUAxfrfCih`VsvEo-StZr!$=KXmJm81Nj_xLPW;PjgMX|I5n z;alL2q{*rCEEPX(KTO2~&GF40O*;J23L470;ZFVw8@#h9N=cD(jE(u^<8VIO8i*Gz z%gDqmlf4&9`W7h<bm`KASeB^`9+CYac7huR?VpCjR~h3P!$UOv+gCbhT`Ml$90>_~ z(}nc$CV0Dbw`kle7rcy8_|&>Kl5KqeF|9sy;EEgXbkC&Esg2}t$pi-_tV1@Br1;rR zF!9eX;aKw}*y!($`qtm*v&0{&8&WL3%o+_b-LoL=r7C}4R#v`#c8l!7F{uw(xkdb| zaD=)BoAJ<VVRSKfBIQ>_Lz~?cn)iMr-ihsko4kU+$+RC%8XCs)?`rVE(FSm7RfIS+ zegy9g43K;j6KLQ51gf~W4_lMpix$!61&wive8=G{gm}!7X6!MvBFF&j-~5(!u8tMg zDO{<zGVB*M==_w{ww`i_Vd`u<L>qs2_rdRGlQ?jH9r^ZL4eL~$@v^f$S$JL-RqET} zgNZ$`Y0*QJ6cRXMr8oG^7=f|-F44@5<0;Zn(zt$b#qx8FlqC7kbKf5aHT^sunH3EW z5~o0<bUdZSGdX;O2?f-}@~8nj5bRD$gA^wmn=pW_Q#-IYKTJ>=n?i<#hxzDse>&T0 zPBo?NdBdeM6x7a}`(5aXfqQi%Hg_@%i^-Py!*(>}yfsgat&+4b0=}G7D-69l98%)- zg{l2E;dci)Wn47I;vW`V?=_k|roM(4p_ZB&Vz^@FC7yiH7Vk%w*_o`6@*A8b-k0xh zaIH8*drs^ZZ`LP^TlGfCEe{7#(YlGeXV^yh`lv?~r4uc0=$Q)kx|OVaW+qIGu8~3h z1xkGvh+11Jsr}uVoH)@_*1GAe>^l$T(C$6Rd26(ItzT#1-IkR+=(h=emg?os&5B~i zTYps2TMA`<TiHx@jzScdvwg89##sz!#opnVJ6z&gI%weelEpCXSr|T28z{bhasd`k z?hlp<cgb(nWAUu}Ioa{_Ycw@=J?48~hn>Dh;Yg>gc;=ZcYCZ0SJ}wbd5V=;Sczr1C zUUyFR-Si&4I@=AS#)h+(N&{5-nDJ?igEYM)hx!=yr`?75LcDHYtlQBh>{A{l-PhfD z^0$>Bf94K`{mwv5>?UshdzyMky2!C-)<fIHEK1uN3gc!UCi8QNLR9k^Vd5bT%*)v! zYxOn3CHanQysJODF6e@%Z=>*JMK~R5X^-P?KcQLgE`!f&OEi9`%ujn?g-&~4(8lA( zXymX)`XKGSqi${@-92;Q<PmFrV>}x6R`#RzyKKenYi>YCBPrA9Vg<)}za-VvYQal> zTRd@K4q9&i4g>CAh2sI&<!W`Yv|r32rSNm`T5TP>C)NwERKn@<wwa)_+mCl?=<~Ot z?mXk(F4SD?!xz>hi@!d-hp@w{+(F?JxE83v>I2PWc)b(VXbqKr-=rk@fa*YPOPi2; zY9t4~A0?_xQ>I5v!|`9g9^CQT9@r7Oifp$O(uD9Kyy%ZQD)0FPW?i$$)A$Xw>(hsR ztjwb)iCuAtZW%p2u>>!5GU414pJ7wGiFn;HTj+aEfwh*Y;uKvQyuEq>Dr}xYY9;4k z(}2;eHpfBkcG8p<B%5-d)CO4aHCMdoCu#lO*kQcEAdbJ^M5BKtf$yaGWD`18^ob<M z4&TAkhg_wYJ-Kk~@kHo-^eX(WdJd<+6<0gxviAE^;<IBKco~PVlT#+%td_%`j;CSS z8WZvKN;lE7lM-95*elpd+5*$~4%AdQ0?ugpz_NbdssGIHP-?!LM$eo=O--#r$1}ln zZAuSZ5OS0ktqjLyC2vI4;sj{x(+N!EcSw8sHvW8}6DNM2$2%>r%WvHO2Ub%f$orE- z5G+jPwL%hD3=HA)yF<C;Xh%9bYAP3;I7F#BYw^cPZK_MK;g*m}A$y~Jg{swFNc=Jl zS5FJ&<f%(3ZA>)#E*K}a`!$ame@*6CD=TTu(8p_oR^Uv%Vf?O>6;`)tpxL<)G|tS! z%fpQ^W2hT9PEp2zb2V74#fFy02FT3BS+H`P0?sUz7@^KK;OsaSuSq;9H$TUU5&0SL z?NLv$TB-+|!c1`W$6wTGwlB>4smRw1Qn*7uKS(irDGb_Til%wPXjx39_}c3zdG$4i z_j!HL+c5_UG%JPGf7E#2j|2R=hdvtyMc|{Jv9PM~BtKpjgnf5@p^a@*xZRAs^uWl7 z?@Qo%rAd86za6_^*vCI$=%tRHQr*1a&;TgBG=RMaMM3#zQ*4UsF5>Du+*f=SD%)p> z+0Eg&aI}q3JaZ$g`;!4*bmqu<4wKe_R%h_6`bTfyf1rKVOXx(@L2B1)i*WbJ67(+{ zi1jn#IHz-l_{l&-yF=5Vtz;}tPqCyK>$Y;^q_b=fs{GLSfuws*Aj{i*S=KlX)dMtG z3IL*_=u@;U#!Hy>G83F99e^2L`GRFpqVWBCG{uhFPMJOXa6q^=YFGUQXJ-dCe(6cF zl-}6gV=PZ9Z3gwnR(8WH)F3J59Xy)066)Js23YSao{@Omihfb>Fy)+`+37h_UTYMe z6ZP@PrcSse!9n)3-F|#z)d&C1J%CR(#=)jlgP0rZg;U>EIr&w#xahqGXL`;jkF;{Z z<v@%aA8o+9?tOXVg9Q0!-9p%vBjCD2%KYsCQ^Er!ymVs^efl{Cr~5qQCw9{HcUoq5 z;JW0iu~`C}UB}^EtzP)X|0hk!+D8(!o{sh2jpOxo0L4|1zM>t+mL~DR4MQMC@r(GN zqCZWNd=c&zn*8i{Z*KQX%AM_PgY7S;g4vwE)L5#6rs+1=|3y#CbFPODEjF-9CS_^& zoFR_>I~i5po8yhSW5tc+iv31Qtd^~SE)NxPi+%{+eS1$Fu|^g4mS>AQf;tL+T_ZRm zCkZ?H<-yg{9VyQx1%vbk%f8&aPV%F6XyQKrthcn$pPgmAQP&wSINW7Noe$!EMLTfN znJe+fztHl`2=TSnAW9#0nY2A`QTFE_@W?wJt>L#gX6+HWtFe;3U5*RGUs}+^WruLy zs$2Bt!Vcli{#f2*d;z}QOpx1dnoR|csT5FR0S{+Il4ab3io@aVV3cTx-2)@It-hM{ zm+66SSUx`8wwzB)^W<ae`*KzBa-6O$<)HO+#8E!(qNbmRZ2KX5{x^S?;OTz2;?9@v z!Z5#`<UeOPREJ1f>qSY>(Mp<St3*JfYCb)2*W;y!r{PqWf5L#jV{o^HJ(bUIhHnEj z+4-{xY77~`f0oRp)uubhqgQf;KP%yu*MU4i+8;mubB^?@7ttK;8C-h$9`#vz2zq21 zbN5Skz$aoNrYm;FE^SJ@+xHeIrnd-fSvzQfi{z;_iIII%x=mTFN5EeBFV&pAES@vp z30%7ZD_*SQwZjJpMhg>h|He@;r-ur+7<c7=!J~Oh<7?6Jl(}&H!gldXTB*>}<2mHc z`U+8HbNJ`-?R4E#ll8WqNA;h(ai?nyc!&6*$J2MTe0^75y-7>9f0hFml*IFjlN&Iv zCIW>97v?i|vbhVT9K?JNy!)oVczo9mT3Km`-?hwGVYeBM`5FnKVx%mgAq6g-Jwy+t z2SM=W3lQ1g9du6XVdQ`$>~nJ|ns-a2H>#s>*ybuo8a)BL|4rb#CN`XH@s~1f6KI5& zHBbCBm>)kGf$DowU~0{N;e*K`QSST-$2%2>3EwTrH~BEs+X;3@HV)_RpOoRc?_N=F z_iTuj=B-r|LOIwqQ;ZFtz$g52NPER1+1}YERM;{NPV^bfdN7RVjH!j^-6Ak3vySwg z&k1&V8r*$edwS5=DrI`_r72!tgj@Rti?1i%hK7xyQkJ{5Fx-6y<j$PS^7DBxCe4G4 z4me=V7#-X+X#flfI)ZPyHj06oZm2g5A=O3~=In}xYfImgs*;9u&q_M_epjH+gXdt~ zIfY&g(!j@sx2PW)fcuxZpljqn!zRm6_xxPprO6XvN6{E@&FCxQI@NKQ{c9e!-de=P ze@4OjP8O6AVl3WQ>PmJ!6XEf^V%Rpdy_gNVXwhmtzVp7AUJJ^+Pghr*DP^UNE}TW5 zzuTk#*zGjF*I%G!9d>IAWUJN)apLo@;^rS=G;?1zs@k{<-x7;R(|8NZYuzcf|0(jw zItrUtT%rrN&Qr<oT(I0K<NOoD(BhyD$tUK6xX?)WnL2=|ewz4Q%07JCT?!?Cp~jJF zCuzx(V^lcLRc61S1DcMKSP@6JiPM#Jg}qa|;n}1cpgLOWHTRM9wI22~?4+a#Zmq@= zECH`eqxq_(EiZiPPX%{OskJAFE*G@X+e_MiS)@ZLH9+KMBaAlm6Pq^OgXEG1*qrA{ zX(y}TmBh^r3CN{0!-`;cYJYB$9TcDG`;$*MX|}E?(1uBiZ8z#@!5_&JBhS<4;9U-! z<0FF^9jxWP8`^WIQxCL>tcRWn0p#{xk?I-_3AY>{$){ZJiuSSpsI5$5>Xug0z5aW| z-oNAVWVgYzO?{SVd-XNcYbHT$`>EucQbQB%b)o4?4Jf<40OyulWH7fUOt=w97dlxA zlX7gt4ab0U3#|B5htcG(m`oox7{TjtNfOuVe^B@4-&Fl!|G#;r!H}6u$xtGRv#%E_ z2_;35G^iBK^YCskr;;fl6(u2+q?9=Ox@Z(lN-0SL(maTw`MdA?*YCZ)-_N?=_gbHI z{{c8>?Q`sFU$5u$@fZu|)LOXrgN3X{fCVi+GMOK6b{MUYTf=&EcS3UATpn&1(vQ*+ z%-Sape=Lij)}UR~{JVoMa+hFtf`i~i5Hcn2GVbMt3j7!_l!dNbMStFWz)edvnXBJO zqF?r`s;nHoZuUmF9n50JPT|%#Co%gkC&bFNA4QJtdEkCvvq;Xakd+vhqU8X8;2!Q1 zdmPGzX(4h{G~SDPKaOF8a0NZMsZLWRHnKeH6L@FpxvGhYzd>{M9noSYOFRE0a%UT2 zVDQf<RP9Q_P6}dSo{P~dGmGBuB~hp2Z*H2_ey-@n3%E9>nw@*F8?z4lz)9ixd@p7| z*H~$K^r-{FdgCGON+c@3dCW=a#X?!6E^Ujnr`sK=bnRX<t{=RVb$JA1qhln0A?+=v zGbMoyJrad8ot$aur4?lL?l?ELX@O||JZUzu`wVoRO2x62&qe+8J=lW7o+y8>n7RTC znB>9B;4?{uHOWn-tkun6_}BuIrDU1z(RzpvQlte|J}`aL7T!7ix;WcconETRkP4G# z8ZO`Qddqg!FY6e{bUU-FS%%oqa0doWZNoISodQgrI-hG8NXNjE)yFHa+-OGO^@=cE z$%?ykY6??*rz{?p_8zQ<4~EflgULNJk$NXz#GL9}x}mm^=^GCMr$>WXj5@)>uTuQL znBRDJz)ikKIUCu&dN?kx&HlVJ!k40K`j7lU;g!hI|IPe|0I>ac`Gf!4;gA1#|Nm$H z?*ED(V*juCwF3X+e~G{SzkL7ynxFT-{QCb7@O%DOU;qE>{GWgO`T3XM|9{O7%6()@ zM?<Et5#j`H?ELZEuXYPzKbuJ&?oJe&5(*paWuPx9pGpq);PcB1=zQfBIG)hRAK7Qc zs+;}Tw41V2@#X-Fc$&pu*;oP#KX5EVrx4r1#MJ#|Fdv{jh;9hrorI-Pbn8VUmKj*l ztBp#`sqa4Ogb(AF*a)ni)#piJ=Tn#!9fMicz9{hKs9${nWNK`M6YdXTrSNR+WE)A_ zMT>|%Z9$ciFgAR{ZXA7N03TX>i(gxz%__CVkjZEZ*j*A2@h^>qJgXs_;vmIRuDZkB z)<bMz3C|^U1~5ytX+oaclbtNwiKY<~z$APjRk4B0NnRdW>ZY)H6E|>PRU!DIHy2Fi zD5KV`P^!v%fvev2;|}G|6Z}wTU|q8T+!}63O(e}qB=6&TZ&jMAC`q#%i!pjhId)zN zgOSr!uymXv4iY+-<>foL2-i_;-YZq|{BTO>kIrTxaDyw}oq`gV<k?``&!U@SSFo`2 zBgwo`%-Kxo<aUb(u;3piY~pQeRvux*OjpUlh$B7NX8(lmT4uwmZmEI3N>z#&v5>Dj zq2Y3DcRby6_oq{F>SXx-A;#vYkk(m4+Ir)Q;4tz5i@4`}*2N9<D>N1DSOH$p??KI& zVPq%FMBAndEIRpcrr!Pt4_6DE-;n*NXE6q*7X|Y^pJU*Ifj8e|H;dI?7G^71bEp~? zOX+tH;6H(1MMIZ5;Piug0Bx7UG~2JZXzLSrxXYeRoH+^<mU@9-VisR_dk5zbnU2QQ zkGLbF&hw8>52T59W7zHCwisYM8JD!HLFv)qG_b`Q7A8c1#CkVadAb&+d>Vnl_fGTs zy?1g(EB)E)o!4-B^h9`aCJQTX`G`*>Uf_mm>#$&vB;@QlO@G=XVNyRmmOVU_WG|ZV zBKh5zuJn#yp*`AVvs^a}!u_nn`3bio`Y*Ofc|uR3C$5-l!7s`G0E$@-{NCn0qD+$( zux|LnHKi5uDItpF$bP_|WP6%$!-YMou;mY1tz}*fF>L?VUAStTAz9ZwgJtp`!E5Ch z_I0=uo^~vRms70i@+>bZty)03M_YOI*KuT|HH4S#3}-6KCPDI_`xv{*k#mrE1t)!` zL-`RS_AShoK6_fDb>>Z&G4v}cp4MQ!9rC0(d=RRpJHuGND9&|bG^>uEP6Nx-*^-MZ znV;1mFdm>owsYdxnYBwOBkzmo=fG1qHMNy%75e1cMlPhushJQW%EqUH2gRe>9^Iek z!`=8tc=?hp$*Vo&*X+{~!Za_@N>Pnvj;rU(j!Lm(W8U%AL%nGBvTdaKX)v=K>qT$f zdtrd)8Fciz!=3tC1fxq{i6S+%*xg}UXoj#03vc@i-UCL{`=M>{I_@Tycj6LE`Km&< zh6Hfk4?9px&W6@*R3^P~HcYZn!o_dC46bsxRAs0z8qQvpqT9aC?66)5RBnrB_NNoX z_kRtdORo&+uW|x6_nRedHuwr}7OOFD{~)+sqe&~nA3}*`37JJ7K&gkXg?`Lh(X!Xy z;f~E_7+39spDv6i-S9|0CSeHk*LPseKbC-6$3rgSat};*UjcWr+wgh!7S?1X&r9u* zWo~;<!0ff3q08YUJY6Xd{}}tSFx`=~zIzjdHF=Q~mqF5MlJx5EZ#--J9Q@VhqE?6l zmjh?G?#z4GlD!nOPVYtUiw;aVUW)ySvt^RDPx+{1Wq34d9o?_l2?~-YF}Qah`n7Cf zm-pv_+qALpkJkfepS&CA?{K4YziMzzkvxmF7{G0;JrB0`j`CZVbm6=AkKs>b4Re@l z!dqVnBKae;$@Fm<tG|`a`^++@5uJ8)-QEViKm87!_m6T9PVB@^kCiOD`V%%twTqJ^ zy0E?7jD0nYMsa%tZTgZ&`3Cb*CU`0&ny$gm*n^Ke%Td>$63(_Har}zaRH2qHw%soQ zGl!>9aM&gIb3KEW-*4eZCynDzEN|g2jt(LTug@Z_bZ0){@hkpIZW}H!@WjHI`PeVe zkck%yXVqFFewn`mOZD`pCmMmYRB;-;pBhV7#6?gqd{%~xkK(kSJp=z=!gFoQVRqx< z19Y!(V}DxOQ2X{pc=I3v$84;}zVil{+p!qN7Z=f`i~U%T`Z%=O<4mfyezaw17g*)h zGQDOqXiK}urT2}a(&5<<zP%BHZXIGgSIG+AR71-OCr<6LwZK+Qz;(ty(6FP3dz&1| zb_c7lnHzLiVAe*MFK13i%j)2x;Mv(>JDgJIg=1V!6w?r#Dfxm6^!1Z#+)%~I^zg@f zD7IH-8d4$Dp&5(~J5<;_XHTe{s7J>yY{G1XaUlAt#7blgSbS~(_M5w!Qyi5F;d*+Y zStsP_=Y7G#I2}B>E{dfcTqAtP?gKY47LDY*Aof2z)Kc5AL*p<$ZA=x9)y!g<PqtFT z@)exXH+4AuV;zKr8PRf|&*)-zl54D)0)e&D*@*rpP_(*&SABDiJ!tb}J%U%WD0l&G z4p+hKqJuD`X+3mxI)kQ%G(=v$jeFPK;@;SgWFMEUCJ)=aEbQt}Uhj4i1&yfyb+=Bw zF7~(hO41Vyy|q`gu*Z{*>BzHeb1k;3L50G-UvL}RHE=wA=6-&w$AKYb{NPKz6!I<- z=hP>owu%{NHvI(dKE9L>mP~^7v!}pL{|db-FNACFE$PX|Ai`n8NbZ*r>-Z;~^Vzf= zLv04KMU}mvI46<f)E2XxfJyweaKY1Ec@+BDeTI;&x^$?&f<T%rWG%S^+00pODBCfc z`c(4ix#T;c^PP#|hiB3K9f^Fjr4$SOv>cQ+CekLU2M~GA58uz5!|Rqmz<S#<STI6{ zAE=tj<=b|Hx#4O&mXgn(`>HN*g4-dE{{TB{rqF!-o$P(AKC_X`fspnBXs#Yl9v$&? zCw3zB`!!rV`(0|~nLDmbXKordYQ|reVH<)V^NTXAD_a5<r3p}Gm%wscFK`O}^GL;^ zkk&T~XNpf|lKabJv}|HN=hxyR`g0?KGP57D2NDapM~<1;daMN={FzB651cSUTj0q( zeFL?3wzJ7iveaFW#$H&yz>t<}Sk|t?iuzvgSLVm@QQ8;b_m6UyP&;$BA?|>{0I<O! zY3(kze#zpI4k>Qq4-Rb~ETW)uzp(pv5MGFIWPj8rBedGlc&`8)ub{wE*2XZ+#`ExQ z;#7Xna8>BJkp_bT%3yNvOBbie1c;pcnEGq(6Ai!Z49D%fsXFBplUKH6I^FVYlvyY( z+@i``t*n6ETm#-2Tn;|f9NjXsBZD96?8@5=xElEZxs(>HyHt%A-!Gy(*%Vg(s)bA4 za{|8YpHKdS?*rT!M6%5@xK%etvel-*(q|7sUV9A`^jF60<$1jQuNrZuu!qm|apo7! zs>VjiXgU!uMc+@JA^QnB6xNVU8!Db)ME?|Yb6LcKelwI(JIHBWT!R;8nV{k<4|G51 z0aqS8#pCO)@`+oE;9A~S{O!}9q#G0AN{utklL+N!5BSJ^@qUkWx(Y&%a|j0QQe|0r zQB?mSlJ2+}GWQTQdRVJ~M>KcvQgff-ufS7S*i*~B=(#T1E;AXoMv3{kAoyP7tyrsK z0=pa-fD4Z439i-*%2^XfD{?oJlerA@`;!27C-#cQpUz~#T{q$B$xRS{O$X&RG~&sP zrFdEDD-Ian4YNe!A)_usv~m4wp4&5u?U1;I)uHYbSrm&4({{1Yj{{-;bWPA)znCra z)u69QGgxxPE;9dgmk+hk5M0zFN%un@ylk9L`WYXfWHv{2Z<j*0+kBD>7C4NO11Ngu zKsGvpu}gfrOUz3HW;yvU|81fjZ1^w-3O`AbO^h#><k^iMQswykiu+<o;Xdr59R{1< z>Qa5tNqBiH3C{l!JZR5fz{CN6QMPpeO+I>zS1`-v-0xJelhbzLca{!HA3HHhX&*^z zS;C0QVs7G}4>;yx9Uk?$Ey}J=VjClOLESV5YRTM7=jJtV=jAV<_>T%No!<+a2altg zy|-}Ibxm0QumP@(h{8%$bEan4%#}^Q!`0TQ10OvAt8dSsTW;H+&$$~6T5iFa^P6e; zn?fw|^hUey39NLc4gFzVd_h4r95I_nyG`9#$<`0RPk98z(pxIk2MXt}B3)SiT<CQD zaK(O6aoBuAlDU5Bq~_+&u+VWL{uKI^!{4W|PiE`j`=K~4r=J5avrUSQb;p73jFIr6 ztqzWz5wSBjYfv$F8%967!n~bE(8~*lu|i=n+#68{5>IY(u1livYx`bmDLw|9oODQ6 z=(tVZnNF*N$Fox>0+?(-1FmXKMDJm0I8%NX|H^O^C06YOx3)oaFREH(BDm<=y<@pr zp$ERF@Ei2ou>o8jB%`dpFH22gZ2zJC?B(EVy!YA;xE!B~gOUgG5w64e19A;qR!=DV zTk;ypa^K=|H7R=H{t7RRti#si4lom0v8%f(!8xh`=4EM-XhRwcnYNmaD!D_YqTn4V zcmNWu9@uh7p3FUaxhwX=*{)~d(0XVt!|SuSatMdn11i8bG8H<%_Co``E71Q)KgMOJ zqJDh}oz^PFvzk_TzP^WBUM<091+B!`a!)$@QI1vbsO9VKY=Ez_$`o~M6IN{v1*s$r z);RAVO`Kc6Ux+ZK<)_xsyEre}V-W?(M$Npjn_$kZTzBE=N>etlZxYMWy~8!zIx(fD zW^B{cU?!N&eCj+{t1y%3Z_%N5dMQk`pO9<IcnJoZf4~=&vFz)(NLV{-CBLLroB4Y@ zhn88FVCG%nd`o&Wh+Y)Ip!imC%Cd{-Z>}m*_>crqIkT9mlm*Iu`hkNdoCB9{zO-iJ zN?v=qh+c1ZhJLcyG-aP9r)3mKdp5e${)Pc;{{mMQ<{rq-g*t<0#a;2n!x7}RehoiM zsu4!GC<_i{9kia72}iu`Kw)(+OA0r{E!pnW8!JIi4Q*JiO9A(!g~P*-+PU;WQKE_V zIo$od!km0-1@7EmjZ$wLQG44YCjC)|`RbhK>t?mVT~kTS&c28D3$$t0r`fFFyugCH zuEM%Qg#MpK#DDhha>k8a^Z)Np`!Dxz0#Eqg?O#`|--j*Fmg6s#1ZaO>jIWOLXXRs# zu+2qLtm@=67Hu+=e)<U>mDNIK<6<Uf^`bvpx#c@v&fLOgh|~FuvzBa<9b@YS|D)-) z;Vkmb4z?!0m>=q&g&FsLLP%{exF#tu$#8dUx-yUkJ#vBXWsmtaR;x+t<Zte4z%M*x z-iY@e3VZNVy1b$LLE3r40`nK7iCjKC<D9B>sCcO&#{4pY&8PZcX{H69{2nFVq`Qs< z3D3}FZ_6R|S0%19^=JD&NT5hbjb%OZW_mulpqHgU-wNXKO|v&nD5%71Cc32XOrK)S zhSL0|H+VcipGwzCP@%Mlp8Xs_O2N)#?RE)AkF%iXwSuq1OO@?gIFY@3=R|4C(m2zH zUbxle0NgrUAj&fv!Y+I6$3^O*zrF>-zmzSy)8+_hcl!g>kzqQ5@<+E3icp5xGP zDgetD^g%+2B|CA(op$kq>8tEE%xSIWlW*){p$GfZ3}#TBq&=G4{fDs1`(E_b&6QuZ z^eD;~45pO6YT^F<8S_+IAiYSIX?-omthPZoZQfKm`gu1UiaUajJx;Kt$1<_uVwI@m zWtP}zW-)a1U4p#Pb68@jKR)Pw$=#UJ1Et>b<R<$V<TML-gIU6};3y4#vSJ{$Y`%;& z_R?(bsRb0#Blx{D+?jHK6pJyb1*cU*Nu%!&eXC(CC#DjNMlZ+0j|T+p&3U-3kk3-D zC{lvfGg$wZ!P`^XT)#0_VcSL{c35r|thoCemz~W4gBND>`c@++7dZhv20FmhoLgM4 zwKJ{P$cFtv6%fwtK$j^p^jl+6mDqnge!MjmFJF{l<;u>`G?If0-*$uDw1H4-u!0OU zm(t;WQ!z!{gL~$Dg}Yr&VEynW3-~^W7Uf4$pJX$LiexE&<OG~ibP0!j(Pjs)E{7Z! z#E}ATwqavB|EVpUJz23$$Y`YC(j~gM=<#=6@`8wRXEuXTn1If38o}2`g|d~dcFcR- z3>^D12tzKo(cqdPZ0vIfxKuERRZn>aww7@sx92}0`CTUeI{pJJn(`2ToRfq*`*Tov z%@VdMZ93Xbddd%u)2AqVb-JuyD&DZY7|^Q&m6LDadw&V4oLtFO{t&X5+p^%)+qoz& z^$PNObjc}G$nh+ESd}!;obAZZ6lW<2-r57-`5^UZZdPeJN>00i_DwlZ<vbMT%iM?g zv3>ZgW(Xz1L3S>F71|w`L`!AI!=l+LY@oqkXrB56bFSB7^o%q7r;A<aFlY^oRJ|q0 z)ko2@lkG4^>pUj<^<#0b$|0!Dkj)A7VTRx0VTP6^AGg#^G-kpOcHn@VD9yIFs!t&o zGd`3<Zs!hoFZY0-EE~xjt^*r0Ne`_pOK@mcBB{xALZ!zEF5hM%+Ph|mrA`N8<lkX@ z`ks1l*h6?o@J*J#@MnQS55cA=2ZD-b(C)TsQOqP;Y@KF9_<So!r<FmwufX_!8&2^z zj47O-k1J**VtPO@Z5h0r9=?eaSe#ZYzgd|MzuAO=)xPYXLjn^yY$tDSwTIaRJ;gCk zt@%|FM>+0+kj+_i6nETehuJo!G)G03@-A)XYGPz*WZ+&ZU8F}V(miozxWH)Glg%gU zDbU%^wYb+~IegOaU}J1eF`?F%`+R8zi19slAb2(&Z#Ko>*1GseN`;cv7J~elZj6um z3-=BUqM{i|%<_o=b&S%W+=Div8AAU%UaNsOm$(VO4=ngy>hb)GeRKJaMb}tk?>^|r zcf;rZTtwwbMY!nGQtsU11$bzUHa%^zMhT@L2r|iH5=Y0=4d+5o>Hk5L=UWfe{tNj* z=PvNGJe-k~mauIsoW1i26dVXC!cL(9m&pZ?{@`1pl(sh@7jYNY$hbm+;eFiiae<xb zGGcemPG+O56u`SvhaP+~z~8=Ute<Q=Xzm+GtpfA!^z1-VKi15*y*A<#<a+Vxq2Ma5 zh+o{5SxUUoL|Lr<D#_M(9OlK<A*9rNi3JT>P0Uc1{kow~KV$yE_CgPkotut5(eX_B zdozDUzYLdX#L>qq68J4W1Y#EU;d4)YJUMk2=RKwf^><$2F7{3rQ{+;%cY)AXTkxV% z;g$*Px+d($+djhTTsN41N<sYaz$l!%;ShWlS<zm-4=~}R7i5^~GTX(i7=L0Q=~^T1 z&|SouuI6E0*<9NHdoKN&ODH`n9>@9r6>li`0%22oY3z$#?4I&`*t+5=7g?@GJLc@+ z19lU4wND-z{xRXaCfDFjsSKJn%><NLGxU=d<Gh2%F<8+UC0+W!`Q;P*yR8!c_}Z`? zIe&PCcX2p#)*0T}C6c`=Z$){1Ur-l3^sekG3{$=hZ7+ZGV_ytrgM?iDf<^woWR-~A z@^Rh1EME1g8oiqk3nq>QXg<!B>khLZ-vOi8fJwa=5v0c=cNNmpwV|A<oFTs?^auX- zsz=`u^I>DtQSQK!NuW1&7d+N{%iY!;LK(p!(E3(|-psoNhWC51_P8r4>AGUITNM1W z;VN&u)0?mCP+^N7xw6>qne6HNcaVE^9}LJ8_Q%gBvhh7j;Me}a{3*HvGeoap$~Or* zXMGz_tJttU|C5~KAVV-%_zO=KR^p5=5+uFrn)pLk2;`oUqHRA3di6b7)dPVmnYN!f z%{qc3UrA8uDph=2dlOsD!f9@@6h*C4;Fj--!MEFIQk}(TzBehFUTT?AOy)Qmb*GyP zQHsR}`G4UPQ)0PsBkAL~Y%*FaIK1bpz>tg2xWbri?A+f|FfY@JG9Uhh@}CPBNSLvJ zgQYkjL5tO2lcGhFTX34?I_}E2F#Mu^6OF#Ki|_vY$_tPhQmx1lJu7u)b-hJUD>VhZ znG^JHf5~b8Ou@ZRr{TJ`m2A6JBTSXukKRo~QPX%fB&`hN-5q;zTwVxQxokAqXz%5t zJy-Hi7xZIc71?m#Wi(a1eG1bI_aeS@pqaLF@y4PQa-Exm%WMQ9-l~tFR|EL@+90;m zcrMNN>qbSXv)t2u{n$XqOh~Y*fz8`Ta%1!VaBOfn29M=fPP87#uNCG?|8p#~uoHab zQuqghKJyws1a?}eOtmSmLoYx0)1!@qujJRkzP}FmSjT}6x6)y?cWl_wWqJ6hb`sOL zSi()Yk;X0hV$aIHl;COMyD&s~G98X^X010=NJ24#&GHuRQ=BF3l{tVn?x;b_@*Qmd zw59O)vK6$2OF^9T5wu$n?h^G%pSsjefz~fecGFItP3d$*JqbD1;1j}6Hs;tIwLVB0 zbrOvAHgIG0Pjl}|ge;oSv$F9q<W+73u(Qo^!hUKdE}ra8Nvwn!lni8>ey+iZ)rORL z{Sv$~&ca8N8}ZwLkI-lu&)CXE^u^VlxhF29pqxAW2VHq`yqLs)m|ws-3+Dzu`yXSc zpX}oLZ^>o#V;z{<`O~1@PXe93cC(AE^LUpzMr6CtN66KjAuZn{yn>4o7}x*6$bb%r z=!_;Cfj{LXFYpSTc2ISe3e5eyUGV80=3LZO1b@IzzV-b}kocrS^^VWDQPyujY}d-~ z@7j+FS>9~+Jb9WFnSj=_12D5|AZeI(y6m51MB8_mVe;T;wxTl*rwxcj$%HD-I^YG& zYB~xZXH8^bO3D0_{ayT*)=E64tqt=h-Gh3^(|GdlQp~7!Cj*@jw!gj~zL@zIT6PSi z#-obNq$`mHPB<*u|EmTrYJCyiKB~?VCwOt2tVg0NnzPYQ`gkb`6*k4yfiCZfqMm!> zNu?=@j2(0N1~W5wUEm3)%En{-MZi~vAxy@8H`H#uivG-&E|waxA5*NTC(VWyg-xOJ z<%7sO>k+QSAQI(!QIDzwH{n<p82wtzV!w=_)&f`dtp6$Ak#(R?bEnYZk%AY~YVnss zDr#?DiK4I^F=}byunSq#RD6SvZrbN^<L7qRT<*j=jtP!YmlKr><rt0d8AyNkr846@ zfgN!q6&tp5kYKukCNx!WVf+#1Q?L|_(@mM#V=ETukPFp?gDCTv2;1VLC{kYqeX3)y zBy%em%N*tgzs;f3NAxJ<=seJ}PU5Ps3Hhi^&a7&5I5$h!<2~{?#6SFU1e6=b;gL#C zG;sTB3|89BPDcjtccyqzvO|IB;=ne3fbw9fnYUkj?wSvC?kYyV3mz<5;BakO@E6t{ zJjNdzbAnCJb>i1pR8XHyHP<m?6KlWr80%tXxh-94Fs=J1Z`-ketu&p(POVIWmtjp{ zx4abZ=dX3yx=h#wZ90Mj-n@f+m!;ItA_hC_cf%p0UW|Gpbf+Jex;%Y(38ikIfGx(c zptS1*Z)q}=1<ee__zpd+9u^MA|DMI=eu!n+pZSGS)7ZJI(a<X153zeL!Nuln@KzUj zb4zEkXkEqD{j*PG8hshEJpSS0!&lOizxH%C$DS=wGGGU%zQ;pjy71-*2Mmd>huZZY zz(S%;{7_;HQ+?Kj-X_JE6Sa`rpO=E&&MuT8!C+#=8JKaC@k?JP@H;zYX{(J3<!mcp z+nEI`EZl~wCr81FHTnF>Vo7d|#4c<rUCUkvrbG2-Wq~hygI~P)D)V`2jP7k3{7xw? zwsk-XYs5za|J9Wieb~ys7Un*U#Qya8%T4}dp*tL17!DU=Rx+`RGvBuIE_j71gPYkY zP?A<)7Jqwj+A|?Hl{blvHa^J>H7{YIqpT@EpdJ++%W#9y4m|Q`F|!@1O>?9ZsKmdX z`3#tf_e^?u|NT-lEJQfxcJZPYc5iWl^bdR^%o*t;hf-7U01WWhf#=iPx#armbY@~8 zHV2NP3G*I<bGa;yFzyAjt4DCEO(?Ux>c;Kt_y&7SgTdV3E4Sp&b2vF`A4Zv$R(aHA z(P*a#Tu?QdZQYd$O)-8XyQ&;=t^|<By-X<iT*}7?`=1}>w=n4SRv7MQ%r;kOu%9;u z;oDMODoh-LQ+HV~MY;8`YQ%ahdMd~K4mi@2qX(!|=LS7D@Ta@igJD$K5T>)hkW~b1 zhJKx;d}~%D`KD^Jmh~saqwYN9)Xr#<?FfC?#Tk>v!sqC^U4|@sR{ZDw;k(~?h5zRM zL2wWKyZytz&2RkMpZ}NnkN@@jqksAS|22O|qbY;mUM|a3oy<V=E`<?~q}lUZJyj>| z<_mq@a)Hy|1|AM3B;mJ$m-4!TV#&D>|51i<OND;sqLHZkK7}U5SHp@%Mf@UN%=wp@ zQB?0Z%CI`kEiUn8^^K1~BlQ}t`&$Z?w;w|7@sp_BAc1jb9^h)P4Scp;A$)l&qJ^ie z*yy_LR9htUtd<O*vEv64-MB6|anA_*ha>Fl2NTjc&S=9teYzPp3dQm7(EE-m<7XVh z<-x!4cv1t{&)*MCvI(4B#&B-gk8T)}t4x)hU%<C|7hJW^2G<I2+#~c+zdH?~FL&L* zOelHW-WTiq^I0`-<~)o|xRVL<$4{r$dsE4%K#pDYHDjutMr@k%WZdlA%cY)ORyEU8 z2Zq%d)BfSRac<QWP))jtsa*mux9FkBHhV1Nw<O}9=K?43@OoT6Y8-dUKmv!3P^0h5 zVyj-h71*1${Yi6O4d;^i3(suG#w{<uaQDoz!T*sYSFT`0gWD|G?-Aj^Pc&c^$~*9o zcmN-BDW9uuafX!*8KSd+DlA!X68bxv!O(!muv_D^;4jU>2`*1TV#GEYc{GdfdRWL0 z{rQK}2z$->q>K}J?icpuo=17<l`?F~W?xts-+&m>iXnsbIQPI8plopvBuIkkz0!ov zdP~yhcZ21qAe`Lji&xoDal6(b=J;IjpG<V%-ikD!VfiUO$8a=!uDb*wRc~;<+f)oE zJvMyGUeK6w3Jgo_`8gxfXiG&jE)4AxFO69W7X}QVIa983lYKimiH>7*+-W%K{4EA& z<5buZIFDtx6@sSDQZgO1jD1M|N4RVj(CzL@Cc)uk?wch-qeEOD#|w<Vi=ZE>!#pb# z@nB&uPQ1E-GV9#g5^qVS=lumPs0)nhdAk8O*I`Egqio-+eRx#DA79VA!&WxAvVlzo zl(I3EE$m9f*TysPxAAvWaXrpO*Hz%gFc*m4c2MBCm!O~EaeTAch)x=7Qq%N<D4}(i z_w0WGKP0Pw{EtPny8A84Zr&x{lC=inO_FG^aIfB=G7YwTzsaBGV{o%#3%|$8k={&8 zVr{cCS(lvfH<wCM@Fxl8q;rsc&3%p)<Nk0tX=-f1&lXYTm>TX~%?Re(B+nbXuH|Zy z@*uAL2IweuKuC24Os#zhyM-C-j@Du}PG13!4JhQ;b4MyGPT-fSE3oEoAE2?n169pz z6iw|qhD&o*IL{A%A$Z6{ocz6#uZRl34C(Q#?sOD9Ta$znqc`z8Yix1ym(lblF_}+( zr~>EbrwVh;TmGqo4Ys*Y#h~y%+<^C!+18mu=-@YRG@HK_8&qQ`@2>`4Ub+B|DW8Pf zb9UgRlXqa>_yX!!S&7$$PX40~9a`I`%sncJ#r9FfI4?<x<ydWlx};JVw(~ODACG~I zgRh`xX9XlTeuwga`Ao0A7Tq!#{MhjsGS%+lS7tzFj#2P4AR64*P41YsExRM`!++-Q zrgi@aoc!JHRR8?}JR9W=JzvCNeRm%>V22Fq=iKEs^o^#KEh_A;;W@Yv{sV1C8Z*mt zW1;;1EAG?iooM266Ro~QL!742>%V>#?yfnERlx(<IiWMRwtXoD*UqGuA2;#K&QD<} zDVn_f%{`1AG^K&jow$5u5c9pUq3Uz&L(zTLLwH?z8CP`8mEI+d!29h-K{H?%e%fM7 zJ=?RvbomAhA32}Wgx@puxX|gjcLvPQ-{l;fF5-^$7cpGYlT_ELu#?|cvG~Jt`S3Sp ztgd1et^DOB@E#suvUZPn_bg44)>8p}|9!MT`v9zSDPZ=!d2mW#KEDxLF$cfRc=r2n zXo?)bW~Astj}F1EogwgMm;tmDEy5kYZMg2&PlUdl;9b$oA^i*2F?T_nz!Z~W+k(#U z?P}>*n(L2wX{t2tCeIJn_kk5R6-Z>)hr>74!lTQp*u~RH+@J$-<Z|DL9$r`=^2~Y0 z9Us&qu%~z7>6QpqmiQ2qr#^=ds}f+Ht|v^07lWw(WGH{40^e<x<Lkso_K&d7r`7%F z@!E0>Gs?oF_k7tq!EZcf?p*%W%`Du~QiFYeO~^M$i83-A@KVS-7s(T?5Wal6STQ;c z-R<9i+QK9nW(YWD#TfXz-3?!4$*_m1>3Gn+0OsG@faeDCeDuP*IC^Sc)rPAbAgg$k zm8BoV+YO`H-IZUUu+5xh4@rX!U0Ye;s|s?Dod&M2YeDvH3cKkS1vO99$#s4(&7QtW z6u;;gKFS`!tS<IvUt4;sMjM^v#T$zF=)^C0X0Zg*^~=K5$<{1WQkE?>eF;4gwoFwf zm3=QBz;=(VVG_P&xbToa{tPpPBY~+@WcnH>&7Z@2EMCZRWSg+Ga}*1td3gSEBCW9- zkNXUoc*QXr$$6SK*t{Q3Tkr3Loj1m_k@t%E+4_^fve%ivDeeTVrl+D8l7s1+?t1op ztv_8K@Eo*M<%nz90WUHN>9Ax2yoYAo7FmqKN*xzj9^j=FRzY_7Zr-m+pPcI>NUAoE zEihkAjz>CN{$u{o(FhLs@A*^h|8@T$Yr|svW|+Y&mYjsqA4{<5`VPu`U&DQRIE?-B z5Ik2SCBP+WBAhrB0qy;QxmjHrOngld{fzz!IgmXpXsi-byf_^<1&n25o#R=pehx0y ze2=d$9mV0Jw6IUjjyHc}gVWy$c_Z;Fc%E?!&J|0rwz*35YPycV@tTFpx9+3KU(?A* z{Ty|*?t!4C%jn#hq5QhwK74wqQ0RJ<!t7i7G0X5cC2!uup8b?zu>pd6Sze00o3IFe z<!uI=ygXd7Bpr{teuFu&c2Hv@Bi?#nm1$o{gn#;%pk8}2Kd*NH1WMn>Qnvxn`RE6K z&SNc=uhxWZbMA{hf5zjrt7<f`U>7&UXd7<4&;`FkMeM_qY_|BPI^41}rR4C{7$e$? zB|Zt%G_VxItbK$$ry9+(3!sd{O;DsN33DqGDRJ#9{_gdIaAf@)@b6J3kB#26EXo#2 zY)jE}%4yo0?1ypfg`(-R)^j7DgyGl$Rh*LDdtRqz8uR!x4W@U01@kmLcJq20_dE6~ zKV$C)m{Y6|@ANw0=cfM5Mscusr)DOT=@$Bb6DqhVqb8AUV<KhqTj;263rL%rQhZ7& zl%Ff%YNZqDjs6JcBd0}AC1x;f{n?aS5yOA8Z^rhur};PAv`BNdCNme=QtYT-E@S*= zQHqfHs&a85eOGO^V~7SF{j>yo(hO;W=M_jEX~&`u$8bl7Z|2U1+XDY-I&NtjiJs3& zX`VO3UtKaZb&(>w{OdJ}_aDG<T5e21){)g5ks;e#KT$KR2CuBRj`sb>g4>09{PNol ze2m}l-xU|J^{JQPzV9LI8^51#cgPf*x~s#_R4ZKfE|%sVm`%l@o^&PK7Q*kF3A@~t zVENz@7kDO#chc3v?y5R0U*~`yj8D<u_~kHTva7IH9>pF+YO!4Da*&8TN7bHAkP|qN z?wwPl4yQMK`^#jqC_T$5KKa9aa<nG3DS{`W{5kxHv80XGML6w-5Ba?61c@zkgiP~R z)OfH9-rBbyA32bD2aSNa!=u^D(IRe4-&WE**vkjmide5=0rJl_ki-!`Hg2dd+uwIx z6r$z`;llY~fb>XGFq%!@q7Lv@&$hwke$(OTtT~w9S_k8<@SJkrI6PPp#?MMIBr=sH zQ9nf}sVe5a6vR@1QXIWBx+0Pb3xJ;0yV=F#j_mz_ldwm!6kRP<F&G%To;(H(!#uG6 zYgamxGLx<f%=`B_?wri$-Ry8e8UJmPA5(K&3;CI^`SG_B*zWV+xX`f(YkNk*a@$Jm z8jys8+++EWw>NpSGE4IQox`UMxsQc<C0Hh$(Jl!HWO3nZc&Erfnv+-zJ0_Na%${_Z zXrzZjX9w|rC*4N=$y`qK<|*ha-hf=?1nBhFXCaBh@%OAGGV0J`ZcE>w=7RntS|Sb3 z@(Ro<LY>W!887U>Em?K$J|=Rm27bRMNC(-oobD^IVfX{^^r*rVnLwuec@S&$P(lBJ z4m39Vk!Zrt{q&DpV^uD5VN;H}(EiX9VB1*D{kB!6^_xs-fpDLeIdur%DkZRG<)-*Z zq7Eh`k7QdPPC+UJZ2o$Yd$4IDmCTf6T6LwMn{GlYQ?7#8sg-wB{ted0zJm8zo>x*I z%1U+9vAh3iIQL8!77h_%Lw=sXv0g*VT)#rn(FrWtv4U^;rh#?U#MNA^#Z7;=<Im?# z_{u?t-I7@d=BiF$Hph!9b{Wv|#0K2sJdUkvJ}36A^h0NC2F-9|(Yunbu<BzO77R3H zX?+2753cfY8)uO58*|WK@t4~(HW!vB9s>)RAf{&hnsfdvWWuw)f=+H2%ActaGPiM5 z)Nhi|b1&g!cY1(ifH%%NWKG}O=ZX!F&wwqK7a)AYcqVK_=*-eP(0tQ`TVLG5cT4^N zd-VhmZ}MdKU(EsYM0ryERD!YX*TqLNwzI<d$=JDQ8F~5WVE<|xKH~gORIC-UGI8ZX z*6KQJ$;o6NuQqWqR);9_?{FF&a~vOJ-v?~^g;pK%V46Dv&y6c(if@N;((k6Q;x*-{ z*su~#=TGHtyZf=Kc4>hfSC3}{Y}oMfePr^bmH%z>6h|-G52`;lupyKCz$nOq;(Im7 za(N_f+telAc2gUUxX5xrW9r$=D<jAwUmf!gS70=5WY+b9$5Qn@jFmmkbYI1j%Ps}x zHNsNZN7+-G#c!yunMYYK3fMRe30!Yf#m%dCW+Q$#Vv6HgrhWTKrH%O*WLw=~`}in! zX3H+-B%4XU?;arK`{{hAQ!ZzH{5Rhs^@&q$vtcEN%(-bETNx8PwvYFzkn_bbc1~v) z$+nu&n6{g6y+{?q-#cQ&6Dg4BdjZ8pQ<-Y&Fg9Oc>nsqwRrhc0!>Q&ovE=w=_%bpQ zBTQVW@zG8+OB#os9h2yEffU?6H=T;^?q%O5-o?gwM$E4;Q&hR=1upCMr<I+_e8GeH zd{9FXl&MVTC(e<=pDY|T1!-`ykdVbgiS(q*gsr=O2YbYOD746&_@%o1J*`t{QEl^| z{&rbWkjsB>{&4so=>Pq@4c33mPyc;?yTkwW{@#E6`u|V;?GFFCfA{~z+x~z1+Z}|y z+yAt`-9hM({rCOtkMjdi*+hv34)}^?adn^^Wyq$Qr(^8u9k|OfmdRSiacXn-a+Buh zu(Q{!Q02K1I}>w;p6gAgV7Yj7p0<U{Sk{S!j`!%~_Cw&k(L!K1UgcGdQ)rCCYBot# zAbyU%%auq?;wJ9i!F^q8jbUCDn3DB!^a@o8y6{n#6@|>E5$6q=Y0MLJ*s2fDg+9qh z)xpfIU6mGxIkRW`5-{cDU<$k&hj%=tf*OpVNxp^R@<roVq(l_hnme<X`xqo%7y3fq zE3xH!0IL~ON?v1~LDoQlh6&wmaef@Fy+4^${cI@B-kXjI9VVZx5qwNc6D+OH#Ww8| zFm1e$$y&XajuggIT8Ry<)RGtYkq@E5X0b3grPHX&xnOZThN~}`T=`a8kuCqxg_|!Q zVfTc7+%^d}#snU-wUj<9c;Q2_w+}N-qruEkYB#W*PB8b$d%R^A%X||IaQHVBDzFrC zZtj!#^LgQvog%`ttOn$54&&aLqv6T=7QX7tJ<Kv50^S=qCe!~fF7CRA-#=}`#WQOl zx66w?ANm8=+3mxc)-)K_{g|IL^CND`sD$|QrsDU{7EwU)MB+Ll`649;s=vD(-f>Rk zmG)HBDmw`DKbx`Xq)$I$H7WiUV=Eq~!^ta0F!n<ZcP_CKbsy<c<g_IE=Sc$JIJr(_ zHYgtrr>D~f`6zO%wZK2F1yEVIimf>$T)&YC5GYlOk8ZSZJ&Ts0$&SPP`oUxQiY1;b zhy4ThZ*=1YiM{9(VoG<_yy)VW?Nud8cC>2ES2#8*1E-brVDVOK*1tELQ(at$3o>Q6 zuhrYQriLg!c->xns(K!hRE~>|p8fy<zJXjvMh#>qFK4f=2{Y)sBHCwbPCsAV=JtJ% z<Kk~>vh-&KeD4Q8)c4n5VO=)#$<7X@%WlWJp<CdJcn1tO@8GQR#?afsLQr5uXxcQ6 zEd0F4;nqiCe^!YH#|Za0lQNLLH<<3(&L?hI7#*BzON*5z(36gGbli9zLj6>k!jBbn z$4ZR9T_ag!mjbKw8jq?Y%tfQrg*mq+SD5*Sp-Jfx)Ch0JFNJ}$D%}Li)b(ibrO~3X z&(p!GKyXTrIm<QbUa7iS?nn7Kk~E~zfqcGXFoUC4QM+>zd-Y%>`G5L?lZ+Gi7uC!8 ziN;QJJ#-{<Huc1Ld5YZ4rHOFuP#FE*e-<WMM4@9@E?fUq3u+|PNJ7JezDNw>JNG@u zHF43{<)T7T(lPvrlIb*5$O!bWDu6ZH+p)KC7(MFtVx^zdA<5mE@m15Ipg<qLfBTD} zi{zPqd@vUt=mBl_H}U6IU4n0S?CFJeo;WC_A9MR;%UbP<DM=-pncXRc7N4(ZI8$JF zFZSW9B@M_?v5e*v1whE&Bz}|6DACN<72xhW2+uvd56{cj@sDRe6YW|VL#K{4<HVJ3 zp~!wQ{n~yP%T#)xD=w9`MY(}t^aHGT@C3?tSEIq+t56w{49jI#QR`$KI(GF0`dNI1 zs~;!fx&tpoaaT6cN$E|@RM8!-lnIW;!SVQB=Q`h<q{qchaAn6_CsA?OU@8m}oM-(@ zSgJ=H<_BAla^pxmA1lRlCLW{Ko2LB8;<I@8*c`Fsg)Erqqd~)r<k<LfDdyMb%BGA^ zV)umC_GjsQj9Gad5_?>j+}W9I*Nu4gT%`>EykCaHq}52%Op_HqT*yRL7A${-B3*WS z$ba8{hO?X`iM(|Qz7ZL)!b_@f`hG0)bBkvY0Y7oXvH<vG)1NYf%J5Er(78Q~_}YI- z)jw8dBq865zPnw;Th?5M#_<|lYoh}@5L`-s?PrsIpFfjUO+e$r0%!T00_M$Jzzs4J zab<>2p>I<u_vcMEY)(!To~d%!fLDK9x{Kv$o_4Ij`PxOR7JtNoI31?gF3qmI{tute z3ht3k2JOG)j@6Qe(7opaei~zpSr3H#k$#D2NX2h>*&N6@O<vB<--_nvC(R@Eqwk<l z)f{h^Ux%ntc^Wp^fvuZug67{5k9@BHnNg>q?V%FR93TtF@(#oD^VdQB<V)C5Hd)+$ zKN0PR6{EavD|$UR0~Xdd!8u8do+hi4dE_Kqy!{kx{q~fLXj7*#i#G}k{{FN<SNQJt z&cc*zZQk7_5gRV}QCRW|s2^O4Lsw5Gy`ghxdC45+-v1L?rOHx$Z7lXa5afxQ-hiLk zC6x8}g;h5dq3NE$?BD8*pF_gopIb_7-$!YB*S7;i{^3-qyq(4$zJrMi;&D!B1as98 z;a_C~+BnvWvwRc5tr%3p+q`fkIF*E9OKqS~$OK5aMWIOEOmGveA<MZZxGMWJ)=VRr zk#!cS{*hxB+!fePmwkN2v??&l7D0*P1@MkO!4)QtWBp23ve0NDKdZim4mJp9$wFPE zcY6XqXxMyM*jqp)+g?}J-&-R}-gq5{6xHImH-nk-8Xguo4Z#gxgs#HHT6#KFnpDjD zlisrlZ2GR(+>9J`kR87Te+@l`CKj5o?4IBYeQ%GC?(i<sKLn?1OFgKMe-4uSR>7<v z!r8(&8@}d!Bi!4v1Am%2GE+YX9G)8s6P^v{nsW=Vd)PZv6Zn652jbycpef;#C`@Z4 zwEWS7f0i4vs%#HdwOoR<FFT-{qastkZ3gosN~liyJX{%X#_hxzq?O+Qmli8SyWqb~ z`J76rD%(XpkNl~A?ojgec80GDh3<Kt7R1Rfq7T0mG0@1LweQqJ`Du#a``DKGbZkeJ zxao8$H3m;vtY>9q2k7gK5}5hVDQFmBMzzD`$Z1Om1=y8y$&(MFSaX<5#_9F&Ep!sO zZW)h3MOLi-Z!cG8`H8RdaAC*$a^UCYpQt;u6Z#JQ#?az{EKKlsIDQU<<-d)|`+EX* zyAA{Y%lVjF50svDn0sNlfDMTdJd1@@@OO6<Q!-jDPPdJ#I_;50xtr8zoWc-+U+T>g z-E+l%u4b}CHRK;Hy9Yz3?}nnRVis^>I(PG6EOd4!@~T`6b^ekQ*~?@wt?opy%oX$N z9E4W4PdVkC{)Xnl#Tx!zgU$X~3+r#R@PF4R3LKwy?)uaiY}>3Q{7)~1_Uahw_hBt= zvbAECdh00htQ&kTuxD1hq{}}wDNs1>I9nCxz%*BKXx4NCW5=ezV~^3S)b|tI?9~#b z-%zK_c>}3QSrgxzD$>U7OssT9o`QpQ@mNt9ulhO-8jqjDd(pCdyRQ;uw)Ur+M`EF0 z_AuC{^bk92?LZ=TFpV@M@$Ms|n0?(b+L7AGue^K^bRGKOX4+6Z{Gk)Hr3Ht;k7Z2d z%QW`BVggw--@${+l~`cTDO}nVh`Xh-p!lU5pAc{b*90hIRMS>wfAbU$Q!r&urv8N6 zr%yo7;%j8b?EquNUg&aCVJ(uc(bi!KI8}$@b_qlA>g`*wVL>yV$W!6Wt&TH;nLFT7 zvac{B1arI8ma#+Onou+RGW}{P!YBPc;aIZ)xY~FeZB4QvDgW2tWMPNuni0&+S&4;a zjHJSyR#Yf?3uZ+7vs0BaDF3Jz>lUBjE+4zcEiJqWpO&14SZzHjcemh{?2{CeC>C|M zMYCOp50O`FA9uaKA=W*Nr}U3$WIi&1DLp+vHR}DCR$3Y#Kg5Rp<9-%~h0TGZ8j|eb zVPB>-P7WTt2X3$B0T{8h5Le|3EWusF*rp@pxMh2ru=g$EeiTRW8c&SbhT}clwSLOL zCJka2-ee0t_$u(`r{G$zaqz`35EYlSai6Ns;(^ruBvUOdxScNXQH2rE9Xgv9MdpIj zvJj^F%bF&PI1gR3$J0{p6>K){<1e+&LECk+Szcc!f3WWpm~LN0n(cq#yxm{aY_O!a zn}XQSxAs)=PUx@SJB`1*6X<<k5*&X%4jY{vnDLhe@sEv}OdG$8Pp^s;JX~r_Lw_5S zQOcx^qfc-nUFKuOD>oP%G>UE0yMkk*s?cHH5*D#Z6>fTj!rMGqCch;GZ+FU&kz@=x z_5TcBLPl{@cs9*U_>7AeH={FK!yjK8!!fTHu*W!+{5rIm#p`&OrsmBS9Xt$E<B}*v z>H=DHiorp372OYyV7ucx_%e-=7;W?nHYr-LsMC6|&rS!^&&Z&acqF8)k7Hw3KgKDW zXEEuDYusP?!E{O_#c5{7;-jrqY)IV~v`mbH7Y2j*=6-5y^Tv8u-Y<m({oy#r>T9re zg$Z@Jnla1P6u#na9?qG)569QmaFOo};iQ}`y==5(+O3PZabCM2AV2|?G(*|=m%6B} zTnEQ9b})mm=iGSD1F&2BDujN!#scbIftr?(VJv;`GVqZjWqmuu_D3ket6f`}Nc4xR z+&%;)ik^v8R2I6lXZWzdG2v|Gf(O_jcphhlyoBg=X_$I%1oqqr!+FkA;Y`CVc(ls| zavcS?Bli$gR_Wojw?CmbFBSj!b&(rAr5?Q1@ACSua?sXu8deB1iuN)Ix*O|DGaGok zd_Dw=&$?8Z{a(WI3mdEShIfFa<vJL#$djyZxWV|vX5hE4m%H|D4nHPv2Qa^8xH=#a z43$*r!j1&;KRS=xw44xk7eTt|0b0?e!kX(=lce4nc(S~g%br}y4#sKGsS~N(lGbiW z+hog|M$7Sr5A@MB)sn7FjNla-*R#=g!fD!9DU$O3hk}<a!!z4^;8iMMf`I{=Oi1IB zEu}#x$bpo${D5sON5S&cD7Lw|7D!$WL$Zq4N&`3kYS<o{do>wvg^cF{f-iv0r{j<| z^gD!^C&Mz$a(uVJn_u%MA1#Hwb$GSlnS9Wn_45#+`_2)l(0CBnZgPjkMa3jzZi1uk zJrVziNypGt3)sS<{;c4!95&NfCiSx(YR>yoK*>+|Q0u}P`+VuDTB6{A^uRH(wWvPf zFN~EdW@gO|+=)-(|EIh+kIU(6--erLFr`V7Xh2j{Qt8^~63IL?AwyCrN)a-YsWecL z<|1i8QVAugYo8~PGDQfbh(zXjjCl9=exCdHd=2;gzR&Oe<9(mcyL~nnI@a3P-sj%? zTx%WYaiE!5H|pse&f2{;v6mNyumGD>CTC1=E~+0dE||+c-=09-KR3dLlp&zrK9{cF z*$BnmN3zEcwz2C9%XyWxr})RW&6tM9c5-wb&0mu3L+iF(hPact&_^82d}OLX?TuIY z^I~iMc}p8N+FXV=PHDjT;m2`yYB@Zp%fKEp1~IicO``X2H;C-4Vt7B*X)L||IFnY; zWck-(De~4<w44$_ALJP}98P5$`3k<#elQu0FUO9H%h}M73NHF>IM*DxLC}ZTjoX{H z!m=I;=>3FX*yJQUBD{zDOZ8*LE6wp^jVlY>+?n+jWr%yc(*kc{?qDq+Nis#Y?EH=C zbZ1^TOnmEsFH1rxrL->_uelEFP7Pq85}ROC%zbP~&c^!(JfLr$5v(xvz|J1WK)3J# z7)?6BiI@jlGEJFTnry%fg$66%T_Ma%+lwzMQ>2yVompkTNjUUiE3NEjjt<2=Si6cT z?E3D)YSt{H*QrjJH$axPzPiF*O3KmAVn23!R1NCtY-3Zq7;$Et6ib76E-NFP?!8@3 z&)5@uFjbPW!gt_drMuV?vY2^14`C(C+ypNwTPEeTklyG-Lu=?I9JOyNYJ^L()5E4v zsPS&5CldwrQ%~a2nk&NE-d6fF`y&=S45Ae05%6ZW8^(qA5YGKkF1mde#t1zA=_#Sq z!Of;mYWKLEqI9OxNrs7()QR|kLe2Gd%ux$vJIAWgc`qq6;@olmxOmPiW;;%3|HP>b zTZE&7QqVl+9Pa<(LRvq@(Sf(YuxE-4iwe@ln3A4Mrt@U#d@+uFsua8euo|_Nb!IoZ zKfqv9W7dDQP>Y-W3a(e_GjabUNW3@BN?lFF{LQ!XW2F{?@<e~8usH+NOFwhU6OV9{ zHNx-$y~hOq6I@gDIKFhNu&xv%&6=}&lEkohXdkee+HDK))t$vS_Q8C<{>vg#JD$#; z$W>xFk(sFS{uqOp^{8{jjHmOAz1($B&~UEAW8ZynnY6$Gei2Ev%aib^-(fVmvzdF| z`x5K9H;TSoP(<zBPH^PKW?ZpFA3D2^r*Z3_@NZ+>>Em}-+;&~??+W<{^&SqCVN}2c z9$p3^aVek?p+g6iZ}TRL#(~p`VwzELgroxW>7kn@^SEflS1L?nKT0Dx`7vT#zwtSQ zS66|aO*}^CJcaK%K_oh{6KiyvL3_#*ym?;4q<r=0-2+QXaxZ7gvXt1SQ8MVS*@eY# zKZkN(w?I{DF=)7Sp~)SV&|NwhdX5!5k#Fm=8{Hx>^6?Hx-`y-e_xwFyGIcpSQMCiz zGVb#B`7=r8sV^xGEMpqC)L1Ovl~vFC!mVpkW%cJ(S;M>DOlG$<jc&7rgQtplUq6v> zE&|ytMNd39V+1&`PVB)<6Q-urh4yzjgG-L}6?9AFX=&;i?yIz{D5pt=MJ8A=pM8zs za*gPbyBLb3+9BC=Ad|A%fk}Q|{MN29xbL7eEELvk9nL8**$Z80w8~VG;%oVnVO3Ui zY*Pl<?&a7a!9#uRUQNuiv*VVjSmO2U+i-HndRCK`%Dp@;Vp}$f`7-%2<Ro~dD9u<y z25*KVmo15F_gPZ8)_4BPQbiWvC3vsq<T8*GcqD=uoMK%bj5CVIWQ{7mu1fH8v6)9Z z^J8f0!Eu}@;3-buorx!NHOQ~(5GEHF@{^w=(7pCb+V8yz@5*GL%r+(JygmxLDD%+l zYeE~eqG_m~J{wSdfS>W%f$5i3ihWfl(C4Pv?Acfg&Rs*2`EQcNjMov^N$CauVO}KU z`zOFD)g1OPBoEGQ8qJDkE@q177BR7XHJ&@QljSOwi_(jg(7v{vANxojM}#}ll2f6q zXP=!g@77}sKAef#*EGe0R#`It9;?Z_tQ~u|UJ!B>Wp>{roC|3v;m^D}%u+K4QqAoe zM3WTiCtSl3i;dVxmxowzR*~BfGYZDdX@;umYWP|qhqo_ofqf@Su#fsHZmV`aXmylx zZl79U*4Kw1Gh`bCc-C7vJZ^+2TPLW0?@n`boVe1JMf{uFj3Sz6p?T1IOiefm+N0fs zn)z`sbMqSrp16>v6z4DpA6pi2_Y&GV?B+h4xCCcHwm~mbCHA>r2Ak}o&-+XmO3$_g zqs+iPtmCLRWv+|hv}@9E*^dVNxG9CE_gA44o5s@Dwh+*fs)IaBeH!%2ikjQ&(Y@DQ z>^9Yj?pI2)5gxT@@E{4fepz4@vIK8!4WsBGwzw#HAUmzO42#T5MY@?!_-DZaA8>*K z${K}<OPUx9Th=ntR#xyXlkLlV#`d5wou`Owyp{+W4*Ox)^<C`i^&Jov5{u*YC5fp` zfN{k$=xoyv`uffUU92vn=Z0>q%%U0n57@A(_eK1Da)&HI^U|py2)1=Kp^>LTNl+xl zT8TPN`d}FzlFHx}Aqoda?c|<plwr|lRaxPovrx5N7i}$kSRN<AfA8@Ie!R${NB37k z>%L0b78Ju3)_OzX&7tI7l+1D_eSrM1arC`HOh;OI_@Sqb`wN`N_f#_7{InMKf1Jy0 znc_x{hM9Cc?EpNCYvKHaI@#qX&SIc2ueE$yJ=$gL#ND%^XpOTxIDDvr+PRu^!XgTF z#@^=2`{nRbxh-6;%rN{aQC`*-rpIb8FM;J+%R#GHi3O|6(6KiwMS~Vhq-9T+V-E>O z(fv-bFzikd%o(3Z^ZUs#laL<luF4r!7aA^BInkGWuT;lQdO0G$GZtW65rXbtN3iZ2 z)L2j_;T@5l$7J4WvQV>NJUG*lYdtiS$)Ag(VG9zt3hyp#*to@P;Eq^0zcP$JkeiJp z-UBVs8ca{^5w5ngA!S#_boO>*dpZuG*ZD`{BNBz>UO{U3^4K6!T)z#^9{7QewUYUp ztyi&X4MGoz{)pWM&?CWjc~#jGc<yY3V|R98F60c}OCo9LqbD32^%!P+)qxRX*Yat5 zFY!sqG3>T@HRgvcAzdj4XnfjLygn<E8iIR5^)eeg^w^hG<Qq_GL@#o)ti*zw73|0k zUDo4bKhmqyWL8~EVax(|43gW;i-r5fvE&_C4L!zHW@xiAZXE09eoOSN#e=giI1GUW z2@qSljZx=l>?0S2@z$I1TE`$Hw^#@?3FV&p9|TwZyRgfD8jYM#3nSV?*>{^R%yYnE z9GKsm64UFrR`dQOtLa0weH1wR{Ys!K)UjV0pN5{FguHrkC%XN8GDxj4U|Bsa!Re3} zw_D2>_h0TtO_AGh#$Z`$Z<|N6XTKKSpJ5pHxf1rwp3a(EAHlVgomqah;DOb51<aBW zYTb<mj`HzDS{0bg+|%~cE@8ZyW?ccJI@Ut^G$6Tk->kYBaHO2z$oDHYBt?T*96HC2 zi8Q}*9~IOD?!_+HA{Pla#C=#@k0~(FehDm1Y=MO#D{<pmOX@Ua8Y9CLRIfaPYj+yb z^Ri4Rtn5d+CoIKhkDur2w_CB|8>w96+%^=P*YN!7x6nOKjjG-#uv1@cxGv2B?56Yy zLC;!vF1yZx{B}W~r@I~OAGDWBbJNM_bXVr}Ln!DuFr7(kNuUCYAlUz8G@09uWY4Dy z@5o*yVCYo|`-X*apWTUjy*(afI!3{lu|vpU&Ki`iJ4;Ky+wc=^9E6rL4ybsb2j#j| z<MRVvRCMJe*O0#gCN7nu_wyWh^Sh~R`oIYOhRH{mm12nXkA|?zefzN6wdy!scQ@|q z)s@Sc8ibL3&3G-Hz3?Vl@S73xV&&v+lwB8rr!V@_z^4vqwy!&XZ|V^|rmjwglmeNT z1U|0f1jfyapoTpuRGgIpvmf<k2X>7oBlR|JzszAaaa=BVOj|%3?CWu$`c<@!&;X;M z3+Qr<P%B)ZL>URUMJD}Yad2i0A2}=nFy{k$k`?<Z=p03@p2j^e>&z|bsZ3RQ-@)wS zK5%=l&COI<h__M~faHT>)|?kzws+8OVLHJFT80eA(Sm2sc|nJ&c67TqyzK!V&Dx6X zF9p8rgP}Abz!Rfg%K7t;`!WxG3+CP~M+4rR#-rLR@WPL?-1W{%%(gxq%&n)8QmHG- z?HEGb0yQ?H;v$;O^Cp)fD;8%Q$ZE@WAU$FrFGhAZz?Dpvy@u9PF&yt<LN*_6fk%p^ zIHG?Uj8!Y)mp!c(YTFy2(qk7+J6aD@$M$F0PJ5uAqy&@fxCr;7a=F$AB05~B!zN!o z01^Xb*e2%_qNR<eajE<hu7_(bjPp}qi-OK`2YdDg9l0lP_kAd@lHHB`BH!Vzg&C-R zKA3mxVhiVdTxoym2)^E>1iN@?(Cqhf*{$Nc{J|AV$aVa6bl5+VP6v0T#soWd%|f0< z-%E#UzJhn*wf;0`<}&7S`?~1ENh@0E_>j*L^nCKpz2V#V8*qH7D|S2Ih%P1ZTn~_E z6OEcd>c$RQGRK_ypOMAV+E9M!sy=l0On>$+V;g8Z?E;S<CgO-ysaRJejhRJpB)L7B zhA=5C-ExfG@7yfbS5TuG$Em!lz+#$Ky9IUKBjDEI8TfwQ3)nSd3@cn`KpVWmVX{dx zswy*4i}4M3uX7v@jf-S1OJ^|4kDXcTQb)F~$e2u{RPbEfagpzmd@vRGz7~t!xV<*J z*_eQ9+#S7(C>SB37Mjz|FlV|~*o&nJ_x1`SPu8vO2}*bVK5|WD8$P>QiN$>!@M~Ns zYj+ujar#Her0;Ql{^&}P&iq)^SQiHQ7h{>JLN~TZPRJqnL;S)w-|%~y1pAbo4NrH! zhegfR{P7t@IC*C=hK2XQPM@5?SX-O2Ce6p*so@x~Z8l6VF{P8;^FY>{qf=v|nChuY zFmz1h^CO<|`fcC1uFZ{DkYGXY?}oszg~ixCbQ{!;%7%oH1+=w&ENkea#9gvK1BZ<> zIP=nDoU3h5XxQS8rLRS-rT7TGj;qJi1J_~ehG_Cp>jRSpB!K<=JDl;mBGJIHU4(aq zAvz6_U~`0d=lVOzBLjSs(Bz>xd$~$iEb*ilTdEXGeD7K0vCj#{o;bsF^(2^TSqHp# zJdS1Czw;M``PW(2r{NChv7uWgV_KpT8`NtnPWA}p9oB}TMbgKSrh^msBNHEy?3xGo zXzO0F)JkDq#vz@Z>T1R$TZZt<t2V%NLEli}f*5VqUInNQ;2)=4!2^wjP=Bm1oZ6Fx zFGq|Zom<+#_9n3d4|}piuV|W^<%qX?O#KIcWzYPRi~f81-NXN5{h0rZz4CuP`Y-;< zf8W1-_<vkKS3f$*Dz|$9C~eW9dj@&1{?H4YBaA6tc^v|Ivs@C-YKASVG{I|rGWLqy z2hVEE*tvuh@GQRtP4TBu`jizZ^pIq`iuQ^%Kf9p6=Sc4HH!qry_YOv#E5+=NF4TD5 z5liOW<-fap;18<mFmb<Vc;~tu*WS0M&8~-W{3$WmCaj^Yl5%{5+b4MaLY|e5U4v|x z61yTT$C*SV(6q5b_%R_h!gy1Xp6U*yXOreIRckY7X{|z^nI?Ge=_weZCuqi9PsD|B z)7ak2eVFvfeUN1|9o0vNVr%4MocDS!H=@jeTQ6Y-x1aCe97u{iO|YjH;r%jt@Dp&? zF=r~RihRhX-Jmr81DMK{;PMzrW>__!J$qtK4a@Xt(~n*(UQY@2okD2Q<R-M@??ds} z2r!R-05+fI(J7TQENc4!*M08uvYJ9}jGISx)1L?9`>N4(qe$#hJ`7yb!<oxKVV-6~ z9#-qsL01D!GHCK;o2N!YqvmW}VLyuI4BQ~h9fs5W&yKWf{xR&9Y0i3W-iGfwJ%>E2 zTG(FG6Km?`vR)HQn0WpH^z6TnvtKRAC|s2f>PW}Ero?-MnbKP~Rp#@i6D2Q0oH24W zHgs2kri4I>i(f(`ZX|(1h9x8n3}^SpzQ^T{YQR#_g|#`4fqn|}#gYkKSXK2f@;KIu zcXsc@9t-O<nvSRW1APohe}EJ<wk1NFf-)S=u;pr(CNP@@M+|j#0lCg9IN{a^sA>4Y zpVe)E_=&HfCVwY2D_XGf-V1pTZ%J_A+qpNFcYx;j$3k6fFdQGdhYyR<W$$$(VSJ1< z#Wf1vl%8TZXgr@g>^B$uEhHhLbu})36Uu+qlEW8HWr8k)Jo*$Cu+oeaTy(M&c5Lo~ z)~g?g{okmOto0+Shny}`nC3(iujLBkN<r(ur#s~y&Bb|}c9SyfBJ0zeaY?#}ynh&R zkH*cRuSLZyX=WOGFmM75lPiQ#caz9|s!%U+rUWj@U*SqTYK8Bx7o^q>W2NfFIM?_H zlRh(A%paAdclUHyYNswx8FCH0k_M7<gC<L;=tom`zeC^s7dh$e3iL%e5O3b<&4Mlq z{yo{!bkWQX-c+6k@s333_31EJe0zuqZ=d1l7-7u3DUmNb*_A1V9mcZPimd$GJ$SIo z6V~eXr=s_7;cT;r&5pHa&p+#uw@}~HeAS3uT*cA7kS085yONSuMq~ZhCh;mOAKdsU zU7Y>Bj&HUo6vknfxowWo%-7uqH~Z~kefXP@u_+YZ>CR_%Mz8t#0kZ78_E5e$Lj}SM zhvF!2F$_2u4TfWc^)RWOwBY3zSm8U6Ha-c#g2&ygg3PjU_a$%gsO|&4)nPQ)bR<<R z2q*dPn*7YCbJ>nJ7VKumXO!%Agm<4Yl}=2&OlhSFOl0~IPVVkOFBI1DTkcu1nUmdc z|6o~W_r#jJmBB+;O(~|;3?Y3x!Fy))Y<RXrh1p7Ea96)2@Hekja@WS`!2-v9g8zCN zDjkfX7d_@uU2*{1SQ}0a>n<U^?Zf6r*|ALxTj5)M99Wn{VA@U>uH|(ZymHls!ck@d zAH;`FcdO#}I&6S?$<xqF<{WBob7J<To!F(E&wSQYd6;5p3Oc%1VSMINreCPdLS<4g zctI;#Un}8{_lpJ5`s0{fq0R5w{R1MdWkckk3$SX{Ram}%CI#0i(gF9+T-N#9te=%E zTX}u}dr^`u+Sb&<@1Jr5hfUjxfk%b@j+#+huO>Vy)SLG7GGq=Xk8v&Sn#_834QF6A zl{pJF-LsYI(CxM^c@-CmjLQ>53oW*?(gcK#KnKc{e+9k$Whp+?lEr(BBkTFwz*>6> zi|k}fn0S^V>tDc}cN^LL2RvT9ufuv@&4zD|H{h+O6m!=*j|Eoc5FTqo;vi3cp}7;f z+~~~?Hx#qG+<boQP$yQn?F^LNn@)|M3u&WSe|XfE3NcrEqsEU=8m5#d1_e98&#ng@ zopgwG%XJ4YX38XFI+6MN>Ad&5@8TqNeOjfe!6$nukX+C*>N4O07#oyx@#|B-^p+~O z^<*mgJ=S9rEEcf|=ccmC=vgo~v6OcabQObk>CpqF0ko_lAGM=CgKR%wdPnrxYGIyS z`rvjPpm)b=!gpCPAN-zgsh*2pmaU~@ceOy_$4FNFRgvrq`;#y!1fPsJnw7UxY*TO! z2U>4{+q0$cY~~4&x^x2Vk{{s_D@mI5qlhg}Hl*R-22g2<z%Mpz;mkVqz?)nGo@rNy zuAzcP+755>u^j+8XDaBO?;tw<I+mC3H3eJ;4`A2I=AhKXeXQ={Tdrs+K_>|<@@m*Y z7s|&`r|5Oi>Us@J){kN#9er{1+A&Zt@GM%_+<+ma@~q8q78}h+Q-g0k=H&X|N-b5I zw<VW$m1qgtjHdWaxfny|T8f`P7twoPj+u?V%+`DwMT@(2#ln}-e2Tw5U6lNY>zgug zO!`jr%dN$b?^^86{<}~RHklpjtHGXZb))A`2k@Eucfrz^H!${=7PCHhkI#M;#60)p z3v<d>g?Z_f!tB>JwyNh)lCe`|ez#;fzD)uSbR@z?RY?}#wUqPr?2Qhs+u7IX^U!nD z30PlZNa4||xa%*6F%^qA*mf$Ae?Fvy^ll%4Lz4<w$$%a(Y2{J0-PBdwtGBw~BR_-P z7xX*BmOF!yyB5ugk7d$fhbZ&e1U7p`SD0sT2OD80bM}>`_mj5qah+rE?e-&3At9^@ z33FGroA-)mD>sT|y*rm@wN8i5tKFf?+Xh(9JFxQ0KJ0R!2i`tBiAk>7!AvK{WA#Fy ztryC;=NX-tYt&%cB5()8y>yxNaCthdHiW*&_6A?W7>IFqhwl4H&{{b!`g&Z3)(xA2 z6Po5hSMCB|om_>J1p>v$ntV)pq{S|16~mZ1MK;2B3%=?8oqucR&%N9u!|Mtpk%~h% zx#yb&4L;AI=x?46&QaA^JhP>I%BIg)JnsgWZ0_KXJPQMlWfeGeT`ta$KTPvBAIErk zL8Giks!&Uz0-mi&ti}BXF0V0!;n$R?OwiAjvXNwWsvp3<+otr8A0(^|`;wVsJ!B0t zW-7^z$T?JC#ls_<^p)P&(YX};CpyujZeiHfqnX?6d=j2uGv8MH5dsce;2%{;!}##G z2qsVX<9R(`iJ<MS-!_4%4jxMLL~(3Hd<<6gOJt`nRDwdq033HR0V6_3F?pRh=5?$V zrwDo6S2IISX=*hKJKvkVxcVOV)HLw>^5(LSeT~`8w9z7o#eFC&^#+Cw>&fD8YZ7<v z0Vqgpr@(7rw9+pGvyKg6@Af*Ay>$W?yLB_WKEjiv2ASiDHDS=$F7Qx4yWxjC4;B^P zmCg;y#<)F;L3#BMkkAMb)^eq(s@rY;#zhU*qvjSDFuD}HZ&_ovEiO<{u!M|H6mh|K zLeZ>1SU2{b3dzsZXzS)1uwYOgOnWsNl4N|rX>AO;xJa|Zy<@o$y-h%VbKpmS5!2lh zNQcy;Sx=wKkoJ5%sg}lI_d{MV#<&Z~U^*{*!;Q5+{f-tN!+Cf2GHCCtN((O<!Ngal z%+Bg88&sDJ+f40P{BC!)D8-B&TzmwtPt0JpcC(m!K_L%XQ{bYY8OFIvQ>4ur)_+kR zCcTcJ_caIM*hj{XaP7<<`C78|bKmooBNJiWXfyUo<vb<~R;TP}X;xkk3;nySf-Bya zt&;M)f#N&C6EeMmZ+Xy$Ufva4>F#hyKW<7<dMbhkKmqREbDIAlc&gsMH<2c2NU+#Z zHYgL`ila{?Lh5Wyn*3@5yEeWPY-}5ZgLjqS%7Y2~DW$<s_v$nJkiG`8U4rP$=UBF= zAe)_+H^!D5UD(|1V_1@Z3AA0>NFLfjsP{M;#t8hDLjRS#W|lr(yF7sNcb$Zq-vu7? zkP=kOQK0#^uarO4y}*}PYH)swTOn}dYB;m03x-{rWaTsNGheaSjhWPqU{XPe%(H(3 z$nF?T&kXDM!pDQrIde0fvl-0db}6vT`%V;49Za{Rr?A$>D6HSQ7{rI?;)jWnl({fM z+`K;vXTP*Y-wPk`+NC(W+}|5*ds|`eV>h|^t!DJWOoJu_&!h=6y0C?P+C?~hDZY6d z$od{Xi+hyJ;kgwL{sJ#dFM3V6`<_&*F6}L_!<o^G(j2r5s>XXhaUk{!BB!<!ETwP} zx2@-MyeCS<PX`CEhN63TGF=)yZBL`~tq@-S>k0g!Dh=YGP(H#!f?cp(N7tiraL{HO z);~NEW;kBw-4dh7GDYBhH_wA32{Cxux}I|k?@m{ob<s;Il@&b*gZ)*Nkb1{~^1c!z z1svt7TXV2|Yd;)6MuD8_J@HH75y%`R$+~4dLQntgwApSiEWE41Vk6_ly01pU7SEaN z!30O%CRho}q~s{rWCc{IPNZ5t1<~NAk1;G}8uSx%j;zcS`MN>%=<7S1zGcdbVyp+# zk-Q1){?{$E)@nB{bPi@C$Ev`3LGN{fsyjJdI|&m74J@T6=fF@xc(y*>$9>2Z^vP$Y z5U<h?Hz|IEuMc%ug-Zf#;eS9@W-#@8#WA_Ccoy`vfXNISk4-PHV#&4D7@~9-^2ko? zrDo6c$H)kpo>CY*Ya+c7*1<{(w{iD2hJvw<I_vT{h5SMviA|&r@$#I)$&x#%%=}7) z)q=jZ_&}}zy4qjIiY+Z%`>F#JeAyl^SnGgun>JI6$$-yM-Vk~_7aqUxfFFj1?CZ8f z^xHm=_UbqAI*pUqnwJ89GeVOrDnrQks20iXcjR8Z)8^khXQGlp1ti-)!nMbA@b-!f zuJE1-vspBS_b^nTLlJ6xvwlyyJz9j$68cQ)x)}|*T?-{HjgS_g&BAqaVZlp%wkT8Z zImmd19vyr6jmHM!gpU`o)3Xp*{d5S_bofy>-#Ku2#|}a7Sb+_jFrPTJ>u|#45~sCQ zjw#v1U`SXPr1gJ}i-)D-H^1quGHN0@e=w)BGu<F(w2An!jRYjGamCkx{U|x903{|k z(NclG7I#XOVz#X3SICZnEAK>TD=Cj%ly}e&u`=6K(2i=~x<kKiqbNYPFVl&x;HTy% zvb{3yOcdP!PLpjhv7!+VY+1sZE4tH%x8ZD)&3eHL(@^Z`VMKL3l33xfKuQZM<*$x$ zApe>@Oy}-sk?A`<X4qvKrFLn8+`juz#?T(4qC>cc<2bg&-~y6rC0OOylGEX#tRf4r z>#2Ci-jIctR3+HNnSzhu#x^+HBNe9}D&+mD2eYHAg!^a=kMF-4qGbw+hJGH2@1_o6 zx?Ww$aNHX>GBXeQ1#4q#BEcMmY>ax-7o%PY>#`p;>FgB`Di^#I#8)Hux}ZZ8)!UtR zF3rG5?Oh-n7zZigm)O|+DshFaEHA0O6AA<`My;1!NPGHj2#|WuliVdp`!JSvEwF$Y zf~V$&r^D$)e`ywJX#l$>Dw1{7cih^k4@(P(<+q&J4k@ZU1XsO)v*#YdJ>yNdlau79 z4$dbCf5V%OtVap=&#-I3R-9u0fSca34-D+RaNUDl__b#+w_4;S=mz&@Z8=4_ae}}t zS*1)Lhd9y5=#Q9~BTuut3cSoGE2uoX2X9=p1LE%_z{NfV_`z=xTPe1{dEsv`Co>I) z95-M$7MKhA|7vVd>lECsy@`!j)`4#;Wyq>vCR^()!>Y<e>|FC+TulPMbn0d{>G(YA zs1y8{4$UKjhpAjZ+yj1^!YVkcTuDV8Tk-KMby_%gIQzUy1_lPl!u5x%xvA+}*`3V| zkmnspNBsPGJC9Jfp>E7=j@P6+CTlTs)B&)nU5(wRi&(rxCF-%+%=)=D3#>Kg3W7hN z#JHO{yIhxddfbW4b^*|DkfK-h`7B@B5K1=)THP+5uxZ>{+I)8(&Nz1$>_-$pS+7*c zE3pvt&1cYKvlsZHRhlx5eDK~Ae>fX921-@T$!OpbR()#$?oGXn>gk)A_8DdRcBm4( z+uq_7yDCm|q$I7qpN|oe=Ma>n*%#H`_(WNf{ct@B8L_ii-DNNKu~#@$xZY&1Y8O*g zwF7+F&<8(S?4$d>qs8gh7tnc!PQp6s1SVG&iGIf#*pGclu>8~`Y~0Wf`m9e8$1mK& z?|KwWIz~ZURv>a$Z>v*rl)X5maSBCctzk`594W_VGHd%%%@?Xn5V;BCooxnV*u)<l z+|?R$QEby-)+4C^6iiQnL97O7@2lkdcG@X6X)M8ap$cSUy@YzI45HWjmw>TPC8oM8 zWp@4TX#mZK&P_Wp?6jcAVJv7P*NCXU(QB@$M<x!f=>UyYV`y;q`Plq?8FZcR!mgS7 z@*)0xxo=NT!os-Dymi45nB=sRcQH}H?;8~PQR%}W-+TeMT;9%5hKH^dQ#iTP^6ZI^ z8e{sKLH0oezo}i3Zo6mUb1THOt|joqWfIPgl4iOKp22N{Bi!!Y^I(0Cqikx)Q2N^4 zhEfMjfk|$}ulSMzFm)hwTD}S<y!!?%BZ|PVa5Q|B)L>GDonc`i;;e!k%3rjDyWio< zTQxW^C4uLC>)a8h6m5t{$IoUtbt+`}W*5A&*Z~tO2eBV}<8a;W43chmj=K!k;8h>F zvW15<V5j32k`)O)I|nrBwT2((IprA4e^Y{%!d%d%JGSVlD#!Z0QDq;#OF;2aJLWuF zj^-|p$Ba$!_^mRZb9|yiYUe|_Chby?U9^=vwuVz!oINaze28)5r_o1$bE-eT544Pn zaB*K3wm1C@&Z^O5ufJ@^1)rN?>&uyBIFbbZ;{s9(Jc82WPQa7qdhU?zTxtxD5OD%8 z#i(41Udkln<d_hFE58nk9b=h-^$x6%=}q5sj&m<hT2fR;q$sgr6>Hy+h<%Nk@M+H^ zFmY}|<4IO5^|&-|cFU07uFi%1^-t08RtzbYz2X<A$>8hInRv{8J;b#K<A)Z3llW!2 z_)53|n><H?y;V8_YqvdwBOSd^F7+z+Dr%o-rPE>V%zz|Jb*l!YXG<U?WtVuu)*$Sx zJBD4^5W{lFl#^&E<KBfN@Li)u0ske31wK}#>g1*9+ERF;=<Gm}=^qHH;!Vu^>n99T z=*;$Lu4VQM5~%AaW0<j8n%sRSvZ40LXzV7<7AIcdj+!mUWNBHla+Rg#$C)f{HAfcn zWznd%JF6Y6APN%J4X6C*12cs6C+jgPwDZ#vToJGw)+&yo*tj3O3Y)<1+Ng^%nhN~g z9+$Y(b;n>vlqQ)^=|rk7O*k}Jk!1$-Vvls=gy*U^sq}GTZP_+7L2V7_rTRnPoBE{j z`Ung(Ur%QT*5ZYtW8(2P@v!M(KUz5HICtN<0HzHwV{Wnm<=B?V-`U#?<6NdN3xPK* z67mnV^Bs7<Ull0Nx`=wajgh_G#Vx$&$sC56gYvEzdNq`zuxTsUCeKQ~UrG{P+Zc}X z&g|d^P2L8|>zvU2UT?DC%W%UhC8pI|ht)n!g~9E@dhLmM{G8p2EL<j?UCJ_KC$6o; zOL@1r#8)>kYV=;n!pCqx!kbO4S7fJ!vG<$!0Gg&>3NBsh*&)4NB<}s37c_{VLRZM; zbE{xo|E+LfPcPOfH=L=;W^t;n@zC_@9)J1PaTvgclW<QmowPx)yk$mt^XiGzIOe5Q zwsSuw+TR^79ul}2ucZau>{6WHbd1xf9m<A3U&truRpW#_P1<J}fc_ma^m#`xeD?1P zC9j`=v#@sCHBL+?kIB;f&WEA<q)kj_Y%}zbjp6mThXB(AFn(5yH@CikbI+<kNva;s zFE|L>V{@VL*m0KB?989|`U3{|_hhLCU0B=WFEC=wT5xbJ#DxoY@&2wpDBazc>2ySL ze4l}E#CIJt`OsOY|KE%D8-+S?{vj-{n#ejdEI?^TA4WZ6$idry-sbM$7Qfd-9g}Rd zi&@VyEboB+&EqKFF@Trb*d&_hw;C%}1mgzhu59KbwetCIRVdPb2+hwG>g@U`ut?cZ zkg6L>v)-=)r{O)sT^1YAJD(74Z)GNPJmAO74#=~EkF&wq;5p6_%dq7|$}lD)5A~;Y zVh$2ppd@<*UD@7=S;^?pTf1HG=u|6u8+E4(_F>F-zzNnRXgcn{uTMUAM7S(y6_XmB zgf*%6z(hsh54X*Ms`*dA%Y87jRFYr@x6klKbN8{#_()JX69dUz^$7M!k;FU)dLX)j z*S;sihE(BwZ$FYur#}HX*Q4AA*^B7&?i{ZW)dP*^mGcI%JGi`p(X=L^5R(gi$y_x~ zw658cR;Y<Vcgj-gH~1AE-!ojO1$YY5V~ttZdzFz575&L<xCOa~dsD=PH&|d4C)7l( z<=2khjti^%;l8=T{NLHlbX8&{zPa}hOx-hKT+LXPGj1#vzx4+FUel?4qbq6*=@4tz z%CNZ#S21CI5dYn*0YesKz^B7Ui65lFeatmxO`gvn@vbzSJ*~ldjjF{Vk>j{y6JKz9 zd*y-DGY6E*4*>fem-*_Bo}_gUXxo)1;_>Q<g3tCv`d($oqI8$S?dnGH*Y_f}cgPWZ zW_uP|KbCNdk6QDA3nIDxo5NUyz{xs2`=a1SHj7M`Yr&!Z5zKX88rWQ%#BOeg!Ve!# z!ibJCe)dpZv6i1EDNow~&z>r?mG9C}IqM*7YMO;|_j<uyt{9hh-$MC~t?(-O4u6WP z=5!Y5k+-M`%zH#z-FLNyVRBxW8B~CFR%!HQyE41h;>Tb5eur0)o53_TUBm#VfjIlc zA=-K_2y7xQh{qaea(dRG6cw1tRhOyLMrgnlj-iw@rZ4tfC#+jv9!4>qnUL;mOq=}; zxuKRS^rgGtUt_$AeO-2$R%WchK#MET+jbM*xOOc_wFbbRZ&TU$Q8Pq!SF~`x)HKv6 zkfH6_cf=hA4*1GblcDN+kw*1gs|AVC^ug*1$W(;m4Ph+M)iDfjUi!lH>)dhj@@igg zUm$9gY~~a9yP?e#Il5HXkL7GW1d0W-Nn+a;UVY;oZo$K$_+41@@tdTGEjAT6T6r8r zPK+aaFKt%PTbhkumWO@g00yq_!VGewF)u=y&NiOID--JR<nkCMzB`r~43uC4PY1J( zMHAWRfEExWu9?o$794zZ)j#~u!!%=9pR~^)F;$*!hPUy#>qEI2<5MZ=oG%4EP^IKW z@)Y9Q&Zk*Ou|q<>5zs9R<_`8pGx_Z>N56sF+p`9QY>YKZOo7(;!9p!}E9e|;w~Dh; zp;Nij>0$d$?1)~DAsV%uv6MQCUGtvr(DH;zQ3#pscH_gGx`NF5O?Y{zKVChh#A>Z8 zu;AW!IG(+gWU75}%Y0P`*>i^bT%UjspC_;ZX*=Meu0E>0U%-hkPp45uk)n9lliY$5 zU98C&!8(ojjQO=8xTCugbt*{^c$_KFeO5fHTm6z-_1TAFZEoP<WI^x8SeM10=Ge5f zAe^Rfk(V==O;_y(uwjEnvMGBmgRNpC8J!%!x2%Z-GuJ>um9KC#Z5C?~wAFVTIkO8b zhr$2r9?l>sOVC-~hFhPx(dWY{ob`KEtU0gC?#e#HHv+dVdgl>1I8u_mC~V?`Z{5Kv zM-}c<R~wYO*j>;D^kyFFr6AR9C7s5v;MbA^?=qJPd9xTopEq-tMZU0S&oGu9H;_Ig zgoqcFE3)jQ$D&mu1z*pm7w8=Oj5~KAlN^_hCW9^<=uF+tyXJ(8YrDzgp0;72oLkGQ zOPt31^`SU3&=L46iSY5uYF3@6O<qF$KNIS;yXsl6yT|%6?P3**er!wW*Di54ha{6u ztf@G0&>FryI19E3b6*=2?P$u!d9dj1V_2AwNcpxRP>R!scy^K-Vy8_C<O!Wu9uxUp zO=dBI*7f*|uyV(jKhQ}zRBT<+84pFa^D&}n#P|P*osVj;%O=gtMlBb--3QQ;gHn)V zI|mjX_yI*KNw}rC38$uTINx;|o-dh<<Ff@m_sTGwx~~`XS>6HjI+fwP%kmVF_7;_; zQaM)_Iq;Uhg??LJLP4(_ZugqkoO4x?sQ;NGc)Ow&H&od&(Yh`)^;RK&D|HljOm@b` zH;3uW2QThg(Oh;SP?KgR$n$|272v8ZO-%g}e=yCP6}TFT-+(4emnwq4_y<y1CN2L> z{>T5%^=pU!$NIJZXZyDg|Bv<a{@-Ts{1yM{?YXCeg#-C(|95&W))F3zJw2BAFY{g+ zFl?b)fZH&$|5(2#e2t8tZKX{*y5Z0>E>zT1YJ#?*%V<O32hcRoV`?X@%Cx7<r%eNs z@Ib4e;lZ_wuKGtqucIlDnD7!R(2}w?zrf{#HJIsS5k8)oTP|^E5^k*brf<Wp@R!4t z>1v}ch4elRElGRG<gTh{uAu4KM}9JBn{L2E#r<KFp(f|i)}K}j>vT1y?Qr?bD?FO6 zLiUYAVDgU@wDR6XPBJqJjNZC%Ym53)bWT3Hw+_d)wSvZllsOr8-p=ydFZ1~qrollb z4?UiL;g9ru!FPT@@cpS8mQUO+-u8GK40l#xV=q@haIO<OOD>$vyVJsY`v6Ftnudd( zHe-#L=iScsWm+@#LDPa&Y-eB=Yb);0iZK;)3x0sQ<v=LPT8XYrKlr%|7Sf{-6VTc@ zk}MAF7Z-{L;}bm_oM0wHHd=#lRdz7G?A?{`q!o?chgY$J!bLPWGa7ux0Fxc?5if@i zCWYQz>C7!2Uf^9&Eqa1w8+t;1KVFnPDHqS_iD9kOIJz`p6b&0(#P1v<<OoleGPAE^ zIo-ZX==-~1w(;vMR4bFE3(k9R{znaJt(gi6Z#gD4-Icu@Ez6AG2)f4$Vqkb$7-xA$ zhP)=1;a%Tu@SwE-zuxFs?rtl1cRrcMvQB*C^7FD`4yiNES}BUqeTSzj&*Au!J#o!? zIl6OaJX`R%fSc`c813#k;9Q4z8WkFXy(e{NJ4z<7v6U0)O!_na%X>e%VJZPWBPUZu zqdEJr-5%Pajj&_gHON_Kz{VL%<C}*<ZGXjKenk2f=CV_rNZO8!w4+g-y5gL46Vm-` z!!|Ycr59C6w5(tWh87uua-}KWj33Dy-{`@^vq!jar4eO*3jNWbu{)$6Im#ElR=}r9 z@#yg7G#yf1&e}w2q~)@cRI<bAP)PssbMqIH&F%_Njo-(8tK7-LtG>gtxm&PwgA$vS zE6g?crDJ1yUy>cY2#iwS!hsWlk4Yy}EVZy?3NsdQt|cZc@Znam_QN`8_i`w|@5w{H zd@x1WWI@fu6Wo=Nw%pO}VhEaT4C%Iqz`)NNTOZ1^cXLNzwB#<_uD71kyFLXBoTP>E zq7!7>$<Qko9pd&iVC5@oTz#t*oc-0vM_qw6s9oVN&9xM2w|0S};5C{iF^sv#r%?8w zUJ!D?k&UfugkxzAaBHv}8LOvIr{Xv$ke-aI+{WO*TY9);$tcnuls&R0)dRD(>%qkx zo?P!fN5EuwAZag(A&aPyY~Zahc46o{+>lm6J6@>Len$y7xpOC2e%p+_W<BC;?-IX! zn82ctoWw86G^PX^N`4{{6S73!ZDSaJM%n`ph4-iH=^8XZe1czkKo7PF9D=bc9*cL} zolA1YJ(;GKEUR8{o+bLGK$@8x+J<ienM?lAXM-xrUtL32j`^@_b)oS7-bk38A?W1> zsxzrci^ybeFv~k5CgY{6>E6LH{3%)Ok?~1_@4^XN_Q-h~8CrKHc4;cr299MJZy#`V zuif#B)J8Hfl45Tc^<=&<miZTJQhj|Ue|Jp@T-{QPGpfwVp#COj+be=KT@1&-yNiW6 z^jOx%vKC&3C5nRf55`K>3>cv4NH@nGCr_W<?7;k6g8rpFfBVN^3LCy0%uZc`0&@pJ z2i$|rN?M09hl(&P#}Y^18BH=9YOt-tkiWQgDrkME;m1b0@irS1pd)h-NoU1jYN?18 zZ@a-gUaL;izBo{f`EeY-V-M>ZoD7$WdNQ>@W$NVShJp3%C~7%~_dCik_+1)n9X5+q zSN0*jb!ph^<85?)Z~(3ZPh{dccPN_mj-67U1gF)N*;E&6HtpVZ%z9NrKd$^hi_!!b z{DQG3yc~<J^}<VML~KHtEt^~}&o*2xgy{1_Nyk=;Gl_O!_f36Z)?6TsP4C6!LCsL7 z{R|f=+KE0j{^grz3m2Dc!M9=w^3_hnQ~t_qY0nF=dBivtohZxP()HQ4wfB)%_=47g z9&)EAc9_&)1EujYIJ3DwtGlEGCwA}0)bobaS>6b<?^wgi7g0ENqYZ29lZamhjgfN~ zorN_=5wfJCV0J#2Ke54x#%&B^idQ|!H}MUAFOXs{r%B;4yBP4gIFm`%_F*&Khe4g! zB4!gIc-p-@EIMsx33GZrfr9WG;8RkC7B=bV+FF8zZ3$?+s}q}-d=Bo8l4j4A?&Qvg zF2wkuPS9{XPtd{9Ws>K{u!t(5#$|{b_LM#fRTob3OM5)V9|2K3R%lbor-g7NR+kNN zG^K!?MEdY#FZoW8q7#F5uw@4?@H+>#!U^kMLjCJ{a8vpK_S)8T$zd8l{mUa>p;?mk z6F45VVX`djaxk0KtpH_PKEdqAC46~O5-z{hz@L8JnOujcvgwOgvK!g87?szL!uw`& zo!p<n(3JjA<kyWIyyk#Gcn}5*eS$WhR*+9aouJ=U1TU_LapvTU(5|%?4Vf0MlHSV| zwPmv&%3dryRGDVx?WNH(hm!5VMBLb?2k7~lQ>c$7U+=JidG^$1lU=|6Bmb5YXD9sk z^aIWQKdQecCnuy*zihwwkFNGyvC@0l(*M-|W4OX!8Qj%9i)Kg({r$E7>kLj?V%Xm^ zxS{_@;Z}OC@OJa{4sr_+j>gq{;kYpd5_A5aZzkr$O-2kiHntpTKElLuxVeR;iJ8d= z3o{c_b7SKXBTOttnvb+FH8J_?D9z4}Y1J2w@UPu&vgBU}EAh`W)i&c@zwNyxB>vi! z|N4)gwqNqjq5eyJe{G=jRa2;>0Qzg!_z8SRX!36j4E0*J!p|+h^?zP~U)veAX8DtU zp31&IwPP<d`}cNM1}t0Qw#d^}ID3nh`gtx5__e8Uc+k7E(8OQ6%1=#Q`?)FizyQx> zD;9dL__YNsQTK>{-WffAYGG!dpAP09m(uuGxNg2pKM5HDC+t7NYjuBz3)kr9Yf_c0 z(LY!~|F!=Nw-YYm-%iCpdN=(Q{`X<UFYv*?!-xM0|2Mr4`5kWdEBxQ|Zty#N#INvW zsVO=C^z`}V;0=F=oBs;eu6rc;XLv35JKW+|_`f;$slUT5|2N>%eut0z6`o&r_SQe6 zPD!}-|7)|G{yW^{x3Cj$e{B3`eC@11;D3v|zw5F7(*J@#;C~Cee~njk`vd;JBJUr^ z{J*~JKR*53f9;>}zlGlI+_M({^RAQl8E@_WhyH(yy~#E*Cnfp|-3$BA_{2Zqe+$09 z*VFurR}B3_|G!0FPUR-cKldLS_6PjG4ZmOd-~R{vZ}Io{JO3Aa&hL2B-{SA@ch4{Q z1AoB(7JvVy|J*;|e~Z6=)BnLg;QuTB{`oHayPx?l$Dj8H{BQC1cR#Y9@v-H9!2cG1 z|K|8F{sI46{QVpJr9a?*i@)A~^RxK*_!VpZfd4K2{;o&;Gd}jlAMn4$-@kc&-2ELt z{I~e~*ZA5$;r}cC{_meZvCV(z|F`)2*Uuk`dw;<H7Jq-Qhy3~cIsFO$Tm1d&=TEJW zp8c9P4F4_u{yrc7bN_Zyf5885#9yt>AMn4$-@m^9YjyvC|1JLhb^O=r{Q>`5{QZ4A z^z-+x9q>Ee?0+Nv?1ugU|6lR<&*#tIM-xByuQ=={{J%({|DNY-3$L;NwZ;F8|MLjM zZgS_&LXP#1O-tJ7ztG12^V2{7?9=ss{DY;j<Uc+LIN85GICB5v?|=PnH@VYa*|&tT N@awOi{~tg5e*p+$F2Mi* literal 0 HcmV?d00001 diff --git a/tests/fixtures/tl_checkpoints/gqa/meta.json b/tests/fixtures/tl_checkpoints/gqa/meta.json new file mode 100644 index 0000000000..05edf76f90 --- /dev/null +++ b/tests/fixtures/tl_checkpoints/gqa/meta.json @@ -0,0 +1,13 @@ +{ + "d_model": 32, + "d_head": 8, + "n_heads": 4, + "n_layers": 2, + "n_ctx": 8, + "d_vocab": 16, + "d_mlp": 64, + "act_fn": "gelu", + "normalization_type": "LN", + "seed": 0, + "n_key_value_heads": 2 +} diff --git a/tests/fixtures/tl_checkpoints/gqa/reference.pt b/tests/fixtures/tl_checkpoints/gqa/reference.pt new file mode 100644 index 0000000000000000000000000000000000000000..42d7ab5dce875dfd48e09932a191d619a20e4f44 GIT binary patch literal 2165 zcmWIWW@cev;NW1u0747`3`MDFsYR)I$*KA&i6x181=%@nPCN`zNT7kKwva(1f(@v> zBtJVfuegvgxg@_RIYTeLv?Md9m@B?0HK{Z+CndfFD4AarUuMKr$fThW!4A|>3{;tz zo?6HZ(Zc1EpO+pC5#uUkiC_dOF(_n>U<T631*L^--Yg}B?1A14-i+QXZG{}(EZ&T5 zg`Azq$@w`ssmUdo`FX`${zWN3C#Se%CYNv(a%lz?a+ehH1QqgvotTrKo>@{{$d>_h zK?clLx19XM61cVeU~3Hv1v1#Y9ZCuXq1Jf|v=<6_J9rCtGqx8BcV>W07AYwd4Js5X z)eG=u=a|@2q`(SvF$f3X3{*)56Qn@ZOUp0HO)QBoOD!q}I^NC603~dZf*B{%5M=lE z_jj5>fd#_2?UrFMMYbCn`0<H3ndy1Csd*)CPKG2~A7y3&a)5n#;Y%i<<sgjP0m2Nn zNDfG<EJ@7=MV6bBCF%CU0wgE1q$DR5<c3z;$C99A2*SACAi!XQ<OWcx);Dl-vLM?5 z+(6qvfDb}3a6xH)C=GI%#`;Ay3P3&x<8zr3y2}jRoa6|&tR;$NzofK?y-y~meaoCR zb}a}0*oFizwzs(vwcl-}#J(8CCHvI4Uf3<2EwN8&Z<4*qB@O$8J=g6+uCLgi_+gvf zv!;ao8g&zFUF7oZRQj{_7nLX3yIak(?=hdgKgV{}UYQ4H_6f{vvpE)h*iQ7Jq`hB< zm;I7o+wBi_9oRqr*b=+h?7sUHcAT<%{(6!f!@6y|pKvhSFVcK&uhJ{Kf59)4eI38v z?oHs>Z`-r;m93i29D9cQZ1#F$N9}8pQ}*k8X0UH4pSAb8=tR41mUHcAZ`-zy`@+%v zd%~LRGyYfD#%S}}X&gGYPmY7b{(|iXTah$1`z<>g?7B`l+8>-TasQUBg?mBa`*W-F z6L8wcoAm@49Ff8op5<%_1TZY2;h>GVKrXmAFFllS`d45uKyrZ|IPn#y7G(l+U{xZh z^oq|+adXlk$w3AN2IiI~MusMarbb5QMiwSUmIekUrj{lq#wLbl=B5^g<_4x_MiwAf z?rHjNq5({UAROS$2y!sI<VP;Pgg_D~0J9!I*N<ETNuuZt1;z?OKU$H8ZVYlkB7<Vg zMO0%jN-uPikc$Ih6qAC09!EF`UJRidfShXuPz<<+VgRHJLD!9(CQ)@40IM^E^<dq= zLL4<VkkgYOirp80wKYOFG!>(3M~-L(6zxH*$l9?d!2oYIHXW!gIc8nBepnj;MoWVe k5Cg*xpqU^5)Ca268HAw<fDr~1V+EDR3>-iRQV&rJ0LE4rZ2$lO literal 0 HcmV?d00001 diff --git a/tests/fixtures/tl_checkpoints/lnpre/checkpoint.pt b/tests/fixtures/tl_checkpoints/lnpre/checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e0d5523d2e2fa65c4f7d88552549c529389bcf8 GIT binary patch literal 85365 zcmd@5XIK>5w!RMof<%?5qDWM-NKRd2qNspi0#N}2l2nvnHH}$BqF@385(E(x5kxVe z&^4wxU|N{K1PEp^BVzX7fNO2sYn^@V^E)5j4=-E=eRVzat}#PBJ!(|-M2{YlVq!8f zVt@Nl5$husG&dw@{({KR2*I%606~D+g85-C&hlb@|N4oQ2nvb}3kwMnghob0$xYw~ zhwwv!M}-Cn<XAByFIL>pM^=Om2@ebjHk;}{nw1C=MDl~?n)ycyLc^lu{P`h)(V=0% z{(_K*s7Sv5LQ6TehoPa5w5U;(NEt9Egq7@SBIgno86cR{^`;yv<s&ZAn6o{7dWil9 zEr@2N$B9V@SQ$?bF%Lx#?Ic##L)$|<iIq#4+YyPpfK~8fd-?PcsTV{>`TrS`=r!AW zuIQL^CFgcAE!aMDMJ#avtJuMk>0l{!v6Q=5Dgsv3i&gVc60riqB7^2fnVFjf2m}$L zE_$+kf1_El{W@rp0=9n#O+o}xOlH+Zv5KM;Pi6;nAqNUr4KH@kKO!fvn!h2fSgl`3 z?JlIKyUDCh2U67IWLCEesV86ud$B|Q0U0QYQvWxkHES?e+Cy8w8vY_Cu|^&7C9%d` zFcSf5>ctNIN7w|`>^GPVJFE+4-T~82WGy_z6IshHkd=V7_F`@R0W{U$gSGt)WXsz9 z>ZpCUv2*xq>>Rt0JORsivG9+`DJ=O7X~#PKLJseS9Pt;@xeGZ`z>e}_NB;vdP!y%h zZ%BLAwX3^se~sOkuCa6Pf_Vtov0m)Be}qk8$NvU%U?+6JCjJHU?1Fi9!6pe<Z!dPT zkMiHvUU)#%e0E9~?bj*e92pt**BLX_M?%!b!kqOH9YIXM`gR;a^qa)`d5CvR^|Y=d zOc$^-yx5sOs((Ae7}xO=yhgMBe?{C0v|wk6fTGhSKm;^0>VgCc*dQ-9`1d}Chb<7D zA)yg$$nOM8c6P_4$Ozau9WiNp3{PU`dJOM~DYOeWPr%OiV#EH93+%v!|Bka_BSZlW z7qF2XwBd>Df{yjac9Hl3Hp+_?{62nNfkj3O*y!I;*6hMxkuB<Eb$G$zE-zTp#ak+1 zmwB<v|IQ2a@8GTYooB<wbWQKd&W<Lsu^pkU>O#c{*my5Cp?fDS{$(qPzmsg)q+bg; zxdZjfS62VEkW;#tsRDM57rXW!nG@J`zccOF^}m=Kx|p5DlJ=Leq<1kl3fN6v?B;)9 zb{NZ+-<kGoMpuuw{z4|PnOzGytBaN`U~|0KZU0D{z~=r=b71qjXxo2{Uw+5<?dT#E z2-rd|cIQ8kI;>^aZzM}|w&+)1cXu1V;=jgkPZzU9z?OQkd;gI+g)RG?X~FLM#oXVG zS^gLEKo|3%fIZ~J9{vYrhq)a2ooUHdboKb?U*mVIYy2v^XjKAM=*4pXNSngq?=&lx zcF~UiMLW?&JK04$C16i`v1ht3=U;2E(_YU0!gYCxr8Rp_gcEtm`3_vCmt5%blIkwp zMFD%si@n_4OMc;gdC8UE<FjF}ihv?7`9lQ!(@Sau>@_d;`tLpb%S&$jPOxQfcFcju zOKx?<)afO)U0zbx71M12d&i5t`*&QYm)!duXUE<b1tjv4`VN}NOBy;BTw@pMfq-rD zVw-;-tbcjQ!{1T%?4w_iJ?><6c*&D4FL~O<dnRC?d$BM6&g=4$mfv{}?8~mXeAU^} zMD}$@Xm7euZw2f-FZR8UyvW3(BmPWwCbO--F|5ql4;?W{2-vo+(NaueKXxsM_AbyT z0sGmD{qh?q(0?-f^>>g3`%M&#qJaJ0!BI?Pe{=-%vy;IW6QybbzPRYG7hgj3H+luX zhv;8tYUm#xA_(ZnF8Pw8w>nZazLe;Np^qqVF@8_c%N2a-4sl0HxG;ntB}z2;GNLy; z#G`zqMDKO>jW63NloS1(_vcPI(ayP%k@LG!J-&RWLZQ2&BQ^ai7v%Tq6!h*c==~?+ zU#dQxD#h-qo+4FO8pv1b+^gJuZ%1zZXOhTQ>D;f{eZNAt{bpei7JRi%P2cXCe*dO1 z>+DxlP(WA+zh9@ke|NdYKguH_`QZU!p-V%8`Rbi*26S)pZ(&&S2X<;Sx@-O|49o6e z4C<6?c9;KK7{BD*!qDn$qusqtukN9Dq||(!PLXbR(cd$Z&hePf*Xz^{{x7sFe@LfR z|G&^K<QsHq4gU-6Qod2A*0{S?^WQtwHE<o>^N$LO<cCD@O*&hdc5gM{pIa>uH4EWK z@P~G`Fzenz{hwQi_|ahk{;<vl=G`0oHt8bc;#+iTEW2yex<fn6t#hmw@I&}konq_m z;=cbZj*JrUZ93Jq-POOLJF%TZ5*op`>lEAnXE7_v{T4(E_zs<7$Nwyz|4Yp46f^%> zyzrM8I>n^B__wv!8MA0iL~_1Ur*?RE?Qf&;ul|YhLjH(OwR3m%-$$c^+#w#>DIV2b z{M)J6i5AUrXt00Byo~OYyZkru@X%nsYp2|;yS#t*Y3xq#n8-1mdiU=7ZXRdB_vqX| zw)_5W>&&9l<HmJr#&_3r>rtnsySYs0luzs~?`AID<lW50v$Ks?_cs3)h9!Ser^dUx z=HJ5j&EqC_%BOUf|63S;_qeH@Z90k-e|<1@^SG{Li0|7e^6M_@w$6TKOr7&EpFgcr zJN>`VviupH+L`}_b|K%tQ#<Rw&@SZ%bZP^;YrA<|_jtQ|Tu^7L;O?!ud0h8ae|lU< zXN%e0TXgfd?k#?K+?>t^bGtYAZPGhDF0@lKue+w(sb^vKo5#)X6o+*ecRQ=Qi#tw@ z@J@9^clB@Re|dn&Fd{p}3;whCPmg0e#r*#){?p^4I>mzjEdJBuqC3S4yNiEYdz~@= z^0-Bv+Qr?qzm3Mf`qyDwOFGp{yQ}{`8h?7+vQF{x?&9B0#ZL6UJZ?p&Jm$ZVcX-^& zPI+v1c{h)<=<=Fh^p1&K)v1r`uJ7=;un7Ozp%DRL{P@oO3ElUrblczEyAnI4N!_I# zhyD}QCGF;0$({Pu-Sr)*@2@1DpVGNIwfpXllIEW|e&;ZW7T20i+1l<h#ebJ|%+b0| z-TH32=%_%mi5}82dTlEX*NNt<W1ILtmWc<5>2xm>o6YV>G6nv>5_K2nLEWpx|Ee4R z`O2c>NJrn??-)<9j_rRw(jYPG-;VUFJnSz@Y3D?QheQZmoX!99!TvmGxBs@*9RWBk zVxM=EvO2c^IRI5Lrh5Q^O9Ua29rbP(XZ!#4asLb<EL0!}3+ad<5gw{{Tu5|m|8opV zV!ZA#bX+$LGk0;e`+p7KhpZkQ+N4i6=qsVG_cBzeX@j8f3sB$74igNs=<<DzFrn3! z`U~A5c7-L}u1#fT^Q?$ul@3vaKDe!UB)y**O|vw2g8GfeFp4dMQmX)3#*Id4(<EZ< zdIx@N+JjNe7r0SrH<_JQW60&o5b{`7mfZLIfD0-P;LJH2d6y5Y$Ni7{z{i6zFoYM! zJlt9dNo!U@SnqIdXu@OAxMYA@lUjMFBDcc&gGJ=SQE&R@TnNp1e+rW?e*>vo{Yjwg zRi->l3QJ!FVuN2Ucs&V530Q?!SJc6C=jmkD`;UZ&Gw7vbmQ?F>JPE|@XwkS6A2>)6 zsd{ZP?2;o9SDZp^9#)d0(kW=-?+m%E^Qnx(c1-)6ik6jW_&9z5eH5)jc6=(}r2U`6 z4R;^z(D!pN`O15|=p^Z|ueKG|RL;Sw<i6y|!U=#Yd!yH}B67rOHQl`6iI8~rqnZhF z^m0%RkM6kwwLTHV{_`mMX{#&Lr)R(#2c%+U(bURT60aNU!e^O2kf(3SFqcNcy_|R$ z<dXnFg<o*Fdo%2^?MV%izH?(e%juZ+hTQ0txkB3@d6MMp4ue@KDzuj1#`_({m*bQ0 zk^UO;cxWg(ui3!RGD9M9Oo}u-aU;|3{=`9h_&9dJa}2yz2v^US!z71f&?f7M*avmm zU~I*mx|75lm)(mEde-#7x-c$guNqyWZi5&7dqdppzF5>hg{hs`3qBSZ5<4RU`1aNx z+TJ|lWVWy7{wQf+z7#fMa#jk>2;Ry#f4hX^#@oV+sTE9}gFl?^tw79@`q1DbX0W<= zF_*Gckt}A@uzkUOAgYI8>kAt=q2JEDQeDG+xI7VdKeNGdTMa57BM;)1?li+$ox8Jn z0?yS7q~Q_c(09ECk@L!A0vm(Kib>6I%UXlN<r<v&!<$%_Xu&PblW<l{k;;H6HL@5= zmHHOo+PG}kbznXer3ZuBp>xdTs_zgpRv)EWRmkjFrrdE8O)B0u2%;?ZLUGM;_*ia6 z-@M)rj!W_|eyb4~|1^MkA16<&16zSzYeVU?-Xwa|25i0JKr6iE$k`?bQhfIW>h3Fq zhr9KmM@hC&@4PxanU=uqXmUZFH9=&}_j=~L+H_cQcsCcn_ZVDxtBXT3hSOV#JGsQe z*=U=w7568622;Km4SMq)B--@J37M5tLTn1%>=Z*4mzgs)mKv15Wg>Cbj^%Pxc0-RJ zr@4{7s@(X1zC_^FgIHCj;023sXi&bJQNEButOf@{ji5rfR_7T8CvCv7p`Mt$Xb;So zYe1v>6>%H4HF1MSxNyhbk0DE%_JaJKHK?!TjWI)q(}S_m<l*`)<YuBYF28>nKK<Mc zm*))x7qu<au{0a>lydOx;4dhB!dEC;W<-v^n+=QXo^UTi45)jg9r8_N$f^GN^vvxY z;OF@ml3Q<ayWicw8;y?KhF85PYox$j$z6x5_qE{rm{;&Q@F?oW@`zU5Boc7&5PItN zA-X<G@pbHBT$%KR8=SHZZC^cvcK0Prq47XksF%#W7_CC5FCR)JH>Se9UY1mTjtiZZ z>`&%4X9?HD@NsW%Rm?fMg`4p6o6tBl3|0=(BOf)^lG0a7q;9zq>94g6liyp>>r#%y z$)*<wwq-%&-kw6O<{h+poHx0fVouLyR6$R--cCLA)3{_+6R;?r0ReW~ad&GUn)Te7 z6g90S)9M{*wPp#myRwCzIM{=HGOI?Xrgh}~9W6?P+aR)jCaHB9M9NlL6Kl^z#v*ne zxi~fn?#*C`?u&15^GRQ3#)n3zSfM}*){4=MLH0P{$Ou&56c4)bX|ToP2WV^!fV4}K zh(l8<*saLM2b=PR`Ti8nyIP~dxdN<_S&LQn_Vi?SH8*BOJ{*X;&W*n=jZyQB;pM9* z(57NXMlO*i#b0y5sc<H}`_)?HHMMY{hZhN3q3U$>jSiXhG>|@FFQWDIA_#N0BEu3Z z5$4t7syKOgeDwr#Dr*OQICKoDSdotRNfbHbrb-)!2#C#8bDFch9%7~sB_G(U`2B4^ z&a`+t+2*{8&}H|k+8cV2B>@BIrE#&O`m_$p&(_Ap&u@d*j4JfKDN9Y<20?n<Dfrx) zNd8!4N^I*&u>O!dxxUGds!c8=civl&<JoJ`d1gFZJTj6joVE<6?%s^NggUOUObgNs zW0+Z16Txr%U3}Uk373LYaM2&ZL}6VobT1l87b!*v<5$HIXMsBH8~l+>KD-kHkEUQ_ z<Z`n0kC$B77(3kU$0OM_&cvzw0ppz=M63?az)Wd((l6Bo+WqFzJ_AZn^HCX0&0YbE zR`_$zCpf`za))!=GoFsxUJ8Dd2Gr`<F5aRMUzof-lJw`p08;4ph(de;+KgC&6MBw7 z`*;4-WVVdZZJq_$d|@`x7!^imRrlaRW>&yEl|is}-#NTFP?J7}C}KJ2C)WPh4Km-n z;FHZ*rp_6tVa8?L=tMb*{R)(EQ=|763qj`AeQsgeAK-j5jZR*=k3O~j!3d-l(ofrg zRBv8DJ2DSVUhgEaj_g*n{in=h#NRRx3m0ejf6Y9)GN5~gdDxgxfeUMn;;JFNNXo!g z#ymcir<}BioV2=*o^lT`f8SnMs%k?gJQ+eB>DEI0G8;1ccr30}cm>SMm$>ujL{PmR zN=^G6qJ?XB^BQ<AxJG>jj=k7}TR7zd$Xz%KHRs!iZ3#=8+BSjYo*`(V<_xVBM&x!5 zj~tQ^Cvp$gVbL`s>ht{^UhlJqZhP#@d>HhC%V?41M(s@}7X3zI`wa$eO60(<ru9UA z?prQ+|2pBgYa5vv`QscHDn0?`&U{Gg?TKe75R;SgG@#c{&{+`9jZ!;<o=xHC6+Dj) z8n_9Jt%^y$x)cP@$-)b7zG3RuL5@?5Y^j~XAlj?%T~J)ggQRg<Sf~u-vS%B!Ib{$O z+V`Zd9-gHiW*6dtqnbGX?G7f>V-tkNYeRv0Z~F7782N4+M0yqj_bk{Q)e>GXg$f2Z z?`kQNq`RM7@EA@sZudtStx0gt`WX4nouYd6<z(FW6ONhV73rZvy~ye#D>2x%ABJi^ zg%{r1H1b3-DcD$#<!LF{kKKT#S08hC{qxD}X?a{vz6!m)|0>h>#AbZ3q!8I3lJs2v zJ9z8kG)_NEi_AU;B(lH=ExjM%r{FW3;?i8IlT!m4zC&P8sU%FUbs%r6T<N~NZBREt zmb#k1V6K{a;GsLY@Xp+y^tw|9TP}~pM|0-E@hRfOWRn*yX*VIeVuxbH=pxd#*a^HB z=aLr@)^zB(!^E1qjN`NQ$yvwkC}H&y>hGqJu`6S6o7qxa^LaIv>|o&54+U!2|0(Q| zFT$-AubHWN?Tosu9GT>K6N)%5x?g8Aoj>G?=+$A^uvtK_UwR2w$^vkMqyw3xv;}%J zkAcf$`%<%8a&%O?B+=Wr4&E*AP2xu=lHluxv?z5CMjWZ*W?NZNf4&JZTF}g$-!OvC zu<uXYj;hkHtM!q)nn<pPmeIFHVW4&G3}<M$2WD#aCm&55$^HKGsOHfIe2C_>`NuAp z7<Umo!<FEPhYma%J&iNB3njZ+*K*SzjiDcV_MkpD-LTo@GCW<gj@KI6j-PsOgXZO# z81ZB{O+MvM(hr5f&a<0ol4mAt-ChQ5yfEBl6AyRa2jY}3*+lKpYi{GGAkIE#E!e)f z3=`AO^D5THVDt6UIR4r)5)v K=dulJF{8SibVl=lueo>jrPa;oH>q$M#6u&E61 z8Bh5c7oczAP%@>A5UsOgaAmz2$=M;q=#`luD3_%pr{&PZ@)=O=8;&|JZ(-q)NJ5Tg zzysP7JN!qlbflJ;=qaR*?SJP#mj7q}u~$li1#1Svr^d<TO~N>cbem7|8%yDA{Svy@ z;4ljJ&8b>YC{8@8)>rv^U*lcAv>SX(L#PocLGxY-oWSrB9BawP{UX1Sk+~wQTAl%0 z2H%IXz0~MO6&o6NG@g4<djJphZU_JUhnNflZ4wltNi0luf$IGSuxZ68x?^8GvPSQ@ ziQCk1iF*_Fy68bPbL!}G+1=a-<BwRKXo}gU*<^Z7Iy5d6qmB7#B(^o0>K2^Bz})+| zJwKHm<LlCaf9%7z`upjWE0R>%<q|l3zl+Uceo*71N7~-H^XNn$a_CeZHK<t5-LVOx z$9C<2ryc|79?MXuz<Kz+tr+*7P{X94{WwqUE;qVej_hnb18x@^@NJ_ZcSF*X24AwJ zU;Ikoz=Q8-@=}fFeH}&<63%cjyNbc@=4!I(mIkyhz5$s54)oH}3Ha!3FK+3J5yaa^ zolbha0H(ih!mWOR!YAit=w1(VhD&TBm#@D-&0gB{-HK~?)#DMI%N|V(SRc6iK#_`% zn&Z$Kn#q}pMKT69Q?Nw+Dpt%~!>yWRMM8f(W@<-2#q_6IFgh-X5vx2z#y=QJkKIur z^9*O<l$svgv=B{fUD%5g*T@FNSK{>ZN`{7Cb)hMZ0wULth`aB#Fg59FRK7xwxDVZh z)moWUvLOIu1KgP?&rhhSqJ<VSW8uytIWp|waFX@85!AiwNxkl2`d~*iRWyGo^e-7h zw5Au4Ek`mzJ77N(H!1^8o-M(z&lO3{1wvhNCpdmzvJ<Kk$I;6V9r1EpC3CWF3SG72 zHSir&NZeFM(r_Y!Ja<ZFbgD|Z_DSMoZqa?zt(!?_*{z~8BCFxk?E|#oh7(U|SUR`P zLO^)ec9Ko&<>-kwS0Q8b2auK1Wm=WzI6h4EpmTy!nPK%#cs#Zh@2pD%+c^i}=&A4A z;<@supl}3h0$zgM>|K~1l*(1FTZHTHw!p>}W9cxnIo#qMLz&9YCE(jI8(7n0==*sr zHtB0o%M+Wa;uA;C?LywM{0Bqm@)7A|=AbLwcb{oAy>uXHa%$m;osXohdlIoBs+^}q zo#?{@`OxBZ4W|}ok?_=~Jf<}cj=pRGox~j+?~6CqEKI=E#XQ=mb_w*|Qt5@9{$zbg zKbkpP0yaK514cu$V1oHLXxA-<*W#tTjr_ylZr=)aQ9tp~33;Nh^DD?@iV?4uQnY~M z0kM3AZf`uHXpa~?-=PBu>9(jgelE(r_{^*s>`1oE=}VVw52Mw)b3kxjgPcFKh`#== zLFWwF!rgK{iwh3`?ZF%e713D4)FzPA+6getT?-l-o-kI|mow>xA28o#KMcM`$Uyah zWX4N#nxt0&2B(zCjMS(2HY1&Tc;+<ZU;PXk1LDY~seW{Knjy2|$1WO`o&yz;XTkA+ z4^?*UP4_F+K+Mk=D4+%SX6y-&8JkJ3*#3a8!LleitBKo@A=LBs2`YXoi&VdMBm3oB z;N5YKQO-Pt8}y^;s*-_pE87ms$CYAkV>DD|SW$%&dYt_NWzIlQ3iB0KVSAD}joLAb z8eLlrMZV{##)C-Sbf1TuTwJrr7jo!UuMyBQUzhg2dk89Te#fm*v6LQpf!}d5eQJIZ zY7U=-YDY8ft`B8yJ&57GnNbWXnlf<FVjvl^U@JyGtHCSbuel}U1@73K1hQuXp;^zF z`bpl01FAPbVaEV2P<#i>zwJTQjANNH#X{cl_(Jq6kR*QQL&3=S3iHt36%QSegcHw} z!|+ua(EsC8E_ay<nHA6Civ0D$pIds<#ON2y(US{6&eM#356|Ydwi)4#T}d$VNfFpR zkm9cFHX?VPOru+d-{%P}k5{?gpFn(XDnWT#EksLe(UX%7G1i;i>G8hEyK_y4_RYKm zbG=TYo^v|$JYpWWmm{)k-|>ci9R>IMe&h-#2a?A#d(fJzfi(K_172HWACfvPi<IoW z!R7sYfJynWr0#S&*7WVc0L{Z@p9`4q6>;0>YM9`Xj24?uadXP<L9K!*d6-p!lh^j+ z9@-xQ&9*e6CuIV<PqV-=cq=x=%x2s}leyrJQ{j1o3a$3iAYc0G6Tj0JxC7f&n2AsA zsPywgnDR^(^$w0_!h_<7liwElI?oYCwhpG(Zd#EOFQ-w1yF<yzm$BGr`-JB-sG18K z+yLu-9)w1Ti)81TrEtPIo4hVOM&|kqg`bLh&@q*P>{;22>Ap0^r_hHk{bb1Vws3>8 zZD~BO*K^=r{z6!w-wFfgTaih)73Lc*pqp#*VEamUx^dtJdSjU-Y3q@PwYRjnQq`Ma zYrKyuzj_&O%}$1Za2*&MK7wo?Z$RF&65NahOQ5okG@bmzm;MNfWHf!Jldnf7!T8tX z$yE7CFuXR3JoQs1hcpj^a>Op8tQ<@N8murU+lL$;r$pwNOr+UI&+}TxM$q5^-ZbL6 zJ9q^}bJG{sVs()LX_xXOJMKJYloYgSRqQ6ZQ!kt3HoWAP?@NZ-ac1O4o()uO4k7Z{ zKfqP%Jb8IA179}mqJuBXQ%{`~syE9G#daJ5sg)PN>`Wch?CnX<`kljDiz{INwY6m6 zr&$oIE=H8}v_<uUp>%0a4<e9FgX))OVEbxq(mUfh^f-NhbL**vlY@pp+gv4*rEDVl zEaQ`cd#flG^Qg=5=XjT&Mfz`=0M4&nK~1WTvmA7c(}^@Ex|X`s@^J!|5FYh4(I9=U zCoy`p10itaY8uij4>KyoAltzJd)+w+*TZ~>>Z}0X5wkR;2XBGO4SSMndx(2dI~YA? zRG`_{K3FvMIqXZ<qOa3xaa-AINZKw-Ge1(8arPWWy*I?wA;~09S(~1a+07l?=!W*@ zuR)V1Lv)+BQ+MRUHE~5u{ll5At{955n)9fHq&cPpz305k&O*QTEnMmCVPuK(VG>?n zhegX5)81k(<a5w=qGnXcc*$LZQ;WPwTbdVb+#E)GUZ}-3M|GksHHQwqdjuS(`qRc! zKbVL7A5dlbO&ET06{h$Zko5-$(GS(2yC(^8%v&Q;U=j#zlG?QG%?D5~PDbUim1xy6 zg_$$Rir6*ZB|S^lqDz?q#OfU3Xug2ruo0v!Af2c_lEz(I2GaHc&oJz+7<m+_hdg#Q zj?G$2B#($bL;Z>{ZMZz`-y)6%W@9PiG=LO)@<BE8DtyIBkg4__m-ksm6MrO-htv3| zmXgj%kM^Wa#v2^=p8|4gbS8$G=Lv0odJ}(}BC=6;7#V2)iO1wl!C}ej@ZKL8wS9N6 z)M*HOijM-93{z5X@4#*MH6vatQV0o}MbxjqXMFBn=k-kBnB$4k+|Rfsj5(}IE}c6J zQ?DArYp)%k<a--mo|0uu`sfm+=eHgA&F#rsJL4LfK7YoED}2QXH?7Itr1uzTECCU- zKcMVtagu5(32Q%}Vffz7@Y*{Y>t%b=>M<?ci#2QDYGx9c`?X_1QXpkbD>xlB8S>)% zUT(Z^5%>0fPp6wLONryRTKv9gH}m<j9R!9}fbWRa=rH&*!}220eRBexdTyIANYNYi z#QF+n6vm<C*zMH%v^O#Fi=xw?>XY2II3m*$NZ+h>C+}uz!9HdLijO;kw}p;Ga%mcT zbk0Hx`9;i?;|kP%xdbd<bp$bf0f}~c$~<yaAm=A$VSsED2}%8kOwndK@zh8vE}bNl z3x5Wqc>N*%!Fg!uGaFo;3czNefJ+r>!`2i1NJ~%>-8<wa&f<Rq$s6}@#HaaWl-nMf z<ED+vR;q!aXgsGlhmiFTmtd^?bZ}q!0nfP<3-|j5F+Wn|$y@10lz(MJ9_Y@YYc*fM zvQaFQG)iE&pCc~OUqxNBl7U$%Msv!YxWPACfVP|_tM#){!QPjoOpT(pgAdc{#pbkE zkt()Lbtey6CNkVH7v`$kDqQS40WR{BY3YG>>|Z&6JlHgoaWgxIAu4m=?ld(hN_)bv z2}e0Cg<4QGQo_o^kGP~O8|b323M_tH4%%M|xyjeh;@cuO5^o9IwV@|*z^X(@y*C$S zj0>S)s5C8&SdB+=`_Xkv7Q)8zS@bL8OI|KgqCbYdu4-aM*WQhuq;_^sIIA33_1a^y z@JEXdaXTmt$&O((%9SAl)8$~gXCfWTkESyoD$|q7t8n#(ySR4v5K@?S6?ds#gd0nS z(elGPxO<ZCxzwG5Ne|bD_^h-U#_0E=uQJWZ6xmzMtGU{6Byj<XUy*|GN;jGJUs%j} zRYFgSTTp((0Q#VO4h~h(q7MEBR3ZbghqE8daaAM=+qB7$dxOa_vk)+twH6-D8cz~} z^68)vebHrN72a!_Pcs$_g*5|HVR@4jvp+zQE`6)S@E2r|aq|tRS=D>|Ho?I0A>WDf z4I4t=>2C)2cbdTQY-wEV9r#lJom1>v0ZqeZ()vf$_<H^a#(icU*p{i$+{dX@Yk~^o z+eFZr&C781h)kLjxsJ$}uEyt2{KyUGIBq8IJ2*Dh@Dv-?VBikX8b6_o<!^1s+4@X) z!Sf--!yA}KdmeH-W@f<)g`G%k?{mwVRxvj&?1WQ|EO|e;o|)BSJ5{^74<kOQIQq8^ zpyN;O0jFiFu~K;-*XFYajecw(K4m__vO!nj_A7N7yiS!%+G$KyzvobOfh}Q{`jLrG z>^OxfuRwu$4;wFgVopB;+Pb;`gJiCN)7?n!%`iinrRq#$a+2xX^Hwy^WfReA$YqA; zrsG2O7ohT!M_jgs3VoezNU>0i)?U7bOY)b%?H_&UkKiOO&fSR2ZLxODJ!D6&25cpj z!xz)n4Vt2Q(*U}0a||5pzYn)(Z>KtU$B{GrWvT6W2WHjogQ9%264cd=;X~Lknln5F z`SX%_x>ZL}aYZuz*yabhT9<H}vLtymsDxQRV?Xc~s)IpNAY6PXPU!t~+UvO|o~$Y5 z4S4&NnI>3(&6kX5NsmL^*p|;2adSTzIP?^@HUQb(<0K5VdIf=GC25bmgT}Y7!I32l zjZrFQUgr0JtO8ZKy*-KKzCK4r#VJ4;XF@gl3Bl^tMU*@#Mve~FA<fF_WWA#^G{o?z zezPiOEKBXIKSXA3EfVXvIWF4%cl8IW|8xC;-<m=LmPb=bQ4Zu={s%gX@;{mT=A7xA zSg;o_!x4|<>3#8OLfv#@@*uMkHK*H>{ETpFc5OU)R@0l*e>ey}BbU;s`axJ|%n;LE z+O)-bHN6?YQ0uaKY@42n1MMv-W^Tap<LYGJFl+K;^>&Oqm4PQ6{K=>4BHrh7Kk(w~ zUUaFcfNVFE1%vH@q_@8~S`56z%+Pjal9$W^-%G~y#QI>e+2kD;l_>iB-F#7)>XXEk z_IV05%M|c%Qwo|jOv4s>qpG|-4j+C}Cs758kX8Ln=*(|{iKTM%%cWuD_8+@&`5$)7 zraWD$D>;rz{gA{(@fA$kVH3Ke)sg<VF6Q{%s}YMLPm=x5*V5rL^GQT3pUj;qhYRke zQp1O5(BtPZdhoV0ofy21l<s&B;Zb8r{{1;{DfuiMwH2dNo2EgA-+DZ;cpVs+xnRHC z1n^9DAxmR#@+!n6Xgbds7QT9nW3N@i0a+;=G^zxPO;WIaQ3S80JPB``R^za?)54ht zBxp3Z2R4NyK}5DWy*}Y6{p=>BwdUXO-kUt;O86RH^#v6&rT+xx?)Y`Mc;Y~!nIVm7 zmL~M&HaASN(||)s4{=RU9jNN|p>g4>gs&U&gm3u1RIg88>R8YhH;QUCY{f+!tf@uR zD%KKHwPBR@*dTm7av!+gpGajCLb(xza-781Q%qy!3rI5M;fBDq^yo?@8fNyG+cd+2 zMt#get#Vy_!@9y#E(vD;NQ2ke$8f=z$G9r*DtH@Dq3+sBxF*V;+ZIyFtu4_Y){UW# zlb*-n*@HVtc^^r->-HvSYR>}8Cvw#OOCq;*`W9I3m%!bsE=JD-YndXMvmmv}9d(#k z;u9oI#wS&S-}@BO=n*6gwygl4CN0eSeA4l=tRboP+s+(%a1EVe5wCAdMHewWtX!~; zS!Sk8^vczUzx_IRVbYTxUs{BpB5uIF<vr=tXVr|+s(5DP6E_<0s)kN9mf;>HWI>Ye zEL!ql1E>w>lMM;B=ycu$<&TWVs6oS-ZHy?Fk8$P<oq>!Fu7NjSS~!(;24tb=+V?`* zQFys#3SCsOln9D6@W^&e^4*tD`*CB*;ro~Il+#a~e!K(^Sii^2Ckb?lL@H>%U(Z$O zsp9>Mc5pVT6-W6#2AlRv81K@|spM$m%erD1_P_*&F4%=f?zKZ+pGBO`!&2NTs{IzU z7J^KV`Q%oq8n@2+FiKDQ3Tnwi@Q2iXI6TLMk*Rr#KjsE-=`Ux{p|_@^;U1Rmn=^<k zxq1R@HO0sX_he9dP=zkRD^aG&5wn{5(5v~cuxzvg?kurFhw~9&m*z=_+%%^19N*#L zfOuGb>J|2xrcCm8k06`<uQIE@TL=T`CVcCd!83W)pO)Tx&B>flAlt%`dOw@T)o%O@ z%a&WCk6R;rJ<*Eu`@|8Sh6d)|<13tmp(9ADt|k+Ult`xCH^_N-9z$wO>BXA2(CAqN zT!=cnE>tB^UQ^I&%stUPc?am%V<x%&<0b4IW<wliCE#rsM74LF=Uool0D8-$skL1d zX5F%<J?c-OUGZZs#x{?b%hm{2*&EWOD@sxGOC4N%WJ0g^e~kts_d|O`AIhFr#*O?o zo_|Xuu@sAg#b1?R+1Q;l{h<zNg?47ZJxjRw{2Fc(eKy&&^{2(neBvx;3Z+Jkuwi@- zN6+79-nMuUz4QVYKlK%ys~$p4KJI4f^p=ROjqK^l7jBSx<vzE4krLPr97P^p_H-DQ zs6sFJ$<j+xFM;w=4bF3^26<SQkE<NV(tT6!<C>FcI742B?tz04zxoGv;E5XVbKnN@ z!e<7F-kCz%R|SKlUNL;AA-rY%qhL~<s74TGO(&dHW=5xSFzKEKz2%$-*Ncaf13qVj z*`o?@icK|mndnmf#}C5cJMTl?y!$v<uNC|GKERtx9%16DJltfei357Xkuk=XkzY9+ zgGU_1A~S{-WL<+Bd!tC%ZC89h*p9g@9|mI<_9baiD^Pt+AL2JA9DD}#CkK0;7u_e9 zqr<jdhu~EWI4stQYMHKqtQkwt+<6Z#sMjXO=+a&|n<j;g!(za{YC349EGA(ym8zya z*-ZUXX45yldZTynFnAGTLRJNxf{Bxr=+omVFml^_xRUf95@eoW>P1g{zi=BFFxeLR zyF4J`We*&!3ZKEGf@%!bA3*1-$U)C7ZH{sg(O8x+g}YMq5uYyqDr}5=hHI{9(`J*; zjGOI5T)J*IpmH<JuH=zm5!Yqq5ZdVRh!?vr94daU<GtFh#69Y>3wy6LqmAw=aDKlE z7d-nO^Hb&xsOVk78k0xB?9rm$s+*YO0}_a5+HTHYY%$%L>Iuu*60zb*4h>1nfmhMR z(B9-k*9H<g;pbL7d!M0Of;H)^{R^qti&OZ@cO#fI`;c#+b5YH81a%2dfe-F$Vcm{p zup#m$x?079g$x6_Q-L<7RN~v|ahPA%2s82>$diOMc+S^?s0mh}!P2Qvw8;U-hbq(Y zWkczKLG6sRcpR}*2?zN}!->T3zBE=&0<>m-W2*Y}q_T%y;jW)29V9w;POe&nQd#Nr z_?R|cjcPCSEcd0oj}${C=SFvh>Jr!SBgjhAs~~@(2}3_jpvK2$(V2WnD14pHtdk!= zHtn&bI&lMNwTT^fqb-RhFJDAwNe>_guiAmij>&LEc^&WWn?M?9rbC;hP7(R?p0q_) z0j~`fBk#;!F)fPfbhs#Q{n9U#8F*cnT8&+RT**#IUecSY%#<RYD^_yexye+=+k&#Q zwJ>FCAJQK42=?XoC+*RmRLvj_V;fh}^RxOAO?3$-e!~^~mb@L7(wU@PauiB_@g}iG zj%Xs>PIsSBBHvX;fwg`GGxX7J(i;ANo5sw>hJbgt@wo(za1DaU2@g5roMwDGNtu2h z*^AbS^4@~aE<`PL8rj`13Km=(M4MJ!W@;|n1AS#x&Zfu}dWLO=rg2s9v0aK9KY9XV zxI>QiUL(1sgQwGnl^Ueq>SB;r97~w@+O*>K61aFMM)cf;65UZ7i+4B5p<9Lze2_j& z3MS2gTTYw6#NVF#VUQ=hBW*xF-nfXfopvz!6{_6RE0$DUdI~5v1jE4Z#&lRkBtB@9 z=d7Hp>D#tC)VL{5?msGE`h5%~fg{4P|KR&VUr!~j;AA6S%{qfCq|&i=e>I(Xp*Pv| zBbJCsijxgbOHo?85A~kYgIa3Nr}HG9f%%>^viH(b(%!`9k~g0OiHKD&zAOROoj-+h z14ohQKg{Xy&kgu>>T}#z+yYj|xvH^WlcDkM1n43Cj&qe(qz9|U<F3WSA;W(?{d%ho z#0`^)zEF-X9asU?m(qkcKOBW=WqQ0^pLryrWg<7`RyaJEu0-5}wP|BDj|ffnaz+Ik zNz<#1pjx;ZM{2e)yIT9w5-BnAbjWSIp_V9&EjNJR0w*#hUWu-DtAOu^M=(RB6v4vg zHI82Ig{;|1q3on@U^6HbW-Ga2_*QX}=n#*as@=&IhkBqZ>X}h7xi~*fok|{zB4aX2 zNK^xZvLkO`P|IlYE_xMR`}rla>(Byv?EXSJ<mmw<TU=qtg*otTMjf|dM<~Rty#-2o zd0_Y4g51t>pu-j@z`HT;VeXd^^v($}9`@3r^B1Lp_3q<UyB@y5CY(v9YmNfD%E9FR zqN!x>ijibUb_zEl$&jSB4WtQ^4l~8J`4Ic~An1DK!PHYpMD}JHyhtYC*6e|~9_D1L ziZ)qU(+rDSbZK>qAx+x)1|L0J4PW}V36~yM!*uI0G&IMZ=XR_JM)~9kr`>TUm(psO z+n@F^t||t!{HlcLn)D*m8hRBnHCCeKAJ%m6?LF8OpAR1n`%qba0JEY*n;Pw~q0gh% zV&SnAPIi|(UbE%EbET*@d~p;+XI>Zf32`9X;`a(4J$Ixx#f)L~aSys|SpoOs=yTlj zLXWAwy@F2H?oFR;6WtHijDy%zectk?EyCIfaimX3J=F<aN)@Oat-10Pb}noaPMQ1) z>>o{F;%&>Ib!KlO<vb0O^mVAV(O~lS<R_F<UI-K3c|l0lAfh9>V{m@BE^Smj?&vt& z8b{V4==$|$7VJ?%-B%Z|QtS{jtNsf%>kh^<b!%{Xv;uKxlfe&R$+Yznz*TlDnx5z> zsvV0{=e7A9brd}Vy6L#EUqdx09^1if+WiA$oSx$p!yGRA%x6eDI~IC+d_Xa^Vs70o z2d36&99gkBi-_qS!E04Mv}LR$T@-o-$Hy4c33`30vZDc}z1a)5q7CSfm`}{zBdRp< zNhP>1T1BVakENfsEkc=iNm|B^rjoCLDwvlGuLbVJve=Uxgv64I-$I!{DKEORel%^A zlz`PaBgw{b54h{e1G$l_?8)u<al}+~Pqci8CaJO9LZ7}b1mg@3;#Rj3g5K`Nj9qoi zwIhL8w|FVDD5jmUH+Lbon`gro+Y>0YMG@96Ppi^+;X%a?^`~DdBuR19NnA8Qg7kQP znFh!h(2=HhP_l4=!%h<?y2tYt$YdvjO#5ma(4a!LZre!458K9Tm?R(p;eC-kaF^*L zIfa>bQ-$WW0fhTraU3@@ji7=Axw`WXD3q;$+jGldjQA{aF>W`OnCFvCD-4L(ZqYse zfhiC<e;N3Q?jMxu70~x=D=ciPW{zlDVCr;5?&yf;Ag1UB!7>e8C?|&|x7N_>I|tGV z-a78+^%uPFr)qIQr6e{hdE%%~{UJ7ICebySOD5j3qG^%=Sd>+cJfG=I(%uYwc2k%9 z*ssAon6RDfNT|gU<%#sEQWaBBVody5dC~Q(3A`PUfeY5BkyV4o;FM)*M62)|=9RHr zY=A0hPMZrK@IEeBeSzC>vJS4l<%#ai?LfKpk8q2-6fN1HLD$ZhDIBRejqcEBXHK5| zj{1&!;ez3Ey7^KMvZtQ}k^Y)1yjm1TA{}EP?j>bBxqc)h&>6SP*#No*D!ida58;7s z9O~HZgkV<{S~rKG#i`~*`}HH-Qx}Tmmx0=aeZrp+6L6WPGbtPBL+ih`qScWL+*Su4 z6b5*3lG_jC4bG4Zdnd|&#-+i7hzZc~{DaQTC*uEa^#|+!bNxY7_4xnkPe=cAzZ9Q} z6-Cjp{qO28HvfO#f7}0a|L2Yw0vg{nIaBi#p`A(_O1s@a{V`$=-*!F2_N6{FY~Mld z?Zj;`a;zCiGd%=u?oXJlF41IS#$AvYk;{ln+{4QK+2p~JIFiHJ(e*bu+&g@_P^Cae z^h|3mT<$fFPB9%v?!-wF@gg<h(Or_HeU}H3jOz{A1ElESk&{qV^#ZjuSGmjImC4-U z+4wzo5AQ+#GF+iLj86QfLB-9hAam3!JUyj1wUA%I$<4K-gYJl)^Vax=&Ev-rrweEB zjouWxWyu@nOZpeQGp+<nns0Dl`fY)N)V{DkdOptdRVRzeS24@a4sn>6JPKYl2{Aa8 zh3c#AFiOUr6y^<~=k(PeAlZi6D4Yg|{$f;K*Mr1unT^vj)<9J7AUagioQ~gn5>jkb zkAJM2#+liRQk57rTHTzC@k_&LzX`3NCAE{n<~mfr$cpq{#Q4HKyfq^V&Zt|{&56b& z-@g|wPBtN$rOw#1w+=b6C6d|qu$<fYd@3=ubHlud0jRic2$?lrmb)LQOsuZO(P7y# zV6r<4D{dqbal=&R&a+&?|04rtC|h!UUoRyZ_vdnaelVn6^hMVPgUI)T#@JBv0MDF} z#20lMv|h!Es>GPk^5+}5EazpU+*z4EjW9<3@lZVOc8jSxc?7KGGfDc41_&9}iqme6 z#j(Lpap0RASn^4h$`vKUE@>g!zqg}{dKXuIa5W=3OZOt&lqX?HTOoPA2VRgm56j!a zcvI(2BPGY<==z*qG}n48*LrLfHJDn3y^;)Q$+k33z3B+7zjX+o?OjFC!J4GJTtj*Z zi*beop&zAcVZ}jTJaZ+C%r8{}(qM-37WD#;13z*2#b%z_(kRk@Kr%h%y$&ZoRirB< zikR&mq-kE+J~UnNl>2bj3`(nF;gw1q_(nWO)15Ck|H0zKV$eNiPWn7DwO}S5)L2c= zOw@;dv>#kdn1Cy5u8`*7O3c$P!Q;hxq|hcE1(M}BZ9pz3f2#t=HJ+?8n_dgYRxPFx zKaW<8uB}E(g+;X0&H`^&<&ZknZ_HPZHMnE(LHJ>3jLY|H!0R=SVdT1EoMbP$en@qt z3DtRUB5^y3tE<P;Qlm&fK^nYy7eFKK>XEqY=fcEudC+j*l8RkV!OL~u@j$sMXxA<x zm5;uoVc##Hcjzg6**B3paqJEbJSfHR9i^#>wlod=sxMlz+N3l^^m%o>25R@6Vm>Z< zi|$desAm!-y7!aA1R3@wZ_kLnACQ}gXJed5!u>S5vHBQl78udK*or=WS@hEyYi_pl zW3I>7d>{})zwmsRql2&FJ?}W&b7?yDy1SXmln=$_XRA<Dq=e^-&tvj1eJYiA4(h)q z3pYGpOqCWIW7xi#By#r((wKJ-r-u9D+vTQItY;yNs7Rn^B7egDz>$=Bnh$NF+S!nA zFE|OS$r#5VuCWoPCduL`eQyQKeLI?-HV&o!-W+&+&`1C8TgmXtFSwNLp`vFe6NtI! z{+vRE0x8M6$@TR<2v3!-k(CLeXJE$HL%$^}v3XoFO>Eu`XI<=Q=9|xui5rDRHM)4m zz!@(3tD||y9nrmEIcC%uDSF#1ox3ox0H>lp2~w1WblzAps<a6vG&T~kibA^g*bjWO z-HG&y6$h=*UZf^{J*R5%nvwgo5@y}c01sUkTBTFYYowAGT`GlNU#&;)$sv?;4&t8f zP{Tas0i4njNzQ5JF6L*G0m<HQ2W4w@NnPRu+9<D%6K$<Q^tdnf*)fmKeXoJnu4<5t zh4yq)?<GRLB{!k(+fdxJ&mLu(3vqs4Im#9)Gt))iw~#YGhbw0UkWBm8_-N!OkbXCj zn90f^9sUj+8sqUx`b{i?adc9^VkXGE1&8*Gp$1nK=;EMOEW^{Fc;OfZO_fB4XNMUs z=@hm_u4i(STEXF-Dezu7!IQUpFfn-~=i;h{hVj{WN$orO=&ZwymxP?5aU#r{+LN5m z?nhpJJIMtuV93`qso)nim<BpLMp%6XzBhm2tuPWjm#6AT-Fp{8{1!)&r!ksRQ4J$X z_W-wN${PALb1Bu=%4g0BQz^H9AG3d64Oin93aO&!cNa{JB?*s|iE2SQj67h@SZ@0a z9^SKPh22RsUZzIdCtqXw`P!0UcJ1gc-h(z@jw6*0KOuaw6umZlDz|L+TJEXB8Mw4x zf!G|)1LbKanAaxp<ipqs=8*kqe1IRBdp53Qc&-bniOQx@)7(jmRTP9*Zh*k5Ih<jM zK6RU6!)@wmM8}Bg$w#zzgIRMiGc-v6{mf=Uo%I2XS9uSA^v+~HWM1UOS@1#3=?IM3 zO34`Mb?B7EpnAzf5~cWvQNKNqjv8}_Onb2&U4!~#oz!gZ;$1boXl{*dZHP|`hGT8- z-el~s(++2JHqi9Dy~wl*Z6avN#-~A-!ESC5S0}j-Ek4@94c-0N)6fjhO!vi$F@CVc zxsVI^VnF)al@j532RITp8+5*$!I~as^fjviGTJ4=v&{;0e(Yn|R4@8YPsRY@#cd}r zXXOj<coRkSTp#nIr}k&sKk0y*p9T#}F#{V{Wpb}6g`53?&pk#-q8hOk*Q{L&D!0t( zl-`@kOtDiq`PgRW)DlNVqG>Adhtvt1EpOt0Q6V^seeM{)_AK|u;mas{>oeY5ZcLrZ zcp%qHk=`}7rNLf{=_<|+MtkegkysAxD~Ho3(-1E%S3#6K!U*o=LfH?31AFcSf#~~d zE~n3-$p<arhm!#yGa(Fxvqpl$&$}=s$C$i->x-GWDcsAJD6Hf!qMIv6LBq-ZbYtKb zMnZWt85biZs>k=H!}j+hu3~>s-)E0-@SwpAlRk#N$u^_bjho2GjF*sZsE>SfB};Om z$R2SO@}fUbb;sLGk1}<dGBk~h;0n2`K_<){fhsu@zna(n^$d6Vr5C+qp9~j`5H-F; zfc5PSAV}Fq-0Q_@?W%=%Q~Eg1a#=2>g!Lh-hRe`b5_aTSX%Z+e7)z&I7(lGcE|VK( zLAY(P0h!rjEPZiVmS`>$qtPz694n`P1Z!_;(kLAON11cPqB4y{f6hnUvUkkkH#_Jb z{q(6qUj;I7&}jg>nM`fqYnU#XL5|G~1atX%c(OABmpyW$@fY$?q1+u`{Sil0_bO8b z&t_14#beB^Y)PLWC7N#HNFR%0b6)-jbBrAT+h3jM<lpOn<P|r3H*OF9A*yFux!&PE zo1TC^V_w2H^G8q_UjvV4v@)L(EU4gSCY-N$!;6{t9KNWn<)%F^KqICF;^wN8l9)XB z_Rx;l?kywVr>();pKd`-!%Jkc3Yos72wPhH9F2lkLQla{SZXW-U+YYH;lX!!raN_L z>F5_i`yp@eL*zkx>@pILR%bG&SM#WCyd23kdJCBsb*Yc2wsqn5VUQlY2UaN=<ALFx z@MuUa>J5med-*>>utS9!%HLq3Un$TJrpn+{KZi6VQV6*|6o<EbhBrU<;Lk~Fcx`Dl zu28F@6Jw9VqUr`Nq4XwLcppWzrv1W%H~Q2zS(MkB_ojcSPbJEImqLKlVsI*|1N&W` zG<RTuaM=<Yxa-l(4AZKD=z1%h^<f%r9I+n7+LqE|Bj>{6*{{)IhCS3XF4Xk(dFIIU z{%BB~fJch6=*HZSIN;tkZlks}^#?7obCW4=O_m%Pu=P3$bEHIHG|9tB18tb*w{mFB z^Z=V9Yhn6cc??@dDel~XqOXS`zwsDv@#sfjaA`i=Fe?T@&ptHdeh<=9<2><J9fyjy zqdDvCX+qxTT4t!eF)mx7L$B@V3nLnhsc!3hpeubL+gt*d7|cNx!?*Zs;X&aTv2Oqc z;xxCZ0@PPcfWBdiuvBe3Pw9##jZxmrc<V6ahx;_#t~#1L*XcpT_z#%z0blWob0$tc za{-Q2TT$m*jaXN@m)n0SoDQ6}i}xW)53D?mp-1QjlC3w3>hv}s8;0hR+q2~v>mL)j zc)}ysH#DInKL#~RRf(#XH!adtq)9U(u|iNL`tE8R8I;YEHjfkJ=qp=B(e4D7Em?@) z&Id6EoD%7#?`l-{;}~$|?lA3>${_sYF0hKr<mQ%Yan}r6U`Bl<{4ACy<G2f0<zj{F zTg8Z}$4QJkyC3FH=}pe>=4hMe2d2U&f;n994Tr4Iq8}ngLQ}>u*tcgjHe2L#dzJ0! zUe9`N((t|Ht+Sb^7I6k#W4!Q2znl2SYBSDxqlk0Do9b<`<wnWn3k48C^ox5^d*v*i z`M%XutVNNuDi0*)h6O0tu1`zqe6aT33CLJ)Kqs-+P%u)NYRo(Z2YSU)VOS4ZDdR#^ zW()p5^4>fu$2jWyrA(DHXd)phm4pVW``TNBLZu{AnMKCPJf}&aNu_}%6(NcWsr%X+ z8Iv(&9?DpTgv|0@&$G@y?{m(1*E;X>$64!~)@rr-tGnyI?(4VrcYnX1jl0dC()utB z;?G@RX_`A|{ZQq~J&z!^;XcgLn@HQD+hbI}y;ODEOz7KJ5w@Q*#qMgkuzKM+%HDVZ z1_hQ;Ij4egmqX%wzl&n*V>Rh4-cRQbZG|rP$BXx-2IB#@Yr>{>2^GKm!{Oc0S8zhA zJ$LLMM9X$Q5}KpbNMT+}Wwfy~D^2l6@70sxYpW4<%o_xw*R<zb11E5vr#~J0<%TNV z!#G>dQ$E%H9;J?7POD!}rp>3)WxHN$u<eQ@-e75jUGKl3b!{#b(`>+<BY-nYFVe1l zHwBlDP1LB8K+#vab5g|-a#fnm-s@*^$hI@EaOn%$`+6jPo-l_d^ccxH8L!0B*+!f? zW|r_&?-|728H0INV?kYdeiY}e#V+p;!jaHSSY>a_Un};|0Mx-lul#VemJSwKe1%y* z!r|fYk7U^EoH(J@83M9S3G$IfJn2d@eA;0E%eM&_dsz!zUs~X7uOVP_P77NW-WQCg z--JS48>;1gJYbM56=z<+PkTo5_fLB%M0psD?V^keGk?LtjoF|#z@u{4>woZQzB?+c ziekmA7f_+|73@65VB^B=+)+sf4*vYg`vV-PU6-ktHnkgfSt9kw2VRi7iV@6RTnr^w zuL!B1Jh86O8A@|9#fEMpWtXl;$r@ec^uewS*Ss=hwa0R(SQ!bwZ_W`ko>^k;;!<JN z=^&o*S{>@Ij^y%Z74SItu~_rZ#@=YSF=o?t+W(+NOjWVR^NaQHKlGR7?ojf-r#}t< zZ|e`!Z#q-0{tZ!Bt<KI(5xh9$7Ukbp$B5MmFzMfZ*4`_x9NOESUMs$lM_lX5i}u*E z)7|#$S>B0H4pimBn_=|L`3cPTJ40%Q`)Tp9AdKvgAmw0!U_$9n*-rh*921aEeb<EZ zr)J=8QA2r8Nk{B*?xc7)-h~&NkK*`}5H7WODyw~{&GN++IO3p@aHsQEVQ%VPDn2jZ z!kTux)o~N8OLf9$h3g@qdp;=`8Su7(v9Q549<m*}RXzzmOdf$&qGGf)UMz3Nn^KO0 z`tPR@;gW~HDpR@0`3=;4u*4Z{v9xA>9MxOgrwi8qMBl$|!iKKnDB{*}UTBdEM(-aB zKQt~&XKo9e|GO6aKIyTg>M(q><2dd4eo>4|Qp9d$eK6_ceOR;6h5I#089xP6P_J2y zCpL5k`JQF)$v&NIhajgYKLUebZ<)>6>-4(jH!V{tgCp1WaQmD?G|10NIJiw0OE-F= z`_s{U{n|<4ikCihIc_JbS^b&r<v9ww$6liM$E_)Rx;LV~JEY9E5og+`2uG*&LA&{# z8Pbo@HvL5KZHl6+U6-(q%|3C=BTHJUX^8IqlIiBCNI0!&34I!>`1oTrmPJYV8uK~$ zwO=3pp7T>wdh86p8w<hApoLnp+&FxLE><;J(4U_B;bmJkK5^UPaNYeWoqIPP|Lwmd z|J`A<Kqhf;DS9Z?MBIkBbC+pxx)J3cS_+ZpwPDM+5b%HZLzvMngI3pB^WiT$q1O;o z>LFP3=FR8DRkKfviw)<I^(l$F;64c!B|H*e1UX>k8fA>Fi)W(~QXO+L0qRtH(sh#r z@>t*n{vde_9?Ip9iuRK0y^%0T(!yiDtmDFS?}f$w+C+(0Xjyb4z0VH9!hd_kk7FLw zn+ez8U%4l*So@gnJ8j^QN@KouJP@8;Z52E8cr8r%rH5DDi^UC%buc-n7S?&{(vO3$ z91KD$Y2U(X3c2pWmuMfnU6xO0o>&O#-HJtJ)jXX1y-0LX?ak}1Ek{|BB3oywLTZYw zd~feOnz|+d#xGJL<@UMsz1V`L2E>44?i=CkrLRywM+N-KU2*yLqtwOG1|z#`v#WYE z-`QM8yLxrRxUzhLOE;x?=Lwl!QfE$kIg*rLZlE&B?=iJ0oZ4?)jJ?}Zgc}}rWg4&c zQ%uupx;#-4svjzIT%0plzsm=|*C|x6KNsVDI%4nIlN7l*ij!7(VWw;dWFA&PYsoWK zHM0Tilu|kO`Xe@(zJcBRi@~!~9FG2|feU9W6vs!F3TqT>amA2)VVP_c7C2PO)Tf8g znB=9jpu<O6(qR?bkDDS}@#7V|y-mXFmBn!N<2zZy&~9iOY=9$+L-3m6Q<`HvnycHS zXIp~;57t~s+6zPY<Kq!nRoasVR%?LKi|O3mR8hA23zArM5AJy<(>37;ow{bq>R->3 zT9;$gp>8I%ZTJO})F-SQGM7@*4p2>Mrr0`Cr1$A0oc4@myY~aI=0PKU?H$bb4jiJq zjBqKla)n%74JbKbGbEkP;ZcvvgfvNet19hB3xfl>KBE*LyonYE8($=|WkrxUU7Mb+ z@5$cZA5oe~HkgaOF?hU<pfo3wUS299^YlG}#{pe5i8ul|!WVe9&4EuWapd1_GpWyo z6f&?i!?0~xWcMvbs7#y%Iv@5!TsKv)mTAJ~DW6C=G8Jb1j>D-p-MGln7SnatlFwyd zoY8Pj&L=GS@~<YSY7FOt@rPt*&X?0l>wRM3pwF~AVK<o<D3iwpZ&3}#IHVqPg`{pz z<X-zvK#r8jags7%-F6KGaMwiTH(sn?S^`_2&lAii<OyR;3gJcWWjLO>OSb$?HO$|n z&mGTrir?RM<16mEIK;0){0=&p^4AixHo2kQw)b*V@3Z*3Pa*}|pM-4FM6uEFFfIQ2 z3RFElkSu)<52z(LXPb|QZaTx<o}+l|1a(kJZjT)WMb>t5Lcg)YX?9=%yp`r|_t*O1 z(Xg3xd9FE!j|&pt1(efg>meLJzY=ei&WDiO?O>_+1>S8>BJFfz>LXX-yn8cHK6?c( z3HGMZ{npT=P-kA&xQhDxVchdFg*uI2Bxn}&5JLuyp~YjGD(}?YhMP_S>{n9<Gj=ZI z?2El2d*el#aB!D=HFu+@1HA>`#^>~L%V9ucD~>DGg&A7m;)UvEq!#;3c<GhI5&aL* zr>1b2A@%>?cTS?_Uy7{h+k+i?^*~Rv2D<F&&doQv3JDuK@%q;hT*Q;<w95s$-s>(n z<*1_3j;F$%ydL;<!gVSuE*AaUdXvtRDBhM5A+GIl7cR_QhWBr^M}scf!tag4*mPJM z{djqkoVre+RiWD0bV!+t`<QdYC3EcJ><O+h7a{9!XH4pTmbEU1p}zM^!O~HYdZmxU zy$+6i$>|Fn-)M&hSK^>YkjVb~#!^@xCoqaoMX#YhgifX7VdWDS-uX?HbCbprNnRM; z0RiBzxdd|`YVeJ|#{@sE0@>jsGBn#8!Y4ntQsAhQq!*Y7`<wHy`AIL<Ie7qX-ce+` zU3%1zwuKIq2E)c>RybgwIol0+L~REW$h^lZxE&EMoX-rSl%KsQDVku8XIEK=a|zP* z%cBoIUBt)VQw8m7!4UYW9nOqAMtQ?>#JUa%kfT$~s$cCTE<k(Mzikfo%cDp&FIJp= zG9Ofa-lK=bVM29gd-x(z7FLb?D}<Yl#`h~;ik>P;)Ng$#?73rrJEuDfL52$0S53gK z10*I&yp-!JQ-YzVtytE)2Xmu}an+P~xH@Jp?H=||jMYho2~BHxQC$Q4JTnM0j1OVL zrCM;_A}9Hi_ON|@8oo{%3oQ#g;HvvRIB|R(Ipv*)Uy`4yN}3^$S~&+kUulDXIfXP# zVsG3Sw1EEIyFjBlDaw^ImXqd)UZPU&98L)^;(ZMr=xy~6a^LKVF<!x39<_+SJyzm< z_OB_c^p@~<SbM%PuZg1F?+IV4I`YSo`?T$8qxgEF7pm$8LRPO5GE#{Z_x)Z@J}bj` z#puEGwxou3FE{`bl?5)JV9qrwo>AM`IlMnK7Ax;}<Sru)u!@fz&bcSmy#|x`;o9G9 zI@OkErRYnU=Sm71yawAj&f+#d1EI-g2G1P7O9(O@PA(1$Xr5vRDlQBMr7+3AJMSfF z+1&w~UVSnCqzWsVMM<78GcK8YQg$U{FAY9u2_ublctS!?F8kn2OZ2Z(P0Cg{I&M2g zcM8Q^vn8~)xu;m*_lu@>SI3r<Gw@Wfq<e45g)2$>MUQ9|ZuemZ4{xZY0EZ|xs5K_v zE)sLW(wk1bK88ytyTImz4iLZTF$C;M7H+Q7;_7z4g;~nophG93y%}BjvsoQIyS$XW zzHh)=+h0=Gj!0a~<dO4RexSU`A<NlX^v|q>sjr7|->8n*ZOskQV0RGOEg33I4|0;7 zd~6}i>c0kyf>($UKkv}h3CG3#DWMdikVY%#6+-YpJuG@`L1FW<dFJv`Se!5qJU;}Y zvc)s`++BrGHntUV7Rj*w$5pY{2o2O3JBOlQMdAWoKMp#7g-U+=2p9HTue`Q020AQr z;+G)>P^WbZ9&4rIfc(SKc{YWzuKa+*!zXc(@o`$aO_ZhB-4dt$y1`q_+oSj=6gRY_ zSA-1OFL->~N`j#dhWF?XIY(S^%femsCpi>;xZR@>$E?NOenpigF{U(f@^3iW>MCxS zGN1o0uZGj_KZ8cvE(l9MBmDVVK=V6%hI1vO(Q<LKY<9{q3R*CPGy@L9`rYQD`|e2? zG_Wssy`qKL*_|ldcPdt2$f0KyzOqoagD_!h3?4rfg^>$ykfK6{<k=Ic#OGV3P;dtl zzv=LzKT24qwi)jA8$@e6dSlYOSh~ICGpwat9^c=W_YHhae|y_<LEKZY@hXRlQ@@G6 zW%2xM?mfs#pG(!ByW_)t{&=eYTe>t#;1ieSWSXLi_I1BV|3DZ_piTUD(OUYkAeukj zh!S7c_r@*@j>`^x-3Mn!9Do~P4dR)nYOHd_fId}q;O?DbXsxy{)3HFY^vP8+ON@u% zRrO-lbXBg6SI3w|TC{AiGyY97hO|saP_)wI@G}#ILGS(Oos`+ASuzZNN_Exb>^{6F zSp(JU6}W@re&Ma_Mlh6e%UMbqxH6y;G<U^<cJc?QezW8)g_^i#SeB6Be+$mURf2}3 zJ3g7LfwBRY#pyBmFt}M2-h45}L+|IJ(e5)8+DQWrUfnACg)FCiWf#TEKOWF_U1dJe zvRfQtn*}~m4tT^ZNN8{|<e{JT(Uvc%s1V+dg5LSTa<vXPQKri?U#AOB16XF6HIcuf z4IRmA2j_mxBI}Kp(Bjj4{1mCqwA~j<v<pPd9pxnP4l(SbTv+Y1U8Ztyy1j|ZSP&gQ zih<^T!2U=J1-B^DM7vfY_^fnJSg$6%r*pvPrqomPyaOHH58|lYcW~P3A1u*Y4Eei{ zL)VU>7#;MURNdF{<p)1ta#$C9x#}mFCQKEly<Lruj{F1tmR5K$d$QE`6qC=~V)EY8 z9$%KdgS!e#gg$OXyz2TDGL+`k0na*P)<OX*$GVZ5yB8^H_U9$w1rPf!r~I~qH11}9 zzVEw}tS6ok`)B#{s^G!6eTFfw=syT27PQKG>{!4-oksGcokMuZiP1RPWhg&OPKGkG z3xaXP8fa7R4T~OJrcE19!ti;IKv$a6{yyT)C$`VR#P+vP*SZk)4*mdcx&BnpybRXu zixP`BnxVU2y?ovQY2Hy|gU{kzaBTKda4X&`b{_B(zO3#?=a;^wHGiw=+$K9X8h=ve zS5ixRsyBmsc^q11j^^mw)>ISr0;<j}WBZ3oc+2=Yak-?~&BfjL=~yo`JX}NhOM4;Q zn+qXPnRGL-^N7Pu-ooPU+h|q!3@rOTo$H-bg-u7}gxI(yIJYuao@BoqN@`QYCqL9V z=fZRB0}XV{DqV0-@5>5-hj7t@8dCC=j{f&j&Mc;lP6+YfKJ^d{o)QZOl?LF2yJIjC zxAKhMp5pJ_`TyyEc`<vG%m1GKVE5nFA0*4k|EpjBJ^zdS|40A-|D;j<m-j`po#X$R zKP90rh5r_S>i^zvq$%B!|7HKx;Gq)ms_Xys_axWSzyEj-A0E6g{Qp4M{4YFAn*TFF z^grpHLnVmK|2{wT+<-TNmv)6PCCEdF$kO7LcT0q+o!-E1{j=a%YRzt;Luv1;BpP5G zO=Z?*5^E-ri&t4<M1~XMT6ex)AhDxjWYA?_3JjYu6i&o!MHllOFspsfQ-dbYg3&vE zL#;GtT%LPUSb1R#)T`@bZT|?)_;d%hYT9yY;UEnDWeE@WWx=Lnn!>}A*1Rp_3w1pd z0`@-ESX0~vYqGC`X;lM6T>eb@F&f<J(o?YcI2QA&_REF^^`J9Hcfx&>HL`h~uZbVa zJY*>yV?<Mj@nYN~KQz+%N<kVC4s)*>h&5_E#UHt)6goRvc-hOBf3E8W3#82BVgnVP z)q6CzgsRY7`5z%$sT6K3@xfZPNAN+a7f$AF$71t`(0pD+mfg7{?t0t?<I4Wgv(R8# z9<R$Wjtj6y_7!qiJXJ80=K57$TC&7iUmWq{lRU^L2Erp&a*w7sRCX+fdvnf0TkApK zd1pn(wS#bK`FHBI$%VDLNnVqFJK@lU7%Vu_3Q9@isPxP^n*AaO2EF)0`)5Ybg*BTX z>59Y~b>1yBKai7qQ!_*<G(+Yj!Y;>i#OA#_ppWe$?p|~NFP^Kc%$(efP2IN0C+M9M zB9!)szd9SEtu*fr^tb^Tt_vvN<0H(RAB^dncEj3P<7n3JcC2-zD`>3wNHgCUirvH4 z!nwpMsvM$&8okwV`T9V3*!#2)xg2B>K9*>ptpZQanBvAOTlu41WaS^(S=jeDjd;x* z(ixP<y)cjTYYpMe*{R&@<<8FD>G*HFE=$A#m?XZ1o59i0aJPpr>P@tK|DIB*222(o znQr2al@DR>`7@Pce#O9+dFnVLXA4<QwZ%OdN?3TtklW2QK!qNg;qI-2;=FWQ&{b~~ z>uut(<CA(=*tC(aM!Ca@v?(a6ZUftv)6ipN8yKB!68x=<SS4T(F7<MgB|f-K8{c>4 zS;J=YhDR@HXjLcLv`gfLm&4fkr4ptb-b2Bm4UjwfzG%0v7JdCo_~%h2>X8)0%_VX8 zd?etV_YJfowl{hl8p<&%6GV@`@t|v>&gCNwurPBuK24enR|8LwwU*>pv2>-2p$uaz z3hC^2H{n~~HxT#uGfWEi<hr;-TAHfOrBhO%=w|{eZqCOeE8dch{ba6EZYNnHjBwqu zTjViSnHRO_V*kELIOEYsKKOGwHZ6Qk+6|H1`Kz-yzwJ1zDL+e}Pdf7MZl7gJi_@rU zqD4h(<xXMC8!5*$YA__$?v%HkD&X6Lqd6k@y7(=}6U^e$MWy*cY-3TvCd=C3v{8TZ zdl&;Lwsy4T;5nG^>ZsUNAqgg4*$vTW%*iqGvhc8CBxfAh29~SUXuSJxus0nJ-)JJz zua7XSbRBIuk;`Ao5xa<q|G@`At06@kwnz)+ju}RC9!%$Fh0)}=M<2|pO!$G(R$g>J zmyJzsQ?Kt6G15T+JtpOm=EfE{eK3b#S3D5Pjt}9dX0ALj{e<vP-3^wWF9U@q0=s7| z;!E0=qUW38@N(-NVdOv;etPzynE$r}s>rvJ$)MqMw&|^4k?;q!7V7hhF-KwQ_bfJT zenN+`3&oK?>@n;_1>}s<<6$#3(MQUVJ!n40`+Fq8-J1S9;FmeKbPR`UGe+}}^=DyJ z*M%%TbYP8Z0|eJar4pC(JRRFq&!d&@@$$q!bb8qZiM2Wr#(z3Y{q-BgTlr^bXLTqp zx0BcoJQ(}AC*nej3H<dTaZqY6ynWIYLPkeJ;i5V*d#a&b<wirA{cH{g%!uTd@rUHj z>xT+A*383(_&?%CJ!5RC*M?p%=A)0Y1snRVhh)3^G`XKC81C-Lk0gJ9M~fUh6K2AU zavwMuPZjIFc=5p8ZLsfRA6h$Q6L-64M)7gU*j{ZJRum{=ZO>U^)=C*Kxw=#cJbj%W z{IR3rCy!u7u@NRVJA&5A6u~ibA$Y7?3OkmUlYh@Tav7t7pKnQh#rd`5H1#U1R*@k~ znFY;;*VL(XI2^g>%9oO_(EyDR@H?^?q5|KF<1<Rhz`h3V3CWbZX9Hi%n!*JgHK_iy zDzDj;#l4jl!ZAZRHJuy5wI6T5IE~F%6WIli6{S=5onZXiGM~n;%@p2hXo>ZTZ|U3o z5;6DlE^$Y4w76LN6j-Zu5Kil~m)(dn5UNMD(UVnrEZC$#fz=dom7h7y)m7&eX&RX6 zzXi31yRd5VXsF%U3;+422qE?+&^D||wpH1U0$wO__Om;D?V~v?`tHbKYZ8UOABOW{ ziIvdd)nQUJP{N_%m4eQi9JJIj#^0T_@zx(*ylPi0Z@0vOTeM^lw<DQWZU+9B*h6U8 zStJhapD(Vp=q$Jle-7Ff=5T7@0&$gVD{U@b522nbA<rfr0z%t!?+@MS&9NFWWz-@0 zLBWzY4?ByWp15P{MQ#2$s0T@qSWMkqcFJ#6J{e1lfp;f0*kVz4Oj-3re$F|R!y}bp zK=C@vn75Wa3I-##=E&TZb{5ulDyAP-)=Hl_MXs4$1r=E@VeID|>Mh$qb!Kt=ExiF| zdesP<wx<fAZ_kiT(I7C?9w~l(IG)~D_T<m%@%(1LGB1uPpp$*w#MkD&Y;XJ(ZWZ>! zxl_7x%_BKPuRSepEPDZlNuKzoQJr7**5VG{x8O~nGHe^$9!_m5#;;?m>2!i2w6ESr z-#g{eil1|Yq@)g5JVo-%ZneO$1T9oaJu2>9Z%p$34*c(i3cDq$;n8?)to#uu{rp9_ z?b2D-)*QcEwLnyYK05}^7mT|pp+em-Sai8R-YC}N`|T9De9;J@<VzQ^XMsCCpIry8 zrx)<EDJ68V*_KX<i_q-zJRD1oP%W>73xx;h8I|zf3r+B+xg2s-6UFYXOL1yP7e4Cw z8bpgC*lV$$&%NC&MD6p&#`h|ea$*Q&^xTTGf2516gYOB;rew*k?Xuu|O_5YErz0l} z%!AfLziE`0Dc?LiOxFI-E8*VGM10aa7Y9TS75e3$qRPM5D7ZXYQ2$iOCkKb|Ri#_t zJuwIu<@duyV`F*7j4Ogu!~nS8M;+@Q3`TLG9R_`D6{Zf>MxPH_c%$<;@$_aX-|3~w z|Hcwz2_NXn<(qJ+LLWBBb+EIKKE58FM<tK-uy)H)`8~T=Vhw9!&4>GPSBYJB?f56i zPj@2iRn9zgO{-9nt&8ne*ON=%Y8ar<0lU3Q;gQFRpv1Hbmg|;LlimWfnfcJ+YGyHv z9IpU=9FA{xci?-Y*Ybpor-j!wcO5h&e#_Uc2Vt(nk15bFroP?Vp<nq_e!qXaaQ1m; zE_l}@TqyPABX9iK?y@J``*#C+>3)XRDbl(8B#fUai*V|)lr8$V9s4{!N|!wx*gqp3 z=b1<zN1q46<nzCU#op5Ed{!$ltp;N8>f7LFZY-R6e2rSx>!V8H8hK8L5&wPBkF|9y zSv%u4Y@jc|QQ>0YPicOX7|xro6w=s`BXXPZ-Jsj?(dcM;f<w*qv03i}M7+KUZ-VdB z;pgwg4d3+m*w`@icDy93cyG%e#`$oQdLQ}{=Em!G<&tseK<*R*{Nl<SygJPu{kaJC zHN;Tx?QmGQIsiv`4}m2AscfSnl0wN@`mvng%D!T{H2f#cxex^dOGTQ0XFSZ;YzH0B zzJvYyYM{P8Mfln4sdz5Mls4<Aqh8@2YDwJ*Ppeaf+trB>JMSZ`boN5e)BY@Za<KB2 zHu>I26;caypy8|?u3Z=kQ<M)=agj0x_nQO3i?)$Aufh2f574xN<@95|5-7?&dF<KQ zeCtT0822lQ-(9!jPd>R6VEdeggjK;X7Y$sW8Z58A9R!UJjo^GjHe@d7js7<bF?xKC z!?Y(y#2rbdyi6sI>gpne;WwiNUsZuyuXn^fLtaa{Ivu=wdmuZX)5gOO{K({nn>adf zHckkP1bIgR3nYYmV3H$saZjY!AO9eGXRDAqWhf>ky@riRb(P=Sx#O$Hfx_612f%8h zsi?hbym+Pg6^!c9ESyrRg2g}05t6L%uB4~l_*YEBms{idNMCGPJ%|-fH^blFmqqnO zW3jwoIeh59TV`RA!xjB@$n`G16`Can+4g1^&VD}&93)nvf4ObNXbTxG|Etc8RdwJT z+Cm{6FA53Xd353B7XA<zfdgkKqs`)v6lM~Iigsaw>Gi3g*0`CT_E5)2<*xi8Y!T)= z%>birWn}R#4CJrA3LmGAhG2u~jAlRR(8%6G(c}bpsxJo>l|`VDyqBl>HW3@-JKSm* z$U&3su~z*YHGHX)UGHH=Y4SH>wMvV4T|a`=BI3aNN)e6dIf8FW{=zDJLkp&rQR!_9 zT-DwX7iae3cU_Ed?vsUBHd<nu|5=E=mo4Bi4zI)~yFxf}%wECtLpQFxBC%ag&lSG! z>5SPYieQMxFsjv5!AUCX#KMOwrO)R;Y8<a3TnLmrJgz4p_P{Z*@`A*436={TQpeEr zYbWUI;D!97%MOekUP0Rn!*J^REO=B`KsFuBXvE!9wC?H*xanI%6Ea;zBb`9JHKz-Q zKRE!w6Kh3fnLVrgHp9<z+v6k4LO4@ZL=)9Jajf2ax_Rjx-L7vSHOc?CTRN+{EtxL+ zQC@|5Bj)k)P<!(J8;XVYmnow|XB<4z7%e?|aRKiUsvQ%B{ONJj_F4nSPwPZ`VsyB} z{;}|P=L-tQu59@6jqKXDn_xc6nUkQHn(j8!i>1FQ2GhA~=Vj1KmH{%a7V&)b65hRN z7rvEzA8+SLwMCOEU!La=-?mH49?M#XRaeS{yWU%2eoYM}%*v~5t#8lgv$R>mEQz%w z1*>4@3t1<huC(z&8h>0W`QZyy@rzLl&pL*ztz^Ox-}<s<?RDCe;DFn-FH-3bIW=~V zLpP6j9<w7895W?OyQVJ=(9oy69$u*SJb+!^-h{sAM&hw+w`r#7KQa7?I*t~@@tWxu z2ptwq5tgTEh<_UTG&!+yWEURQ{etKmp~t(YT%mWbx<HM*Q0)F9ln=*+fr{dJxGQ@n z97_xX(M+n(&)$L8-%ilsqa%bbt(IWN^}<SndJv8kkWSTT-kwEVRQN+Eljg>Uo~4Vc z&fJFV{S{C$w1QHc-pQVat`{cUPb8gN$uKfA3eKJ0gqDL7D0iX`PP&{4){fF^=(YpG z)T`iNehl=#_XSQX_Ja#AmEdOjbkuBjT7>u&qT#4*@=w)WY1IAptQTN~YcHe-lMRx^ z#MS+9tO-;0z$x6fbuaTP1=MnJ;a17BaHGBiF737AVUqrt@v=gU8*9WPx8`zd+dx(` z>V{>r6VSl3Ggw}jhcxA@xXF7QI945ybn5rw{8dI+{@R#MeHnypUfa3Tiwf{kX{F&; z&r<c@5^D9zl9#1+<#GL|Lf{=`-nHa@MPsv!HEOy+r|Lbh;Gicz(pZkCb#-xm%|MjR zyiUE28-RDS#J*Y@%@Y?crHJ35xa8XyoW1@5xynw`x%x}!?-9mb-|vC%OOhbWE(j-e zze1UDeRzF#ADYxZ3a%FI7jmyofkf+MT9%?qwii`#)mDG%?Vd(gR+w?Uy5vuG&7nmz zwD8B=!`NUx1^#Yu6sHeg1^qjp$EMA%q@L;|^;DUSqcf#?1?Iq}tsO=TGLOVwq2aP< z(^&C`o`&c+e7!h8$p^IVXmO@di%_t2wvcEU2*)=+6dP~q;qIDb>fswiKAP%e->nyp zyLw)zom@^f(<h3yb#rL8#J36=@tx`oLaFMBDkV-?0#8g92)WZ2p;}BQHuy9OZ=oHR zcT|Ae?-jV$fwj1^uSgRr1>E%5g8S8{W3*K^^$wZBokq>Yb7vot?_@(-DD~*=bo$bT z6ZOJ1Gl{!0q$}Djv%txw`Q$Oa2-hFB<LD9FXwrFeZa%jY6Z>|=#*Rt6@pT`5ZXdx< zr_6^V{;u>b^#`5Nh^L^V!PEu4_=eg}yz}Tgy<53mFkXKHx_*8_(H(L@t6dm(Y?z19 zjh1XO^9FfrN@V?6U+Gmt24&WNqqwx=lvvRK-4pt7lfzzK{X)*w`a@9lr#c-U9|pe5 z{ON^-2S)7I5<N!cLq)qKe7MmW#D@tO{o)L)v)#qtp60-urYa6NlfWLe8$dT`F9msb z=cbuiaOR4Pdm4Tv*UgG34S{%L{dUxgkD}3A8)#$w39-pLoRxDfz+*)wWgmOC%5H~| z8cj0y&NJ}lMHQMje<WCZek*L8TnXDfm3jEZ(P%ZdgeDElh6~60(cFv!kY4{5?3A5k z4>J@nZ=n_j2uN{mlWCUqJl>(|1QRR_D6w6X!zHip@cVfS^p!lJ?Q^E{gPmbCHK(8C ztxpz{)hzJ5v4?PM+;pV#K0M*(TbMrm4?TG(@p0$$6;+g`lh-?epF}!hd$SSr!7!Vy z?fn9d<9CUxo!u#Uy_q;9+l+kcorR_M^613;`xO`W&qnc$B^><Q6CJLG!+`qTkiVuD z>aEI1Vfhf?J-x|pL?+F*T1#oaEhOgFJ6PP<53hE&f<Inv7`AEz_{E9XqLTvAsoMDZ zTsjUc9l+5y?!cI}Ay`sTLY|*A$)JM=pGmk5LoTPXM_)jBd@MYU>%y^aY2^RMi~44U zfN#qlxRB$H5vwlKP3t7`Q8dTb0c+&;p5?G}Uk>$I*OA<PB4CZ5G6tTKvhnc(%Y!wk zaoIkx--~zjIC7~tw{RjXU5Mn6>Of8NlxW`5HZj6_C731G!2DH#cypK%9^d?kCTS`P zi%o8dQ^MxppF=xnpjrg9vvvVt+zWAj^#U+F-VLW*3*d=1%1l1${LXP7)tUCdZeeMt z&?kh_BZ!|ln$xqTmb`cQd`NR1f@;2<u>1Pyq*B~0Zg}klkN3>OG@%!}Ykm=jZC)X9 z^K<y>s;BT_o<ha9X<sQ|<Pi$`Tg5r4&7zqsSynyaiWn2wo-d|%r2Fkh$xm12gG0N6 zVyag!<m!7+JUf)9d)e`-P0r%eh1bO&Vg!c<bd;ai0;rO)m1hRa>1wCJs1e}=&pcF+ zyOdGL+n(6NCYqnl7{FK3lkmu4M_#m9^3f^Bi7Pg561!h6q9MKlS1DeC-uXJbU|6Ks zJ<$Y}dq<Fh-VjVFYJ{KN(sAgG{uuee1wEWo=-MBoqjkz$)b*)@mw61_vzrJW*9--< z#V?_;n*o}n#o(a0y>Ke`r>r5gf+}XmVaa+0T%D}Itx;>}=lf{faKZ?3>H_ew#dfO9 zxksZKm+<GUF1-D&7EiYlDP+J){&7;#VUfx~m{S#skGJ0@)16H)>5hUB7uX#h)w?p? z^M-4eN62s1M^w5N#-rwe0X(;VB<zvqbMJ2s;wKHNynS~cK3&q0J2ZB~b|H)CMbF># zvhOxZTDgyYjqD3=q`abrR7-v-Nfa+cbc8FT4Cu(U6XM*cCYr8pL-+fP6D6RpI77(i zlP#g}C2c5vTYF3>2{2*Xw~uIqpDL=KyFwR@q?)}sibFC*niOmzp2&Da`nS8{Q3Zxg z#_@Q2i84kE>;yjRZDH-<G6-(<#>#WW5O*+!!Y9>Jg~U00Ym!TDea7OVS|zw`(V4^b zjIh9ToEYf4l|TC^pqk{n3%cEt4tq61n_-~5y6zI_qbp~1cH-WTMa)0<36|NYaahm` zs(rJ8k`;78Db$meY)|8xx~E`|gFpXh{vr1L;tj12Z;;YZEz;FXr-|3<C}Z4OIw1K_ zw)^zM0SD9R<iKb;pnH=Z{k~0~?)AnjStZ06eTSp=ufXn0EpFHDPrWz_*WJ>h3k@c; zf3Yg;`nCe^_I07wiSaz|%mSF5zgReA^+u>&oP>J&J_0E0r;1qVIaD2s7aW~fwmk#h zt+gTt$K#bL#znmN#7IeJ*JCH6a`H<XE^CxOpkB4Mg74j}oTlz7+GkG2WZnC)p<q0g zow!7bvz6eGPoglWPENz!U%;4MA(HMslupcU&sxV`fm=}_7#;D$)Z}K^bukKhPgBNa z&Uvus)*#&ZN`+IF6w#JI>Abu?kAj;j#3gDbc<JSD*?~cuA<wK-t~lWzEy>h@6W?8U z<(1E1Y9KL)GnQ8V4Sy%LTQ2SV(!C|kMcwi7kl}Rxi5|vIj1}dyPO!c>3a7YgW7;nz zqJY`rl8TO)Gxw4h>#rs3Tk8OYvm<!s%n~fWbq8c)?Vu@nprn~b(}JMh<h{CvE<bmK znf5*S@QB$McJdw>W_gG`mPE*Jc-!&L2X*AULei`id*K}i1$lK_D$ks!1pU*bdgjO| z?i?)bor_umo5s|F();)Fn31Jom+VNqKJPSW4kQrv_Q1bPaPN7j+(3OY8n-RQkbg<s zEi{3ShhG=8F6*GzaUaw!zbFQ&FQaLR^XR>`3LSrZ4aU8C0$Sx#4}D5YygXFksbU_i z&-g9sZixrkL;<IdzAvnld}h6*`@GBV2y&?J%sYQ=fvli8P`%0=e+4WNs+aD8y)DND zPq)tS{!kKr=nzFNK1~#Se1Z79`Ht-Br5mzG@!MD_-4%y4XUcBvZkF~T+^0inv)S^G z2N}85Q|vHTIF?ibhyRpP_m5Gyvxwv~ZtlhX85Usj?iLJ?a?fk<7<5gUi@B>^xn)Ry zmK9CJyO+PohR-|&x^|vu>emSmVQ;+F>j7<=DFa)zSO~w-o>!J@<LB+WS@EEm)KACK zC?j{ajhaYNVXCM-a3y{m>BnZ1Kay6H4Xnt~;EBO~_|AjlvgbSRljeg@w5D5ukomWW zJIH^+n4@aY$2V2LgZH6do4`kYD4~AE1u&|L0_!fT$y(|G%f=PsNac>Sd{tN8d3GCp zNMyXcQbC%>rQo?Ane<)VUeYo&*lCUqo28tkpygL#n};UNxtmRHw-xxoU>oRad6wo# zdn~Lg+Og+6BdQ;112Y#XO1>Be7PZIX@J)>{D^?pf5B7#>cUonAd>nAtqa1p$`Kst* z*cT@*2<5R~wuupKhq+6%Htt<HSl+mMA0}uVkofSuuyWyOx!Q}#f=$o^LHqs*QB^mH zQtV|gZebYBdshd~SKGjoH4*IU=!{oeZ&ca^OV2bZ%RAfu3>*zn<Sp%op@G*{kUunK z`v^}wnQewC$@=Uu#SlJ4uE&GJ++g${A5<tyqqnKu_>;C0YuPTw9RrQ=<W9m-eS0G< z`3#f)=-|fbc7n-3W1cY54v%@>r!{MC(BA_a;Z4DLY+v3;&lmn9%ePl5dw#q}{vSS( zo!<i>&-rL&Ks2fNo$TgK1HSO<FPO|ZEOeCbhAtkL>Gqoh-rjw_utZzHUR`zxKJo8m zc{`KQJz_U+c0_i{)1$gy&0urt4y^rqo;2@%pupJveB)<4l_xpSz7;bnA4CskQSmw% zm}kKBm`@a>Al>`BOIc%76<l;RqU^X{SnoHGlX}j?4bKXotlCY;4zGootL)+53mK<L z?9bu4mN={8kEkc@j@Ug+QDW9fHOHw3qA*mSXSQ_`&n|r}hRz+w11C%-L&pTl*t!N5 zJ#3(`<I&Xk<2oIkHX0x69%8Tkozb({p3j)xr&EoM@X~uDcIvAFA2dS+o@kBxBpo<l z>j-Qy+r|YRhe^A&KhAIIglo-$dF7i!a8>&k=$=oMSa4?eYL^I5X@lP_4~q7?M{wWA z(q4d7zsSzEHwB~z!0#Wybb8GJm~?*z|GKXR{|=}?=N`AfYJ;RrMP3sUe|E-chhr(6 zFVo?ZA9zc1FAUMshc>@(JZTST;&2S!#_M2T>uBuIU@pWYjOBhKR<YbA3bbOJ#REes z#Bj9{6zpn@mA_WO>gHx)&Yx#cH6jptkI9C4u5YNH_HuX?z6I_`o~AmVlVZ#ELsT-* z9N*s2q(iT)p}wLU?&QC)VL~=VDJgQUu`wTi63$0jgYf(%8JU=6@q|T^KS<g~w`9oy zEYH#g@5ufTJJyqf_f0|DmBzTn@F4kr`$`9FYQ;sHBOxJUkFaNq3Epl^6ODW2fv-^t zpIFyMvaJswrqz%3U-smkURm@xwSk;3n&6;>b;#zC6hG4y#{Kyv9BsM?8w0&i-{u>A zmiRYyLrTO~*-roIpHV2-F{@o)$u1#%|F8TE{}=uV31Iu*^H2O2lg9t=e*F*c$CRz9 zs{gsWR@z_t|M2=t0H6Qf^^fVE4e75``TNrHiuE&_W#^AdI%L%r@vp*R>blB|hh7V# z3wh(HpgJ1b9467655w_JY!}?*8w{?d{c!xyFrIr?gXcRLz{Qmj;?VeEoDsB8${~%V zz55cV^1@zhP5vNSM4uBhMl13i=dTdrJwuwY$I$X%18{u%Th{qXthi3$a^+>KUsSL2 zQ>txx%AKv$*?x#N{+Q4QznhKcpnY{Tz2|CJr|O26+#Jcm=enr!xE($i*8>|DJVePM zfx}izfPksPFm~@nnznHaMY>2H@sA!@akhbyq#S(Shhw0opU=ayqv1i~B#4w=&&iT$ z96rp17S_e`hygng9F9wa6jvOTFo11RJFqx6Oi&q>LWV_$_{jD^I@4-SH7DEihKr{u zxZMQqcfKbEW#~w3@MN%x$&vJ72O4tDh9}2XOI{cOUyZL7hF-OWlz4q%a=%UZ-C0hV z7mTswhXp_Obz<+yZy-jfrN;UguAFv}CmyiJ`_bhNCd;M$5pEK1Z2E8Ts60r!%l3&k zA18}j^+w1o4+T^4x^cYQYNLF8)FX=0iI&&*Oa({XDpo!{4aP;+$e`dnrM?eBtu0m5 z{_ZqR9Oolz-E>CworiL0_a5Z7HCnvZue0!e%L*R!+XO#L{Zi*9MKSYTAS&rCf%1T@ zY$iKPA&Se`u|yN&ENoe^cR1$RN?c864Ln!62&Oy_!$)cZ#WzpS!=j1(!BXKa1+07` zp7A;>JGSQ<O-@~p1rx5r&gn<saHp+!`nf%7J?Vvh?h#ZNxmKoleJG`^J1hHcdXHY8 z>4s6G!r51)9;*Dz_>{&0@-NM$K8F1%t*AhV*X@gSJKBW3%2v{Iy*p3*wgTkOy}+>F zX{d?a#7%!sQ4h&iJnHm%XuFV2>03i#^vpwKel}5vYC0{9JE(#AxjSU7(@k)3feRb& z>W?1ty5OnXC_G&rP6wOY<Cxn|X~z3Y;P=K7jo&Ntv))&sQ^rf$c<d+*w`!n|lJ*jH za}(+Ao(0Db+wfasC&;MkN9}jni`&=SfR0AePNfT#95>+=sis~LeC4;rvi-Bra{G4} zaQ`YCTX<crRu@bA#9UGeKMQZv*0EROW8t+*I9=K{4Rq21c$bDge=F|JQ}69U%|(8E zeoeCYtK|cP9a7~E3N7GKs0OR|H<97>PE?~cRQ`RFl9Z#X1GOz}Lf(nt9Q0v?s4_*F z9yQwHzkWTq<F(zeBXlL%Zz-a&;X`=AA9YmT{R_;xW|NQcTWZ&*5B*q?PfruO;$q!$ zdRDd=FLpBFyt2=*sogld?vf+)J*&W4OI2}_t{vW9Jr5N&Pa?I_bFgWE6RXW~mU|vI zrFqGw+$Xgj=6%f*F9b+l*|!cDZ!n1C?>CauuOyg0elFRCjuQPM339@Bu>X*&6tg=I zjy@R&y^mal-_<YR1bE<TXI<9*a6){3R0A*J5O#IV!kbs*u)E_aSh~hUJhj48^y#F; zmdi5)d&yg18sC8$i-y5zEk9V=?>qIM_8m@|r;*dNNz~ZbDs()(impxSf%8I+@PZZL zxU}@Gs9KT$ZGAd{iTn;}`)}jV=R0xY=Q+I7^1A%S{eNIRIf5p%NCd;8R9-72fyKZO z-g9>-mmcXzXGTos!m@*us<Re<9M`701UqgHsS<KFI##M$XF%eYDY$w{C?`)|Lg^!; zdHTH3V!K~+sNvT{j<vRyY6gA0HfT9c)3f6DoviUnn+BSl4MF3qe7t09jG03{xnYtr z4xFvQYRz`EEOw#HOq>BLMl0a7lM<uU%?{jLM&UJyXXP2-QaP+36TUs_DPEEKfyOWs zT>a@6b(%RH=KNITYX&LYp<e)`7`_q)Z81gDd@EWSQzgFfJwm>H&EZ3SADrNl3x%3h z!s<V2y!Xd`e$zvr4TB@_QO{Ud*>IenEDgrKJHODzwn^M>Y6d+pGUEFZIA3XeA2DFZ zF0lIa2Mm4H(MRf=HXIxPMHdI~gh5eI@!1p`<GPEuIv@9zoPnzLnPN^;IL>#n6H2CS zgmr&1;fu~JSx+nJ9B6d|pXz_~?!!mgYqOZjq7G2IUR#8_PZwig`9OR;HI8#TXNoNb zB03!OhqlsD=$~RqQ`c?fhVf_E5mfo1@dGJ0kU*BV`?9QIG^#JuU}*sn6-S?-Z85&W zj8|FUHhw=$^(_!AixY+K*P|(R^mfYX*@qW~Yom7cUvP7CX5&{rBunXy-MvTg_>)bb z{>0kBwo(m}V&21}DJ$S{yGsD;r;Dd09=Kva6g*5h>tJ?jmb5Q2iqDGrcz9DM+>+od z``K<EKC<qEe`oK<ryJv7)5<~24UdHr-&HyJb&j~;g9c~$%q8#i3c-DUj2s_rz`I_3 zdE<iw`DfiC*p(~bx`WF6?EzE510}q8V>h+@oP_><5BaHsbpM@_IqbhK<<jgH!)A}s zI9sb1z770Ild|`c1kI<Tz0+`vz7C+c680=_$FV1q_`rrC5Tp1-d{Eh+#!I;(FAGh6 z{<}A~`z7tY>}`kbFQtOntiRN7QU^`<*kS*dJu%<yF?49QgOxIAmv7IhqSN1rsPe%a zZ_FMgZX^%v=OnRWwgS38RKzX%A$a%QJ#p9?RmiBw5qAW46#jZda71nrb_~detEW0r zzIzG=>kXEDxp|%BM;y>3Z~)kBX`??o%Xy=&8=iN*%Pu+}#eIqn;H)!S;-7z^Wmys8 z8?8aK$LbPk``n_O&p+VNgm|=p-{Q!%hv}}y3ZCG8Ot5`rK@XQ6#5pT((cAMoggg6U zd6V&Z_;xcvZog?J6}qI-!cq%(I3toQ;~rEV3ikq|L__Qz6v1teuaN#yJ<ts+z-QZ* zaoH3fKDxdySC=e9e{E?GTTd4p;pZi426)T1A9Un@b7u%XUWY30eEBX|1?;52S+;N` zM9L~GNP>>m(kxpg0vc2c=&6?;FF7;?C%gX>2K*g~X%>!DF}Df64bWt_&nBobWB~tJ zJeyXV?jY}8$(4bugj?PO@mNV8dGhBh>0enuv$Uu3$y4{J&ys`CBg>e(U%Uf;5#w-= zVrT5qro?H}Z-L^TW}z*62hDSrGU+BUvTsVaDZBMBI4b|8nlqQgv*tU2YnNl?%XPfg zc7R|sKN0tB909X>sBp7!SN^xkiAOfP5nWE03&+lH7r&&R6nc8UfV>%BA*y^9|6I16 zuA6GI-qv%d{xc1Cdep##kO1_4_MVol@5-w;Y037@aOT3&cwT;d1LoI6piuA5eA+=a zd%m=XvA`SezU?m_+qHvMR2kxTEi+a~GsBTzBOz3blqJ-sz{N8M>4ASRtlE4YBKv!R z&M7^N9FT;4ZZ1LdZi)0(bp%>%u7)J1u`uD^SiWmw$2k^%Da$^AhWXm?xR$~E<moU} z-<<-JYxW5rO%953x7RqvwNOm>Zb{RV55Z#x!Qt>mTkhVX4A-Y;h<a%=Ay%5VR*wzk zRUTPlZ1`9%3&<ty<qKpPGfk+dc?y*E8O(ZM#j{4%!i#PZ7@So{`fg_h2R#k$KBqlB zXlRvo3TMzH-!H<gy@SO!<8DL!#!zXOw2ffvwFB~|&1U(zd>EPTO-B2jv1X(WZW=!T zh6EqRw_O{=AWcuyvqDI<(}h{P;^Eqo52UK3A$?}097?~-(C5Jmu<4vauLo)1lcHPH z4-LTU%WTj!awe;ZGSodcTX<#iRM=5GQe5M7SzM<&8gqWl!PZ*~xa7|WSl`KlGDD2T z`$}EOp=TmInNtGWCbt)JU>7Y|t;cshl+YVNnfL1IiqoWBZB9iq=<|0+3>>wc#`O9N z)TG0nZ9#0^8X=B*@m1XXBaEi)%|TT=FX3BaF=-laVR@|=#r8iz-q}ZB^YV*y{?<7v zwao*|tuihsvqFmlIwT)g0OEWj;b-aqqQ_Ij@6ztYcip9>$S>76Qtdb`etMLO=6J{) z=XF5S5fUro$TrbmSy#xI+zrnp-2hc5Y1Yt3^4EGhlGSm^6Wn?QOR*GuFFNs6OM9OG zERYKCm{My`5Z%vf;{;!+|F+lzC#fDHHydHJVSw1U=^iAP*2Cs}A4)%74X-6`Z^*(t zI&D=9X{r6WQFcIlt{+H#-K5#NqCgwQFS6gLqXmDYOqo1ipI7a2=3GA+OzmJJpT40z zhr0GayU54TGhrcleo&;k`h&tPmq+qR*Sn%)>_2KNmzcgMtLR?;-D2<G@p!!3VA`fW zL$ts81|DlBL2diVG(Dw;#yRRj<Chvx_IwF$&9}&4c25|4BZ$s-vJ}SW+KC&E0_PQ4 z^N9{l6sVX?pEel5o6$)USIh-YbbBPcH<Y{@6HItfp0}t`X35Jst;DZAw?lmOED>sY z^RJ^raiB*o{R&#hPp0hP2Y+9R2b|jB4gV=nKZQ`)y+n9*s+>MfFvHow%ek%b1Fc`8 zjU&d{F#oZ_%97*Iy3w8Bx<7`y_y{ZQ<Iw8!VY$ksk1|`QTyV<RDC;z&0E_w_B~uMw z5bmYRU30Qw;<QeDz-<hF|LBamw2bdw?at%dt;5`bhv~-n6P2DZe?fc34cQ{>$lKdu zgtD9AVE8+fR9oWcHBZ5LqZZMa)HHs-i)F8E{|Xaz_6P^+>*1pFX*_W^meMkQ(bxs~ z;&)1c78gan-~1BhecuX+=eLkb<3m9~KLSd&^yH8&R($>CPQF<CfYul;!4_A4x^25f zJeAla^zeyBv%FB6I^3R@R4rrkcZGsW-CS9~97Xi}QVOq&calrRbD3H%SDd?d6m_mW z#4QsHuzlt^@EH3)$b0i|uHx_S+dR`?$V{drN+|I@`-MtENl_#XlA?K@%N!+BGF6m> zRFYESefFkNlu}Ab8fc!AqWM0b>*s5&`(EGA5BIv(^$*~zxAQ*x?AP=8c&M;exoMQW zx(y5;T40KlEYm&G2=Rf6w7|+6W^CHZJ7ruG=lZDA3pE*1VbV;)`3GKW-@*E29|f7O zPV7pyAvQPPf<e<dF`exof0M^fXPXDoQLto<@d_+Ino)S8BFs>-;%=Rs%2eMei-)Jb zhhZZI!x*{2<d&60y;Ck=UTr>IS6j&RjR%3_gTX9DonYZNDSlwgAG|%_2H&Hci!7%R zj>&7Yzb}mNg(#QuAK1{5;Hhk+IDs1%I)VG$Wg+Z`vdG=dkz!LrV1u0u^d%Qk*?}H> zdT9ZjtG*1!5?c7fIaaK;&6iETAxl+n_Opn`+5F{=Ww7uA$1-(_u`^6eU%LnM{@R1+ zx&Tf|SQ<r5&swm;z=~dNRAP>OcTp#NIKRYNV6Ch^M+!S1!|do7%pT^00`H0XH5NgZ z#x^+ab{|#>vz?<&B<&C_BKEi)Rm#HHhz+}O%;5oiNNE$lwo02-YmFt7F&419EFR)t z7z=qSLpIf3iltt0gWDYk*}^iOOMdCkEY+q9c|s3%qIf5oMoa{g@P$;v1~Nx^d1!B# z%H~hnz<JgL<Fnp;Fqx~2T1_ETQ}7H|z3ayvESxX+QBK3UHUnrHVMwhc&B`S2;(9Mt znx-g8vqzO;^pZ+^c{vOmW~gBKctso}bml566S)YN(QN)JRr2_7Qt1E9VZm^nE8U%n z5*Ov!V4KgP8{<~6uyYP%-Xi9#r@rKNiwCfvpC)Y5&0(xE!ibr!l7o?ld$7~)5&vbG zHLtq04*IH9DPrV8zTvor^U>Y$bi>V$PR6N|;rsg-o2NorXAEiE^=`r8;SCmXPx<T% z8|ZgP8rrfVJg?t_nlZ!4R=6j(&k$Hp^5IOq>j56B6*!l{`%uqfEX*hg;=Mn|zy|{_ zzSVX%tG^`N+p^|RGdz|uZtus<0pCPJm)hfu1A72%mcw+LZ@6gNBe=iIj!l|18Wfg# zf^Skb-*7XLvyaR`<Jt$@;nC;#2d4(oB-^p<<_H_~H=cq^y40Zj$OszPJ`5HnM1aJ4 zS6F$f9;SX8i9vTx@%y}Xaz-ot*z28FaYpnccyu}&tDC&V$CJ);L$!5SkVq2p_MD=> zosuxQpB~E{5kj&TOn8y}Zp=`6$FI;H<Ge-gD-6PY?77n;ZbkGzY?tzYo+J-kG0%cu zQuqNBv+eo4ZF@vnChah+`7hU+Ud*QkE7B<T6aJ>y(ZuV{>`9dkf5>Vr^K6b``?l@E zRpSk5Si=)oCjSvUSB_=hMmXZBQN{3L>M*)A+mp&`7Le|d4qp9r92sd1;bmWjGnHkN zA?5E~j9oQ~vzK@UC%k7s<zXZCJ<Ntadkn*2SvO$j&~K=CN`v)2mnY2;gHScY3C8(G zaV{I9S#A6b8d#yumRwlLe60?G@c<pNnH$GWuU$f!1>K@w15e_#v<|LA=mT$eSV&XS zvLINLi;o2lfP1YSx;-s~+wl+Z(nVd8SG&)z$<YxG^IxEqq8iH@-^f=Skzz;3zT<0$ zdeWR_+e!1&U}ibalis@Z!T`<FILh-Dck){aj4691iqzC%w})?~nZmnRc;`Rx8Zd_5 z5AB54aW}Yv;}>D-Hx+6c;?I4({~WdCtZD5=WzrjO%_LhSoP9%OaFzYV8bgIKaORQ} z-Slx{hxE#zdV4&xJCz{5`+E>wd}T=gloPml-z{;A!8drbSdDr41;WibO<Ea#AIdDt z$Sit4O5J}g<g?d`mc9M~x2(6o_*!rLbbbQqhDY)-2}78lzCCOExdhan-{&GO^}r0b z6>uxJ6Q6$F%37`Dd8s|J%ysW^n6vgXe6c?Pk5|gWW@8@~rt3iKzixuCR!@@RGD%uZ zl3pG9gJ*1>f}h$v)C#ue^58W0HR}$x=Pt$UQ+v_tf<04?mtw!;Y?!3YV?HWH86J#Y zM|bOXf`a4;4C>86-}bHS(!P9fojwjWd)|YtDZ4Q=(Us2ruERAY@+{V30JpLJ9N64B z!f#vh1>e1Y2!A8%nEgBxe%R$el0Q6$OdnRT#-?1}d$u`^d}&M9?5yF(;~((y?h)?Z z@tye6eI?7S{e;a@UE*YkFWA*(#=e<Gqqr-AHgy+Jp+P9h1Wkh^(>3@Rd+>orCF&Yf z!<qJEj$g5ws?-X_Hv1%C)`)Zp3cCn@uVvEmyY2j#<njFR<?Z~1F@YrE`B|iu;lw9A ze8qR?cj6KQ4=kQli2VW#nRu~qR;DH5m-*SVG!H*|q!B<%6{pktX|Z%!Tmp^4|H_aF zQJnUZC*b#6n5)_kvFjJ^p<A6R``g}$+BYx2n|l#Bc4H&<oio7v=Zj%NX$f7t(2oVG zk4LLLPNZt%OIwG20jq*~rq^Z$o#_|2jK1+yJ|Y*wceG$&(?P~_)vV}EEwrz2<kTJx z6WE3cxX$<|8a^-K-ljyd-9ajB)&?CGki8K?<;>|wMFV^kJQImFBPeZtIL75gF%7{P zP$;<YUO&3Z4ON^%_kX^JQafd)Ar(x|HG{A@QH9NS@__0|dUWjkCd^eB529~MtW3s$ z#pnBDzj>=U#nEXHuBQi@4MLuCem54!>EMZVQ7rwy8sQ$h3*5k1G?Md#*#CG~rFP(R zjYIgjB~3g|Gn-{S+C~w}S8zt()#237br2S2M9aNDqqFS^uBC1&1k}%9Bl{mm(dsH* z_03s!uhWC|2wu68par-kTm^GW4#1Gs_3-7T6KJ|iL*%8KxOZI>_r}hFeO$Jh+->%< zuq(fKy_?AtIJOGZU0?DIv46yulOJJ7(_YcS9uGRIBhPZpwb-s^6$<xy#%<`*zzOu3 z`}MsM2L@O0gD?6}@ViKy+n9pdDrTJ7jN`cb*it@7G8wwgoCI6_%k-+U7_Ppzq(>VA z35O3Sx!*?Y`Q{AHd(#dKu^z}4RriA8+$4%qTg>wOC-d9G1y61D5$I?88G^Uz(!u@; z0_m}swdW6Hvu1aq?DIL)r&35yCEp31!z>IxG@I@wCh^ghQY_@ta!}fsM4P1ULF8Fq zd_R9KuUmN!8*M6J!AKc?plTXdX!8}!4OipQ)I$F3H+6y2*#&X@2iRFRl|uD*viGt2 z%vv%Jg1d^Kt#$&rKaZzdv6HCZ?-AlT@6xJI-*RC(^U}G|Gygdc-w+5{-O9ACVhLE3 zCqRX50?Y3>&nfuLCl&i*THhw@v>(kPw--lg*`z|wx7}Ox_j&|n<=$uaBo=ZHMrC2g z(RR4^cNUr4bHoU3fd}^Z4b<P-!KSpz($}JN_RR7b2De|uiY^^i()WzN92&<*X<vXp zKP#O>Y|Yt*xcx#0)f$JScR4rxmc_%*rMQhhIkdUAhyu_4#;<<@@qEN6_E&uhLWeC) z@bt$C3JNTBZ4A?FIS211P2&fRP=%iB=`hH@0;UALaCVGLfXFEislR59Xv9q?IA$w& z&{Iz`d1Xtc^HrXWHVdJJTUB|hl~u5tYsOoHE5W;#qb5UJGWe;^F2Bu$E0G_NOKr!7 zi?w*+{UR!mO=Xp@+PSnn$Km_FQ1Tmm7vR<)l5LyGt-9gBR+|FLm@^1@?KM!;Um0_k z7w~q!>%=dGJy4dn6TfhFEw)HT)A4vI`f=hk*-g};u;yIaQ1u8S`lq6+^CA}bhoO|( z0Z!|}8azMS1QlnyquT*@xP0$19$R;XPuf-jR|~%3AMgGo-I4^C>zru5L<m1;z(=mz z>peE;DhNHIAsDzzm1P%1QR9b5y5(xf+=A8Ue!T)7)=cE3<~_mR0VlDzr=EM(b4|2E zW(sbL67%yw@Eyoou@1!qb}7Ig7aY|ST!opGw<eBO<ZmHIa~bCQHvw)>>J?2mmBoU- z+<?a?HbMMV9hBS9f+seX<0YwYIAFq8m@S$BnGKnujq6|Y+@8@aQKAWJL)<8`Bo-H@ z?_wbz2SVr!P0(Dwm@V_spl``DSxVI|GXHd&53$w|T&xbH`=J0{w1kp=<_9R7!%@TA zrI70yN^(I0M^AD9MeiKQ#zZi7k?(Sjd11gTr~Ko;PqKv#ALc^wCrPr7@!^s^zT$^8 zIli#!u2@od4m)dy!IrnW)L3!?UNj}cxxa!3=;<?<G~ge~b_}2?M~?CeX8D}ktr~V> z`Y!yzGC=9$ON>&=A!#j37+GD)P5S!*$6jc_Bi=VfxwXk`W8^Mqm~Kz)S$pa1{ATW) z{6!T1RpF%zdtuAq@l?0B31?r^gyr{};p)gJtX4H=YKCoG#f)2AeS<pi(F3sd=1gjG z-41<DU%{Zg2~MBeLd)M2V~K|s+Wtsj<+H5mFZ;q573IQVvq`ku)RmQO`vCmZ2T&@# zwOW0ku>Tk7!t$p=r{Jdx_KS+cw(FA2<<m=QYx@ifM@_<CLceFk`*ilnY#sbK7{}%H zv*%^DOVQD<aiBZX0X}p#z|qqpcKSvgD&}v;=qHz%m*Yr!asCiiDJ+INBa1=e(M`@} zNi=@z+Dq-FM`4qr4(SRVizz!ZXm!v8cJjDCll5=LRUJv_HCzp6$?xJ{8E&Ganw{X< zIf(8=)rw367ju_aELSh|5cd>+hkl70!1-PZ%If>Dv}DHi9o)xW48F>Ht$hxc;?r<Y z%0NECWdy%pu9?g331R=rUPDFxTU@RtMUUKG;YEiA>_~YIW+E$gWmh#gMHRvPY%LOP zNN2&*SJM$CH>g$=xcf!-K%&DP+Yic<xkoQ|*=_{e^&}iR4$fnEZ8ld4;V@@F75GG^ z!OL&`&_M4p^grB>ak*)z-<V3Lw94^}rWKxR?BSNzO0e01D>1gxgU)=EW3`F(e8a5` z@J&{kqK<CDnr$H<m8`*9<{zL*^NRTM5yrIq<T`p6=Sh1kq9DbnjW>1`%#qdW&Ocsh z$_DmLX4$&8xHcO{rqtSoothfV1ap~pgFEXGW)l6aI`mF2m8td<ay6MRz+lTy=vEoW zzKxHBwX;|9OX{_mpZim2pM4Q#-4^z%(py0EtON$dcZgG$T|hr`RguDnWQfX}%~Yi< zQ1;VL96V8Q(|`A&H5*s*+A~D-dWRGAlg*{6IhLH3Q2_1P=tldR2e5q$Tv(V}06QDv z1Rhnl#TySrkn8$2{A{Tf80oAmI7D@D*z_zo>}3lIt9w~;xFK%Mb)()`33_a3&GMa# zxJT_A9(vHlWekcEO|r}5?(P-l<faw4b6+h=y=_75?UR}GM;+#)bB=GA-3hl%B{4Vm z4&E)&rrDq7u%dGU%j=p7`x-3tPc$O9|Ktx=4ZXPKf0+Le1Umm+{^0-bz{h{C|Nomm z=zq%(5d=E_-Sw|ppM&jBmg8@g1n7ESimwj$XO&|Qvn?f2tmedY7Hu+=e)$R>h1J5D z>4hxL>RErba_bMgl(m)36ld_6XDr!dTgKK4{x{R@BUt3EM7E}|lppGsjhT0TL2!K# zxFjnu$#6Gpy*!WxJ#dB}6%Y9}R;x+t#2@aP|8G2K-hy}T3wx-Oy1b#=0or-p0t*+U zi=00_;T&sqsC20!#{4#cEhqb6X_f_^_z@-Eq`Qs<3Oj^~x0R6gyBgP-`mvl35-3tq zW7!Y9n4Y&T=w&O=_o8@w)8>T}i>mRei7qKT(Wh9mp%mKs29NpcQ~6p6DwY<}lV2lA zDaeV2xn9IE<1OfEz2FP*RAo5}C$V?$94UQSI%j&{6Sq0<ho(bCq5`ua?2^YmT%`U< z{87OJBSZ7>*`|%)rtL!K*GsTx{q^bhKH^UF9E0YQ{#d!74-(2O+40kEw2L21-(<IA zUPmpTa(xdAInbYGGK1P=?J?x$H-uH+^`vjEF8r#cM^L_KFs1g@3iHcnEKqHSj1pO< z^{o`MI|t$P`P1mg=iPKL?l3-dKhBmO&BEpjHKMW?*<zzvrSQD(A{307%aY3d@ZQ%K z-1VtFQ0^s9uCfn7PP2$Nm@T{ukJjL)C<ajbmP=S?C(Y)aTtE>$g5NjOjVb#}u^5wj za9lN%H2MzG_d3S%VyeMt%yKOLxL?p-oP(PRg)Hr|A|+@&f%X3wygjMS^&5Kywr@0I zhvZhlirY_d*_k{rcxFbgn_4)z$cgAa&>p7cHF3SeoM^p9F6;}ef^aSoou|ssAC1X1 zV!sLav1uA!x*)?Um7So~frInkcZ2QpflzO-f($g5(xHCSFjd@xd**(F+g}`E*!>&K z|HmL&R2WHpl5HR=k)=Y1i8!<5A`b7?W(Te;hdgIQ2Z5K`yfK6S)EUkmt=KM{k*4C( zCAzri;SXN&yomB=wSiHXP+fN%$u~xYu$3;h%xm3D9QPs+gU`Ft;JP7f+*5nFSTu;$ zPJIG4mT@B2r@tWOT^9d3{sSzUdLMtDm4sXS@=$rr61FOR2HH-3%ny#!rzkshx};w& z-ms$-(DOMer(DPPeiBqYrJAe$=>|#Lv*G01c_=US3JQ93$uUbfM_+iqCV8MaODxP1 zXDbNa%Kbn1K=o*Dc6kO$PQQ$Ht$9%6G!#N*?m}p6A3muYLP>CdosD0Gw)-d3QrQWx zXpRaSXz&l(rai*EYxNjC^EChI!WXn3v<4hhn*_P?XnJy@3+8H_!zAB+Ebdh$1a=y- zxdGnH@OwPW)UxE`mb!|@P8`Da@0Sy$+w|7-Ddc13hf2tQnF#OY?(tJ(Bbog*U}Goi z;V{cG9Qq}R)MQ>lwfk|d(0UTuxnzo^P6c4(zu|nwo<^|WLwHc|%~d}0V*x@A+`1$W z0!wDn?#^0K%w!wvm~Kt@bQ?&gS3sAKz!-lUPVv``DVz_*6*H4C!#{|&4qi_8-^2+l zNh?;^rc8(4Y{GzAAGZ0R!1N8<$(vj4Vb+0<aqQz^{3?kf9CuGR8((w;6PvnVj<qSx zRnet_i#xcw7+G=%*h}S$^k_wf2hIu?I?{V``6N9BI`g?6_qs2KPa5uQtc@uq)cbIs zFU|xpzUTG_&B0@Brub)=E<TV_q2#s2Ab<KR#z*~wI|m0*$;@PC`N)8tkJh041J<IM zLjN*etC=^KxB))*EcjjO@%*!#dHnN5S6NGM4m>Y(#iyGupz`DrT=Z!vcXsgtJh(=i z9=8ue38g>?G|6TXM<&p9r(#g)|3OsX(+IVG3;983&-1ed&o(J7VcS_ad*>Y>FxXRt zok}w<lk+G2!A+vn&Nm<zaU0jjxIlv8UEJq>o}K<;#BQCL!bV#ufY(bMy7$Qd|M;Y{ zezNhPnKO_&1ZLf-IRT`8w2kk4ZNw+Y_2T1$K{Z+tzq!k^m3X5`vRL~~lC5z+#EWZ# zNvZ833mmkXn4v8DeO;e^#cal|Vt0_8mw`Re@l5(h8-H290+(pS(Z|aY_&p;SVixw{ zQxAPSF>M#;HMRuxcb?}i^v)1d<WjbGfzTIP@T^*)$pm&?70w4bKf>yKR|q|+AilqU zG|t;_5I&2nXs_M}n0UeyGEH@v&EgJ>KR%Fjhao2FE@G`$3b3GJ9_{-xkABZ1l%5@r z<Nf}LHxzY4*tA|6_iPutqZ|s`Ry^h+E7d4*?jGKMH*r_`<e_=93FkSb4tGjr()8&j zpv>B!pR^d~A2^0VipD7E+y_oC9^t?3)%e%PnkDA_<rUt=;jG!Gc_-&c_NuZ2<@J3) zUGPx5uq!ZJ`6hI}_`{ETHkb_(a?J}C`2mwvB62OnbvfC*>SHx}H!&7WMirs?c%d5` zW<fp!MzaBvdodzVk45e(rpIeTI8`}Aeo4qr{Nve(J|jb6W9t!a|B}g|H*Oa^)O^d` z)Ez>ZLBY`RR)yZoZvw-+y;y(Dg_Lw%u+}vSHgCAX8}IbutDmc|#SdIq?AKZB@%wj> ze<cS7WC{E8r<2%(o+a>m-(dbE-GZ5-*D&?F1f3mr6HlpFvp&BQ+^9i@V6gBvo+z%y zncWg3z3Zy@!<S&lKPg4qe-ZTRd$5{&0v9oTA9I|27#&_oQ28oVd|Q75JIumqUWyb& zty18Y?~1{<J7!UX#b>@ZIhtN*nNm#Fcp81{D;KO3i}wou!6&A~^5Y!n<M>=MS}Qnw zLsekNg(qBb%yxG6-$|ICWkp%{|3T%i1q>w2nE!!toS2}+8m~&xqABe--Etjwd3+dl ztKUGQ?k@4|U*C8E5<#j}d7>xfPOPD~1nQ-xq8D?7{#`FP?O&<5_wjUG*SV7Iuxf#6 zvis1hbtq~Y&w=EXVZ7U@UL0Q#%vCQNL)O}R`Dl-o{Nn}vSXfmq+;tv9Rc{}|bi=)f zFYIZS%{;umD3x61CF3${fk?ILBk0uue!e=0?KGZ8p}t>HQR)o$xL-dua8wo~Sk=Ln z9S+>M!oM6FT!}&BIF=W!2l8u$`P%O+3n_jH-g2q@y+NOOjh_O$C`6{#l-HpbAN=US zM#5L}>mcW!JwDX2=fkaZSp6+)wsctmKB%9}G%l2JQ?I9Ui@NPtMRysV67Gc|%2Vi2 zgcIwyu0j%unQXR~@SNf-X|K$Fynag!+LtG?ebbl1!%J4s87>8JPKVKULAZ0&D}DN+ zb`rFHTe2Ir@@(qMQK%;&$C|x^`6<R6o2%9bsiRMTvEBx5oc<~9U73&t5qd7x-iEwN zlRrDt7ANfIX5r!~Zj{W*m_gY<w&~XzoK$N_S=TPYE8}c@Fr@{*@BaubrtyreTtwY2 zcFZklAqD2$;y>uh)2Iu{{D;sY&Pg~o`PKg@JN0N6*MDn1YaD0K)Xtp(^?nlQ`0XpZ z&@rEPo@+!l3%!MN?bD>?bC_3fRs!S3pBU-?93oyuleNHq@st;MVUCGZo2>%#KJO5G zT8B7ibrpeszmxBH{{kdF=}_aSC*0^^Z$NC@!SDOB4;8Y#*qr(DG&wQ>ht2WFtS<ve z!}O)|zR5<kBhd^~21m0MFXM3ffLN4FsNshBKZDt=N8sb^Ni0k$g@3f~3*X&QjYqY$ zA$0N`XdHD4PyAbonYC_YpcBmYHTJ`2v))2`;y`LSqR33XB(Z>rheZ2+*TDs?Zqdyn z>MUuZC%0*s1G=C&8}q1-my%FnQ(f%o(w->lxif)OTBFE#R6gHqW(Kc|Jm6%-1dP7` z_{uPt$=L0N`fXRxkJ-?LawGP0sulI5ThpSjsdTP#5P4-kz_l1iqC!vVQI+5(9{mDF zzZbLE?vd0{<ieixKgo|`&(XW>rO=^}f@jm~@wY-6YHwMIqOd$MYH8u{^V!r|dYzAM z&2hf|YX@wpbY#zu3XVbN<JAl07>)EENdI!um~ny7-9Ma$%{w?qFkL|tTdTM*{xI_{ zS_;M)rp)Z26$`M>huY#nl=Vb}o$*l=sjq_GwJ}(hwGE7A4snCu=FzFcdK7$QK4=X~ z=4!7B`GieQtY%C&H(S_aK5#$C-|s#Q%FW~Pa5X0yxMMX2DeY#bBK`SWQ#~ogzDRUo zU?)F7c`((@-zPqM)tfneDMjD&?krm1;A~y+57r$x${!tjoXyC0<kwhKQJ-}!_k89i z)^+tEHpI$uTfeBm^sm2oo97GIO4GUQ<jQ1t5!MQ}%gga@;acZy%Y<Fnro%Yk%{wS` zUP}EeV(?|-Za8Svi&1Zc?&`yG=f^KDqSVdfu+=ygly)8GElq~9z*!*}|6C7ihlj(l ze`j#HFJeXRXMUm7bawViG<1mfLF}H3aG`BGywyeC+|r3GI##iDn{z~_(U&0GeKQvy zzLFmOv!gS4c5I1~0Xs15JsuqU1#gVB$KdEjs9*m9EF?O`_a(+M)hA!j%cK<Zq84)d z3R3Z_lQU&XFql+z8fM;L{L<G6{LYuMw9VR?^0pVT9n69i7jH+^6QkkynnM0WsU){X zVi$InuVt?TGNAUevcPw|&M)3_g?T?VMz>B4ey5Zc+cqGTwcrDRzvx1XK5XM(3v-`F zQh)l~eS<$y>;^{`hQs-ol}zmH#CNW|4W1#&;A(afl%y4y#lK#h{zS+v6ijAgj8AYw z&C6KG=wVdo--wE%DsY2QA|8IQnAtdJ(_HBUD)Vb(-UFuL9g|+(Z=Vzm4;Ie(ojvK9 z?OU8E{S)5^b4G^4P-+btfd1}@crK%hOKH4DrzZtqTfk_VIR7CyRm##x<6bbkau}ys zhcL@4uH4S&-(inw5SSZ$<Cgq=3MXdgV3cWjjeA2jjd6^?1vO*Xwq0q^8skf{t12Py zvOl@s$%3-a<$R2=|N2?kgn_TO!3bYtwxvpg{klE~-<IoAancZ+w#$Mk%B_c0BiCce zV>#x#e-u4BvY*OzuG3QkKe~M_2u7z5VLA&8S(X15==ZXm@5qiMpEOO@zW#)G^sW1x z+G$O)8L1DuIAhXS_!M1s$dF~vivP8L`Qdv`;s2dq`@h{k2m+n|ZvUXsn#u2|l;vtp zWFmT%!^j8H>}gX^%?aC3p)Xe{aK<~q-QI*Gd{^*No|jQ9IS=AL$}n!J(2rW=fV%Hf zX>xomtY}fhZs}6aufmL?ddE|y)hTXqnGb7hc?cS5S8?6Ha;Uy}AL@^tK;>o$j5~b~ zS9@;Yb8U;E`>lu;p0Z+N8g@{9iI8(&GJwWS7)W&en&3n{ExdmoW@kQ_kj^ni8|LfN zjkwV$j(>+<w^SKF^C&J4`h&-ko53!0AGFFQaJHEvxMe@T!jODrs(#rGKDE2xid`<a zRC(bZp%3=MaR_zab_FvbWP39w*6G)iTHefQIGcDY3qmK%ppHA!$f!t;UGXtvsvbsc zx|87K_UYx)&Md2$<)H(^8;oh+h}}4^<}#=z-@vpl0<W;-zQ`tb9OJhp;oqkMr|Qsp zTt0d{chW!thmKUEAIoBEUc439XEyywb6p+hob?+|Z^*^1FS@xqX1U<^K$5FeFrvYo zmh8{SaNs8yuqx$5JSZN($6PGrYTKP)Wpk$JOn?eYQJjo^PG&IF{~_$w_$>JIvT>sG zV~`lRog9v2^Iz^4^Fx3A<ut-xbKa@rMIQTveX++8UV5bro4UmZ7REOt26tfaU_H() z;29`e8~_QDV0y1K;bo&G>GQk6a&#b0Y4O1;Y^b<P>mVETRPZ-Uvgh84G@yC;Nj}eT z418|52*EXPFw}J#hLau}F?BC!Og#yP<#znsk?FLxDjFAt^of_oEQRv}2GHE8SGXxY zFFA?lN9ma32-Nvk3Qoppur**l%XBRUO`WA=I%paDkg-|#*eu|yn+ur)g_F5Ywg`<5 za(x^xFqSTWeuxh9s8YlO#l1M`$_C16aAQlnB$=L9H=I`&7@_lb18!-+%>GAM&Z`_e zBH@Rx=ig#0TV2?|Rs%}in8p@<Ny690v+$4c4^(kE#zi+&;l?m$h~9oc;8B;Muiz1T zv&D!`7;94Ni~}g4b({C-e;z-isDS*>MYQ_sTa?|hOT0CE4aA!y(_rDddV|Vz*!trJ ze~OR6EsE{@9;;FGW_mK~oRh`A$O*r5sU!t`l3<QH2iUj#r&u-qFPE3D#s>Uq7gdj~ z<KEScWInC(yus^wt}eL%;<~Pbj^c9&uFZsL^)Fzz@C?4CwU~|9SHPnKiaGXl6jhWa z@JrPdSljmx(9++YYG$>FrhPezOY>DYj}QMKXvid-@}rusit@)y=?SdiR1`c}lZ=z1 zH}N~`Y;a2V7<!YG!l&F<fpej0!gK8{|JdFdJKd&XVEA8d!22m|+pHmU;JX)^g>J)U z)fg)Hr-7H2E`X!TC*bDXM7((77UWDQqUS5C@tV+yeehg|*7hlL56WV(Yji2jPnKeN zR@<Q=xg3V?yo7egVj%OtE9lu-1t~2*pwd5-={44)YZil_iJu`$?KXa624prW3V!)V zgDbni9o4pBx5Rz8IdnIz+bnR3ce_#Jk9+WBv={Vri(%O99Bx3O4C?3I<~H<=p_T0_ z?6%=qI3NBKZ5)i5<=JsidG{6fY0OSEalV08-=iT;Q|Q%Ry8^e@9KxEQf$Xf%Sy<b( zl!EGK(Tk6p_+{s&veZ;f-tNX8#txX$!04B_d}Sc>xxS(1bL@T5U6+G+O?eqta@B?2 zC6C0rT}MFEe;0n*YC}Cca=~=@1`KxyrF7xvOgkoIs_&cz^K-X3d&diyxc&l$OL~y% zS`~KU$0`<oXdWN_#*8&ot)i8`J%zsYJxtN=5$~R@Nz!^MpzoJM3$*vcI_Dy0*INK5 z1?J`(u@$rT-GXO+jDXh20c>WfKJ@4i?AjR&Z-yH{d&we9{A101ef>!2D+t~Q%{<aS ze+}~�g9vDYiZEG~cC`f#vyrSdgwt<8Sc%V0~{`aYKPbwtYBaV?8{$w2ECgmCOy= zA4kr2jp+XQ1tO2UC)}|?Jp%i47oKX5U=>OCL3!F!_^>Jg*6Dh{#CS1?`cHw%M=J2c zdO5yMie#IGeJ-u;M-SIlVwh1j9=YSg-U)uqvGeBfuWn@H*7iE=`)5Kvfl8E_X^$6! z-#JSj?||^-GsKF~>F8$n2GkZN({Mw;u`9;Hza6glEL(=%Ps_jqZbcA!X9J!c$n(() zZ{wJ01vMM4JO^3DBdj9h0N!jK!)~wahT=|hmOCUJHhkH}0$x>-TkLdjd0h{(Z&TR~ z-zcbiq)slOK{RK^DpCBRqxc|qB(u8Ep9#L{nlVNvc=3i3K02uzPcN2Wy1v=CI%OEk zl9XjjO<zDygbh=bNn<}s2e94a>X?L21ui`3hrhy1;c!42m6*Q9$)R(3_r(iYo@^_Y zzZ}g1Xg;31ltgQ6Ct!|2E3Y_qBRNgi2J81DXxrVLu=Dx^=5VK!pQAq+EPI{!8{(Is z)%sZUOmZ-N*Imzkto5U71D=AGsvL3ciSR75m<~xc!+U7M?UAJ@tW<H4<$hjTVHM<t z@8*45^~tF*f~4vT*aGv_H0tnk=j;uO@w;IrvsiKh#(XTp)@zBB^}de#bbmPe?Jjt7 z93;RwY7!hj7y(`Vg1FgVG?@6RBKjKr6LJW9Sl~D%rg&inZt@?;#yQ2adi^|Htoa^a zUp#^%Mr&c8nk{es#u{h56Y?(NSMW5m3C@;Eu+Di(^lFBVz_FQ)%eUpwly4biq<)sZ zbnJn^rOW8->7o3(pgw$ju~^8um&2T<eVA={j8e92Vo!d_uvmY=y(urn-c4KtzYDg2 zbwL5HSdxLqT)xBHSX-#GmJx5etID*`C&A|aWvJJc!q4v=00Gi>vD|e4ynOJJKkL4h zDpza5_PKY(9>3!8>J>E_ShR~9VzeDMp8o=WLPYGtqg=N5mpU|Ano>&mYK#%>#WL>% zY8_aPVZ*$IJfRxRxAmvYL#<GvDhczdlPGEJEB^Mi18{i#T=45rCijhAv@FU7%52Kf zbm}SEo8pUcUB#jqv)6MDkHT==fErH8_C2ptH=VhEnhrC*egpG#J$B<-C-*1z3O{r2 z2bf!`5AXDz!>>*KnYH3z@lMSwCi7M3KTNFRrjDLWHZ4h%%WtJ4y6qrsZc6c~<xqLH zjH{PUqBr^@nYWx4J(ifswDspuT2&1H-L4J0)}G?uY}X>qIhxE|WJ9r|e>;!$olU7i z=BUQmne<(>S>g~4I`U}=_M{uqM32jm;$X|755;gtMr`5EgxdiBX$EfXbU=?M<uu=m z;qNapG;NV0yY%}tiudiu@mj7-L3R|YJ1j#sO}|hxtPU@)xQ2H9$ARnlM(qA$3*N?W z_#cXk*!r|faM$M`_D$HwciCr&P2JStSDF>BdlyUd_Rpcx5D&VXYXjkT&4gXrO0c|l zkqbDT%scAp;n$i5tXyZ0AB<1Zzxd@abBc?w*Bj05MQX8p=}M4@JWI76j*u5HknWsS zr00%r_^uZzWKn*GQ+)K7`!s48sZA9)*_BV>XN)Cn99DwUuX~gCtCt|Lb*_-9-G&<X zcEMY_cH|=mGOxgqFmFUOdof1DjqTe;ng@FMKpPS3RV+gO$p(@*?90Xv^<n$^u8D%x zMnSl6KI1R#Kng~4=zG+D-s;JAxYTb39GN{A3p*NM{AHd~?i-H>s>1l$sfI+RvLx!K z2xT>;Tz63{`76cI3!}>-xiEj|S-qQGI5vvCA8-QpNS33Er78viW7kr~qG6aj_J8d{ zr&DLq6@mHtKF^Jl`MjGQN~qw!PxfVMqt-%U)@y!3Qv%z4?mHJU4q<JN11z_x#xDbs zagbXqAN=+PZ&qPRUVrlV^dW)^Uat%*gfp@w{sAm5d=2jy89;NBN+EGl1<35lfJsJr zICM@R|8Me5<R8uBL~kC0zT$PrS5APJe)=ppX$1b6olHj0wV3PDH>kOwKZ%w|gOj`h zvx-n>Gi4?SJ4j1bo1eo(PPM@A^8o2UJC^tLGHe)e4?Ns!FjXdiDSsZsI^0#zZ=gMm z%Y7i4_-h|+c5SK2XU=Tu5og*LavW@0YPml)%CvryDJ>A5lQJg{;#;Kzwye?=A4oL7 z#1sd%?fz7xV!*a<7r1+yCQ;ceNv73M4!Rj8v@-Pyh#folQObW{*wJs`b%y7a)Q7Ti z-3<KN{}h~kq6-U$h_Jb^K;U?;p=B=LAo<8d7CowpZ~v}=4YkD8U8u)R|90T7r;hl_ zUWYZwECh2^M=+b~NmaWH=vY!S?r|E=*0r4#`&9d)6Sje7xUuM6**94Au>y+*nzHmh zf4T!#__&QT$@q;q=&$(4?HQL3%ae|Rg-jq*8}^!W`YdFUv%i5(ei+K1t`ahzaa7W8 zve5G@<79WbgQUL~&ObPeess+f8y=epTP@E+_=X8gcq66LOK(Bj4HIsCX*>T_@+a7- zCxCd92fO=fE|@3Dlj^53jP1H6KAgFO6^Ewa%SFq`(_07o*IM%t=YFALy^z(2s}!<6 z*I;X27W;Umm6NeLNLl|z(3qHGcrW)ZVC!$RdM*#9`7`nC_;RNBb~q>fZYnEXQ;CYr zE8$e(H2$WWFRSU27T8IRc*ftFji}5alkN`wkI7>kvuGcv{@lQZOz8uoKnsfR)ga5| zk+^-+7xDHR+HlxemJ1x)$Yxz0N$!Q}Sa_%kqj4h})+l)JRNuom*<(!iRXjQGQed7V zEror69d%m#fyTP|l>MxTjn|OC^+q+^{6;4>@=pt<jyl7%Z$7HFHXn;@n;Yzy5XDY! z-NhVbv*^$5{iJ+1gMaCm&kZ~Fhi{kq#Hn^#v$BKc+;sPCj0qmhhdC<bbRmqL)frB* z9cDDP^9EciQN{50qcGx;6iD<vgHoeuOf_vd3l-P`3k2`Y-KHFzW<Cqcj$MLohe(Vt zaiNw6JJBq8JbFBzOs9&Z;O5yGRC0SS`#$M5w#+wTzQtLh>P63R*;hYW`7(tsx);g^ zHkUw!$_#$eTq*p;!ckL@j;06+O+1)Hk19;qy1TcqN4$qZO3aC0s>|QeI*Asw*8j`j zyLf=N_y2JJAP98+yZyue;rS0ip!0w8_1pc|{+x$}{-|uCL<0wW!-}{DP>wQW)66q4 zc6B1|vW#W2mT{cgyuIAyxjO93RV!3^YQ#>*oTjIGGbl(d9-XFd<uaGO#NttR=){hL z;I+|0V2@qlRgF_=to>>>Sydp`jk(R0NloS^?M~#rtsRD8o>iEd{bI}tRS7!(QJ0kj z&!Lg$44G-nBedV94^M?Yf`jT{=Gvu7i^H7Qlbi%hJu#R9ZpYy*_i3O8BWbcvvAA;4 zcor!U1vchR>_rZPq-#Ro<VQ8O|L|vZW6Q~NoD;|zD9~`BdnPW7qqTRZkgBgWrQ3PY zQK19ly)A-|iD`wUwfWeoeH^Aw5Hc;R_tN3wcuFs`rj=Uq0{`qjR9P<;=AH~1T|Eyh zj>T||WmBr(YAdqkKfmCXONZGVp<lIK!j&<BCpt_@pA|jxrr4W@n5NNSHcDzYu$_)D z@6mf~vW;av2?jXgy9yOq3OOOS$^5y3aLP>;VS08m^45oN@2oNKXni|hbNUWu8xH}m z4IGo{{|^^`xr0AGZO6s4>LCA%Cwn^dC$6*2!Mcug82<GkKY7+i+>}`j@#jp%@1HCp z|I$gsy^Q2bl<cYT_6~T*Ig)4kV^N3fAkhD8#%7Q{{fyP5_$J0yJj{R-mk(p?hdl0V zQZ?#6(51-f$+Y=V0^c&FL1Z?l5DjN!&<6P^8dYzBe_e{8x_A{^b5QvF91<WvsvI9& zZ|8azEkTpSL;U)|<M^s29xRV-hP&6l;(3X^=pAfIx79r9LidiEG9_DDwdNZf9i55O z%X+YM+c4I@H=I*lT#O5{WVmm&+qu@}C_ZT2UVN;24w6-liH@B50RBD!-1E#j$W2+! zUR@Pt$af`_V`EOgp55efKFD$LH#AwslOn$NgD>j)X|S*_*7V8N7G}uqz}q2P;j%ap zMwmb6tO~}`+u~wSU?phUI-V?iJ;}c5qp*Lf#sg!8XOu|=NZ%PucWgq58y-dn=Go9< zrHS<Dc_oh8cn(5*Rhh!i6?Dr=jDK7rS>zW5R_!?fRY#hOMym^RYJ0vgvkym;^24YR z-iF=90kkT^1S-_^Xz|4{qH#|%z^X`a%8ot5wdh{1xl!p$g?W-Rq{W`RyEB==kt?YE zax!~$&w>0tb>n2?1pZm=a(<GrBV7w|U{0nUIKM!VyRkG0t{x1dKl{$WB#S5<Rguru zf760G2{n??FrjXVLHx^{r?@6A8oxNJkd$-`f4po44HeE<`_~k~njKx(+cKOUeD!4I zpVT4QZ5ZQgW<XJqKK}Ur4?`BoGr#yCE<C^;I`3}c&#t-%-*4H`GwlL#U}`_+`pJfM z*p^bVN-i_IRSxam-_UTDz@T01&DTmA&?v<Unp@%z!F!YWP2QtLvtCz$o6jITd;cyx ztz5@HobyDqYh?_bJlcknR=$N2yT$Z-$8D@o>47hCX|z4c6%3>AVb#4yP`SGn4fbAv z>WCCrF1w04rs&YoE635-;u~D~I2qUNe<6yyyopXoZ(^p3Zg9Cma7+!3$M-td__ky{ zE_R{|JL)o-O2Y<IaiHLA>0idu+~cq?$bytx9PnJM6w{e>lsayh@(!hE@X*n@V#)K_ zFw0wmh8xMT36)aJx6g%59jV0b2!Gq(rJ)$J@){)dI5W93v)Hcd@$9Ke1^#`%42Mgr zk*1j@E4{yviL5MG;R;2%<a(d~vEwvnIaw0<VP*J6WWb6qs=}$evCP*so<;co!ja4T z;gfZL$_lK&TmC|4^AO@|za=%Bt;|S5z5{)ByNI{0xdts0G`Nlyd$vEQoc`I(A^ko- zCas!)#)kw>=vf6Un6-c#WG3P&3?D<^rgHA@n_SqEk|ewn<*@;;{yKjxm8bdIu>z-L z7p+?S5sTt<m|~YSyZrh;K7tk81D#CT*W`w^l7{eg&j<W8))=$z3Hc5EGSQH#Kk%Y0 zfODL(oSkcm=0lU`llqZ&P^@Z>H!H6}RJlA2pJLC}%`rjqABcy4RDsOsQ_y)|31<zE zg`)+BVEMUgpnl>7Bvwojcil}wyWyoMuiJs1_fCVwup8i%tVWMh)X6+@GA`b661IJR z%tdsn)7Zru1%`Bg+Mp}k`@OR<HCLN=b56qM^S%_8@(daWm*ddYQ%G;<Tv}c>m$~)- zgjQ*?)L0*jz4rw9)227zYjzQ3-G5`v4Mk|ZBQU48dEw{aaM;|W#Bx4L)4RSz5c!2u zwek*{aOf5$Er`dtArZ_)Lxlg74QS&yPtNj91h-;P9dG^2h2Uf|hAp**V&P0x$~6i_ z@@9ftVGUW%JI>YErL#72U`E5TN%gNBJMX5zb~@+qRnu$0C|3k!is!*A`Z!mdGM@D- zU&%tEh5VuV8amJ{oW%)sgWeqp{Gj2Xu&}p?%C^6*ZoIQbl(O*}4k@X}@oxq*<uyDk zavXvix`nL!g?f5CO`259`;*?2iEPHM*WAoJb&#E~6@L#siY6ACu<VZDJ9}@34{q_! z(mw^KV|yd0Pk0KFIjdmyPhmGd-kPs_-vW2GCgLyCQOwlW9!KQI!o(*dxVHQv{5t#{ zstNp}g8lJuHNcecQ52@P5L*81!N1E5Sxv4xt645V+L!Fnb(A7gzi9^ZCCaEl`W#%I zV8-plnWR<N3>OzGLzm#cO#PfjX(~HJJrDe-ao$k!@p6K13x)1-ffmHcFQN~>6*0ib zk9F<TL;2~7;PcRic|YHQDseODVp<HIv{=t7D)!U2>t!%&^GRqPX-4%U<j8SrF!|e7 zaw$^|pjdObbLOe_@I7QQxon+)fhAU~@n0|3VEKt}aCc_M`tsn{mS3nl^d<Bi{DUE- z16i2BpC9!(0G9tTCa)g}_|;`N_+2W*{6?UR>_gl$%LQymgy4xPu7Q8MqnMJ>YH@~5 zT+J!>bjshPM&lKR2>dlKmgJT%{(B{hC8;6*VA&lQI%7AKWS27k<1@G$2V&vn*Cbw* zi=mgl<wSNenM~_z5?JPo`E~X}E7-e|3Qm1TbK%1p@m_<?`Be|=uebC6)+h=bf-dgb zv>5E%q9uH%mqS-=4E6i47B|^gF-yI5lyt@wJ{Q?BD_+uhb6qMFk3Ys%#o05>l^mM2 zUdPyRsqoN!3@i8f1UGuML>brBDQo^fYE{<6_oj-paR(DCotCGdAYD9K62_~(PKTCb zXYo$7EZ^m$L|GmE>Biw$=$AVjwkzGo=Qg$=kw2Im3`xBE@Mvb&aFi0$TKJWh4uGzG zAKXYEiibYD1Z`=7f&FtCQ|X@0-d9Z|i?&;MV7U?t$UBKkTLW;nR5p~paOD&HFXI}2 zWsGXw#_VpK#Ni62?9sGeaP!o0=vjP~Y`H`*R_ui@jw-BO@-^DnPX))?5ZobQC|<o| zD>g4^!{Y@ioVnF8W-u!e9;EmPGeQuzOKllD7_JF*BQDYJ_7Z&5?-PzQ8-S~g$J4fC zYm)MN4UQJJsID2oT%D9yNTvf7@3f*~$tIW??Z-}5%b@&&UTj!=oV#@N4!5-U27Fp_ z3SzbOsM5`XTaqIwCQ&TvZjWZW4jm-V*go!Be?x4zA5R${)5+W+fhj%SPj%}3m{xi^ zA3wyJZFV~Y!^7sn5e-Rp;E)f~8ZQU;-UGMSazBjRR*b6(1s2k-;cV04O5D1mQ`p;< za6d~Uc#TKKY{Ri0?rJ|}V3P;2^KWtmA9M|P@l$cF=XmHg3_!&to!qC|Gq^vkKgrZe z3vP;wd{l7+d<~gHiz4&Eaak}^{XL8(jywln=1icaUMtuf%;7I~%tf1Zb67#&Oa4IL zCotWyh%~$Y!8zN1sM%~uZ#M<9UvKTG>YdPkzH<tHdnM5OzGOJ|bUd~=*)!wrX7SIB zSxg&$h)=DG6g(noOhbP=lTpf|jbn~;4$h%i^~x0n2aaaj^)BPMs2a4Nzl23>QiU7t zA@H_9mdS5T#hWi>$Vf7V9Q%I;Pa)&ADLj{EC49!ki`&qNt>KTYjp3N*GuUICM!wIr znZ@gPn6Bo<79BVQ)8djTRq8xiycC1I>MFV$9>I3UKj$kn90ce66WF9^!J<y-L5{5s zW}KEmE3pHlua9HnRzJk4TV^xqs;k^T`N4EjB*kfF#o~i)HEc*jH(Dme!83!wd|N*? zwq;`@Ebo`f0{?Q{sM@Qrc7+LjaWP|-9jScP?E;)TCkH3g*Kv{W4B>>F4ZUcwWZE5z zx$&O6!QWp2l{7=xgcrJ~t=s^|G836W*i&wT$9~wYeFZ|kUuFIcuRu*p$bgl<cOLj) z6lH%u$o54j!K+={m`L=OtKKmLB}$%%Ra6!_cV&9BfU)6h<$`<IEO@GB1;2pkb?KOP zXC(Gq55xIR)8KS-6Fk^u0{No^Hx+juR95NX)wjQ(w;&BS|GvPDnc4_m>bH6QS9xgT zF&(Rf8AW@U1l^AHp;^s5UOE?yrDvRL%>FE4g~cs3dLy2LrR6#pxyXZtU3Y~Ei_O3{ zr<c3>eJ(#XAQ6~v8(bNX1cpkgbUrbG{Eo~gS1m`x-6fD=x}R2jQDJQjt4UIC4Ln-j z%jHffX9wc6=;ZM<Zb`>iNZ(|`n?}p=hWGT*CC!qqPKw|aTGq2Mx58=qHz|_x+Dt*q zmf`6gJ@6_GFu}k8O(v#uDVEZp6KGFLTYtj#_9I|<ax~l0Ru3dEhr!t;Y^8xKe<f@W z&AXC<O~Dg5|Df|={plE_5B&jQ<|(jDvl8EJ@Z#6}EksLU?;Kt$c(U&GXZ_qo=(ckt zDzqHHwVT{vaY-r3n493}JCDRaV=^#g)dIG#q(3WqD2Ht{j!FG$gt~J+<X`p+KGZw2 zmOdZ4qLw6htlV*IY(1(^{0HOYN|{+(Gk5%x7#$o`X+U5+`?)ZZeQdO1Yscj<1zUn^ zyNr18iCL`g;Y3pFYlm&wR-pZJ7PWM2hcjwU>~+^})}pkE*VtIczkcYzbaj)-Z>}4E zLtc-z?7j)<wMRiuoWvH(Tm`K!3v1q=apm84zT(_Q%kcl{-<UXVS@=KOKiLUB-T!X? zVw>HDbK=k8%$yqNx_c1&O&`Lvf*y%_y0?jZom2Q_nm#P=!8s<aq|c7uNu!<j6LHL> z9rR6+VOx0)+s<F$+oujC8;=_NbA1&X9({pJ>W=50?A#{s4Gv)PqeNKIPYEO55RCB7 z#8bi?U@T?C&a4@Y-<#&LO)+vzM|4o!@2eq12)m1o{7#ZN?amtRO`(q9c<}5F#gAuW zDZ9#mdFXEe-#T*^D-j8iDbKJi>o7hm2?c{=Hn7?*6y-uI!SrMo*m#w2A{NS)_^7in zBe&r}r8eiK`=Z&x!Gcr1x<9QMEypget%b6#L|S7s8mFD<$9`(q!M?w<S<{9U^f|{L zkD1G}=bvt|kCF=1dS)4W=+cCyCcD{WWgE_clVbZ}4_9z7o}PC9Kjgi6RE=@;@1F(@ z#zrJ5g{Y{c(s}k4i437KWJoF*Qe-AWr9wq%6p11U36X~L>`kPM3892YWS+-}`+V<T z_pa|<-``sI``2B+);epo`s1l{PS5##K6}6S>$QyE;46CO){e8I6X=B2V`_?6i2iRQ zuxP2T<Z^LA#rgC2Q=d3!j=WC8bGDIwv=UwzJb@#v_M)kBEY!K3qf<9-O1;5t{A0#< z%6}fpyS#?Nr@g+E7~MsB?N14Dt$Qe5%3)9170GSF4E~|_MA$9ML>(1nlxcNm7W+vv z!#gNfFA{f;(&H=h70FWYqdB9K1l#x><l6c}&>6gdMuhGnyZFm==$AJe{u|AQKZnD< z3Cb87YE1D(nyB1<Jh#7=h(9VM);3j<;nMbaxAQX!x3NN_)zU2Kfe&z}(hTKBsgUwy zv}1QY8HU*H5JxG_1MRUPsIhrJbT9rXXpcQ9jMtB%tNfLcLrR2)aihiJZBkz<UI`ly zXtKiKBxvosnp<7+>BFOiH0s$LvF_Ib);pUimgH$+ZcG;Gd_98@zk&MPv=#XZ;`=>E zBnD;$o%tI`OO>Py=eroLS(Zwtf=`g;qs_wG9@kOxU@ZT-+Les=PJxr}Hq-L0X3*Ye z43FOMO8gw}$KU_@(Dpl$|0d!))cJexe#?Ae-HD|Tk+=)=xAfs7+7CqQ1*2g~$!VTi zev%ben(=dA1N6UUDOPAq#DB#xg8E1~ZP@e{qN^&w)H#V_a$m#WKA|it*-bZ08o_wN zE4p_@hKfO^{N<Sgr}~xQ(rhgZ8?H<t1|2YI$7NFcy%j3cPlLX92Oi(%0G*Y>L35Pk zlzd=<cROvNn3oBVxwlb%`R!M+$ZZ*xR3?z`{>P&Gv1zRQI*7aWD@FbLdbm^Uh*h(H z3F{x~V%-&8tpCyjmG>&~h!z((a{ja!6fBcoi*<OvYXBXYIutxm1)oi`Mm?<#e5k`k zT70IL#NANmCFvK1-%6^o+=t2-lkA9rISt@_oB4&G910a%A<d>ADmo@mYVdq<TgQ0H zIidvfq@J(GWerrl+JQ&txXHSHR6iG0>Bwio_Jhko0gp=#+O-D_=;-9h!crXvx^v(G zoJ-h%H!{+N_h)6eb(35yRUgSyB-en})HQ7WX$T1i+R@q^2QD-GEB;#26<5xe+;zEm z2x?LeM3O0Wtvw2(Et4oszf!ENlza%zvw8Qicy>E7T9B=LO_TO!(YZW*4z4^-X{QUs z@voBkNoxfkTC|EDEAJ=e?ONP^Lo9UA7NIf7nl~E8@t|Nc>|1qMochxP%}OieLAtK| z^WhA9Gs<4@({G0%VXCzM;}%lUdMAFH9RtTglHt5=E<TSq3YWu1VBxfd*!A)Pl)G2a z<@38SuWOks^RyPZ*R+bGUYOC)Xir{zJ`y#1?uOa-UsCvqEHb`rAn(7*0Ykd1=0&Bg z)T8;Tv{%x`r`FLzM17HX@xuvB&+5lF9^4?Z-o?G8_i*S!OFZZOoboSq6*k5XhtV?| zp|Yw9ewVA!gKJwM=Ufr>?EXR6W_%0`+sXvrA5Ad*_j6DlxE)po)H!;*Y=BsoDNy&- zk7wmh5sFt7ik}`JZ)u!PcA;M>J^36MkMNadzDG-5=1&klb{<bSor@lUF1Y3Kb#n38 zD|{=t4i_V~LpK{O{Mmayjt?{w16>F4o2}ub+;1PYom#|M>$eETH!^7Hzk2!}wu>hj z>2OKdDE{3N0euu};i!Wd_y6F?jjeU$*X=QNa+|_WE0l1ke+`*GOC_OqHaJEsru*BX zIBuW|ElBH!7YvqCp<S`eB<q#<CS1z%xoVKAWt6<=A!5|hreQ9slKV%s7Y2^%!Xw*H zkU7s^EHMZU!QeZ4@b{erh>F-rW6aty>bb(`(^L7<!-4$!i#2&WR#U*nPFQN+NFj%v zvGQx7_>}!1TVjSysSkzi9j$rT`AC+Cj#Q&iD<~Z)rQ?eG#jX%b17ddzuQn-T+$CKs zIDQE#cbJfieIOnc6vV$>KEc0t+5F<^3TV!$;O(LDLP5<UD7ZI>7Zs*q?znGoENV3W zEtm7jW)c3G8q=ZtDI9b@jqm+f3x~eX61Gn8WlxJNKAUkEo+ma5!P1=LvXV=*PO7Ud zn^;Ga_wS~?(_?v!mpXWStAv_a23%qvOMON?5XySzii&wnLbt3a`mIn_+7e}oHPwq@ znc*@pJgtS{-Ie*wrxmjPbI0=1*UPAjf~V}M%1#*ks1Rn3N#QxYmC-t)3qIDlh_#W? za-EW1__v}vshH-<f-l;GRe1#Y{T_;)H|k-iinK;#9!2HP1{i4@PDiGB3eCp{q59=S z9y~W$C|}e82ajHe{StP<l@(Fq;k*OH@_o=0r;nz3FKG2-XV&&X>~pXa?rS?v^RK*+ zpHwI)n;)u2@6Yt-t{b+~rNjT|rD2+QulW{Lt|926U_{iZFTaqym8(h@!&@&)8nwFv zdb1ZSS{%cJUc3@;_)D1jyAKQ<wN}hHcwJ0Qi^m7@)pRUsF`FoQK*Q^f@(tNBTpz9p zRZE@e_{$(HKW5J9Te`8YLj~pEE60-wCfMa#Z#JzpK*x^7FmkRRg{tip<<foQS@Z=Q z2b~cr_8ViVFdBRN-Ix7o@)z9mPr$nTWY}4}9l3oR^;C<cB&W@EyRAR5?@m}}9VxsH zIRZXrk6};9L>}f^14COQ@vn0S4CuR%`W@@RDVeoGvz-yE8U(USPYuESkQSIobI8}n zWKh5lX<t8Hg&+JK4~lEdF<a9CrW~Iy>@^IcL)D%5Va#@#IzW|MTW0f&86TzfGl~*_ zR=~a)ld!S*1>8Q@9*<Q?4w+udVY;$3t867@jL)X<sTmvM;qNay7Yr*atHwiYrc z0;{e6<JieuVC`g2vG-{U?rOf12F;v|GK1g3cMUx$yRiqhs>Q%vc~7kEG6DLzFNP&4 zO)xKF1#MdEz$ybLB3taD?iCkl?QRQxTbcz06}{P{#6f=P>=mJIha;Z8n=Zu6Y9Yy7 zL{~n3hR%t4T=_`@&;NE7IyA1tdrBn|4^;YG_L&aHS|z?r=gDxW|3NO!%Vf(79dZ6Y zX&~e9Bvja%%=z}AaOl+twsRSVZzoA>WVa%)m|p=ogQJ9>ek^?4kwnUE!{OJcfowi= z4Jp-L;w68bMc2DWpy{Fqbv@jL^L(r5?cw=cc=Mc4e{4C7U82TcXM2ctkJE8dzb)cj z>+djqmj%_m7>L!qdg6nc?lj3{FXeRWDCD|@QcN#f(Xh`!_!KAc`lWrbc3LMsP`iaL zTnplUuRX{%r?dFP?IfM)-kmLY7i3+R^0ZxDQJB4j>-X*A)7iUV#*1Efc+VKN?A{_A zQa*uWN9Te6#JRlDy^i{IzeP@4^ucn_T&}($&3fi*@&4ooGHauqG$8AS7&CYaQ0_Mh zU`PBdae87`PZXZnwigy_YIEh$zhHYU2YkO83)6Jw(f#zf(C*o3Y&;rQda(aqshklA zO#_F}2+4VHMdB#cjA)fdw>+a$+1se~y_9!-Hi%tU22h-LnRw-8FZ4IFN557z?)&Ki zoibidSN~lS?zGoJm%2=_bDF?f#Xh8#Fp!10dN{TG8rjZX#NLIDm}s>QYf2M{Um&Q< z2_LWYVe6$Iq4|8gAo^Rg^SArpzso_s#i$fU=@p4fUsp-9$@Ng-zlSECssp!CMtES# zKIq*}0o%1n?%>!wq4}ANPt^9o@zsYxp`S8_d6mePG+dx1>aT<@K6x-YSOXV?UJ;IH z_JBTWui)|5NKxlNCk~GJLVM=zr|wt6Mb8c{a5>P252X(k>%5Do!+d?7@pTs7KmAxd zvV1Z7jJZP|hlcTm@Q&P&JQ;7>t7F`gOt>8+xxa23@yuyU(f`36*|&3!yu$Ojm@DyU zj$ZyGwupD(>=Ga9bftm3i;{#cppIiL8$t1I0xzCv$3_=bskkOmT(YVsKfY*$U-oYY z{ns7f<?|F8x+<M&3zaCVFp=Bsh~t5%NX1*v;M4Yvax)D*zTxR6`ba6Li8Wiv#BU4S zKQWcQ&VC1brjEpd_2#^Bel(1?ZX{i8lr>r1g|B_i!tv2D=)GhrI(%=B%}YFSeW4ZG z#Ol!H#Iv%X#mB%#%In%M^c4;|@5PZTZwrr1uaP8@AVad_dr@Beq@WvSNcZ+~%K+?D z`-+s>{~fj_riI?zs-V+7J?M90B({1FrbM$-Y~AauIA=tKtk0aCq`y82j$PY{HX5C9 zfts|35RZ%VKK-G;84CF0zyWx@_bV)DtP;;oEu`_gPg7KM7gG7*1y;t!oIP$1^+=DV zmD^{)q#_$W*ZC-@E)w|s$XL`pUjY`LDdMp$Z$z_}KSIaG2Fg#i=dX_=VDP-t)H!lH z)C@lW$q{pTTk9yS@2MqRcfSZHto93b#b*Q`7fq<&>PN*NW!QB3Bz;V*qx8dfVB5wx z4%F)j<N79p`<zFD)t5q9zfm2ewZej?3{=3GQr)}mQQEMTL8)Z@+z#KbGLb91>V`|S zcCy%GI{W8Lfl(zF(Zo~%bxYgetLIrN-SJnvCe?qZJ6(WBV2Xpbj;D+iE$rWI8;$pm z6g}2Pl6~s;VKxJj#gk)Su<Dv;^kUmVx#9|`F65Dk=R4YByQYESvdWDxN#gTqT$Pja z+FJlsE5(<)Zqng~0;oIF3(oJ$ruRdKa-aLgfCp3Y@N-Q}nIFfqvOVcR7q|bJe+h3G zFaN)+KiK?t>kt2L=U;68pZEX#pZPBeYXA9t`9IBH+5ETtJO7pMr>lbh!T|rpAxjr6 zSvh!~?@HgnHWF{Aqr%Mp<x4>!{NH~(hX;9=?3JZI`ad53)is6xd`*dA<M6*-^MASg z|L3=LBT^mnI_HDd);|2i{3vWV{*Go!XR;6TkAu2x9?PdU!q!y=Fn>-Ob=#Q(Z*JJ) z<>XxuP<9_4CS4$<^Ny_1r5)}mJSaE#=}jR4!-SWA=CkY3FEIRaF&${@zztVCsc7b7 z@vrwc@rZ68l=qH<FFreH?NfK&>~n&~oR@=3@*3XOPED-${Q)1}sble|HH3q;@TQWQ zV7(=oCyp8@j*Pe=o!8s(Ym<KbX538Fb+Uz~=1K~jW=&6CpND~_5;N{j3e8KLhzF~C zqSCM&$hMqB-A6=HbIePc{qdkMwA4e`pkNCR-X;hhtcb6Z-ML9xpGORM1%7?(P^Y=8 z7!kG?wB~#R8?_=@7T*pnD(B#vS9V;#)QrRab;Bf6Ei#)D!3)MeBuDWnoF26W?2?{= z^N-nlUMGVJTmHeFz^9_BfpoT@&BHpG&4w|(^!Sct40R|Q3_h9B=-p4Mb8S3IRi?Gj z(cFN|9|qxOw>W4pm_f_khx5#S8>QNBG(Y|6$$RFUp-x$L*lqK6`l9j{jyl%BjvJbE zqjnZ{8(W0(Ifp60C`WK#-3~cgR}5{-q=Pmr`bXLDXJ1_m{G`HZONpiqTTS(yb>Lz0 zI!;Vl%tP;{g2sLaNbVPnPe*;FWiM`kLsxHX@fr!eHRj0MC3nEes=@4krkm*Hr-Cn~ zKAVB(1@UlC3pVSk$PFzi(4wIYC-%DtHA|Awx!#i^y}UuKy$-qF9|||>|B07O>LF?D zN4RlpH#c^*$Fd&tME^zYz(Z^mK2;}x!I+oQTxd9)9kox4iZ{WpCNVH3UWpSMBsX4w z9FADc5l#fpf)M+5u%&r5E&CKH{xnphcT-9wE`d4)7UW~`{#~@-TrnhU?m$kfpUFc$ z>9MNQ3&-bz32IE7!eejeN#}BD)<3W_=bg%<*<pKG8}_i%h0U}$Q^t$_Sqd*k&*a~Q zr!jR}20rWO%7fJkVEE%yc6XEJMJ^V>b@iJ<k$;W!91eow+QC@d{WQ(8I*CdbN65uf zs{G|iA52&202L9pVSZ{q_Nq6)<nrF^w)YDK9l9nc?a<&~+Uw}v{T>)vE%_r3DDgGh z$?&P-3dj>upxciVVE^YiC4YWHBjTl+Ls*JfdafgOjXFW4AG>1NpC|BaPXMenG2+6n zpW#xY3}@_g$G1OCc#$;U)OgDhudNdJNyJ0C?6QK>R>V==sE6`Zj)Ao4N2dJ1-&(QJ zzCb$1Rtwub<1olCkTwVJ!Jgth*dG=NUrgrUWXq4@oRzA0#dwfdwO<FK3kK2fMRMqS zBn~V_O8rd5-8}dGFIXPbk2k%Fp!}De97Amn(BA8d*uSbL1XV@x0GnZ4IX9Zs{~CzX zUeCgWPxg3k|4(Yy>7?j4)s0KWR&z#iGRkbe!@0d(_?^aDaqANYoHpK<4h>Mn$*-J* z`};+Rx=~CSHwLoVWXXN7dIr2%tb;BJ`-NM7lEr(sC9nBtQ<&?SBl({*Nb5)}zw0uK zYtvTZrkZH3Uw@tWb5EQTI~l|3x51yfM6kEsLK(Zgg{F_C@WICz3WnQCd7ePN(5X^9 z=&=#%+FgKd%9qJ_`xJC9R>AAJKgDb}b(r8_1AR<x!I-QiXjWj1k;=O$d~P#2-7XT( z_TCAy4QDB>+*sVR_aAJzeE?$mUxihxZo#rc(>T1gD<AgzDP&)HfV~}6amAIs_^v2Z zw*6s~cxb|18a#0utve<CzEif`((NIglICqS=Ubr1xidmjs{uOAxFMK3x}lde(>z0~ zmV6(W@ch#SGOMy=**yDgSe#7Iw$6jI)IUIv5LHf!bigG4(d;y5J2)9nz!()Prj$z@ zQ}+&Le%XXipNVwsX&>xy>j3=mybGTL6w%N03gtVNLG(^%mWKw2^X#UO_uU?NqW&~K z7Uqbf22H_&?H8f+$s}$Fn8%xJjo?K~I>g`VLHhq9d9c<|IcQ9ld`eyT)VSl=DbEk) zqYWx3tFYbIN#dd}f90v&&3KirzL+*&gVjQpa)-WG!OFZ$NZPOqZ0_p{+s>s^@Jmy4 zwO@d)m))=;ZaU0LDHgpYu43pOQ+}q^mzS0wBjebgpxPVI^rRWCmii1zM|MzO(?^c3 ze^tS5z*n)UY8L%kx|Yv8HUy1-!?5aiS9UKjVyUPGKP&~Fess6oIsY>CbJ_?GW+>97 ztP)VXUP6=8UeHO$c0B!GAuda^;30qda&eKA@3v?XY*o6@Jt3Jcwswb(krE>;VG#$q z^o87u<@_b6KcD@$Q&jIZ0elAZ#oMJbNpWlr)?WK86fR*<Q848B^$C2nY&5IHt%qix z+f=k+I7YPfq7iFHLVmwX<aFaM3@ld17SHK8LX6}3pgPLU3#1i>x;%Sp9`7kKl-P_m z^hf(NMb2`NzkMs?uR#LZj;O{pKZf(dP93S>eVn)}#Eh@C`%W7k?x&HNyD2!Yh9dqN z;-f>4AwOz79`B`(ueSN}+t+=?tV4TX$@{ys^S&WE9eE-i_z;Q#`;JL9`CC$5e}z;% z+>Wa>2eI;GT@1djDu^u#aJVf6HtDv*q>jbHqJSRc(YynH$6W!<;U%!4$bzHeRtb0B z4@MpPMA&|Qo%nWO5t}|Z3CG73U{T*LFmA;uatZ4w@7ANc<guQLPbGe5)G{xy^fTn? zNjp&~>Nsb;am5+SJHl-9N7Mkj(JM%mzmD53Cbo~K&pS>+xq{SFlWKo1n-9uoXgA1J z7qu_TZk`0~SNlPS&-JiO^uV&}K&)QpPY+Iv!*;6@&}M8BRm}t5cC}P^yI%!;Vh8Y6 zDf<~c-vph8sPhHAf&5Fg2LxHfL%g3KbUwrqYvhXT_gQ6LKX?MUKAa65g{xv!S|yE_ z0x-jF9HZ11hIrNRG>ok6ibI38(x=XU#m|#Ng!lWDMH4CHQ-1uO@OHDrpbHp8A$G^W zE4GSGPirch5cZQ!&%O)Ro7=>bZ=%3|X*s#A&!eg8CwTVevy`MRF}k{>OS3FG5YU{8 zO@4Q2*$oRAa$Ac_C4R1=b31%g^$c<z*zj|)ztr0gVq4ET$R2EkI%y3gc$8E5^OJ(o z%^uX&zL-MBPT_H#qNt;Pqp;cQ9K51NvE}r4Sb6xW_@Z11#zcQ6uzn?;J=z5pOKf+u z7FX0AFo<W%5^-o!JXQ8i!3$R_K%=}bjXsx5TOx;}dY?p`f2M{eNc*_owibexTNOrK z>4EQVeWiUj>cyO+v+#Q_E4-I6LZ+~=Cr72<rKrK0nDoGah0D)CLtzK6yB)<Vf+HyV z%s~8d(2L!jl7*eyHshV40j${Hj!M=<K|`yQBmU`2-;VlYY;;Gy-2VV2?pp}js|SLD zeuUJESK`V}55&9I^s&p0`@+f*#jxnU6Ls3^4f*+t*{Y;a2!9kww)s-ue~}xcz0u=s zoA1Kh{zqZrhY^sf90XI=#*??V5}xR>Q;0AP0}h@E|5jR}$-Z@byu0Li53Giaw;Ncu zIG#EmpARFgI<PWjimG>gvGw&|vi}|}`uUYYYkOUucg+&Uez3vGj+d~1Z5nL1nT$z$ z{cyo9TRgJxB;6UiA6+I-N5A|65ez55HHn!e_$YCV^BOc-aFkL%ZsD&tj=-7kC=T^$ zk1v88aKo&xV#Tl&SU<uRKj>Vc<N@9JK%5em<?jTe4y)kiqH4#~W1XPu7s*MMSuQp` zYoYmz%7x;+(U5u8hGR{2BnLn~9o%<8{3kj69y}S#uKEhNbGS1pM>o@mk`zdvVZh@* zY{c7RRA5s}e;Tl-h*lg)7SC%9fZ7i~;h)lNQ0)-P7k}==1^EZ?in<jw-R*$0c8tW- zkRoWgzKQ*fL&@}I9E_CmNd+M*M1yQIzFpl{2=N(527jd-`oJR6%hljH4{nydHn}Pm zIp_<)3!7oxu+?xetOG^e9_JW1`lnca&=;+1hoWL=3I-U}gKEMEeq&K97Q7rlURj&z zvhx5;+@pb6Pp9z8s&KxqGy$6%VySN1LXe-BMc>A@<E(jG<c){2X~uge3cC7@ZeLHN zYNJKu(!-H@oVh2=X}0BWw)*TEK8;<ccEEW(TV*t934Qv!4tt%wMEkVt;H{$wAyTf- zG;U3q-@bIm4y{d);D!9IIF}qktLRB!BFKY7c}hzO?kZRyY}b5CPh{!z<49kuFMLAh zGL<O6<pO!#j}Xm%m(V|5C6I?kid*az@T$vtz7v~E{Wm+KQFIDS^}Hkcro^(tE-AO) zI2%qT$I}I;I>9r#GvD+wq4|pGSokao4pmk_`XdiM`kNtj<td@6IhR_u^`<c+HF!#0 z0R1XB30cG2VW;dD6cDn5H%~qY^B(Kt&X^>*$%kREHDDS(bM+LR!?mbXQH{f`mqVrQ zSgr}ykPUeKlA_`#LT`!l<7nGetnFV%K|wS4PnNnY-e~}zJnD*1e{bcrj(cgIS2zwE zr2`uz-l?muA5Xb`4#r9hD6LnQ!9qd$Y<-<0e9M#g<kNPssMDLmx_*b>&rPu0I~lf$ z{~$XnoO^!|P%SD6Lx1O^^58M_@ck_+y1kkrv`)ZLo-CiQ=Z<D0l_jR7B85*M%b%n^ zVsXKC;mM{*u<FwtJG|V*!I3ZI)=I}kbwT4?(W7*<yIJlyx0ee&Q_F{r?suqsYm?Br z>M(~_yVF&tKH$}2jC%3=;b-h3h<uO-FW>pYKZ^qVy*-73cl6_fX7ys9hH<#&y_CP* zV!-wl5gc^Nkkt-(3Lm~0i(kF6NXxt&(%fIr+B1FV!Sek=!4qqAUNAxQx6t6@TlB<6 zGfjRlLPlN+W~g|_mIpqlfg<k)$XIEN(I$B?_q`b|$dWt<``?g%+d*;DnSSK@{Tivf ziGbCw2f~fEVD1z&6HX)~NW5bW9PB!W1-&~^VtrjO+@^+F&hZox6$KeaZ)xG+O!^Z% z2`gg9ve!2|zBJ7jaz|LpUpgy5+8Q7FxUM&+rRI}@>l9uh<qs3j>vH_o4dQau;c)Yd zj9l8OQwQw?9w^twu>4li`_mbEcN)$sO?qLUxN_0$SPC9g_Cr}*Jxm$zLMi19ba?Ax zY%K50-#$lUnDYk7#bzN7@VDgJE~!{>W*uil6^pk<dT_{%ec0#m2${_nQ?%$Xk<&Xo zguGrkq-^0%v2hW?^U(q>Hor=&TLF%_E<EMLAS}-Y>Ucg04s6V(>$(a!cADhT+SCG< zx}=la@d7cpY5<;ECEZ6OMf&>Nf*f|SY|zhP^u=u;n#}LW7NbAG$!SNScepV%r!dUa zI6$$VdQt2LsjvOrfG^$j=Q7EqAiuRmtPMTRu|52F_mce-W4s4c*CoQP=<7J@Sf#w& zMOAEPyc_Z*7ntGu4s1MWFRWDjDze&j$oMvj_sq42sgl!e<Le<@Vx)vI4(70DY*%)Q z{Y%?adSb@Po#NJ#9k5GRgz(CDaOv`Mcw!YsyM=b5+kj&X(Vs+{VRfY7_Y?Na-9{7K zp9zzia=_euKCOSYhkk2@3#(=GC2npHY{@O8O|DWlXO%X8A2@}F#eJugTy>t&QOec6 zTFzw$x`<Yl36S(C8Lsurr*FXvaE08SW=DUb+^h^5c-9>6&b5>H|9aTJc>?V)4#S~K z+vszJGCSr^!?j+@SXnB=%Z&$VHB0$mx6L^2>}+nUm3&OcXS4b9bYW%UGjXEED!8Ow z!G&$x=;ie8Ja5(z{JBRN`i1X=JI_}OZkgNg(dK$Mx@aAr3=R<|`$xjv?pDI)Bm;hA zy_T|u9|p&o)zo>C43q3D$P{Ow(_3R)S7RsShkql5(f4RZnTa^%r3%jQ1~98v<PUYn z@R*VX6mOJR-QEEZHhL{@ew;&7FFywNq4`kSEgg;)*-L!$sr=IR9ldK-;w;NRdh#j+ zE=7%mVjVlS?6(-J?$4!z>DAOdb2A!W)aF0OD_~K}XPPj%QZN|Sj@LdtMqAolCeTvC zU%EZ$m3BM)=W`17@0^ac)${Rtw`eH$xrZNW7II~k2mIRDlfK*M@YA3X^2|GP`HF{% z)Q5IOwbB?0K2wkXa#CU0`4`l%u{ZSGuuGmaZ=bm5ML74d3>C81k#OrlcRn5KF5lHK zfn&4R;KK=?oEtbETYgoE1v;)WU+KKF-FzgD{nsYkx?v~V`EUSsNzDfhoAY44Qy+|T zD#TtYyXDpmMf4?7gPolgv!+ge{&;9HSOr#4y7v;CY~;>;c@DIHxSOIbNIViNi4A;1 z#zvMOg@;|TXwZ!|&|fu@2Xvl8jc=Di$2s13+b&3q2<avKd3_G%CAJrx@=wCJDZ52) zYaRN#v8y;da|j%>n+x96JCKw`=vY2MP`jXxuLAWD%{GJTvn^s+Ygc~YmrZXSi84AC z!7J}^G$U3CP3FFV2j(Y*y?bZFhAyYjt!NPc?(EFz{U^XUUly1D+6Ca&4^)<|0@p8p zplN6!SQLza@9p$av7kN7D<GPlpUcM<BnVI2f<(u957d%!`u8uNM6EarIyGhn=GN-4 z!>2v)!9D@TR`kbz2NP-igZ->j|CaVxtf5<hYNhjz>%(r(t*k1OJUfRC_@jQX5HR5k z%=uJA4pJ>B?2!uv=&E7wPrCT+uL7JtH5t8TsPU|2NwhyKiT+d^6FgsOvEG$P;h}Lc zs4m#X{@bECD$yP0#XP6PF%$WFh#l8m$pJ&FLR#3%8xLk)r0F*d@Z+x?H1}sCY<oYA zErzj_U!BW(>rRr==n{C<SSK8JnZ*s!TV#Ti>#-~|<oC*HG(J8;nsrzYr#*L~hEoES zEBD|(ea;H+&pB{xTZ}BFeigQEOrc(u59zgLDp-3xB&%_bn0{7Cw7qY^pI7I>p}N;( zaX+5BmVOWyW+>Cg$Z2%OeFG%6hSRquDO37ulKf`0IgX#HfS<Ka!rJZ6;bdDkQcJ%j ze2C4Ft(bB`xY#$9(tWEy>&;?_*tJLQx-FF2n~cPp8{;vTZ3Kn-QsGNPve+?pIEcS; zaotN@u1Z@%K1~HBg_ruVve7!wm51S?-#;iyqdo33Sc~p+leyz?E10@kiT#4c;vn}l zvhr2Jg(+8sQ?|<}O-Yp<eN?&eWfmr`7TA7{Dp}Ta#+m^dvQVk-I^kbWm@4(}oJQ*K z?jMV3`O0Omw(D@-nfOoCL056lCKFON&=4PYxh|xyKLb-^4cKmi3hR15q(Nz2F>7Tv zd|{F(eXcHIot{&$<$yE0>a79O^bqKE&y4jyo`imO8~9Sc8oF9|Mn1+l3BsQD=6U1J z3QxWAVd6kr^i^G1MlD(5ql1kw+Is@pOSx~EwExh%(ne2vSAzERYh=3DitzIuVcwGf z^cZXh+I!;p!ytj9CN9UYfC{nqu2jCgDVk<qOc48z-wxXAr;y*19&9g`(#8*3XxO6< z*1S%K0j*NMyJWUFb8lCSR?fuh*%nxGdj(xTdS6KSaF=379E5Co35OLHp<7*7yda&u zKP9c?iDt#%-JuSTn|5P)kGG=4V1jZJX)m8w3G0ov!Qp+~P$e%Kbyc$kU7sX)_~D6I zeg7=<#UU)+lh`MtKP+pSTGqIFEH{jN?|8tgH_8rmrfbKgY|BR_i5p!^a~_@%`qT`< zA#dl2$);80dene(99B|Dn==1Q2#22`y`bphEAWzf-yIX>eC~`Y&uM=GI*$uO<x!2$ zMKxYD+Yte10ATgzG~L_w4lci`1TDonxH9($?AVzH4QI|`YNMA}^7|k34bjAO^A6bZ z@)rzUvlcvj3TWQEJz|JYASrbY!ai*=g4nYkoD5oz*5BGo{Dp($zDb(X7oWqj%CXp{ zZx31tJ&`rz*<+D8f6hw~7JfCLKGp|la{LC|@9+rB?wuv|w!Wg;riZez!K<l!c{pwK z>WI@`=#|a+tiv%O19?uKG$+|p17lPpL9uoaPyf0KrVQyK@37FEzXV1I2P?AB^KdZQ z9#+RAFAsp1`CFPPSH@+9+AwneQ8JsTf*uN6q3FPJzPUpM9hFV_^W;77;(RkLvh2)P z-J>w5ZwYn?okWM8nsMMG87&Q6g^D9m=|=h!u-1|C-z_tta?UH5?>7J)v=q?%!9~$> zRt{z*#emkucu4DL%8;YT3bQ@<nd~Or{+k9H)1~#^eHhzJdIf4er-W~+*C_DIWl<xx z3t7!B6U}!f2uJfr@S5ZTN-GFrJKaRt`o;iWt|teR2}`*5fDd$b-w<h*<25LavcjmZ zI>YMAjo5aGJ^Mv3;w@J{QNCrOG}E?LTsvY1&8zB7IkTkt-=)obOJN0ldh#4>{IX#5 zjZv68dK8`hya>#?P2$!~KBPaeO>SJHjI%UuQSydR@vm(?Ma<m~KTe!tvA@3XJ<kds z2E2ik$4YSNf<AT|UPA+8Mhj=gz7r01I|_<#JV@=>N^no87OUDc+3*PP_M5NdW4fnE zp4V&qx6%S*O_soess{P*uQEJ1@Fcx)xdhGMi-d)zoWyl=V+5nkQMg6Qv|X5SP4baV zXPae)aNKAM`s8GQ^R;n!Z(}Td`*se7wv~!A2ARkWgAG`F;zoG$S{qk<$sq0QBM|m* zI;lPB29Jf)w5;=1KGx6-AJQI)=Y=Yv&s<YpBzp*UUE&;{`Z&R0wfU44nopA*Gx*mI zZM@wSEMEWnNYqiAiuz&KXyufCG~?ZI-gY?@oVQ$+k1{t9Or0V*c3rwqRob05K|L+^ zjO5&ry-0Jt)c3C*%<%zPkm+T`n?o#wK@K|ntFz><v08<{msazN{cC8Q{Y~iM5+*jR zT?>lMD`DRsHyks3s;u^=A<a>oNc#E8yyL(ld7HTheF!i>()}vaubSmJHzkh0Io<^2 z@@Tp%odr62M$x_NztF7KkH#;n64i3nkzvthG3AgiIZsgI>jk|rck^-Rnm>aTwr>@? zZ+axmeLjf(N<E+8ab2m&xtvC5kLH-MiR?b#81s85;h3dIsaGOEzYQJGJTH!pZqepT z4VUSrYaN|i7LW4BqtLvc0`|KQj%^FZ;)s<^AR(Ty&+8@{aB9{6^zlX+#ADBlpP=BT z&iA5Q#JmlW!qhS8oO(HkL!aq#+5&Zs@M#q@>=p62wBK0SDGFu{2q9bb9Wc|ZUO1?E z1Ek#;HYiMh=A;4AEPgZeIo0Zz=%~Zz^Ct20*4@+=w~Qk6YXmFB?znT!SFz1704ih= zY`fQ2jGEFBlsAM?^`H>CbzTc=oXRQx$rw0$U>hq}1<}?yx)8DNqVTgWnV!E*#=aQ| zaLvSw^uEp&<kgdScwvkz$>*Ffx5$KU<PJrZp+D(ZO#~%$)?$^SWGM%`3p!6v!rImE zg;hTTd8hMTI*}&vmaI%L>4Jb0GeT*i#x+sRd<Nf|+!qJ;ABGe5Rf9{{2DUucS8Q4n z1GYZvm~?)_sf_7ZFR{h<T6*EtrV|kId!JyQnk{jLx6`&azWnpVF2U)mF5S3df{#_- z&?hOI7`OW*92wRQ-xWL*!|y+$N>3f(M@MH;yVhA^BQ8S!?!}<kX$4=P-w@oC3tzI9 zNc(0vM80hls%1g2Z{J`%kl2sEB}d2?ly$`esV`-#hDlz}hwsR1=NsYj;VkxCGJ?%J z2++rEhv<_VEwAaMPWxI0gLYnx*j?cQ9orB|)7E){cryjQ|5}YzM~!*D^#A`xnz!$0 zYLAc4^g`p)Ivn@Xg)?to7w!#A<32lW<SG5vh^^t-uwANsZR|RkCw!j`3qHSudC4h! z%tZ!TiDr<5=Y)Zijah>Op#6$7vfx{37%#D&$Lx<P^L+o0rf5gXor>Di@t9UIUN(_M zqwm!Ils;BlH=?s%9xU?f%ZraFLaxh9n0NRe6zZhX*2ahAwo9NnJ`?Fm(RdnjK+40f zh$6R~ZqRdC8_ZTIrP<Z$yd~o^X)C4+KHh4uNc}zqZ+#E>-ExJ!Yd#8Im4z~+izn$p zc@1r>bV1qr4(xWnK)jzm9Q?<7QNyPbeDT|S;dbFHELms3(~{N2b^FV~$5@H6`wQ_% z#v;u3v5<cP1DK>(2>+2iP|UV&`oHrB+5ETpwf{M%@&B)%{KxC>6`j^ep`G-+rN@7j z{c!rv*^mD=zbAT)vc$GB<~}CTpqUscdsu8uE|E)l<GOEPU~Y<f=NwCoC(PlneyMc0 zSz>?)t+HDoanSA5E=Wm!59Q>**_+?dvH|*NGhRk7r{$F?93MxU>K5^z!8gU~Xl=gL zV8RhSE<jW2KDK_WE1M-TReP$B2V<L!bo{gt47V^4{9BB8wbUoNVbcoL7eCOcOdWP_ z7zpG4E$0<at_kh3V!`sWx3IRb7suruBfsV$)UsA$WGLFPMf)9ithHJ^c6A~gL3QZz z_Lq24^PSlK8N=V#dQ>)chkX0X?J&ek2S-&`LU`U3yu{vu^Op-!zqc=>yJb-S*Nt>T zE{eXFdZFRe9C$c)74BY_jV-5*@HC}UUj9Gm?$8ekvsaMM!++wex%2o%gf$rM9>(^E z56KJU1L&2hGr8I-v$J7;S|$0&-}mS!su;%6q7$nyzhD86&x(V<k$|dwzf*Pe0M_Wy zkuTmC;T=5&-NIK?x=|C3^%iAm<MQaTsT|fSj^^vG!+G$4LUH#<X^ncd1Z{th5=?q6 z=D%OUantYVq*toSSH1SpobUSFe8UYiJ`1Qg$p_z$P(`ax5_foRJPgT*5*!{W^ZfCp z^f;&!JZsLU-*+|3{9Gh=<g1C8UGhgbcJu(u<nCxtqsUuKzR-n=%QWVkCau|^#*ZG2 z!MQK<g&F=QXz~*en&pwi!y_Z8$GFazP~?iED#r4~%s1k%ufcrRMgancjpy<PJN&o9 z9a`cnscrpj$X#!aqpg(a({qUpP<}!jnz<FdcXwx2n#`8QanzkV(#%Y2Hu>p{VGX_b zU1cgS&0kEBg%+S)VMF(lhN0&tQ+R&qq!6t&v@}@5h|C*0L*~g-V!=lZdaad29=|T| zaouIuBFkVy@7=6(AexUy7?oX~GmoA3mV<6mj_{{qH%3?fg*UUdQt?JDoSrA0b%Qgh zA+r~&j#vPe>7U_niR6(`v7usn2h^ClK=3KD#&yrP$&H`aLhF2wvZnze98(YHEza3+ zV{D0VbC`>8YKI&`XIMd|%W*IdUPR5$)$z-$p%m9{5A87BAei2n0OnJar1Q!YI51h6 zKX~_HA*Y@yJ~+|p`_14LqQ`;VHLzaqrg(jpgEV`!2RtR$Wro6F^h?^s2l{t|h{K*Z zs<r{nWO%^+0cvd3eHW{oPK11=@wCc!B=x&*N{bf{XXE|{hTTZ_r|cc3a4jJ~=+W~e zSPxmp#tY)vK6V)PyB~#D2YsQ98AY7%PL~gPD!{qjyTRelX6iQmh2Zj-#btw~E`Gal z;({zIPUb-zER&(MOA&oHMTr-c{ONeK5#Pzw=f3h1amis**e><3M=gITPk20w)vPqp zz)%&d=3c>+pk0t*t41!-VW50H1bS}NCG}ft_~w~Fyj>d!U!M$v8T%z(?Yi!$IBo%3 z9}LH%7v*fVWHmoIGEzLRYCJ3{RpQ>4xZn$~?QG%Hp7FXH*Q^_b`#(PuYCrnXFU3u4 zZK;T#7ieM-jKYx923%K{B|ctL1h=-Hrm2;7Y+iRyaOt)MA6|>5b&nTHHQb%p)1e05 zN2SO@4-KFS-Tlzlz?1KdI?Dlpd-3p``x5`sU3~Cw07ng32Day~L%yAd#NqbG>8a~U z`FJ5k<vP%aM<ZBy;|*$Qvk<SXbpyk1H^fmfzM}J{WN6Fk&q~>elwK_3h1>56FV}YG ziN8EJ-tH`oN!W)S!_(k;p(g6B(`FT4Us_k!O0uRS^t7#%!oOr-^Wf=NRne18*Jn_- z?+?i9*<rXDJ{IM*eo#353!d*j4lZ=pMmKLKocQDpWq-KA|8D*x`{HC6@DA~nsD^Ph z^Xd9U8M>Cb;P^6i+*n-zaaRU$9~VQxI?e;1+62P%S-|>XU*%<?jZka+h8A?4Ec?;$ zAKo$-xVCsJeU>Y5kZ}r~57EXYnpa`-(9sx|qKdwmX1IOrQxY|Pk(0zDR(Um<QtO?e zI7ykNH5y^<buB2_dx+AnSa5rFOFHn#30Ay|rBR!lv7u)Q{gxOdm#=wAJuewy>M5|j zk|&mIwB*s7qOj|&01is|M1S)Y@%=<aIx{&Q=3kqJ?P_}BG{3=6JAVN>Z;_m0?@!1s zOm={ony(-~`Yr?(Rg%4PCiyfMQ9(;GS?y85*=d*I@o*)4vt+k$C2}4m4VnV=XOBu8 z9TRMKc_ePBlx9>0`jV#7DX6@9PF&LECH-3&D^j^J@A@$hPVO|pfu1(JGB<_4y*kK2 zu8LgJKLM8>xhn4N*9;|2-K6=|4dAQw4cv{L_`1hLani3BqDEso>@8&sYNAvzsyZB} zcgiQ#rXMilWsz8xno7&=*NYe4wr8J#x;Sa!3cP!uhGLKQ=ICBof{Nc87_`d>3WGc0 zk=q^=N=KmYpjYJlV>t)b*Gl}aLU?ymPSeI;gI2?XWPygXO6j0b*m3~7XwS#!NNt{W z^dOIzHi%u0q|m0GUBEQRjw1sN#5#|S7@%p4<9+@@>xU!o<x(fKTxW{y@_J%vQM7n( zSsh9^H@IDC136u`2-8DCafQP`Ow3NB7w_xouJ<Ewi*X|JfE|tlBR){<eWH&_-SA3e zKAcrvN0|pVK+Y#Ma4J>B1l4Y=Gh_qh7JFm(+tqN~y9>5=oW^<kzLQ<%`Lf@G%Y;3; zTJ-)$qj)|2H@$zzQ2WmpS{uI$Dc?@Ri;-$9tf{BI3r^6I8>3<Wa5eP(szkHz1f#6y zP>MP#LSmm7y4BuY$|JGRqpeEl<*m<_XW}6;`GTnP_bc7&9K$(dJM-{KCuIZHmgBRD z1v29g({SF--B7T%J=>kl5H_|(u+q{SAhwvo=1PebGFKZ-LKWF{GegKKOS~~)0ZcFN zMyA(Gg^3j<64!nNmQU8__V(u3uj^pUf~_*GJWs4se+xI=yP{Km1_mZrqJ@<w)+a2N zIlVUH2Z!ne*|Hd#-Q_Rny*CH@oh6i7eGrOH>=oWetl+BOJ7v??w&$_G+aSoHHyZ6X z<xs;)`XR9~OvI7=bJ_*M34Qpyx(fUl9Yp<&r|_fgOQ=cu+Z#6b;sVW~G;HfgTAooU zSlQhdZT+u6$<{#F6X7Oqv=|6iPixZpk&fKqsXvX1F9xN^pJi?x-;wXt@8ZJ0>9l@B zPt4LN6wkSyL7g{q*yC#?%a?^wc8|y6_#Yl{#cL)k)*r}UhV;cx22AIZmWm^6aydqO zFomyBgt~^EIO6Rh@s6*%04aNf%wzXxz4kzS`@>28b;CCpTkpz)TwSSG_9(oy-vUl{ zhy&vf6=Hj(c2v{z7U=nHpipB?u)iF^CHbZj{As;(3R@-JKRfB^h;;a-eOXxeY^yk5 z>lrPMX@hQcx2bafaL#Zs5yfXqVUPARDE6B!z|D2Cw~4Ki<IaRf-|-Q%ERT}K^!})& zzDaCbIhuR#o{ByFve5TwG0g}+K(zcDyjFh-FGeV0Sl0oVex!!_y(b|jTm$!148-~~ z*<kjP#B1Xgb8C};3vVT1mgY2CZd@d~_#dV&?G{11*;0ONaYvqTDUqJ!o`Al4`}6nq zirD?|LbkV1=IdNcJ;I)esgy5dw`}GKi*rD6od-P2zd}z!%y5pKHtXCMP}^b_uUWg2 zK7DTzVs90*{9*=&*R7_aj|U0e-qGN{qgikK7#z`hm%6wmiFb|ssfWT@KGrjxR8w-m zKBt;>^^90wsxiL#{zTaJYbctE`S3_J3-o5C(nibeEb~4ic`P!~dc|D+*8Gf?TAQGQ zYaXr+9FF~7=RnquI;e(he2{%y%2UTs@8_q$sU??w{yYYAP>JJP({NRiuP8H-+!eQ3 zoc?|cJ$buRY#g_b4&R>(w<UjPTct844e?^rH7DTR%*)`i%3pl!x{HpKCc}}OF<|ED zPoXLI;KP+;;(*}Wj=jCgAXPs~Y%(>(J9P@&oH!2^I^Tz{FJ|Dj(Tj=Na>#$vX2^={ zgs~Fu>XYV4+F@hM21?y2<%}|yNxkX`cfP^#e+1?kYS^Y`$zNVX3Z6x;=~c!ava=`I zs-_;Wy6+=cnYI`WFB?Hft{$J>>y9^j{ieNLB&KWjC77`yUDh+Wl4g!RA<ftr!78VZ zka*e;dw%G|jjdm3?c}du_gI4`ItRnw+)|o1ycI5fl6=1wZ^dmb<uGXVUtxuPU(QWv z&o=2zFx9w(Hh3JStuG5vW&S8}e1|!_=0^iOH1&o~GyR1>S=sPgA((FOek~@Ro+Y?n z4aQ4VWe}*ljqNfcXqIs|h>W#FU1LKbvGk%$_S}<f(!)UYbsxmcc<PiXF<_IT>F}il zF|E2;6k!~8G7F`HJy+B0+S~Ma+*>hu%T4i`jv;OvA1`)$c27`}yz2cPynqvR17U!5 zDQs^IVE;p-Xcu3Cu$$7kZs=m(?a%=tm6Skns-|FOR4l&uXvAxjm+;*kYw*Rg6|^^a zDD5$klR~&9$3&hGUW?y_S10<Trma0!Hf^Mo*o*SrRgyzG&I)@C*5`fN|HMv8dtjYs z0&MR)lVcveCRxsNi4h+qJX6Uf_hI%JcH5HQ+?j-#&vl^N{jPY>$U&ChJAx;@DImq1 zWo+KK6;w0EbM2<ZG(}~*ICzUCd>=oWwXej&@vGbE^v@C4VscX$R*=K;UJ^bgViL@Z zVd1f>3u>*8<xlEoM86DG8a-&GuzO;rP}9E}1`IMHA5w%Vufy?*Sq?RR7{`4+tcCc% zo8&&zMjr7zUE&F*Q}Skaj2*BZDn37e@wFf6pX)#R@HI>jKRL3~Q+qUe*G!{Z_R*0= z%UQE?FIL&QnxkwN@xJZeTs%7n_Lq00PA041S@J@`c!3Xh-!K=o+YOgFo;(c?e*Y2M zCS0Le`ReGq^){R})Req*zI0WuCkKf;(KgSOm8OMY+WJNOXj=m`ZK|TC8RnF<Y66+) z_oA!u<4NsgHp;13IL^63w|zY@@aY!rxnn2Xy}k@Or^itAx91QzWGtJSJ%E|r`e2s@ zVbJNb3-;7#rd91OgPlp7aB5!(jM2)Kxj9PvzI(fQy8BAF?)X<&u*8kr0!K0rX#}sV z3|`*j7|A13%f2QL;4D3LT&CcIid|iW!eY5#vg0-dly4SI8k$8;c7=OJ{dq@-fR-Z< zljf>6ar+<@9DB?XHB`)CbBqNkdk(=d#=~e_+HknAKm(U*$3t`QEl|pGlD8z(k@XM< zTzKfbkT9|*^-ptP@MGD%^b&ZRsL4W?J!tzV7HvLt#8sb$VCc#tFnIVdT))&7);3te z@3lqbIerrzUE71NXwT)9owI3-zA7kLZ<72)QY}%GT)Zd!>6z?<5VmADPjjsVE5*a` z)2fnYOtPSqxN4|bwNH$7XwM@}&WfSWhrvI)gOulgMBcpfDwxz4LD%P7DPlt}e%0zl zS%;9KL_HQxsqwDIXX)HS2P$*6!2v~@Og@jrS&t^tks~hLcTiup+xL=Own=mMDG!kz zeW%^MlrgQn<be#hDtp>_o6z3wFTKM6UfR~FtaU1)eQp@V417)Bt=qx!(BH6OiXx;8 zZx**6P{yS>yWr`~Lt^z4D_lQiA$1rs1SdaR4*NeR;mgU@;<Tmnuvn)W{cIl7uXQ_E zL)(sAf*w=K(_!>*rYdG84Mh97>7-&k6d#vabJY8rvig8Ne0746<J`GLbbo>?JXrBv z7}T<c{zQ$$BOba~m?VeT?*wa?l@5Cr8;p_hD){$gocN;28K7e`oYT7r$CAgAd0{m? zKI*{N_M731MS*A&dr;^jaY>g=3ZeUsd32>8!u92eWTopU+gMt{H!9nxvx&49tz1BT zv-e=_k4&EZr7!+CZHP9r6UZ|?9TmF8;#SRFTzj-L)m%{p4U;hXxnL9Kw(kr-Iu4+p z>2COUX0G^dUORrVRu`;8RcYrR2e{axC4cc{oUqp}o;;(b$zG@1vE#%`vK{X$K&v#K zwjA$<63hx`r}XB4%odzqFdg-JFW|Q?4++t!x51=&7lf7g!tno5-kHZ!wT1s*i3}x4 zC`n~1G9?N7Sqf2<St(OQsWeGSQb}eankW^b+)AoTarUzsG>{BUNJ2Cz&6Fm6w>!Nu z-2408@BQ<8ytXsG*IIj@XYIZAKF|7mB#toiA}la~%k3Uy$mz<QOUJjHjPU8Sr_d?l zLdu2?BSSJ`@oA|dkykwl64IlvedlVJ>F|V?`(`%0*qDj|;|$@I_e9e8Z8Xn9y#*`| zmSe>c&TnhyE@J3DlD13;rJp|RV2(Z(BK2P{gO3^?-6ySqQ`E#rtNTmX?7(R%h(^Qd zgEqLh=sVi%kA{jG7L%raWd}+w1y{#3GLzG25Unxi_Bl(a%(Es8Oc&<-YosWhZ^ztk znn9HnM^SZ7GpXg98#(DPlt@l`%H{Xekx`C!c(--<*)OMulM7$2f$70PAY(d`3TOX< z#YZy9vQL}`+o%Iz^RW$A33T9{=ifnZ$v|A=d5JYr7a;BZeCQV2D9rSfp^AK0KqiFC z6P4S>&3nBEkF_?_6U#!t;}~LtYbnHTrG=p(`-q%LT%p%jN?wKUfajVDfcmAZyMHb` znWsxKhh(!aBkh4SYEfAi6I!z|6i$VS!!HdZX=mb992FG6%)A=S`DGquhQu6YuS@b% ze^(QleP5e6hMi~52%W=uU%7L^>BGsP8~#+M`V3fouVj7|ZUJ#+Q8LTjowU5yqbr2u zVWeCIys>a3E1yVF^{HIuchzp{%LvjXzLL}`s~B(17)?`-6S8GNI|P5;z$!?$z&hQI z!mq!U8!q$HheDMjpzU%6<{{_FbNd~tz08CI@AKG$#y(8Z{veXV{l*n&<x2Qs6B+9x zZ}HUn4)mD!h>nSV0YCCj=zlLbjE#Qpm{EKpH1*~_7<4&^s*jgt_@BhUMMZPc{OuH) zMdb0`zRG2WizgDTMaHxxK#_doay0Eul*41MK>Gc*3!T~D2GeIxB=(E=Xpm_-dWyDz zdVUL<b-H7yLp3@~k3nAAHCWbk65}|JlF&hJB-ZRJW;j~23ln|7VNfI;!7gLm1yb1B z_9N`Gl}6-R@Gi0;rW%J{oKJHve`MYoA7%BA1z@e4H?&MFKxh8zXz6bccb)9W?IJ=) z%;JFtyMmruR}Y)3^tsIG5b7<bK$06psd%w0bk@~^SfL4NAGICj4;Nytm?}9Q%uiRD zsFF}qHM(=a9@<>76ZOk>f%>#Krdc_PnP+57gjb!Xw{B^HonbquPtIl+&Shb)73YzW z9%qofVmZCHT^LfPYLFKeLZDI1&v}_Fq)ysGoQD^S6F&}tFyRt7D>{?F5oL1!!WHJh z;Mw#S{Wb`3(Zt&ex3R2d28OSSpgE4>B<cBhA~K^A=FDz_&fx=S-BtlIYMm~Xz30N^ zigO;5SqkuIoDaAagff;ZrV{0)lhCMoK9r@(z^cq3)R3)4v-YhRUZ72Rn>o)GXIp%J zEt4BB?j#ol(x~5dDYE}ZIgFBcME3b<(2%5Ecu=07x;BK8f{m^4N<JRlI)9-H?ZRNJ zy&rAQ^FSrtW~MV`8ue3?B$*LuRJdq0J#~C6jTGcTdL5y%^K0-Dek@E-%p>u~%}JAn z5zW8w6?PjRq<g)F;?8G=V6pck1YCNEODk`qd2<^G2pm9_U!7?5t;Hlk!48~jB0wNT z0Sxs@iRwp3GC^u62^v0;yw30=l}|olSkWUka!3UGTGp95nBPShv>LAMRV4yX(y6cg z7`n9Bg(^QiMtD23(ZbA>XwTGSOQ%hRjX}9gsC_F+a^nHHLzZOMg6mi^c^`c^a|*c4 zHG|pqu4GijJfig?0p5#l#cvBXz{R%{$gv}`r2Ye^i@r;a-C@2K?Vbjcg+^9%w_P?q zIPS~yIxN6msdc3*o91A2#CNtxGZPEG4J9+U{o{i6D{QN120b|P2#U!yvELaJqTqUl zE}nV_?yaz-)5@h8&6<1|3zpP&;0%1NcmvIpT2Vcx5xsxC3(Zf=p>lIGlr##HMS~GO zTt16V{B9VQahcUJ9YAsv2Gb!6y{Tn>f2wp*4L(*Tpz%IV548U>rfEPMN_D(|>9TQ{ zKaZbib6&7!x2Mr#{-^P(x*0j=Cxc$xwHbv;CortC2)BfLlm07rz@%s1RO5yoF`c{z zMaLV_Ro44BUtuBccrAy1aiZh_tijPgwgL^aFnD^dmQ@iHBe4z1;Ffm-cNWgX{$GpW z!JQOV7!zSmn5SVwNDj7SOrxRX2bO)VgYY*AOhJYksS^IdlOJ{s&}tLPbnv6drBg&h zs~L9AA49ZCVo~;o42=o<#FTm71-tlqP}WWc{r4@fEhUWTFH{2=DL!gpJsK?=4XH*= z46c10fivQrXl)$JxCe?;g@JdNcW#o*tNX%mL)MwRe62&iKABCEbw-mZwUN}?JDF)+ zK7ts@H>2e7CNzZ|G}&)8)A+-Ou1yxihGm;kqWUg7M5GMfdfa0Mw%lS$&j#UJSsgra zqCd!v&!+Y<(Wp8-3w*W*!x~*5Oca$u9gk3Ww_%H6UP>(Hm?RR#SW&uNHW$}?TTgVi zJ_G~76cj$dgNgk@sQ3#3D)YjZ<UCwS+iG+nMDq|@ZaxE%4rNej9)UM+7T}JGapclK zCD2q0!<-5^_C3EN>2%h>-RsjyLMzZ+i5D<w<RjKqX$)A#IO3f8aC~s}A>5p^7*}w5 zXvv*1hOXs{=!H#|B=JoKRJD!6(|0*N-TD#4Zm%L8ad0$VY869+4J;dYNr|ox+(we- z8`3xN7MM4;5y*Z$xSJ)%+-x=>u7`y&)mok?&X2)J?t0q&)908Qq7?+DR)hI|Dek(H z3v)tmFfQ|n#KRe!Hj9`Yo%CTFero{wxxN+S4&_2A;pWm=mw~QQmj?4Y#?Vue%co9a z^_%@cIe!<fuhXKbYctsqF_)nA_&N;P90<p<6zDJJ`#^W7nBjo0Mbz&{8*{-c4IP(A z!uv-<N$&Q6+;zSSZ2v7u)Yx}9+<$frie?!UCbb(;=7Ko3DQ8nPAt`dSUkHv#tw6)G ziP-;p9El#g9umT2=!vvIGGD}zRCl`Z<`p*K^SV(~IYAZ;CRx!>b#2U^d+l&g;xV%% z!wL8;@6hX<Z&cpMT3jZ+1w0-&lWdnF7$Lcmc=1U<tC=c!NUmdbrUaF~7-10K`5MiG zZa}8+YVZ=a1N#q6%-4G^)bpe*J>9;Qd@hZIPrMww6lOxzAQx<oS~E|KD-DeTZ0VqS zP5Q&aiq`5>K}~iev-rCaeYGxtcl6K%60G)`y1&W+zxJ1fb&Azo-is+Mi5kXwmrS7E zA9mw*RVh$6`+$MtTVU2n6|(R8Qrd4xG3uUChQ+?yAuHu5*e{A_Z@-hH?w@Yq!LTgD z!;vE3wp|%E)rn9iuCL$AuqByE`)T@3YbyGMkQZ6@<a6pAT4r_{`e~|=HDYt<^$R7; zT=^iJpSb~#>r12SfgmPTCjdfb%_0q*cKRm{PbX4`Io;sIVpwy^oGw~*4z<It<BqaA z5P#%GqFYWgTT%j`Urv8cKXV(~BBMsl%Ol8Y_Xc#9N~h;{R$*?A7ICVTqi-*c2KC3Y zX!)Xpuyx%YkSo~@whq_OeUbw+`7)P#p3=$awPc}&C_h_1PnfE0&B0^tR-oR#3x5pZ z=6bxz1=|D4;NCeBS34@v=ucDNh1)KeTpmHEk^MNQ(FYVfleoBwVl?ZtVPByfkroyp zv%}M%R6m}zyr)WeI!>^6^j7jt=K@nbSBZ?cmdOTRDP`?BT{mY3d92@foG$q_5x1Kw zFgt{|@<g4|NWAbXmgIOFHh<ki_lLC^dR3Z1+{&{kaqm6o7vF<PiVMIsc{(c;DGR5D zj-kIw>O$U$EvT7hLW>kvf+Q`+z0&7UT8i_?ei{M8J;lkRG66bTErvE+I}T%>-eRlw zpFzTAqxyw#n6dX8?)E;<%@<A~RyF!C&(#_7p432$qb(gU;jw=5{j12ht%l8+dJrjA z0=k2>NMn-}QF1GQrT|}d-XKwuQ8I+&e~zTnCP>1FNxF1!Ya)#CBJ@cAcg(<7+o8Zn zla4PrVG!tJN1imXY}EHmTyUrejqwUP`m}Lgd;`%b*b`$Hd}Kb(qU1uzKq_v$jomXn zhRmEP{F{Et{g4eJ|I7T3|Kt8SI{(=}ry;?OI(5dt8Lz8`auu5hw+m)h@N9V%dGfHN z(-bG#Po{o0PjTtRu~e&Fish-Q(4p~Rc<V$Id#+xC2tCS1Z3%CDKTnMMNi3m451QD4 zw*_h1*y*$(vIf$>dXOO+9pKQ`hO?HO0i~l=?4Ax6qVlAbr^fv@Co7X@$j>K6GOxRV zw)O`I3$h|_l2xc_(<7$ZD*}s}ybZpJg<;kGwKVLbG2No7hz;7V#6|85FQ6oeP3HW` z9&`R&+9#jjSogh{oMBFv@q(!8r_am+x&=BlOMw~D$W(FH&T>PxqxC{7cxks7YZm2W zNMIopVjCKF%E5^vdgR2uy-dLa4NOvE*#T$O@qNe|jP(JW$jvu2-XKJ$8rhI5^E(+{ z$Y?5a$PHW83gefs6A&J1%5KknjA`5fPUFQB@Lc^6l{c;=sk{=X<cX2ci$h74a;br| z)=eVj9YqQg>J1a-m2;X(4EfL@OrsKI$V6!~vaM8*RE)Eu&$Vx|r#{P*9~D`QSQJ0? z)lh(FC4O2ob`xDel*x3HW>~wS5DOaksCDNxGTi<$E~<_}<HzsVr^?(MG-m-4vn+); zv{i9F#)2erl@%-Sb|T8u4aTw|;>3Py6_l5TL8JN*o@bURJFcOCEe-Oc9htW<+0dIA zFZIIklT<yNY8XQ=XNb^1ehISXp(koduVhue%cJ7QTxNjjJB-#4AxYO1$hcE1<6H0^ zou5QuO>8bZ;kGmDpd?7o49dr%>@l=j)(boX<)DC@*CfC60gRH4fhZSA>g;oeWJ}F~ zw|S~GJ~fKY;Ix%$&z_{2(fO$FwS$^Erob@!5EPw}f>pJ#bcy6)NcUPuLgicxUE}45 z>n%wdJ@pAbJZ%nJTvJgf*p{q0wUe||9R|N=(J(->23H)IjW>$xQ9IF7|DBc&>RRyA za(_)ys}h38mrW<TD|6X?cLvZOu?tz-+v*7J>14|03(%^~X(le`yuRc57oHS8P1gxm z!m|DP#CD}BiM+H7J*8qumdtmI@G+pJx#Ot&kX<C$&Vt4a{({2;4}<0H1@xe|Dx36u zA>(kV$>4bWL~_@Hp9Cg3(el>*bouE(ZoYLj6Z|M17HjLk>|-XNl6f9nBSWBgyfK+L zU5mHOT#E3DbMRgL0T}M~43(rdfqL9PI70H-b#{4h`Q0uIRX$8NPm<wv?oh+H#9R0w zdICW+Lamyo<Kb^v*eM}NuW=fQbGdopBLy@`C8xKr)x`>Ug*wz%`T_ZH+mHx7h{V?` zra*8N4@{?h26;UXs^#rZRmuWr(hhrgE*iy53Yo%r&?-@3pAulU%+hy{mWOI_G4gOL z=T}lTh1f{QK-B?Ryup79ia(}e&Dm#IC+^3yDZUJk(+AQw6Y`k*Xh6?1gW0*oi%EUR zdlcP1jof;;iLJg_hHC>^FglcHu)lH#I7jV(;pM@w|4RebTz`n?)-7kwJ#rz#)5B@Q zcOhaDEJ9cGC*ZWhl@^Uoz_b|xWajmqBzmL^X{*}|(g}lUc%&jdl5_?;^%-c%=A)B6 zlZZsRHZeZTjgM3NF&8DTK)@%&M;#kr{qqJ!Wm7Ih)$hfn<4xeq6+66Fww!$S8~{g~ zGa*Q{5}vB>#-I)x(k5NT&B1eIm%qsZYnuq#aY>IJc%unL+Y0c^u28r-4RQS1m!Lb$ zAC5(t8Cum&XR1$SL&vA_w4e7~jEWTlwc{Hg(m0&)9wm$QT4nh9r~{PhGy`^6(vs$f zm_6HzHd}M^q-MNB9b*YnZhObTQS2j|f5`(pqQyv2Rv7G5V~KM6ID9@Jk@$)#lCohU z)VcN{40GB@bsn9kyWef&l`C8W>4}wWnxGi1=6qNSEk3ZdcH0=Oq-^S`=79@G9A(pl z2Z8IyJ3J3T5qLkm753JuqwZ!`5+8C2yu}`1Qb#C`o#I0`D7llFF}bYN5w3lEt(mIv zYE0@GH#+~uRO)4=kH_@Z(Bk=Sw8C1D$_9OdSk>L^8KeHRmOBUgB6E~}@|r^J<HYF5 zJ$VILH%cKOBp(i>KZmN~YG!y_B>Fxl*m!a?R*1cXe(w%Y;kkS?c!M>x_*s(CZe!`$ zvuEKZi%{`01rAhHLgsuw`oSidm8c&;XM2U<jPVSK>CDHA$vWh9{6u!p43=5-ax8h= zC;=BARKi9Nfahyv$vF*xQKHfqTfUc!^)rV1A$xFY_-0J$$iQ=XlR-SV06Jxoh;Gvc z(qc4?Ogf(mt|R9Xk$C}l^SCn#bf$rbWB{GK<s~!4b``iNJz)0tv!gM^YBbMGf=20x zqn$w<Ojf_aTW5C}Z`b=V#Zr=3uwp38T{wt@iA7<~iEWS|>_s2ygcJYIdc?YP08vtO zr~Wy=fC_G7?zXCsZFR9I996)C+?qp!7VG0LpE%#Q+?|Z@awQnQs)ELzy^rTVPoVuC z+<~;PQE>JJ4{UkE>9mSiI>2`=*yaOXotZ{TUWVZ$H(y@NxmG+9twP&M1Po0e7}(Nc zuxd<ay7E8w9Q7Oan_?NKTK4bqKXv}Ie*S;Ww(V;Fltpptrf~&zE&t!`5AgO|yL8pc z|Ej;IBhuA^TL+go_UCHvTK>KTWBD|H&Y;!!O_FV`x8G6^-=!Np{JGY+FZD7rR{J+% zYT7z_dQ&EA>2beP>FVif>rBy}s-vx~ucfUuRco@oj?Uz%`r6zR{jOGqWIApc&$X~? zv9ave)mT2EKa}be+Jk?s9?r+twG8WequZ^Aj-1Bo<-R=&IQOP5^p6~3zLDKv6)C=M ztG#%Y-wF?Z_dmXXp5+*=U(@pErj+Ye&dwg?to2{z=dr}wo!fd#R<7_~>EE-cFlZhj z$Q9AGNOmhqLb_X%JOll`SNVB)`}Hhgw4qq|pFLw}w-WYrFX6YZQmZGfaNI=xH$^>8 zRj)fPKBij(e#5nU;;Z)U&H7VUvOAuj*c+eJ6CZo`X}`X4aoygyPES1b%dyK_xpnSZ zdelCnH$M4a;AXvX-JbXw3pm|(;}gt#<5PO#%^v0^edFRzy>Y#s_&=<>Pj7tczrdIF z#<{-vdtdBZHIm!>f87$6^~SY(h276ujuL;dzdcyq2j461`e#Ki{PTRgJMYjJ-z)Hb zo=?!7uZ-!V{$7!{X2Rv5zU#M%>x2Kt(CbnE$v*gAvG>^~U+B-^>A~mp!S@QjpZ$=# z^EMay;QulDy7QG4eenMnem(fgKKQ@l@6Thw&we^R_*=dCNxkClXFc5RyiHRde6RTX zx%YMFE8q9Q_lm!N;6L=i{}q4#@BUZW)(77!{{FuIO>6Ii?-hSP&xh*1{=53(d&S@1 z_dju|em@_If7gWV6@NcpkLzB4f^;AJzlcBaseSOh;_v72X!rWX^?UO=z2fiZ@nCm8 zVR|2YulW0UG}oQCG496y%@F42an{&w``_E$`MyVhHkN{d6aT<Q^Q-?=fbt)Y{=DBW p{QC<VOuEKhT|+v-fnE2*=KTJ=>%WbqfMC~a%G_Vq=YRj~{{UDTr?3D3 literal 0 HcmV?d00001 diff --git a/tests/fixtures/tl_checkpoints/lnpre/meta.json b/tests/fixtures/tl_checkpoints/lnpre/meta.json new file mode 100644 index 0000000000..eb9096e15e --- /dev/null +++ b/tests/fixtures/tl_checkpoints/lnpre/meta.json @@ -0,0 +1,12 @@ +{ + "d_model": 32, + "d_head": 16, + "n_heads": 2, + "n_layers": 2, + "n_ctx": 8, + "d_vocab": 16, + "d_mlp": 64, + "act_fn": "gelu", + "normalization_type": "LNPre", + "seed": 0 +} diff --git a/tests/fixtures/tl_checkpoints/lnpre/reference.pt b/tests/fixtures/tl_checkpoints/lnpre/reference.pt new file mode 100644 index 0000000000000000000000000000000000000000..8596d239eb026b8cdf887ee13c9e63e19a234bf4 GIT binary patch literal 2165 zcmbVNe`p(39DixkG-lPM&04IMS&EK|ZF-kp)<m1THkI4(%1onWu?QheZpqnOuJ7{F zp(5;urd9{emF=&gFotu2(rw}|<~>k4M8~wCn@GWe2>RF9KW^@y?!7c=?^3f;zv1zY zyZ8BiKKZ`S`@V=&YX`vT1nZ&+xPdP3lXW?plzUPMl<-bYsA9MQjG5wK&1(>J#vBBW zv<W$zgSI5n^kl{x&m%?6x#GG!o>$aV9FdQj9-s2LU`=OdtezlpWMv{P!`c;yYgo&s zpIdq7g7%n=EMZ_>td`8l$vkvOb_DB3Bp}%&djUEnyJRcC`(~0!O;zP2QnYN&^{k#E zl2cDBN#ugA$D?oyf(=o4e^H{Ur4^Ke4`hgfOqo?wwFFwF-CCq&p*!P{1`up4(Mj%U z*dz@|Zpk(cJu?}ac^iVwQP`6AMx=V@yGQl*I-;14NY$dY0?t%a?><f6pFr^`S<eyi zVwkbi)>P)Ik1XZ>{Kn!DS}Z!MbGHG%nY&c{ctTau+5K`BiD7or^kX|YDnK{@|E?kA zbW|7M0U?us@t2XT(ajRW!OioQ3!*9rsWR2@M(B@LYC}4zYjA^plLqSQ9!3lY{<nYz z!bYP!mk$!H1P!VT44$*KlV>{arL3b;87qdjZ=meVrFo;!cEz}@_8T94u#6oK9>Q0z z{fiInNZ~z8uk*9ZFY>2v?&VKzxfFW!&4S=O;t37?IM1K@2VnH|DgNlf4*cz(9e892 z2xI3L@$=2caN*)C9=SY?dt)~Z{nk6io{z_bmuB3?Uz2V8hXYp)mMid|whD&$)CD8e z+k%^WMse5iZsVJ|B|QB56Zku67QgU5k9RHi83z|1<(CdSjN!)~;ZJ<BAj~=4{OH8n zJow_X;O_oB&!{!Vu8ZB+dF#4y<M17P{;eK-cJ2(``sHyWdcDIq_PtBEaDd0#E}!F1 z{_v~NG&N*=1)dbfe;(u?;-4{SeSaC;J74tvJ&#@^7&O&)^~i-bC~(<mRpwq7Rj}}0 zdURv|w*$tcz*{t5PSzE20$)kc?iE*3Vz_Hlf*6MB3vxb|WBoo~pD)1qf(*m?gB<7c zF>D~<4+I0Ozjr59`CH4`E1krHbVMXu(W2$fZ*E^r<hwQ2-vgC+a|>#<fJe!$tl=wK zUS%3{BWbgwS+J&A*S#t;ncIQKk||35_?n>AHdL9wd}`g61lKGHR(eP!+U!XydXijc zYxG4lY2w!1F#D;|!u=Dut=G^c7gxf~8{KY!N9)Y+|F&R6ayYt5sO`1gtN3zjE6)!X h4Fte#l9L9((Muh8N(;z_k#}{p9|I@((e*29?*a_B99#eZ literal 0 HcmV?d00001 diff --git a/tests/goldens.py b/tests/goldens.py index 8c24f5bd63..332a327144 100644 --- a/tests/goldens.py +++ b/tests/goldens.py @@ -29,7 +29,7 @@ # Set once the dataset repo exists; pin to a specific commit revision so golden # updates are explicit, reviewed events (never a moving branch). GOLDENS_REPO_ID = "lars4776/TL-Goldens" -GOLDENS_REVISION: str | None = "130db21b365cbe286a5473c3267725733eefd085" +GOLDENS_REVISION: str | None = "e680e6756d2ae3faf322ecc12661dda904470719" _ENV_VAR = "TL_GOLDENS_DIR" @@ -108,6 +108,11 @@ def __init__(self, model: str, config: str) -> None: self.config = config self.path = golden_path(model, config) # raises early if the cell is absent + def has(self, name: str) -> bool: + """Whether this cell carries the named safetensors group (older dataset + revisions may predate a group's introduction).""" + return (self.path / f"{name}.safetensors").is_file() + def tensors(self, name: str) -> dict[str, torch.Tensor]: return load_golden_tensors(self.model, self.config, name) diff --git a/tests/integration/model_bridge/test_bert_weight_surface.py b/tests/integration/model_bridge/test_bert_weight_surface.py new file mode 100644 index 0000000000..76760c94b4 --- /dev/null +++ b/tests/integration/model_bridge/test_bert_weight_surface.py @@ -0,0 +1,196 @@ +"""BERT weight-accessor surface on TransformerBridge. + +The bridge successor to tests/integration/test_hooked_encoder_properties.py +(#277): every ``W_*`` / ``b_*`` / circuit accessor must have the right shape +AND carry the right underlying HF parameter in TL orientation, so +property-level mech-interp work doesn't silently read the wrong tensor. That +legacy suite certifies the surface only on HookedEncoder and is deleted with +the class at 4.0; this file is the coverage that survives. + +Uses a tiny randomly-initialized BertForMaskedLM (no download). +""" + +from __future__ import annotations + +import copy + +import einops +import pytest +import torch +from transformers import BertConfig, BertForMaskedLM + +from transformer_lens.FactoredMatrix import FactoredMatrix +from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_from_module, +) + +D_MODEL = 12 +D_HEAD = 4 +N_HEADS = D_MODEL // D_HEAD +D_MLP = 4 * D_MODEL +N_CTX = 5 +N_LAYERS = 3 +D_VOCAB = 22 + + +@pytest.fixture(scope="module") +def hf_model() -> BertForMaskedLM: + torch.manual_seed(0) + cfg = BertConfig( + vocab_size=D_VOCAB, + hidden_size=D_MODEL, + num_hidden_layers=N_LAYERS, + num_attention_heads=N_HEADS, + intermediate_size=D_MLP, + max_position_embeddings=N_CTX, + ) + return BertForMaskedLM(cfg).eval() + + +@pytest.fixture(scope="module") +def model(hf_model): + return build_bridge_from_module( + hf_model, + "BertForMaskedLM", + hf_config=copy.deepcopy(hf_model.config), + tokenizer=None, + device="cpu", + ) + + +def _layer(hf_model, i): + return hf_model.bert.encoder.layer[i] + + +# --------------------------------------------------------------------------- +# Embed / unembed +# --------------------------------------------------------------------------- + + +def test_W_E(model, hf_model): + assert model.W_E.shape == (D_VOCAB, D_MODEL) + torch.testing.assert_close( + model.W_E, hf_model.bert.embeddings.word_embeddings.weight, atol=0.0, rtol=0.0 + ) + + +def test_W_pos(model, hf_model): + assert model.W_pos.shape == (N_CTX, D_MODEL) + torch.testing.assert_close( + model.W_pos, hf_model.bert.embeddings.position_embeddings.weight, atol=0.0, rtol=0.0 + ) + + +def test_W_E_pos(model): + assert model.W_E_pos.shape == (D_VOCAB + N_CTX, D_MODEL) + assert torch.equal(model.W_E_pos[:D_VOCAB], model.W_E) + assert torch.equal(model.W_E_pos[D_VOCAB:], model.W_pos) + + +def test_W_U(model): + assert model.W_U.shape == (D_MODEL, D_VOCAB) + + +def test_b_U(model): + assert model.b_U.shape == (D_VOCAB,) + + +# --------------------------------------------------------------------------- +# Attention stacks — value-checked against the HF parameters in TL orientation +# --------------------------------------------------------------------------- + +_QKV_HF_ATTR = {"W_Q": "query", "W_K": "key", "W_V": "value"} + + +@pytest.mark.parametrize("attr", ["W_Q", "W_K", "W_V"]) +def test_attn_qkv_weight(model, hf_model, attr): + stacked = getattr(model, attr) + assert stacked.shape == (N_LAYERS, N_HEADS, D_MODEL, D_HEAD) + for i in range(N_LAYERS): + hf_w = getattr(_layer(hf_model, i).attention.self, _QKV_HF_ATTR[attr]).weight + expected = einops.rearrange(hf_w, "(h d) m -> h m d", h=N_HEADS) + torch.testing.assert_close(stacked[i], expected, atol=0.0, rtol=0.0) + + +@pytest.mark.parametrize("attr", ["b_Q", "b_K", "b_V"]) +def test_attn_qkv_bias(model, hf_model, attr): + stacked = getattr(model, attr) + assert stacked.shape == (N_LAYERS, N_HEADS, D_HEAD) + for i in range(N_LAYERS): + hf_b = getattr(_layer(hf_model, i).attention.self, _QKV_HF_ATTR["W_" + attr[-1]]).bias + expected = einops.rearrange(hf_b, "(h d) -> h d", h=N_HEADS) + torch.testing.assert_close(stacked[i], expected, atol=0.0, rtol=0.0) + + +def test_W_O(model, hf_model): + assert model.W_O.shape == (N_LAYERS, N_HEADS, D_HEAD, D_MODEL) + for i in range(N_LAYERS): + hf_w = _layer(hf_model, i).attention.output.dense.weight + expected = einops.rearrange(hf_w, "m (h d) -> h d m", h=N_HEADS) + torch.testing.assert_close(model.W_O[i], expected, atol=0.0, rtol=0.0) + + +def test_b_O(model, hf_model): + assert model.b_O.shape == (N_LAYERS, D_MODEL) + for i in range(N_LAYERS): + torch.testing.assert_close( + model.b_O[i], _layer(hf_model, i).attention.output.dense.bias, atol=0.0, rtol=0.0 + ) + + +# --------------------------------------------------------------------------- +# MLP stacks +# --------------------------------------------------------------------------- + + +def test_W_in(model, hf_model): + assert model.W_in.shape == (N_LAYERS, D_MODEL, D_MLP) + for i in range(N_LAYERS): + expected = _layer(hf_model, i).intermediate.dense.weight.T + torch.testing.assert_close(model.W_in[i], expected, atol=0.0, rtol=0.0) + + +def test_W_out(model, hf_model): + assert model.W_out.shape == (N_LAYERS, D_MLP, D_MODEL) + for i in range(N_LAYERS): + expected = _layer(hf_model, i).output.dense.weight.T + torch.testing.assert_close(model.W_out[i], expected, atol=0.0, rtol=0.0) + + +def test_b_in(model, hf_model): + assert model.b_in.shape == (N_LAYERS, D_MLP) + for i in range(N_LAYERS): + torch.testing.assert_close( + model.b_in[i], _layer(hf_model, i).intermediate.dense.bias, atol=0.0, rtol=0.0 + ) + + +def test_b_out(model, hf_model): + assert model.b_out.shape == (N_LAYERS, D_MODEL) + for i in range(N_LAYERS): + torch.testing.assert_close( + model.b_out[i], _layer(hf_model, i).output.dense.bias, atol=0.0, rtol=0.0 + ) + + +# --------------------------------------------------------------------------- +# Factored circuits +# --------------------------------------------------------------------------- + + +def test_QK_circuit(model): + qk = model.QK + assert isinstance(qk, FactoredMatrix) + assert qk.A.shape == (N_LAYERS, N_HEADS, D_MODEL, D_HEAD) + assert qk.B.shape == (N_LAYERS, N_HEADS, D_HEAD, D_MODEL) + assert torch.equal(qk.A, model.W_Q) + assert torch.equal(qk.B, model.W_K.transpose(-2, -1)) + + +def test_OV_circuit(model): + ov = model.OV + assert isinstance(ov, FactoredMatrix) + assert ov.A.shape == (N_LAYERS, N_HEADS, D_MODEL, D_HEAD) + assert ov.B.shape == (N_LAYERS, N_HEADS, D_HEAD, D_MODEL) + assert torch.equal(ov.A, model.W_V) + assert torch.equal(ov.B, model.W_O) diff --git a/tests/integration/model_bridge/test_bridge_run_with_cache_incl_bwd.py b/tests/integration/model_bridge/test_bridge_run_with_cache_incl_bwd.py index 22a39bce70..af7991f00b 100644 --- a/tests/integration/model_bridge/test_bridge_run_with_cache_incl_bwd.py +++ b/tests/integration/model_bridge/test_bridge_run_with_cache_incl_bwd.py @@ -20,21 +20,6 @@ def bridge(distilgpt2_bridge): return distilgpt2_bridge -# Lives here, not conftest: single consumer, and a second resident distilgpt2 -# is exactly what the golden fixtures exist to avoid. -@pytest.fixture(scope="module") -def distilgpt2_hooked_processed(): - """HookedTransformer distilgpt2 with default weight processing.""" - import gc - - from transformer_lens import HookedTransformer - - model = HookedTransformer.from_pretrained("distilgpt2", device="cpu") - yield model - del model - gc.collect() - - def test_incl_bwd_caches_gradients(bridge): """Every cached forward activation gets a matching `_grad` entry.""" tokens = bridge.to_tokens(PROMPT) @@ -149,27 +134,59 @@ def test_incl_bwd_gradients_reach_compatibility_aliases(distilgpt2_bridge_compat assert f"{name}_grad" in cache, f"missing {name}_grad; cached: {sorted(cache.keys())}" -def test_incl_bwd_gradients_match_hooked_transformer( - distilgpt2_bridge_compat, distilgpt2_hooked_processed +def test_incl_bwd_gradients_match_the_golden_snapshot( + distilgpt2_bridge_compat, distilgpt2_goldens_processed ): - """Cached gradients agree with HookedTransformer's, not just in shape.""" - prompt_tokens = distilgpt2_hooked_processed.to_tokens(PROMPT) + """Cached gradients agree with the frozen HookedTransformer goldens. + + Anchored on the captured `gradients` group rather than a live + HookedTransformer, like the rest of the compatibility suite (WS-9). + """ + golden = distilgpt2_goldens_processed + if not golden.has("gradients"): + pytest.skip("golden dataset revision predates the gradients group") + golden_grads = golden.tensors("gradients") _, bridge_cache = distilgpt2_bridge_compat.run_with_cache( - prompt_tokens, return_type="loss", names_filter=NAMES, incl_bwd=True - ) - _, hooked_cache = distilgpt2_hooked_processed.run_with_cache( - prompt_tokens, return_type="loss", names_filter=NAMES, incl_bwd=True + golden.scalars["short_prompt"], return_type="loss", names_filter=NAMES, incl_bwd=True ) for name in NAMES: grad_name = f"{name}_grad" bridge_grad = bridge_cache[grad_name] - hooked_grad = hooked_cache[grad_name] + hooked_grad = golden_grads[grad_name] assert bridge_grad.shape == hooked_grad.shape scale = hooked_grad.abs().max() max_diff = (bridge_grad - hooked_grad).abs().max() assert max_diff < 1e-3 * max(scale, 1.0), ( - f"{grad_name} diverges from HookedTransformer: max diff {max_diff:.3e} " + f"{grad_name} diverges from the golden gradients: max diff {max_diff:.3e} " f"against gradient scale {scale:.3e}" ) + + +def test_incl_bwd_gradients_are_the_true_autograd_gradients(bridge): + """Cached `_grad` entries equal torch.autograd.grad of the loss wrt the + cached activations — machine-independent, and exact by construction, so it + holds even where no golden gradients exist.""" + tokens = bridge.to_tokens(PROMPT) + grabbed: dict = {} + + def grab_as(requested): + # key by the requested name: hook.name is the point's canonical + # spelling, which need not match the compat alias asked for + def grab(t, hook=None, _key=requested): + t.retain_grad() + grabbed[_key] = t + return t + + return grab + + with bridge.hooks(fwd_hooks=[(n, grab_as(n)) for n in NAMES]): + loss = bridge(tokens, return_type="loss") + loss.backward() + reference = {n: grabbed[n].grad for n in NAMES} + bridge.zero_grad(set_to_none=True) + + _, cache = bridge.run_with_cache(tokens, return_type="loss", names_filter=NAMES, incl_bwd=True) + for name in NAMES: + torch.testing.assert_close(cache[f"{name}_grad"], reference[name], atol=0.0, rtol=0.0) diff --git a/tests/integration/model_bridge/test_encdec_weight_stacking_parity.py b/tests/integration/model_bridge/test_encdec_weight_stacking_parity.py index a84b3d00e2..17b25703d7 100644 --- a/tests/integration/model_bridge/test_encdec_weight_stacking_parity.py +++ b/tests/integration/model_bridge/test_encdec_weight_stacking_parity.py @@ -1,41 +1,75 @@ -"""Stacked enc-dec weights on the bridge match HookedEncoderDecoder. +"""Stacked enc-dec weights on the bridge match the raw HF T5 weights. -Deletion evidence for HookedEncoderDecoder's stacked-weight properties: the -bridge must produce the same tensors over chain(encoder, decoder), and the same -head labels, before the legacy class can go. +Deletion evidence for HookedEncoderDecoder's stacked-weight properties. +HookedEncoderDecoder did no weight processing — its stacks were pure reshapes +of the HF tensors — so the raw HF model is the same oracle without the legacy +class: expected tensors are derived here directly from t5-small's weights with +the documented TL orientations, over chain(encoder, decoder). """ from __future__ import annotations +import einops import pytest import torch -from transformer_lens import HookedEncoderDecoder from transformer_lens.model_bridge.bridge import TransformerBridge MODEL = "google-t5/t5-small" -STACKED = ["W_Q", "W_K", "W_V", "W_O", "W_in", "W_out"] @pytest.fixture(scope="module") -def hooked() -> HookedEncoderDecoder: - return HookedEncoderDecoder.from_pretrained(MODEL, device="cpu") +def bridge() -> TransformerBridge: + return TransformerBridge.boot_transformers(MODEL, device="cpu") @pytest.fixture(scope="module") -def bridge() -> TransformerBridge: - return TransformerBridge.boot_transformers(MODEL, device="cpu") +def hf_blocks(bridge): + """chain(encoder, decoder) HF blocks — the order the bridge stacks over.""" + hf = bridge.original_model + return list(hf.encoder.block) + list(hf.decoder.block) + + +def _self_attn(block): + return block.layer[0].SelfAttention + + +def _mlp(block): + # T5 FF layer is the last entry (index 1 on encoder blocks, 2 on decoder). + return block.layer[-1].DenseReluDense + + +def _expected(name, blocks, n_heads): + per_block = [] + for b in blocks: + if name in ("W_Q", "W_K", "W_V"): + hf_w = getattr(_self_attn(b), {"W_Q": "q", "W_K": "k", "W_V": "v"}[name]).weight + per_block.append(einops.rearrange(hf_w, "(h d) m -> h m d", h=n_heads)) + elif name == "W_O": + per_block.append( + einops.rearrange(_self_attn(b).o.weight, "m (h d) -> h d m", h=n_heads) + ) + elif name == "W_in": + per_block.append(_mlp(b).wi.weight.T) + elif name == "W_out": + per_block.append(_mlp(b).wo.weight.T) + return torch.stack(per_block, dim=0) -@pytest.mark.parametrize("name", STACKED) -def test_stacked_weights_match_hooked_encoder_decoder(name, hooked, bridge): - """HookedEncoderDecoder does no weight processing, so these are directly comparable.""" - expected = getattr(hooked, name) +@pytest.mark.parametrize("name", ["W_Q", "W_K", "W_V", "W_O", "W_in", "W_out"]) +def test_stacked_weights_match_raw_hf(name, bridge, hf_blocks): + expected = _expected(name, hf_blocks, bridge.cfg.n_heads) actual = getattr(bridge, name) assert actual.shape == expected.shape torch.testing.assert_close(actual, expected, atol=0.0, rtol=0.0) -def test_head_labels_match_hooked_encoder_decoder(hooked, bridge): - """all_head_labels is a property on the bridge; HT exposes it as a method.""" - assert bridge.all_head_labels == hooked.all_head_labels() +def test_head_labels_cover_both_stacks(bridge): + """EL{l}H{h} then DL{l}H{h}, sized by the real block lists.""" + n_enc = len(bridge.original_model.encoder.block) + n_dec = len(bridge.original_model.decoder.block) + heads = range(bridge.cfg.n_heads) + expected = [f"EL{l}H{h}" for l in range(n_enc) for h in heads] + [ + f"DL{l}H{h}" for l in range(n_dec) for h in heads + ] + assert bridge.all_head_labels == expected diff --git a/tests/integration/model_bridge/test_nsp_sentence_pair_helper.py b/tests/integration/model_bridge/test_nsp_sentence_pair_helper.py index fa364707b8..29e9be5d14 100644 --- a/tests/integration/model_bridge/test_nsp_sentence_pair_helper.py +++ b/tests/integration/model_bridge/test_nsp_sentence_pair_helper.py @@ -43,15 +43,43 @@ def test_pair_tokenization_matches_huggingface(nsp_bridge, hf_tokenizer): assert tokens["token_type_ids"].unique().tolist() == [0, 1] -def test_logits_match_a_direct_huggingface_nsp_forward(nsp_bridge, hf_tokenizer): +def test_logits_match_a_direct_huggingface_nsp_forward(hf_tokenizer): + """Reference is a FRESH HF load, not nsp_bridge.original_model — compat mode + mutates the wrapped model in place, which made the old exact-equality + assertion self-fulfilling. The reference must load eager attention like the + bridge does: the residual ~2.4e-6 against a default (sdpa) load is entirely + the attention kernel, and with matching kernels the match is bit-exact.""" + hf = BertForNextSentencePrediction.from_pretrained( + MODEL, torch_dtype=torch.float32, attn_implementation="eager" + ).eval() encodings = hf_tokenizer(SENTENCE_A, SEQUENTIAL_B, return_tensors="pt") with torch.no_grad(): - expected = nsp_bridge.original_model(**encodings).logits + expected = hf(**encodings).logits - actual = nsp_bridge.predict_next_sentence(SENTENCE_A, SEQUENTIAL_B, return_type="logits") + bridge = TransformerBridge.boot_transformers( + MODEL, device="cpu", model_class=BertForNextSentencePrediction + ) + actual = bridge.predict_next_sentence(SENTENCE_A, SEQUENTIAL_B, return_type="logits") torch.testing.assert_close(actual, expected, atol=0.0, rtol=0.0) +def test_compat_mode_shifts_logits_by_the_centering_amount(nsp_bridge, hf_tokenizer): + """Compatibility mode's unembed centering moves NSP logits ~1e-2 relative to + raw HF (uniformly across both classes, so verdicts hold). Pin the band so a + silent change in compat processing shows up here.""" + hf = BertForNextSentencePrediction.from_pretrained( + MODEL, torch_dtype=torch.float32, attn_implementation="eager" + ).eval() + encodings = hf_tokenizer(SENTENCE_A, SEQUENTIAL_B, return_tensors="pt") + with torch.no_grad(): + raw = hf(**encodings).logits + compat = nsp_bridge.predict_next_sentence(SENTENCE_A, SEQUENTIAL_B, return_type="logits") + shift = (compat - raw).abs() + # centering subtracts the mean of the two logits: both classes move together + assert torch.allclose(shift[0, 0], shift[0, 1], atol=1e-4) + assert 1e-3 < float(shift.max()) < 1e-1, float(shift.max()) + + def test_predictions_distinguish_sequential_from_unrelated(nsp_bridge): assert nsp_bridge.predict_next_sentence(SENTENCE_A, SEQUENTIAL_B) == ( "The sentences are sequential" diff --git a/tests/integration/model_bridge/test_positional_weight_accessors.py b/tests/integration/model_bridge/test_positional_weight_accessors.py new file mode 100644 index 0000000000..5fc8bf9065 --- /dev/null +++ b/tests/integration/model_bridge/test_positional_weight_accessors.py @@ -0,0 +1,156 @@ +"""W_pos / W_E_pos on TransformerBridge — deletion evidence for the +HookedTransformer/HookedEncoder accessors certified only on the legacy side.""" + +from __future__ import annotations + +import pytest +import torch + +from transformer_lens.model_bridge.bridge import TransformerBridge + + +@pytest.fixture(scope="module") +def gpt2_bridge_np(): + return TransformerBridge.boot_transformers("gpt2", device="cpu") + + +def test_w_pos_matches_raw_hf(gpt2_bridge_np): + """Unprocessed bridge W_pos is HF's wpe weight exactly.""" + hf_wpe = gpt2_bridge_np.original_model.transformer.wpe.weight + torch.testing.assert_close(gpt2_bridge_np.W_pos, hf_wpe, atol=0.0, rtol=0.0) + assert gpt2_bridge_np.W_pos.shape == ( + gpt2_bridge_np.cfg.n_ctx, + gpt2_bridge_np.cfg.d_model, + ) + + +def test_w_e_pos_is_the_concatenation(gpt2_bridge_np): + b = gpt2_bridge_np + assert b.W_E_pos.shape == (b.cfg.d_vocab + b.cfg.n_ctx, b.cfg.d_model) + torch.testing.assert_close(b.W_E_pos[: b.cfg.d_vocab], b.W_E, atol=0.0, rtol=0.0) + torch.testing.assert_close(b.W_E_pos[b.cfg.d_vocab :], b.W_pos, atol=0.0, rtol=0.0) + + +def test_w_pos_raises_cleanly_on_rotary(): + """A rotary model has no absolute positional matrix; say so, don't guess.""" + import copy + + from transformers import LlamaConfig, LlamaForCausalLM + + from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_from_module, + ) + + torch.manual_seed(0) + cfg = LlamaConfig( + vocab_size=128, + hidden_size=32, + intermediate_size=64, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=4, + max_position_embeddings=32, + ) + bridge = build_bridge_from_module( + LlamaForCausalLM(cfg).eval(), + "LlamaForCausalLM", + hf_config=copy.deepcopy(cfg), + tokenizer=None, + device="cpu", + ) + with pytest.raises(AttributeError, match="absolute positional"): + _ = bridge.W_pos + with pytest.raises(AttributeError, match="absolute positional"): + _ = bridge.W_E_pos + + +def test_w_pos_refuses_t5_relative_bias(): + """T5 maps pos_embed to the relative attention bias; W_pos must refuse it, + as the legacy accessor did, not return a [num_buckets, n_heads] table.""" + import copy + + from transformers import T5Config, T5ForConditionalGeneration + + from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_from_module, + ) + + torch.manual_seed(0) + cfg = T5Config( + vocab_size=128, d_model=32, d_kv=8, d_ff=64, num_layers=2, num_decoder_layers=2, num_heads=4 + ) + bridge = build_bridge_from_module( + T5ForConditionalGeneration(cfg).eval(), + "T5ForConditionalGeneration", + hf_config=copy.deepcopy(cfg), + tokenizer=None, + device="cpu", + ) + with pytest.raises(AttributeError, match="relative attention bias|table"): + _ = bridge.W_pos + + +def test_w_pos_slices_the_opt_offset(): + """OPT allocates n_ctx + 2 positional rows; HT's converter slices them off, + and the bridge accessor must agree with HT on both shape and values.""" + import copy + + from transformers import OPTConfig, OPTForCausalLM + + from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_from_module, + ) + + torch.manual_seed(0) + cfg = OPTConfig( + vocab_size=128, + hidden_size=32, + ffn_dim=64, + num_hidden_layers=2, + num_attention_heads=4, + max_position_embeddings=32, + word_embed_proj_dim=32, + ) + model = OPTForCausalLM(cfg).eval() + bridge = build_bridge_from_module( + model, + "OPTForCausalLM", + hf_config=copy.deepcopy(cfg), + tokenizer=None, + device="cpu", + ) + assert bridge.W_pos.shape == (bridge.cfg.n_ctx, bridge.cfg.d_model) + raw = model.model.decoder.embed_positions.weight + torch.testing.assert_close(bridge.W_pos, raw[2:], atol=0.0, rtol=0.0) + + +def test_w_pos_works_on_bert(): + """HookedEncoder certifies W_pos on BERT; the bridge accessor must too.""" + import copy + + from transformers import BertConfig, BertForMaskedLM + + from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_from_module, + ) + + torch.manual_seed(0) + cfg = BertConfig( + vocab_size=64, + hidden_size=32, + num_hidden_layers=1, + num_attention_heads=2, + intermediate_size=64, + max_position_embeddings=32, + ) + model = BertForMaskedLM(cfg).eval() + bridge = build_bridge_from_module( + model, + "BertForMaskedLM", + hf_config=copy.deepcopy(cfg), + tokenizer=None, + device="cpu", + ) + torch.testing.assert_close( + bridge.W_pos, model.bert.embeddings.position_embeddings.weight, atol=0.0, rtol=0.0 + ) diff --git a/tests/integration/model_bridge/test_tl_legacy_loader.py b/tests/integration/model_bridge/test_tl_legacy_loader.py new file mode 100644 index 0000000000..2e46771671 --- /dev/null +++ b/tests/integration/model_bridge/test_tl_legacy_loader.py @@ -0,0 +1,98 @@ +"""boot_tl_legacy: legacy TransformerLens-format repos on the bridge. + +Deletion evidence for the last HookedTransformer-only load path (NeelNanda/ +ArthurConmy/Baidicoot repos: TL config.json + *.pth, no HF model_type). +Value anchor is the frozen solu-1l golden captured from HookedTransformer. +""" + +from __future__ import annotations + +import pytest +import torch + +from tests import goldens +from transformer_lens.model_bridge.bridge import TransformerBridge + +SOLU_1L = "NeelNanda/SoLU_1L512W_C4_Code" + + +@pytest.fixture(scope="module") +def solu_bridge() -> TransformerBridge: + return TransformerBridge.boot_tl_legacy(SOLU_1L, device="cpu") + + +@pytest.fixture(scope="module") +def golden(): + if not goldens.goldens_available("solu-1l", "no_processing"): + pytest.skip("TL goldens dataset unavailable (set TL_GOLDENS_DIR or enable network)") + return goldens.GoldenCell("solu-1l", "no_processing") + + +def test_logits_match_the_frozen_hooked_transformer_golden(solu_bridge, golden): + """The whole chain — TL config derivation, .pth fetch, weight conversion, + native solu_ln forward, tokenizer wiring — reproduces the golden logits. + + Bit-exact on the hardware that captured the goldens; ~2.5e-5 absolute on + other platforms (fp32 accumulation). Same 1e-4 tolerance every golden + comparison in the suite uses — never assert exactness against tensors + captured on different hardware.""" + with torch.no_grad(): + logits = solu_bridge(golden.scalars["short_prompt"], return_type="logits") + torch.testing.assert_close( + logits, golden.tensors("logits_short")["logits"], atol=1e-4, rtol=1e-4 + ) + + +def test_long_text_loss_matches_the_golden_scalar(solu_bridge, golden): + """Cross-platform: observed 1.5e-5 delta on Linux CI vs the capture host.""" + loss = solu_bridge(golden.scalars["ablation"]["text"], return_type="loss") + assert abs(float(loss) - golden.scalars["ablation"]["orig_loss"]) < 1e-4 + + +def test_mid_mlp_layernorm_is_load_bearing(solu_bridge, golden): + """SoLU-LN's defining structure: zeroing the mid-MLP LN weight must change + the output — guards against the LN silently dropping out of the forward.""" + prompt = golden.scalars["short_prompt"] + with torch.no_grad(): + base = solu_bridge(prompt, return_type="logits") + ln = solu_bridge.blocks[0].mlp.ln._original_component + keep = ln.weight.clone() + ln.weight.zero_() + zeroed = solu_bridge(prompt, return_type="logits") + ln.weight.copy_(keep) + assert not torch.allclose(base, zeroed) + + +def test_attn_only_repo_boots_and_runs(): + bridge = TransformerBridge.boot_tl_legacy("NeelNanda/Attn_Only_2L512W_C4_Code", device="cpu") + assert bridge.cfg.attn_only + with torch.no_grad(): + out = bridge("Hello world", return_type="logits") + assert torch.isfinite(out).all() + + +def test_checkpointed_load_stamps_cfg_and_differs_from_final(solu_bridge): + early = TransformerBridge.boot_tl_legacy(SOLU_1L, checkpoint_index=0, device="cpu") + assert early.cfg.checkpoint_index == 0 + assert early.cfg.checkpoint_value is not None + final_w = solu_bridge.state_dict()["embed.weight"] + early_w = early.state_dict()["embed.weight"] + assert not torch.equal(final_w, early_w), "checkpoint 0 loaded the final weights" + + +def test_checkpoint_index_out_of_range_raises(): + with pytest.raises(ValueError, match="out of range"): + TransformerBridge.boot_tl_legacy(SOLU_1L, checkpoint_index=10_000) + + +def test_non_legacy_repo_is_refused_with_a_pointer(): + with pytest.raises(ValueError, match="boot_transformers"): + TransformerBridge.boot_tl_legacy("gpt2") + + +def test_boot_transformers_stamps_checkpoint_metadata(): + bridge = TransformerBridge.boot_transformers( + "EleutherAI/pythia-14m", device="cpu", checkpoint_index=2 + ) + assert bridge.cfg.checkpoint_index == 2 + assert bridge.cfg.checkpoint_value is not None diff --git a/tests/integration/model_bridge/test_wav2vec2_bridge.py b/tests/integration/model_bridge/test_wav2vec2_bridge.py new file mode 100644 index 0000000000..54275a8c70 --- /dev/null +++ b/tests/integration/model_bridge/test_wav2vec2_bridge.py @@ -0,0 +1,53 @@ +"""facebook/wav2vec2-base through the bridge — deletion evidence for +HookedAudioEncoder's wav2vec2 support (registry ships the checkpoints).""" + +from __future__ import annotations + +import pytest +import torch + +from transformer_lens.model_bridge.bridge import TransformerBridge + +MODEL = "facebook/wav2vec2-base" + + +@pytest.fixture(scope="module") +def w2v_bridge() -> TransformerBridge: + return TransformerBridge.boot_transformers(MODEL, device="cpu") + + +def test_pretraining_checkpoint_boots_as_encoder(w2v_bridge): + """The checkpoint declares Wav2Vec2ForPreTraining; the bridge loads its encoder.""" + assert type(w2v_bridge.original_model).__name__ == "Wav2Vec2Model" + assert w2v_bridge.cfg.is_audio_model + + +def test_forward_matches_hf(w2v_bridge): + from transformers import Wav2Vec2Model + + hf = Wav2Vec2Model.from_pretrained(MODEL, torch_dtype=torch.float32).eval() + torch.manual_seed(0) + wave = torch.randn(1, 16000) * 0.1 + with torch.no_grad(): + ref = hf(wave).last_hidden_state + out = w2v_bridge(wave) + out_t = out if isinstance(out, torch.Tensor) else out["last_hidden_state"] + torch.testing.assert_close(out_t, ref, atol=1e-4, rtol=1e-4) + + +def test_masked_frame_entry_matches_hf(w2v_bridge): + """encoder_output honors the padding mask on wav2vec2 exactly as on HuBERT.""" + hf = w2v_bridge.original_model + torch.manual_seed(0) + wave = torch.randn(1, 16000) * 0.1 + padded = torch.cat([wave, torch.zeros(1, 4000)], dim=1) + sample_mask = torch.cat([torch.ones(1, 16000), torch.zeros(1, 4000)], dim=1).long() + with torch.no_grad(): + ref = hf(padded, attention_mask=sample_mask).last_hidden_state + feats = hf.feature_extractor(padded).transpose(1, 2) + frames, _ = hf.feature_projection(feats) + frame_mask = hf._get_feature_vector_attention_mask(frames.shape[1], sample_mask) + out = w2v_bridge.encoder_output(frames, one_zero_attention_mask=frame_mask.long()) + real = frame_mask[0].bool() + assert not real.all() + torch.testing.assert_close(out[:, real], ref[:, real], atol=1e-4, rtol=1e-4) diff --git a/tests/unit/model_bridge/supported_architectures/test_wav2vec2_adapter.py b/tests/unit/model_bridge/supported_architectures/test_wav2vec2_adapter.py new file mode 100644 index 0000000000..2b5142b44d --- /dev/null +++ b/tests/unit/model_bridge/supported_architectures/test_wav2vec2_adapter.py @@ -0,0 +1,127 @@ +"""Wav2Vec2 adapter: only the deltas vs the inherited HuBERT structure. + +Wav2Vec2Model's module tree is identical to HubertModel's, so the structural +mapping is covered by test_hubert_adapter.py; what needs proof here is the +factory routing, the ForCTC nesting detection, and that a real forward matches +HF end to end. +""" + +from __future__ import annotations + +import copy + +import pytest +import torch +from transformers import Wav2Vec2Config, Wav2Vec2ForCTC, Wav2Vec2Model + +from transformer_lens.factories.architecture_adapter_factory import ( + ArchitectureAdapterFactory, +) +from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_from_module, +) +from transformer_lens.model_bridge.supported_architectures.wav2vec2 import ( + Wav2Vec2ArchitectureAdapter, +) + + +def _tiny_config() -> Wav2Vec2Config: + return Wav2Vec2Config( + hidden_size=32, + num_hidden_layers=2, + num_attention_heads=2, + intermediate_size=64, + conv_dim=(32,) * 7, + vocab_size=40, + ) + + +def _tiny_bridge(model, arch): + cfg = model.config + return build_bridge_from_module( + model.eval(), arch, hf_config=copy.deepcopy(cfg), tokenizer=None, device="cpu" + ) + + +@pytest.mark.parametrize( + "architecture", ["Wav2Vec2Model", "Wav2Vec2ForCTC", "Wav2Vec2ForPreTraining"] +) +def test_factory_routes_wav2vec2_architectures(architecture): + from transformer_lens.config import TransformerBridgeConfig + + cfg = TransformerBridgeConfig( + d_model=32, + d_head=16, + n_heads=2, + n_layers=2, + n_ctx=512, + d_vocab=40, + d_mlp=64, + architecture=architecture, + ) + adapter = ArchitectureAdapterFactory.select_architecture_adapter(cfg) + assert isinstance(adapter, Wav2Vec2ArchitectureAdapter) + + +def test_forctc_nesting_is_detected(): + """ForCTC nests the encoder under 'wav2vec2.' and adds the CTC head.""" + torch.manual_seed(0) + model = Wav2Vec2ForCTC(_tiny_config()) + bridge = _tiny_bridge(model, "Wav2Vec2ForCTC") + assert bridge.blocks[0].attn.q.original_component is not None + assert hasattr(bridge, "unembed") + + +def test_tiny_forward_matches_hf(): + torch.manual_seed(0) + model = Wav2Vec2Model(_tiny_config()) + bridge = _tiny_bridge(model, "Wav2Vec2Model") + wave = torch.randn(1, 4000) * 0.1 + with torch.no_grad(): + ref = model(wave).last_hidden_state + out = bridge(wave) + out_t = out if isinstance(out, torch.Tensor) else out["last_hidden_state"] + torch.testing.assert_close(out_t, ref, atol=1e-5, rtol=1e-5) + + +def test_hooks_fire_on_wav2vec2(): + torch.manual_seed(0) + bridge = _tiny_bridge(Wav2Vec2Model(_tiny_config()), "Wav2Vec2Model") + wave = torch.randn(1, 4000) * 0.1 + _, cache = bridge.run_with_cache(wave, names_filter=lambda n: n.endswith("hook_out")) + assert any(k.startswith("blocks.0.attn") for k in cache) + assert "feat_proj.hook_out" in cache + + +def test_pretraining_class_is_refused_with_a_pointer(): + """Wrapping Wav2Vec2ForPreTraining directly is unsupported (its forward + returns quantizer states, not hidden states) — refuse with the alternative, + never die inside component setup. Checkpoint boots still work: the arch + string routes to AutoModel -> Wav2Vec2Model (integration coverage).""" + from transformers import Wav2Vec2ForPreTraining + + torch.manual_seed(0) + model = Wav2Vec2ForPreTraining(_tiny_config()) + with pytest.raises(NotImplementedError, match="without model_class"): + _tiny_bridge(model, "Wav2Vec2ForPreTraining") + + +def test_stable_layer_norm_frame_entry_matches_hf(): + """Stable-LN encoders apply encoder.layer_norm AFTER the blocks; frame entry + must mirror that order, not the post-LN hardcoding.""" + from transformers import Wav2Vec2Model + + torch.manual_seed(0) + cfg = _tiny_config() + cfg.do_stable_layer_norm = True + model = Wav2Vec2Model(cfg).eval() + assert type(model.encoder).__name__.endswith("StableLayerNorm") + bridge = _tiny_bridge(model, "Wav2Vec2Model") + wave = torch.randn(1, 4000) * 0.1 + with torch.no_grad(): + ref = model(wave).last_hidden_state + feats = model.feature_extractor(wave).transpose(1, 2) + proj = model.feature_projection(feats) + frames = proj[0] if isinstance(proj, tuple) else proj + out = bridge.encoder_output(frames) + torch.testing.assert_close(out, ref, atol=1e-5, rtol=1e-5) diff --git a/tests/unit/model_bridge/test_boot_native.py b/tests/unit/model_bridge/test_boot_native.py index 8792865dc4..fbee8522d5 100644 --- a/tests/unit/model_bridge/test_boot_native.py +++ b/tests/unit/model_bridge/test_boot_native.py @@ -197,19 +197,15 @@ def test_boot_native_accepts_dict_config(): def test_boot_native_rejects_legacy_config_with_actionable_error(): + """boot_native reads only type(config).__name__, so a stand-in with the + legacy class's name pins the exact error UX without importing the class — + this test must survive HookedTransformerConfig's 4.0 deletion.""" import pytest - from transformer_lens import HookedTransformerConfig + class HookedTransformerConfig: + pass - legacy_config = HookedTransformerConfig( - n_layers=1, - d_model=32, - n_ctx=8, - d_head=16, - n_heads=2, - d_vocab=16, - act_fn="gelu", - ) + legacy_config = HookedTransformerConfig() with pytest.raises( TypeError, @@ -776,3 +772,64 @@ def test_boot_native_kaiming_gain_scales_weights(): # Same seed, only gain differs -> std should scale ~proportionally. assert std_2 / std_1 == pytest.approx(2.0) + + +class TestPatternHookWrites: + """hook_pattern must be write-capable on native bridges: the hook runs + inside NativeAttention (pattern_fn seam), before the value matmul — + post-hoc tuple firing silently discarded edits.""" + + def _bridge(self): + cfg = _cfg() + return TransformerBridge.boot_native(cfg) + + def test_pattern_edits_reach_the_output(self): + bridge = self._bridge() + tokens = torch.tensor([[1, 2, 3]]) + base = bridge(tokens, return_type="logits") + edited = bridge.run_with_hooks( + tokens, + fwd_hooks=[("blocks.0.attn.hook_pattern", lambda t, hook=None: torch.zeros_like(t))], + ) + assert not torch.allclose(edited, base), "pattern zeroing must change logits" + + def test_observe_only_stays_bit_exact(self): + bridge = self._bridge() + tokens = torch.tensor([[1, 2, 3]]) + base = bridge(tokens, return_type="logits") + observed = bridge.run_with_hooks( + tokens, fwd_hooks=[("blocks.0.attn.hook_pattern", lambda t, hook=None: t)] + ) + assert torch.equal(observed, base) + + def test_cache_reflects_the_edited_pattern(self): + """The cached pattern is the value the model actually used.""" + bridge = self._bridge() + tokens = torch.tensor([[1, 2, 3]]) + + def uniform(t, hook=None): + return ( + torch.full_like(t, 1.0 / t.shape[-1]).tril() * 0 + + t * 0 + + torch.softmax( + torch.zeros_like(t).masked_fill(torch.triu(torch.ones_like(t), 1).bool(), -1e9), + dim=-1, + ) + ) + + with bridge.hooks(fwd_hooks=[("blocks.0.attn.hook_pattern", uniform)]): + _, cache = bridge.run_with_cache(tokens, names_filter=["blocks.0.attn.hook_pattern"]) + cached = cache["blocks.0.attn.hook_pattern"] + expected = uniform(torch.zeros_like(cached)) + torch.testing.assert_close(cached, expected) + + def test_hook_fires_exactly_once_per_forward(self): + """The inside-the-computation seam must not double-fire with the old + post-hoc tuple path.""" + bridge = self._bridge() + fired = [] + bridge.run_with_hooks( + torch.tensor([[1, 2, 3]]), + fwd_hooks=[("blocks.0.attn.hook_pattern", lambda t, hook=None: fired.append(1) or t)], + ) + assert len(fired) == 1, f"hook fired {len(fired)} times" diff --git a/tests/unit/model_bridge/test_gated_hooks.py b/tests/unit/model_bridge/test_gated_hooks.py index 2f4eb058c6..fcc4c99aec 100644 --- a/tests/unit/model_bridge/test_gated_hooks.py +++ b/tests/unit/model_bridge/test_gated_hooks.py @@ -150,3 +150,163 @@ def test_add_hook_callable_filter_leaves_ungated_points_alone(): assert fired, "Ungated hook should still attach and fire" assert not [w for w in caught if "gated-off" in str(w.message)] + + +def test_run_with_hooks_callable_filter_warns_and_attaches_nothing(): + """The filter branch must warn and pre-skip, not silently attach dead hooks.""" + bridge = TransformerBridge.boot_native(_cfg()) + tokens = torch.tensor([[1, 2, 3]]) + fired: list = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + bridge.run_with_hooks( + tokens, + fwd_hooks=[ + (lambda n: n.endswith("hook_mlp_in"), lambda t, hook=None: fired.append(1) or t) + ], + ) + assert not bridge.hook_dict["blocks.0.hook_mlp_in"].fwd_hooks + assert not fired + assert any("gated-off" in str(w.message) for w in caught) + + +def test_hooks_ctx_rejects_explicit_gated_name(): + """The hooks() context manager raises on an explicitly named gated point.""" + bridge = TransformerBridge.boot_native(_cfg()) + with pytest.raises(ValueError, match="use_hook_mlp_in"): + with bridge.hooks(fwd_hooks=[("blocks.0.hook_mlp_in", lambda t, hook=None: t)]): + pass + + +def test_hooks_ctx_callable_filter_warns_and_attaches_nothing(): + """The hooks() filter branch warns and pre-skips gated matches.""" + bridge = TransformerBridge.boot_native(_cfg()) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with bridge.hooks( + fwd_hooks=[(lambda n: n.endswith("hook_mlp_in"), lambda t, hook=None: t)] + ): + assert not bridge.hook_dict["blocks.0.hook_mlp_in"].fwd_hooks + assert any("gated-off" in str(w.message) for w in caught) + + +def test_check_hooks_to_add_is_the_gate(): + """The documented extension point itself performs the gating check.""" + bridge = TransformerBridge.boot_native(_cfg()) + hp = bridge.hook_dict["blocks.0.hook_mlp_in"] + with pytest.raises(ValueError, match="use_hook_mlp_in"): + bridge.check_hooks_to_add(hp, "blocks.0.hook_mlp_in", lambda t, hook=None: t) + + +def test_add_caching_hooks_warns_for_gated_names(): + """Explicitly requesting a gated name for caching warns instead of silently + returning a cache that will never fill.""" + bridge = TransformerBridge.boot_native(_cfg()) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + cache = bridge.add_caching_hooks(names_filter=["blocks.0.hook_mlp_in"]) + bridge.reset_hooks() + assert "blocks.0.hook_mlp_in" not in cache + assert any("gated-off" in str(w.message) for w in caught) + + +def test_gated_paths_work_once_enabled(): + """Control: with the flag on, all four previously silent paths attach and fire.""" + bridge = TransformerBridge.boot_native(_cfg()) + bridge.set_use_hook_mlp_in(True) + tokens = torch.tensor([[1, 2, 3]]) + fired: list = [] + bridge.run_with_hooks( + tokens, + fwd_hooks=[ + (lambda n: n.endswith("hook_mlp_in"), lambda t, hook=None: fired.append("rwh") or t) + ], + ) + with bridge.hooks( + fwd_hooks=[("blocks.0.hook_mlp_in", lambda t, hook=None: fired.append("ctx") or t)] + ): + bridge(tokens, return_type="logits") + cache = bridge.add_caching_hooks(names_filter=["blocks.0.hook_mlp_in"]) + bridge(tokens, return_type="logits") + bridge.reset_hooks() + assert fired == ["rwh", "ctx"] + assert "blocks.0.hook_mlp_in" in cache + + +def _tiny_bert_bridge(): + """BERT-style bridge whose blocks.N.hook_mlp_in is an alias OVERRIDE onto an + always-firing point (mlp.in.hook_in) — alias != canonical, unlike boot_native.""" + import copy + + from transformers import BertConfig, BertForMaskedLM + + from transformer_lens.model_bridge.sources._bridge_builder import ( + build_bridge_from_module, + ) + + torch.manual_seed(0) + cfg = BertConfig( + vocab_size=64, + hidden_size=32, + num_hidden_layers=1, + num_attention_heads=2, + intermediate_size=64, + max_position_embeddings=32, + ) + return build_bridge_from_module( + BertForMaskedLM(cfg).eval(), + "BertForMaskedLM", + hf_config=copy.deepcopy(cfg), + tokenizer=None, + device="cpu", + ) + + +def test_alias_override_onto_firing_point_is_not_gated(): + """Gating keys on the POINT, not the spelling: on BERT, hook_mlp_in resolves + to mlp.in.hook_in, which always fires — refusing it would reject a working + hook. All four attach paths must accept it, and it must actually fire.""" + bridge = _tiny_bert_bridge() + ids = torch.tensor([[1, 2, 3]]) + fired: list = [] + + bridge.add_hook("blocks.0.hook_mlp_in", lambda t, hook=None: fired.append("add") or t) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + bridge(ids) + with bridge.hooks( + fwd_hooks=[("blocks.0.hook_mlp_in", lambda t, hook=None: fired.append("ctx") or t)] + ): + bridge(ids) + bridge.run_with_hooks( + ids, fwd_hooks=[("blocks.0.hook_mlp_in", lambda t, hook=None: fired.append("rwh") or t)] + ) + bridge.reset_hooks() + assert fired == ["add", "add", "ctx", "add", "rwh"], fired + assert not any("gated-off" in str(w.message) for w in caught) + + _, cache = bridge.run_with_cache(ids, names_filter=["blocks.0.hook_mlp_in"]) + assert "blocks.0.hook_mlp_in" in cache + + +def test_default_caching_sweep_composes_through_hooks(): + """get_caching_hooks() with no filter must not emit gated names — its + documented use is feeding hooks()/run_with_hooks, which raise on them.""" + bridge = TransformerBridge.boot_native(_cfg()) + tokens = torch.tensor([[1, 2, 3]]) + cache, fwd_hooks, _ = bridge.get_caching_hooks() + with bridge.hooks(fwd_hooks=fwd_hooks): + bridge(tokens, return_type="logits") + assert cache, "default sweep cached nothing" + assert not any(name.endswith("hook_mlp_in") for name, _ in fwd_hooks) + + +def test_default_add_caching_hooks_does_not_warn(): + """The default filter matches everything; that must not read as an explicit + request for gated names (run_with_cache's rule).""" + bridge = TransformerBridge.boot_native(_cfg()) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + bridge.add_caching_hooks() + bridge.reset_hooks() + assert not any("gated-off" in str(w.message) for w in caught) diff --git a/tests/unit/model_bridge/test_tl_checkpoint_conversion.py b/tests/unit/model_bridge/test_tl_checkpoint_conversion.py index 3cc9d93049..23bc54a3e0 100644 --- a/tests/unit/model_bridge/test_tl_checkpoint_conversion.py +++ b/tests/unit/model_bridge/test_tl_checkpoint_conversion.py @@ -6,206 +6,134 @@ key/tensor format `TransformerBridge.boot_native(cfg).load_state_dict` accepts natively, so these checkpoints can be loaded once and re-saved in bridge format without teaching `load_state_dict` a second key convention. + +Inputs are FROZEN fixtures under tests/fixtures/tl_checkpoints/ (captured by +scripts/capture_tl_checkpoint_fixtures.py): the legacy format never changes — +that is the converter's whole premise — so its test inputs are captured once +from a real HookedTransformer, with that model's logits as the behavioral +reference, and survive the class's 4.0 deletion. """ from __future__ import annotations +import json +from pathlib import Path + import pytest import torch -from transformer_lens import HookedTransformer -from transformer_lens.config import HookedTransformerConfig, TransformerBridgeConfig +from transformer_lens.config import TransformerBridgeConfig from transformer_lens.model_bridge import TransformerBridge from transformer_lens.utilities.tl_checkpoint_conversion import convert_tl_checkpoint +FIXTURES = Path(__file__).parents[2] / "fixtures" / "tl_checkpoints" -def _cfg_kwargs(**overrides): - base = dict( - d_model=32, - d_head=16, - n_heads=2, - n_layers=2, - n_ctx=8, - d_vocab=16, - d_mlp=64, - act_fn="gelu", - normalization_type="LN", - seed=0, - ) - base.update(overrides) - return base +def _load(variant: str): + """(legacy state_dict, reference dict, TransformerBridgeConfig) for a variant.""" + root = FIXTURES / variant + checkpoint = torch.load(root / "checkpoint.pt", weights_only=True) + reference = torch.load(root / "reference.pt", weights_only=True) + kwargs = json.loads((root / "meta.json").read_text()) + return checkpoint, reference, TransformerBridgeConfig(**kwargs) -def _ht_and_bridge_cfg(**overrides): - kwargs = _cfg_kwargs(**overrides) - return HookedTransformerConfig(**kwargs), TransformerBridgeConfig(**kwargs) +def _converted_bridge(variant: str, strict: bool = True): + checkpoint, reference, bridge_cfg = _load(variant) + converted = convert_tl_checkpoint(checkpoint, bridge_cfg) + bridge = TransformerBridge.boot_native(bridge_cfg) + result = bridge.load_state_dict(converted, strict=strict) + return bridge, reference, result -def test_convert_tl_checkpoint_loads_strict_into_native_bridge(): - ht_cfg, bridge_cfg = _ht_and_bridge_cfg() - ht = HookedTransformer(ht_cfg) - converted = convert_tl_checkpoint(ht.state_dict(), bridge_cfg) +def _assert_forward_matches(bridge, reference): + with torch.no_grad(): + bridge_logits = bridge(reference["tokens"]) + torch.testing.assert_close(bridge_logits, reference["logits"], atol=1e-4, rtol=1e-4) - bridge = TransformerBridge.boot_native(bridge_cfg) - result = bridge.load_state_dict(converted, strict=True) +def test_convert_tl_checkpoint_loads_strict_into_native_bridge(): + _, _, result = _converted_bridge("default") assert result.missing_keys == [] assert result.unexpected_keys == [] def test_convert_tl_checkpoint_matches_source_forward_pass(): - ht_cfg, bridge_cfg = _ht_and_bridge_cfg() - ht = HookedTransformer(ht_cfg) - - converted = convert_tl_checkpoint(ht.state_dict(), bridge_cfg) - bridge = TransformerBridge.boot_native(bridge_cfg) - bridge.load_state_dict(converted, strict=True) - - tokens = torch.randint(0, ht_cfg.d_vocab, (1, 4)) - with torch.no_grad(): - ht_logits = ht(tokens) - bridge_logits = bridge(tokens) - - torch.testing.assert_close(bridge_logits, ht_logits, atol=1e-4, rtol=1e-4) + bridge, reference, _ = _converted_bridge("default") + _assert_forward_matches(bridge, reference) def test_convert_tl_checkpoint_places_qkvo_in_correct_head_slots(): - """Independent check that per-head Q/K/V/O land in the right slots: read - the converted+loaded bridge back out through its own W_Q/W_K/W_V/W_O - properties (implemented separately from the converter) and compare - directly against the source HookedTransformer's per-head weights, rather - than trusting the converter's own reshape math.""" - ht_cfg, bridge_cfg = _ht_and_bridge_cfg() - ht = HookedTransformer(ht_cfg) - - converted = convert_tl_checkpoint(ht.state_dict(), bridge_cfg) - bridge = TransformerBridge.boot_native(bridge_cfg) - bridge.load_state_dict(converted, strict=True) - - torch.testing.assert_close(bridge.W_Q, ht.W_Q) - torch.testing.assert_close(bridge.W_K, ht.W_K) - torch.testing.assert_close(bridge.W_V, ht.W_V) - torch.testing.assert_close(bridge.W_O, ht.W_O) - torch.testing.assert_close(bridge.b_Q, ht.b_Q) - torch.testing.assert_close(bridge.b_K, ht.b_K) - torch.testing.assert_close(bridge.b_V, ht.b_V) - torch.testing.assert_close(bridge.b_O, ht.b_O) + bridge, reference, _ = _converted_bridge("default") + for attr in ("W_Q", "W_K", "W_V", "W_O", "b_Q", "b_K", "b_V", "b_O"): + torch.testing.assert_close(getattr(bridge, attr), reference[attr]) def test_convert_tl_checkpoint_raises_on_cfg_mismatch(): - """A wrong cfg can't be caught by a downstream shape-mismatch error -- - merging per-head dims produces a validly-shaped result for any head - count, since d_model == n_heads * d_head for any factoring of it. The - converter must catch this itself.""" - ht_cfg, _ = _ht_and_bridge_cfg() - ht = HookedTransformer(ht_cfg) - + checkpoint, _, _ = _load("default") wrong_cfg = TransformerBridgeConfig( - **_cfg_kwargs(n_heads=4, d_head=8) - ) # same d_model, wrong split - - with pytest.raises(ValueError, match="attn.W_Q"): - convert_tl_checkpoint(ht.state_dict(), wrong_cfg) + **{ + **json.loads((FIXTURES / "default" / "meta.json").read_text()), + "n_heads": 4, + "d_head": 8, + } + ) + with pytest.raises(ValueError, match="head|shape|mismatch"): + convert_tl_checkpoint(checkpoint, wrong_cfg) def test_convert_tl_checkpoint_raises_on_unrecognized_key(): - _, bridge_cfg = _ht_and_bridge_cfg() - with pytest.raises(ValueError, match="not a recognized"): - convert_tl_checkpoint({"blocks.0.attn.totally_unknown_param": torch.zeros(1)}, bridge_cfg) + checkpoint, _, bridge_cfg = _load("default") + checkpoint["blocks.0.attn.W_mystery"] = torch.zeros(2, 2) + with pytest.raises(ValueError, match="W_mystery|nrecognized"): + convert_tl_checkpoint(checkpoint, bridge_cfg) def test_convert_tl_checkpoint_supports_gqa(): - ht_cfg, bridge_cfg = _ht_and_bridge_cfg(n_heads=4, d_head=8, n_key_value_heads=2) - ht = HookedTransformer(ht_cfg) - - converted = convert_tl_checkpoint(ht.state_dict(), bridge_cfg) - bridge = TransformerBridge.boot_native(bridge_cfg) - result = bridge.load_state_dict(converted, strict=True) - + bridge, reference, result = _converted_bridge("gqa") assert result.missing_keys == [] assert result.unexpected_keys == [] - - tokens = torch.randint(0, ht_cfg.d_vocab, (1, 4)) - with torch.no_grad(): - ht_logits = ht(tokens) - bridge_logits = bridge(tokens) - torch.testing.assert_close(bridge_logits, ht_logits, atol=1e-4, rtol=1e-4) + _assert_forward_matches(bridge, reference) def test_convert_tl_checkpoint_supports_lnpre(): - """OthelloGPT (this converter's motivating use case, #1588) uses - normalization_type="LNPre" -- param-free pre-norm, so ln1/ln2/ln_final - have no weight/bias keys at all in the state dict for this converter to - handle; this just confirms the round trip still works end to end.""" - ht_cfg, bridge_cfg = _ht_and_bridge_cfg(normalization_type="LNPre") - ht = HookedTransformer(ht_cfg) - - converted = convert_tl_checkpoint(ht.state_dict(), bridge_cfg) - bridge = TransformerBridge.boot_native(bridge_cfg) - result = bridge.load_state_dict(converted, strict=True) - + bridge, reference, result = _converted_bridge("lnpre") assert result.missing_keys == [] assert result.unexpected_keys == [] - - tokens = torch.randint(0, ht_cfg.d_vocab, (1, 4)) - with torch.no_grad(): - ht_logits = ht(tokens) - bridge_logits = bridge(tokens) - torch.testing.assert_close(bridge_logits, ht_logits, atol=1e-4, rtol=1e-4) + _assert_forward_matches(bridge, reference) def test_convert_tl_checkpoint_supports_attn_only(): - ht_cfg, bridge_cfg = _ht_and_bridge_cfg(attn_only=True) - ht = HookedTransformer(ht_cfg) - - converted = convert_tl_checkpoint(ht.state_dict(), bridge_cfg) - bridge = TransformerBridge.boot_native(bridge_cfg) - result = bridge.load_state_dict(converted, strict=True) - + bridge, reference, result = _converted_bridge("attn_only") assert result.missing_keys == [] assert result.unexpected_keys == [] - - tokens = torch.randint(0, ht_cfg.d_vocab, (1, 4)) - with torch.no_grad(): - ht_logits = ht(tokens) - bridge_logits = bridge(tokens) - torch.testing.assert_close(bridge_logits, ht_logits, atol=1e-4, rtol=1e-4) + _assert_forward_matches(bridge, reference) def test_convert_tl_checkpoint_drops_attention_buffers(): - ht_cfg, bridge_cfg = _ht_and_bridge_cfg() - ht = HookedTransformer(ht_cfg) - - converted = convert_tl_checkpoint(ht.state_dict(), bridge_cfg) - + checkpoint, _, bridge_cfg = _load("default") + assert any( + key.endswith((".mask", ".IGNORE")) for key in checkpoint + ), "fixture lost the legacy attention buffers; recapture it" + converted = convert_tl_checkpoint(checkpoint, bridge_cfg) assert not any(key.endswith((".mask", ".IGNORE")) for key in converted) def test_convert_tl_checkpoint_supports_gated_mlp_and_rms_norm(): """The native bridge's gated MLP has no bias parameter at all for - gate/in/out (matching how real gated-MLP HF architectures like Llama are - built) while HookedTransformer's gated MLP keeps live b_in/b_out - parameters (pre-existing mismatch between the two implementations, - unrelated to this converter). convert_tl_checkpoint still faithfully - translates those keys since they're real HT parameters; load_state_dict - is the one that should refuse them under strict=True. Here they're - exactly zero (freshly constructed, untrained model) so dropping them via - strict=False is lossless and the forward pass still matches exactly. + gate/in/out (matching real gated-MLP HF architectures like Llama) while + HookedTransformer's gated MLP kept live b_in/b_out parameters. The + converter still faithfully translates those keys since they are real + checkpoint parameters; load_state_dict refuses them under strict=True. + In the fixture they are exactly zero (freshly constructed, untrained), so + dropping them via strict=False is lossless and the forward still matches. """ - ht_cfg, bridge_cfg = _ht_and_bridge_cfg(gated_mlp=True, normalization_type="RMS", act_fn="silu") - ht = HookedTransformer(ht_cfg) - - converted = convert_tl_checkpoint(ht.state_dict(), bridge_cfg) + checkpoint, reference, bridge_cfg = _load("gated_rms") + converted = convert_tl_checkpoint(checkpoint, bridge_cfg) bridge = TransformerBridge.boot_native(bridge_cfg) result = bridge.load_state_dict(converted, strict=False) assert result.missing_keys == [] assert set(result.unexpected_keys) == { - f"blocks.{i}.mlp.{part}.bias" for i in range(ht_cfg.n_layers) for part in ("in", "out") + f"blocks.{i}.mlp.{part}.bias" for i in range(bridge_cfg.n_layers) for part in ("in", "out") } - - tokens = torch.randint(0, ht_cfg.d_vocab, (1, 4)) - with torch.no_grad(): - ht_logits = ht(tokens) - bridge_logits = bridge(tokens) - torch.testing.assert_close(bridge_logits, ht_logits, atol=1e-4, rtol=1e-4) + _assert_forward_matches(bridge, reference) diff --git a/tests/unit/test_deprecation_warnings.py b/tests/unit/test_deprecation_warnings.py index d5df94ef0b..b3219756ca 100644 --- a/tests/unit/test_deprecation_warnings.py +++ b/tests/unit/test_deprecation_warnings.py @@ -105,7 +105,58 @@ def test_bert_next_sentence_prediction_constructor_warns_once(): ) -def test_direct_hooked_root_module_construction_warns_once(): +def test_hooked_root_module_is_not_deprecated(): + """HookedRootModule (with HookPoint) is KEPT infrastructure — the supported + way to hook arbitrary nn.Modules — and must construct without any + DeprecationWarning, unlike the legacy model classes.""" + import warnings as w + from transformer_lens import HookedRootModule - _assert_single_deprecation(HookedRootModule, "HookedRootModule") + with w.catch_warnings(record=True) as caught: + w.simplefilter("always") + HookedRootModule() + assert not [x for x in caught if issubclass(x.category, DeprecationWarning)] + + +def test_hooked_encoder_decoder_constructor_warns_once(): + from transformer_lens import HookedEncoderDecoder, HookedTransformerConfig + + cfg = HookedTransformerConfig( + n_layers=1, + d_model=16, + d_head=4, + n_heads=4, + n_ctx=8, + d_vocab=20, + d_mlp=32, + act_fn="relu", + attention_dir="bidirectional", + tie_word_embeddings=False, + positional_embedding_type="relative_positional_bias", + relative_attention_num_buckets=4, + relative_attention_max_distance=8, + ) + _assert_single_deprecation(lambda: HookedEncoderDecoder(cfg), "HookedEncoderDecoder") + + +def test_hooked_audio_encoder_constructor_warns_once(): + from transformer_lens import HookedAudioEncoder + + cfg = _small_config() + _assert_single_deprecation(lambda: HookedAudioEncoder(cfg), "HookedAudioEncoder") + + +def test_from_pretrained_warns_exactly_once(): + """__init__ is suppressed under from_pretrained — one warning per entry point, + attributed to the caller, not two.""" + import warnings as w + + from transformer_lens import HookedTransformer + + with w.catch_warnings(record=True) as caught: + w.simplefilter("always") + HookedTransformer.from_pretrained("gpt2") + dep = [x for x in caught if issubclass(x.category, DeprecationWarning)] + assert len(dep) == 1, [str(d.message)[:60] for d in dep] + assert "from_pretrained" in str(dep[0].message) diff --git a/transformer_lens/HookedAudioEncoder.py b/transformer_lens/HookedAudioEncoder.py index b2c96fc9a1..e81fb45134 100644 --- a/transformer_lens/HookedAudioEncoder.py +++ b/transformer_lens/HookedAudioEncoder.py @@ -7,6 +7,7 @@ from __future__ import annotations import logging +import warnings from typing import Any, Dict, List, Optional, Tuple, TypeVar, Union, cast, overload import numpy as np @@ -38,6 +39,10 @@ class HookedAudioEncoder(HookedRootModule): - There is no preprocessing (e.g. LayerNorm folding) when loading a pretrained model """ + # Set by from_pretrained while constructing, so each public entry point + # emits exactly one DeprecationWarning. + _suppress_init_deprecation: bool = False + processor: Any # AutoFeatureExtractor — HF auto class, not typed as callable in stubs hubert_model: Union[HubertModel, Wav2Vec2Model] blocks: TypedModuleList[BertBlock] @@ -50,6 +55,18 @@ def __init__( **kwargs: Any, ): super().__init__() + # from_pretrained warns at its own level (so the warning attributes to + # the caller and survives the default DeprecationWarning filter) and + # suppresses this one — each entry point fires exactly once. + if not getattr(type(self), "_suppress_init_deprecation", False): + warnings.warn( + "HookedAudioEncoder is deprecated and will be removed in 4.0. Use " + "TransformerBridge.boot_transformers(...) instead — " + "HuBERT/Wav2Vec2 are supported via the bridge's audio adapter. " + "See docs/source/content/migrating_to_v3.md.", + DeprecationWarning, + stacklevel=2, + ) if isinstance(cfg, Dict): cfg = HookedTransformerConfig(**cfg) elif isinstance(cfg, str): @@ -414,7 +431,12 @@ def from_pretrained( official_model_name, cfg, hf_model, dtype=dtype, **from_pretrained_kwargs ) - model = cls(cfg, move_to_device=False, model_name=official_model_name) + _prev_suppress = cls._suppress_init_deprecation + cls._suppress_init_deprecation = True + try: + model = cls(cfg, move_to_device=False, model_name=official_model_name) + finally: + cls._suppress_init_deprecation = _prev_suppress model.load_state_dict(state_dict, strict=False) model.processor = AutoFeatureExtractor.from_pretrained(official_model_name) diff --git a/transformer_lens/HookedEncoder.py b/transformer_lens/HookedEncoder.py index a5fdb80cfe..ae155387c0 100644 --- a/transformer_lens/HookedEncoder.py +++ b/transformer_lens/HookedEncoder.py @@ -49,6 +49,10 @@ class HookedEncoder(HookedRootModule): - There is no preprocessing (e.g. LayerNorm folding) when loading a pretrained model """ + # Set by from_pretrained while constructing, so each public entry point + # emits exactly one DeprecationWarning. + _suppress_init_deprecation: bool = False + blocks: TypedModuleList[BertBlock] def __init__( @@ -59,12 +63,16 @@ def __init__( **kwargs: Any, ): super().__init__() - warnings.warn( - "HookedEncoder is deprecated and will be removed in 4.0. Use " - "TransformerBridge.boot_transformers(...) instead.", - DeprecationWarning, - stacklevel=2, - ) + # from_pretrained warns at its own level (so the warning attributes to + # the caller and survives the default DeprecationWarning filter) and + # suppresses this one — each entry point fires exactly once. + if not getattr(type(self), "_suppress_init_deprecation", False): + warnings.warn( + "HookedEncoder is deprecated and will be removed in 4.0. Use " + "TransformerBridge.boot_transformers(...) instead.", + DeprecationWarning, + stacklevel=2, + ) if isinstance(cfg, Dict): cfg = HookedTransformerConfig(**cfg) elif isinstance(cfg, str): @@ -428,7 +436,12 @@ def from_pretrained( official_model_name, cfg, hf_model, dtype=dtype, **from_pretrained_kwargs ) - model = cls(cfg, tokenizer, move_to_device=False) + _prev_suppress = cls._suppress_init_deprecation + cls._suppress_init_deprecation = True + try: + model = cls(cfg, tokenizer, move_to_device=False) + finally: + cls._suppress_init_deprecation = _prev_suppress model.load_state_dict(state_dict, strict=False) diff --git a/transformer_lens/HookedEncoderDecoder.py b/transformer_lens/HookedEncoderDecoder.py index 7bea005548..8ba8b53f21 100644 --- a/transformer_lens/HookedEncoderDecoder.py +++ b/transformer_lens/HookedEncoderDecoder.py @@ -8,6 +8,7 @@ import logging import os +import warnings from itertools import chain from pathlib import Path from typing import ( @@ -55,6 +56,10 @@ class HookedEncoderDecoder(HookedRootModule): - The model only accepts tokens as inputs, and not strings, or lists of strings """ + # Set by from_pretrained while constructing, so each public entry point + # emits exactly one DeprecationWarning. + _suppress_init_deprecation: bool = False + tokenizer: Optional[PreTrainedTokenizerBase] encoder: TypedModuleList[T5Block] decoder: TypedModuleList[T5Block] @@ -67,6 +72,18 @@ def __init__( **kwargs: Any, ): super().__init__() + # from_pretrained warns at its own level (so the warning attributes to + # the caller and survives the default DeprecationWarning filter) and + # suppresses this one — each entry point fires exactly once. + if not getattr(type(self), "_suppress_init_deprecation", False): + warnings.warn( + "HookedEncoderDecoder is deprecated and will be removed in 4.0. Use " + "TransformerBridge.boot_transformers(...) instead — " + "the bridge supports T5-style encoder-decoder models. " + "See docs/source/content/migrating_to_v3.md.", + DeprecationWarning, + stacklevel=2, + ) if isinstance(cfg, Dict): cfg = HookedTransformerConfig(**cfg) elif isinstance(cfg, str): @@ -600,7 +617,12 @@ def from_pretrained( name_or_path, cfg, hf_model, dtype=dtype, **from_pretrained_kwargs ) - model = cls(cfg, tokenizer, move_to_device=False) + _prev_suppress = cls._suppress_init_deprecation + cls._suppress_init_deprecation = True + try: + model = cls(cfg, tokenizer, move_to_device=False) + finally: + cls._suppress_init_deprecation = _prev_suppress model.load_state_dict(state_dict, strict=False) diff --git a/transformer_lens/HookedRootModule.py b/transformer_lens/HookedRootModule.py index 13d89ffdb6..9ba1f3ab4a 100644 --- a/transformer_lens/HookedRootModule.py +++ b/transformer_lens/HookedRootModule.py @@ -1,15 +1,17 @@ """HookedRootModule. Base class extending :class:`torch.nn.Module` with hook-based introspection -utilities used by :class:`HookedTransformer` and friends. Lives in its own -module so that downstream code (e.g. :class:`ActivationCache`) can type-hint -against it without the broader ``hook_points`` import surface. +utilities: wrap any module's tensors in :class:`HookPoint` wrappers, call ``setup()``, +and ``run_with_hooks`` / ``run_with_cache`` work on it. This is permanent +infrastructure — it survives the 4.0 removal of the legacy model classes and is +the supported way to add TransformerLens-style hooks to arbitrary ``nn.Module`` objects. +Lives in its own module so that downstream code (e.g. :class:`ActivationCache`) +can type-hint against it without the broader ``hook_points`` import surface. """ from __future__ import annotations import logging -import warnings from collections.abc import Callable, Iterable from contextlib import contextmanager from functools import partial @@ -55,13 +57,6 @@ class HookedRootModule(HookIntrospectionMixin, nn.Module): def __init__(self, *args: Any): super().__init__() - if type(self) is HookedRootModule: - warnings.warn( - "HookedRootModule is deprecated and will be removed in 4.0. Use " - "TransformerBridge.boot_transformers(...) instead.", - DeprecationWarning, - stacklevel=2, - ) self.is_caching = False self.context_level = 0 diff --git a/transformer_lens/HookedTransformer.py b/transformer_lens/HookedTransformer.py index 96e8fc1e58..5d700c6841 100644 --- a/transformer_lens/HookedTransformer.py +++ b/transformer_lens/HookedTransformer.py @@ -139,6 +139,13 @@ class HookedTransformer(HookedRootModule): as the sum of parts — inspect with :meth:`to_str_tokens` when in doubt. """ + # Set by from_pretrained while constructing, so each public entry point + # emits exactly one DeprecationWarning. + _suppress_init_deprecation: bool = False + # Same mechanism for from_pretrained_no_processing's delegation to + # from_pretrained. + _suppress_fp_deprecation: bool = False + ln_final: nn.Module tokenizer: Optional[PreTrainedTokenizerBase] blocks: TypedModuleList[TransformerBlock] @@ -166,13 +173,17 @@ def __init__( default_padding_side: Which side to pad on. """ super().__init__() - warnings.warn( - "HookedTransformer is deprecated and will be removed in 4.0. Use " - "TransformerBridge.boot_transformers(...) instead, then call " - "enable_compatibility_mode() for HookedTransformer-equivalent numerics.", - DeprecationWarning, - stacklevel=2, - ) + # from_pretrained warns at its own level (so the warning attributes to + # the caller and survives the default DeprecationWarning filter) and + # suppresses this one — each entry point fires exactly once. + if not getattr(type(self), "_suppress_init_deprecation", False): + warnings.warn( + "HookedTransformer is deprecated and will be removed in 4.0. Use " + "TransformerBridge.boot_transformers(...) instead, then call " + "enable_compatibility_mode() for HookedTransformer-equivalent numerics.", + DeprecationWarning, + stacklevel=2, + ) if isinstance(cfg, str): raise ValueError( "Please pass in a config dictionary or HookedTransformerConfig object. If you want to load a " @@ -1324,14 +1335,18 @@ def from_pretrained( """ import warnings - warnings.warn( - "HookedTransformer.from_pretrained is deprecated and will be removed in " - "4.0. Use TransformerBridge.boot_transformers(...) instead, " - "then call enable_compatibility_mode() for HookedTransformer-equivalent " - "numerics. See docs/source/content/migrating_to_v3.md.", - DeprecationWarning, - stacklevel=2, - ) + # Delegating wrappers (from_pretrained_no_processing) warn at their own + # stack level — a warning raised here would attribute to the wrapper's + # internal call and be dropped by the default DeprecationWarning filter. + if not getattr(cls, "_suppress_fp_deprecation", False): + warnings.warn( + "HookedTransformer.from_pretrained is deprecated and will be removed in " + "4.0. Use TransformerBridge.boot_transformers(...) instead, " + "then call enable_compatibility_mode() for HookedTransformer-equivalent " + "numerics. See docs/source/content/migrating_to_v3.md.", + DeprecationWarning, + stacklevel=2, + ) if checkpoint_value is not None and checkpoint_label is not None: raise ValueError( @@ -1465,13 +1480,19 @@ def from_pretrained( official_model_name, cfg, hf_model, dtype=dtype, **from_pretrained_kwargs ) - # Create the HookedTransformer object - model = cls( - cfg, - tokenizer, - move_to_device=False, - default_padding_side=default_padding_side, - ) + # Create the HookedTransformer object (suppressing the __init__ warning: + # this entry point already warned above, at the caller's stack level). + _prev_suppress = cls._suppress_init_deprecation + cls._suppress_init_deprecation = True + try: + model = cls( + cfg, + tokenizer, + move_to_device=False, + default_padding_side=default_padding_side, + ) + finally: + cls._suppress_init_deprecation = _prev_suppress model.load_and_process_state_dict( state_dict, @@ -1507,18 +1528,30 @@ def from_pretrained_no_processing( Wrapper for from_pretrained with all boolean flags related to simplifying the model set to False. Refer to from_pretrained for details. """ - return cls.from_pretrained( - model_name, - fold_ln=fold_ln, - center_writing_weights=center_writing_weights, - center_unembed=center_unembed, - fold_value_biases=fold_value_biases, - refactor_factored_attn_matrices=refactor_factored_attn_matrices, - dtype=dtype, - default_prepend_bos=default_prepend_bos, - default_padding_side=default_padding_side, - **from_pretrained_kwargs, + warnings.warn( + "HookedTransformer.from_pretrained_no_processing is deprecated and will be " + "removed in 4.0. Use TransformerBridge.boot_transformers(...) instead. " + "See docs/source/content/migrating_to_v3.md.", + DeprecationWarning, + stacklevel=2, ) + prev = getattr(cls, "_suppress_fp_deprecation", False) + cls._suppress_fp_deprecation = True + try: + return cls.from_pretrained( + model_name, + fold_ln=fold_ln, + center_writing_weights=center_writing_weights, + center_unembed=center_unembed, + fold_value_biases=fold_value_biases, + refactor_factored_attn_matrices=refactor_factored_attn_matrices, + dtype=dtype, + default_prepend_bos=default_prepend_bos, + default_padding_side=default_padding_side, + **from_pretrained_kwargs, + ) + finally: + cls._suppress_fp_deprecation = prev def init_weights(self): """Initialize weights. diff --git a/transformer_lens/SVDInterpreter.py b/transformer_lens/SVDInterpreter.py index 4d0f3e397e..87de943c05 100644 --- a/transformer_lens/SVDInterpreter.py +++ b/transformer_lens/SVDInterpreter.py @@ -1,22 +1,28 @@ """SVD Interpreter. Module for getting the singular vectors of the OV, w_in, and w_out matrices of a -:class:`transformer_lens.HookedTransformer`. +:class:`transformer_lens.model_bridge.TransformerBridge` (or any model exposing +the TransformerLens weight surface). """ -from typing import Any, Optional, Union +from typing import Optional, Union import torch from typing_extensions import Literal from transformer_lens.FactoredMatrix import FactoredMatrix +from transformer_lens.model_protocol import TransformerLensModel OUTPUT_EMBEDDING = "unembed.W_U" VECTOR_TYPES = ["OV", "w_in", "w_out"] class SVDInterpreter: - def __init__(self, model: Any): + # Base protocol at runtime: beartype validates via getattr_static, which + # cannot see nn.Module instance submodules, so the WithWeights surface + # would spuriously reject legacy models. Everything touched here (cfg, + # tl_parameters/named_parameters fallback) is on the base surface. + def __init__(self, model: TransformerLensModel): self.model = model self.cfg = model.cfg # Use tl_parameters() for TransformerBridge (returns TL-style dict) @@ -24,6 +30,7 @@ def __init__(self, model: Any): if hasattr(model, "tl_parameters"): self.params = model.tl_parameters() else: + assert isinstance(model, torch.nn.Module) # legacy fallback path self.params = {name: param for name, param in model.named_parameters()} def get_singular_vectors( @@ -37,7 +44,7 @@ def get_singular_vectors( This tensor can then be plotted using Neel's PySvelte, as demonstrated in the demo for this feature. The demo also points out some "gotchas" in this feature - numerical instability - means inconsistency across devices, and the default HookedTransformer parameters don't + means inconsistency across devices, and default weight processing doesn't replicate the original SVD post very well. So I'd recommend checking out the demo if you want to use this! @@ -45,9 +52,10 @@ def get_singular_vectors( .. code-block:: python - from transformer_lens import HookedTransformer, SVDInterpreter + from transformer_lens import SVDInterpreter + from transformer_lens.model_bridge import TransformerBridge - model = HookedTransformer.from_pretrained('gpt2-medium') + model = TransformerBridge.boot_transformers('gpt2-medium') svd_interpreter = SVDInterpreter(model) ov = svd_interpreter.get_singular_vectors('OV', layer_index=22, head_index=10) diff --git a/transformer_lens/__init__.py b/transformer_lens/__init__.py index a3d3723894..8da45f2d13 100644 --- a/transformer_lens/__init__.py +++ b/transformer_lens/__init__.py @@ -9,19 +9,17 @@ tools, utilities, ) -from . import train from . import loading_from_pretrained as loading from . import supported_models from .ActivationCache import ActivationCache -from .BertNextSentencePrediction import BertNextSentencePrediction from .cache.key_value_cache import TransformerLensKeyValueCache from .cache.key_value_cache_entry import TransformerLensKeyValueCacheEntry -from .config import HookedTransformerConfig, TransformerBridgeConfig +from .config import TransformerBridgeConfig from .FactoredMatrix import FactoredMatrix -from .HookedEncoder import HookedEncoder -from .HookedAudioEncoder import HookedAudioEncoder -from .HookedEncoderDecoder import HookedEncoderDecoder -from .HookedTransformer import HookedTransformer + +# KEPT infrastructure: HookedRootModule (with HookPoint) survives 4.0 as the +# supported way to hook arbitrary nn.Modules; it is not part of the legacy +# model-class removal below. from .HookedRootModule import HookedRootModule # LIT integration (optional, requires lit-nlp package) @@ -34,6 +32,35 @@ from .SVDInterpreter import SVDInterpreter +# Legacy names resolved lazily (PEP 562): the deprecated model classes and the +# train shim are deleted in 4.0, and importing transformer_lens must not load +# them eagerly. Each still warns on use via its own module; the deletion PR +# removes entries from this map and nothing else in this file. +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Static bindings for the lazy names below, so type checkers resolve + # `from transformer_lens import HookedTransformer` to the class, not the + # submodule. Runtime resolution goes through __getattr__. + from . import train + from .BertNextSentencePrediction import BertNextSentencePrediction + from .config import HookedTransformerConfig + from .HookedAudioEncoder import HookedAudioEncoder + from .HookedEncoder import HookedEncoder + from .HookedEncoderDecoder import HookedEncoderDecoder + from .HookedTransformer import HookedTransformer + +_LAZY_LEGACY: dict[str, tuple[str, str | None]] = { + "HookedTransformer": (".HookedTransformer", "HookedTransformer"), + "HookedEncoder": (".HookedEncoder", "HookedEncoder"), + "HookedAudioEncoder": (".HookedAudioEncoder", "HookedAudioEncoder"), + "HookedEncoderDecoder": (".HookedEncoderDecoder", "HookedEncoderDecoder"), + "BertNextSentencePrediction": (".BertNextSentencePrediction", "BertNextSentencePrediction"), + "HookedTransformerConfig": (".config", "HookedTransformerConfig"), + "train": (".train", None), +} + + def __getattr__(name: str): # Lazy: model_bridge is import-heavy and importing it eagerly here would # risk cycles with modules the bridge itself imports. @@ -41,11 +68,46 @@ def __getattr__(name: str): from .model_bridge import TransformerBridge return TransformerBridge + if name in _LAZY_LEGACY: + from importlib import import_module + + module_name, attr = _LAZY_LEGACY[name] + module = import_module(module_name, __name__) + value = module if attr is None else getattr(module, attr) + globals()[name] = value # cache: subsequent access skips __getattr__ + return value raise AttributeError(f"module {__name__!r} has no attribute {name!r}") def __dir__(): - return sorted(set(globals()) | {"TransformerBridge"}) + return sorted(set(globals()) | {"TransformerBridge"} | set(_LAZY_LEGACY)) + + +# Five legacy classes share their submodule's name. A submodule-path import +# (``from transformer_lens.HookedTransformer import HookedTransformer``) makes +# the import machinery bind the MODULE onto this package after the fact, and +# ``from transformer_lens import HookedTransformer`` would then return the +# module instead of the class, dependent on import order. The class override +# below de-shadows on every access, so the public name deterministically +# resolves to the class. +import sys as _sys # noqa: E402 +import types as _types # noqa: E402 + +_SHADOWED_CLASS_NAMES = frozenset( + name for name, (_mod, attr) in _LAZY_LEGACY.items() if attr == name +) + + +class _DeShadowingModule(_types.ModuleType): + def __getattribute__(self, name: str): + value = super().__getattribute__(name) + if name in _SHADOWED_CLASS_NAMES and isinstance(value, _types.ModuleType): + value = getattr(value, name) + setattr(self, name, value) + return value + + +_sys.modules[__name__].__class__ = _DeShadowingModule import os as _os # noqa: E402 diff --git a/transformer_lens/factories/architecture_adapter_factory.py b/transformer_lens/factories/architecture_adapter_factory.py index fb72d05967..87c718bb13 100644 --- a/transformer_lens/factories/architecture_adapter_factory.py +++ b/transformer_lens/factories/architecture_adapter_factory.py @@ -149,6 +149,7 @@ T5GemmaArchitectureAdapter, VaultGemmaArchitectureAdapter, ViTArchitectureAdapter, + Wav2Vec2ArchitectureAdapter, XGLMArchitectureAdapter, YoutuArchitectureAdapter, Zamba2ArchitectureAdapter, @@ -235,6 +236,9 @@ "HrmTextForCausalLM": HrmTextArchitectureAdapter, "HubertForCTC": HubertArchitectureAdapter, "HubertModel": HubertArchitectureAdapter, + "Wav2Vec2ForCTC": Wav2Vec2ArchitectureAdapter, + "Wav2Vec2ForPreTraining": Wav2Vec2ArchitectureAdapter, + "Wav2Vec2Model": Wav2Vec2ArchitectureAdapter, "HunYuanDenseV1ForCausalLM": HunYuanDenseV1ArchitectureAdapter, "Idefics3ForConditionalGeneration": Idefics3ArchitectureAdapter, "InternLM2ForCausalLM": InternLM2ArchitectureAdapter, diff --git a/transformer_lens/hook_points.py b/transformer_lens/hook_points.py index 529d097621..b7559c0304 100644 --- a/transformer_lens/hook_points.py +++ b/transformer_lens/hook_points.py @@ -375,7 +375,7 @@ def forward(self, x: Tensor) -> Tensor: def layer(self): # Returns the layer index if the name has the form 'blocks.{layer}.{...}' - # Helper function that's mainly useful on HookedTransformer + # Helper for models whose hook names follow the blocks.{layer}.* scheme # If it doesn't have this form, raises an error - if self.name is None: raise ValueError("Name cannot be None") diff --git a/transformer_lens/lit/model.py b/transformer_lens/lit/model.py index 2270fb2088..a717ac743f 100644 --- a/transformer_lens/lit/model.py +++ b/transformer_lens/lit/model.py @@ -559,7 +559,10 @@ def _compute_gradients( # Add positional embeddings if applicable if self.model.cfg.positional_embedding_type == "standard": - pos_embed = self.model.pos_embed(input_tokens) + # W_pos indexed by POSITION. Calling pos_embed(input_tokens) on + # a bridge routes to HF's wpe, which embeds whatever ids it is + # given — token ids, here — silently producing wrong positions. + pos_embed = self.model.W_pos[: input_tokens.shape[1]].unsqueeze(0) residual = embed + pos_embed else: residual = embed diff --git a/transformer_lens/model_bridge/bridge_core.py b/transformer_lens/model_bridge/bridge_core.py index 08ab2fdc76..c52a800daa 100644 --- a/transformer_lens/model_bridge/bridge_core.py +++ b/transformer_lens/model_bridge/bridge_core.py @@ -732,9 +732,23 @@ def check_hooks_to_add( ) -> None: """Validate a hook before it is added; override to add checks. - No-op by default (mirrors ``HookedRootModule.check_hooks_to_add``). + Raises for a gated-off hook point — a targeted attach there would + silently never fire. Every explicit-name attach path routes through + here; filter sweeps pre-skip gated matches (with a warning) before + reaching it, since a filter was not necessarily targeting them. + + Gating keys on the POINT's own canonical name, not the requested + spelling: on adapters with ``hook_alias_overrides`` a gated HT name + (e.g. ``blocks.0.hook_mlp_in`` on BERT) resolves to an always-firing + point, and refusing it would reject a hook that works. Driver-fireability is enforced separately in ``_check_hook_fireable``. """ + reason = self._gated_hook_reason(hook_point.name or hook_point_name) + if reason is not None: + raise ValueError( + f"Cannot add hook {hook_point_name} because {reason} is False. " + f"Call set_{reason}(True) first." + ) def _add_fn_to_hook_point( self, @@ -763,6 +777,16 @@ def _gated_hook_reason(self, hook_point_name: str) -> Optional[str]: return "use_attn_in" return None + def _warn_gated_skipped(self, api: str, names: List[str], stacklevel: int = 3) -> None: + """One warning per sweep for filter-matched gated-off names.""" + if names: + warnings.warn( + f"{api}: skipped {len(names)} gated-off hook name(s) " + f"that would never fire: {names}. Call the relevant " + "set_use_*(True) setter first to enable them.", + stacklevel=stacklevel, + ) + def add_hook( self, name: Union[str, Callable[[str], bool]], @@ -795,29 +819,16 @@ def add_hook( # match is skipped rather than raised on — but silently # attaching here would leave a dead hook that never fires, # which is the failure this warns about. - if self._gated_hook_reason(hook_name) is not None: + if self._gated_hook_reason(hook_point.name or hook_name) is not None: gated_names_skipped.append(hook_name) continue self._add_fn_to_hook_point(hook_point, hook_name, hook_fn, dir, is_permanent) - if gated_names_skipped: - warnings.warn( - f"add_hook: skipped {len(gated_names_skipped)} gated-off hook name(s) " - f"that would never fire: {gated_names_skipped}. Call the relevant " - "set_use_*(True) setter first to enable them.", - stacklevel=2, - ) + self._warn_gated_skipped("add_hook", gated_names_skipped) return - # An explicitly named gated-off hook point is a caller error: the hook - # would silently never fire. Raise naming the setter to call (filters - # above skip with a warning — they were not necessarily targeting gated names). - reason = self._gated_hook_reason(name) - if reason is not None: - raise ValueError( - f"Cannot add hook {name} because {reason} is False. " - f"Call set_{reason}(True) first." - ) - + # Explicit-name attaches raise on gated-off points inside + # check_hooks_to_add (via _add_fn_to_hook_point) — every path below + # funnels through it. # Fast path: canonical registry names skip the alias-map build (hook_dict + # map construction cost ~ms on large models; add_hook is often called per layer). registry_hp = self._hook_registry.get(name) @@ -967,6 +978,12 @@ def save_hook(tensor: Any, hook: Any, is_backward: bool = False) -> None: for name, hook_point in self.hook_dict.items(): if filter_fn(name) and id(hook_point) not in seen: seen.add(id(hook_point)) + # The default sweep must not emit gated-off points: they never + # fire (empty cache entries at best), and feeding them to + # hooks()/run_with_hooks — the advertised composition — would + # raise. An explicit filter keeps them so downstream can warn. + if names_filter is None and self._gated_hook_reason(hook_point.name or name): + continue fwd_hooks.append((name, partial(save_hook, is_backward=False))) if incl_bwd: bwd_hooks.append((name, partial(save_hook, is_backward=True))) @@ -992,14 +1009,28 @@ def add_caching_hooks( remove_batch_dim=remove_batch_dim, cache=cache, ) + # Gated-off hook points never fire, so caching them is a no-op; skip - # them rather than trip add_hook's explicit-name guard when enumerating. + # them rather than trip add_hook's explicit-name guard. Warn only when + # the caller EXPLICITLY asked for them (run_with_cache's rule: the + # default filter matches everything and must not read as a request). + def _point_reason(name: str): + hp = self.hook_dict.get(name) + return self._gated_hook_reason(hp.name or name if hp is not None else name) + + gated_skipped = [] for name, hook_fn in fwd_hooks: - if self._gated_hook_reason(name) is None: + if _point_reason(name) is None: self.add_hook(name, hook_fn, dir="fwd") + else: + gated_skipped.append(name) for name, hook_fn in bwd_hooks: - if self._gated_hook_reason(name) is None: + if _point_reason(name) is None: self.add_hook(name, hook_fn, dir="bwd") + elif name not in gated_skipped: + gated_skipped.append(name) + if names_filter is not None: + self._warn_gated_skipped("add_caching_hooks", gated_skipped) return cache def cache_all( @@ -1103,14 +1134,19 @@ def apply_hooks(hook_list: List[Tuple[Any, Callable]], is_fwd: bool) -> None: add_hook_to_point(hook_point, hook_fn, actual_hook_name, direction) else: seen_hooks = set() + gated_skipped: List[str] = [] for n, hook_point in hook_dict.items(): if hook_name_or_filter(n): hook_id = id(hook_point) if hook_id in seen_hooks: continue seen_hooks.add(hook_id) + if self._gated_hook_reason(hook_point.name or n) is not None: + gated_skipped.append(n) + continue hook_name_to_use = hook_point.name if hook_point.name else n add_hook_to_point(hook_point, hook_fn, hook_name_to_use, direction) + self._warn_gated_skipped("hooks", gated_skipped, stacklevel=4) try: apply_hooks(fwd_hooks, True) @@ -1232,30 +1268,34 @@ def wrapped_hook_fn(tensor, hook, _orig_fn=original_hook_fn): hook_fn = wrapped_hook_fn if isinstance(hook_name_or_filter, str): - # An explicitly named gated-off hook point is a caller error: - # the hook would silently never fire. Raise naming the setter - # (a filter matching a gated name is left alone — it was not - # necessarily targeting that name on purpose). - reason = self._gated_hook_reason(hook_name_or_filter) + # Resolve BEFORE gating: an aliased HT name may point at an + # always-firing override, which must not be refused. The + # message still names the caller's spelling. + actual_hook_name, hook_point = self._resolve_hook_point( + hook_name_or_filter, aliases, hook_dict + ) + reason = self._gated_hook_reason(hook_point.name or actual_hook_name) if reason is not None: raise ValueError( f"Cannot add hook {hook_name_or_filter} because {reason} is False. " f"Call set_{reason}(True) first." ) - actual_hook_name, hook_point = self._resolve_hook_point( - hook_name_or_filter, aliases, hook_dict - ) add_hook_to_point(hook_point, hook_fn, actual_hook_name, direction) else: seen_hooks: set = set() + gated_skipped: List[str] = [] for n, hook_point in hook_dict.items(): if hook_name_or_filter(n): hook_id = id(hook_point) if hook_id in seen_hooks: continue seen_hooks.add(hook_id) + if self._gated_hook_reason(hook_point.name or n) is not None: + gated_skipped.append(n) + continue hook_name_to_use = hook_point.name if hook_point.name else n add_hook_to_point(hook_point, hook_fn, hook_name_to_use, direction) + self._warn_gated_skipped("run_with_hooks", gated_skipped, stacklevel=4) context_level = getattr(self, "context_level", 0) + 1 self.context_level = context_level @@ -1541,7 +1581,7 @@ def stop_hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: if names_filter is not None: kept = [] for hp, name in hooks: - if self._gated_hook_reason(name) is not None: + if self._gated_hook_reason(hp.name or name) is not None: gated_names_skipped.append(name) else: kept.append((hp, name)) diff --git a/transformer_lens/model_bridge/generalized_components/attention.py b/transformer_lens/model_bridge/generalized_components/attention.py index 06d01f7d5d..8e153f6653 100644 --- a/transformer_lens/model_bridge/generalized_components/attention.py +++ b/transformer_lens/model_bridge/generalized_components/attention.py @@ -822,6 +822,12 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: elif len(args) > 0 and isinstance(args[0], torch.Tensor): hooked = self.hook_in(args[0]) args = (hooked,) + args[1:] + # Modules exposing the pattern seam (NativeAttention) run the pattern + # through hook_pattern INSIDE the computation, so hook edits re-weight + # the output; post-hoc firing below would be a silent no-op for writes. + pattern_hooked_inside = bool(getattr(self.original_component, "accepts_pattern_fn", False)) + if pattern_hooked_inside: + kwargs["pattern_fn"] = self.hook_pattern # try/finally so the captured tensor (and its autograd graph) is # released even if original_component raises. try: @@ -839,7 +845,8 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: # For T5, second element is position_bias which should be passed through if isinstance(second_element, torch.Tensor) and second_element.dim() == 4: # This looks like attention weights [batch, heads, seq, seq] - second_element = self.hook_pattern(second_element) + if not pattern_hooked_inside: + second_element = self.hook_pattern(second_element) # Also store for potential hook_attn_scores (before softmax) # Note: Most HF implementations return post-softmax weights self.hook_attn_scores(second_element) diff --git a/transformer_lens/model_bridge/sources/__init__.py b/transformer_lens/model_bridge/sources/__init__.py index 2d29d57199..dbb1db7203 100644 --- a/transformer_lens/model_bridge/sources/__init__.py +++ b/transformer_lens/model_bridge/sources/__init__.py @@ -9,6 +9,7 @@ detect_tokenizer_bos_eos, ) from transformer_lens.model_bridge.sources.native import boot as boot_native +from transformer_lens.model_bridge.sources.tl_legacy import boot as boot_tl_legacy from transformer_lens.model_bridge.sources.transformers import ( boot, check_model_support, @@ -19,6 +20,7 @@ __all__ = [ "boot", "boot_native", + "boot_tl_legacy", "boot_vllm", "build_bridge_config_from_hf", "build_bridge_from_module", diff --git a/transformer_lens/model_bridge/sources/native/model.py b/transformer_lens/model_bridge/sources/native/model.py index 30ba363099..1174f0bc69 100644 --- a/transformer_lens/model_bridge/sources/native/model.py +++ b/transformer_lens/model_bridge/sources/native/model.py @@ -29,6 +29,11 @@ "relu": F.relu, "silu": F.silu, "swish": F.silu, + # SoLU (https://transformer-circuits.pub/2022/solu/index.html): x*softmax(x). + # "solu_ln" is the same activation; the mid-MLP LayerNorm that follows it is + # a NativeMLP submodule, not part of the pointwise function. + "solu": lambda x: x * F.softmax(x, dim=-1), + "solu_ln": lambda x: x * F.softmax(x, dim=-1), } @@ -249,8 +254,15 @@ def _rope(x: torch.Tensor) -> torch.Tensor: class NativeAttention(nn.Module): - """Split-QKV causal self-attention. Returns (out, pattern); AttentionBridge - fires ``hook_pattern`` off the second element.""" + """Split-QKV causal self-attention. Returns (out, pattern). + + ``accepts_pattern_fn``: AttentionBridge injects its ``hook_pattern`` as + ``pattern_fn``, applied BEFORE the value matmul — so hook edits genuinely + re-weight the attention output instead of only decorating the returned + tuple (which the wrapper cannot recompute from). + """ + + accepts_pattern_fn = True causal_mask: torch.Tensor @@ -300,6 +312,7 @@ def forward( hidden_states: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, + pattern_fn: Optional[Callable[[torch.Tensor], torch.Tensor]] = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor]: batch, seq, _ = hidden_states.shape @@ -332,6 +345,8 @@ def forward( # Fully masked padding queries softmax to NaN; overwrite masked entries # so those rows contribute a zero attention update instead of poisoning later layers. pattern = pattern.masked_fill(block_mask, 0.0) + if pattern_fn is not None: + pattern = pattern_fn(pattern) attn = torch.matmul(pattern, v).transpose(1, 2).contiguous().view(batch, seq, -1) out = self.o(attn) @@ -377,9 +392,19 @@ def __init__(self, cfg: TransformerBridgeConfig): if act_name not in _ACTIVATIONS: raise ValueError(f"Unsupported act_fn={act_name!r}. Supported: {sorted(_ACTIVATIONS)}") self.act = _ACTIVATIONS[act_name] + # SoLU-LN models (NeelNanda's SoLU family) apply a LayerNorm between the + # activation and the out-projection; without it their checkpoints load + # but compute the wrong function. Named ``ln`` to match the legacy + # property-format key blocks.{i}.mlp.ln.{w,b}. + self.ln: Optional[nn.LayerNorm] = ( + nn.LayerNorm(d_mlp, eps=cfg.eps) if act_name == "solu_ln" else None + ) def forward(self, hidden_states: torch.Tensor, **kwargs) -> torch.Tensor: - return self.fc_out(self.act(self.fc_in(hidden_states))) + mid = self.act(self.fc_in(hidden_states)) + if self.ln is not None: + mid = self.ln(mid) + return self.fc_out(mid) class NativeGatedMLP(nn.Module): diff --git a/transformer_lens/model_bridge/sources/tl_legacy.py b/transformer_lens/model_bridge/sources/tl_legacy.py new file mode 100644 index 0000000000..030374aacd --- /dev/null +++ b/transformer_lens/model_bridge/sources/tl_legacy.py @@ -0,0 +1,226 @@ +"""Loader for legacy TransformerLens-format HF repos. + +NeelNanda/*, ArthurConmy/*, and Baidicoot/* repos predate the HF model format: +no ``model_type`` in config.json (AutoConfig refuses them), weights stored as +``*.pth`` state dicts in HookedTransformer property format (or older layouts +converted below), and training checkpoints as ``checkpoints/*_<label>.pth`` +files rather than revisions. This module derives a TransformerBridgeConfig from +the repo's TL-style config.json, fetches and normalizes the state dict, and +loads it into a ``boot_native`` bridge via ``convert_tl_checkpoint``. + +The two legacy layout converters are ports of the HookedTransformer loaders +(pretrained/weight_conversions/{neel_solu_old,mingpt}.py), rehomed here so this +path survives the 4.0 deletion of the legacy loading stack. +""" + +from __future__ import annotations + +import json +from typing import Any, Optional, Union + +import einops +import torch + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge.bridge import TransformerBridge +from transformer_lens.utilities.tl_checkpoint_conversion import convert_tl_checkpoint + +TL_LEGACY_PREFIXES = ("NeelNanda/", "ArthurConmy/", "Baidicoot/") + + +def _fetch_json(repo_id: str, filename: str) -> dict: + from huggingface_hub import hf_hub_download + + with open(hf_hub_download(repo_id, filename)) as f: + return json.load(f) + + +def derive_tl_legacy_config(repo_id: str) -> TransformerBridgeConfig: + """TransformerBridgeConfig from a legacy TL repo's config.json.""" + cfg_json = _fetch_json(repo_id, "config.json") + architecture = cfg_json.get( + "architecture", "neel" if "_old" not in repo_id else "neel-solu-old" + ) + normalization = cfg_json.get("normalization", cfg_json.get("normalization_type")) + if cfg_json.get("shortformer_pos", False): + raise NotImplementedError( + f"{repo_id} uses shortformer positional embeddings, which the native " + "bridge does not implement." + ) + cfg = TransformerBridgeConfig( + d_model=cfg_json["d_model"], + n_layers=cfg_json["n_layers"], + d_mlp=cfg_json["d_mlp"], + d_head=cfg_json["d_head"], + n_heads=cfg_json["n_heads"], + n_ctx=cfg_json["n_ctx"], + d_vocab=cfg_json["d_vocab"], + act_fn=cfg_json["act_fn"], + attn_only=cfg_json["attn_only"], + final_rms=cfg_json.get("final_rms", False), + normalization_type=normalization, + positional_embedding_type="standard", + tokenizer_name=cfg_json.get("tokenizer_name"), + architecture="TransformerLensNative", + ) + cfg.original_architecture = architecture # type: ignore[attr-defined] + return cfg + + +def _convert_neel_solu_old_weights(state_dict: dict, cfg: TransformerBridgeConfig) -> dict: + """Old-layout SoLU repos ('*_old'): left-facing weights below 8L, and 8L's + W_pos alone left-facing. Port of the HookedTransformer converter.""" + reverse_pos = cfg.n_layers <= 8 + reverse_weights = cfg.n_layers <= 6 + new_state_dict = {} + for k, v in state_dict.items(): + k = k.replace("norm", "ln") + if k.startswith("ln."): + k = k.replace("ln.", "ln_final.") + new_state_dict[k] = v + if reverse_pos: + new_state_dict["pos_embed.W_pos"] = new_state_dict["pos_embed.W_pos"].T + if reverse_weights: + for k, v in new_state_dict.items(): + if "W_" in k and "W_pos" not in k: + new_state_dict[k] = v.transpose(-2, -1) + return new_state_dict + + +def _convert_mingpt_weights(old_state_dict: dict, cfg: TransformerBridgeConfig) -> dict: + """minGPT layout (Baidicoot/Othello-GPT): unconcatenated QKV. Port of the + HookedTransformer converter.""" + state_dict = { + "embed.W_E": old_state_dict["tok_emb.weight"], + "pos_embed.W_pos": old_state_dict["pos_emb"].squeeze(), + } + for l in range(cfg.n_layers): + state_dict[f"blocks.{l}.ln1.w"] = old_state_dict[f"blocks.{l}.ln1.weight"] + state_dict[f"blocks.{l}.ln1.b"] = old_state_dict[f"blocks.{l}.ln1.bias"] + for name, hf in (("Q", "query"), ("K", "key"), ("V", "value")): + w = einops.rearrange( + old_state_dict[f"blocks.{l}.attn.{hf}.weight"], "(i h) m->i m h", i=cfg.n_heads + ) + b = einops.rearrange( + old_state_dict[f"blocks.{l}.attn.{hf}.bias"], "(i h)->i h", i=cfg.n_heads + ) + state_dict[f"blocks.{l}.attn.W_{name}"] = w + state_dict[f"blocks.{l}.attn.b_{name}"] = b + state_dict[f"blocks.{l}.attn.W_O"] = einops.rearrange( + old_state_dict[f"blocks.{l}.attn.proj.weight"], "m (i h)->i h m", i=cfg.n_heads + ) + state_dict[f"blocks.{l}.attn.b_O"] = old_state_dict[f"blocks.{l}.attn.proj.bias"] + state_dict[f"blocks.{l}.ln2.w"] = old_state_dict[f"blocks.{l}.ln2.weight"] + state_dict[f"blocks.{l}.ln2.b"] = old_state_dict[f"blocks.{l}.ln2.bias"] + state_dict[f"blocks.{l}.mlp.W_in"] = old_state_dict[f"blocks.{l}.mlp.0.weight"].T + state_dict[f"blocks.{l}.mlp.b_in"] = old_state_dict[f"blocks.{l}.mlp.0.bias"] + state_dict[f"blocks.{l}.mlp.W_out"] = old_state_dict[f"blocks.{l}.mlp.2.weight"].T + state_dict[f"blocks.{l}.mlp.b_out"] = old_state_dict[f"blocks.{l}.mlp.2.bias"] + state_dict["ln_final.w"] = old_state_dict["ln_f.weight"] + state_dict["ln_final.b"] = old_state_dict["ln_f.bias"] + state_dict["unembed.W_U"] = old_state_dict["head.weight"].T + return state_dict + + +def fetch_tl_legacy_state_dict( + repo_id: str, + cfg: TransformerBridgeConfig, + checkpoint_value: Optional[int] = None, + dtype: torch.dtype = torch.float32, +) -> dict: + """Download and normalize a legacy repo's state dict to TL property format.""" + from huggingface_hub import HfApi, hf_hub_download + + repo_files = HfApi().list_repo_files(repo_id) + suffix = f"{checkpoint_value}.pth" if checkpoint_value is not None else "final.pth" + matches = [f for f in repo_files if f.endswith(suffix)] + if not matches: + raise FileNotFoundError(f"No '*{suffix}' file in {repo_id}; files: {repo_files[:8]}...") + state_dict = torch.load( + hf_hub_download(repo_id, matches[0]), map_location="cpu", weights_only=True + ) + state_dict = {k: v.to(dtype) for k, v in state_dict.items()} + + original_architecture = getattr(cfg, "original_architecture", None) + if original_architecture == "neel-solu-old": + state_dict = _convert_neel_solu_old_weights(state_dict, cfg) + elif original_architecture == "mingpt": + state_dict = _convert_mingpt_weights(state_dict, cfg) + return state_dict + + +def boot( + model_name: str, + checkpoint_index: Optional[int] = None, + checkpoint_value: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + dtype: torch.dtype = torch.float32, + tokenizer: Optional[Any] = None, +) -> TransformerBridge: + """Build a bridge for a legacy TransformerLens-format HF repo. + + ``checkpoint_index`` / ``checkpoint_value`` select a training checkpoint + (``checkpoints/*_<label>.pth``); by default the final weights load. The + resolved values are stamped on ``cfg.checkpoint_index`` / + ``cfg.checkpoint_value``, mirroring the legacy loader. + """ + if not model_name.startswith(TL_LEGACY_PREFIXES): + raise ValueError( + f"{model_name!r} is not a known legacy TransformerLens repo family " + f"{TL_LEGACY_PREFIXES}. Use TransformerBridge.boot_transformers for " + "HuggingFace-format models." + ) + cfg = derive_tl_legacy_config(model_name) + + if checkpoint_index is not None or checkpoint_value is not None: + from transformer_lens.tools.model_registry.checkpoints import ( + get_checkpoint_labels, + ) + + labels, _ = get_checkpoint_labels(model_name) + if checkpoint_value is None: + assert checkpoint_index is not None + # Negative indices count from the end (checkpoint_index=-1 is the + # final checkpoint), matching the legacy loader's list indexing. + if not -len(labels) <= checkpoint_index < len(labels): + raise ValueError( + f"checkpoint_index={checkpoint_index} out of range " + f"[-{len(labels)}, {len(labels)}) for {model_name!r}." + ) + checkpoint_value = labels[checkpoint_index] + elif checkpoint_value not in labels: + raise ValueError( + f"checkpoint_value={checkpoint_value} not in available checkpoints for " + f"{model_name!r} ({len(labels)} labels, {labels[0]}..{labels[-1]})." + ) + cfg.checkpoint_index = labels.index(checkpoint_value) # type: ignore[attr-defined] + cfg.checkpoint_value = checkpoint_value # type: ignore[attr-defined] + else: + cfg.checkpoint_index = None # type: ignore[attr-defined] + cfg.checkpoint_value = None # type: ignore[attr-defined] + + if tokenizer is None and cfg.tokenizer_name is not None: + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(cfg.tokenizer_name) + + from transformer_lens.model_bridge.sources.native import boot as _boot_native + + bridge = _boot_native( + cfg, tokenizer=tokenizer, device=device, dtype=dtype, model_name=model_name + ) + legacy_sd = fetch_tl_legacy_state_dict( + model_name, cfg, checkpoint_value=checkpoint_value, dtype=dtype + ) + converted = convert_tl_checkpoint(legacy_sd, cfg) + # Several legacy repos ship no unembed bias; the legacy loader zero-filled + # missing params, so mirror that for exactly this key rather than failing + # a strict load or silently accepting arbitrary gaps. + if "unembed.bias" not in converted: + converted["unembed.bias"] = torch.zeros(cfg.d_vocab, dtype=dtype) + result = bridge.load_state_dict(converted, strict=True) + assert not result.missing_keys and not result.unexpected_keys + return bridge + + +setattr(TransformerBridge, "boot_tl_legacy", staticmethod(boot)) diff --git a/transformer_lens/model_bridge/sources/transformers/helpers.py b/transformer_lens/model_bridge/sources/transformers/helpers.py index ce96b3ff18..4f95c337ce 100644 --- a/transformer_lens/model_bridge/sources/transformers/helpers.py +++ b/transformer_lens/model_bridge/sources/transformers/helpers.py @@ -172,10 +172,11 @@ def _resolve_checkpoint_to_revision( ) else: assert checkpoint_index is not None # narrowed by initial guard - if not 0 <= checkpoint_index < len(labels): + # Negative indices count from the end, matching the legacy loader. + if not -len(labels) <= checkpoint_index < len(labels): raise ValueError( - f"checkpoint_index={checkpoint_index} out of range [0, {len(labels)}) " - f"for {model_name!r}." + f"checkpoint_index={checkpoint_index} out of range " + f"[-{len(labels)}, {len(labels)}) for {model_name!r}." ) checkpoint_value = labels[checkpoint_index] return format_str.format(value=checkpoint_value) diff --git a/transformer_lens/model_bridge/sources/transformers/source.py b/transformer_lens/model_bridge/sources/transformers/source.py index c05952509d..18316f9b55 100644 --- a/transformer_lens/model_bridge/sources/transformers/source.py +++ b/transformer_lens/model_bridge/sources/transformers/source.py @@ -440,4 +440,19 @@ def boot( driver = TransformersDriver(hf_model, adapter, tokenizer) bridge = TransformerBridge(hf_model, adapter, tokenizer, driver=driver) load_modality_processor(bridge, adapter.cfg, model_name, trust_remote_code, _hf_token) + # Mirror the legacy loader: record which training checkpoint this is, so + # checkpoint sweeps (e.g. induction-head formation) can read it back. + if checkpoint_index is not None or checkpoint_value is not None: + from transformer_lens.tools.model_registry.checkpoints import ( + get_checkpoint_labels, + ) + + labels, _label_kind = get_checkpoint_labels(model_name) + if checkpoint_value is not None: + resolved_value = checkpoint_value + else: + assert checkpoint_index is not None # narrowed by the enclosing if + resolved_value = labels[checkpoint_index] + bridge.cfg.checkpoint_value = resolved_value # type: ignore[attr-defined] + bridge.cfg.checkpoint_index = labels.index(resolved_value) # type: ignore[attr-defined] return bridge diff --git a/transformer_lens/model_bridge/supported_architectures/__init__.py b/transformer_lens/model_bridge/supported_architectures/__init__.py index f2e7111d9c..770bb5a853 100644 --- a/transformer_lens/model_bridge/supported_architectures/__init__.py +++ b/transformer_lens/model_bridge/supported_architectures/__init__.py @@ -104,6 +104,9 @@ HrmTextArchitectureAdapter, ) from transformer_lens.model_bridge.supported_architectures.hubert import HubertArchitectureAdapter +from transformer_lens.model_bridge.supported_architectures.wav2vec2 import ( + Wav2Vec2ArchitectureAdapter, +) from transformer_lens.model_bridge.supported_architectures.hunyuan_v1_dense import ( HunYuanDenseV1ArchitectureAdapter, ) @@ -368,6 +371,7 @@ "Gpt2LmHeadCustomArchitectureAdapter", "GptjArchitectureAdapter", "HubertArchitectureAdapter", + "Wav2Vec2ArchitectureAdapter", "HyenaDNAArchitectureAdapter", "HunYuanDenseV1ArchitectureAdapter", "Idefics3ArchitectureAdapter", diff --git a/transformer_lens/model_bridge/supported_architectures/native.py b/transformer_lens/model_bridge/supported_architectures/native.py index aabdb98547..9cfbc7b4d3 100644 --- a/transformer_lens/model_bridge/supported_architectures/native.py +++ b/transformer_lens/model_bridge/supported_architectures/native.py @@ -21,6 +21,9 @@ RMSNormPreBridge, UnembeddingBridge, ) +from transformer_lens.model_bridge.generalized_components.base import ( + GeneralizedComponent, +) def _uses_rms(cfg: Any) -> bool: @@ -71,12 +74,17 @@ def _make_mlp_bridge(cfg: Any): "out": LinearBridge(name="out"), }, ) + submodules: dict[str, GeneralizedComponent] = { + "in": LinearBridge(name="fc_in"), + "out": LinearBridge(name="fc_out"), + } + if (cfg.act_fn or "").lower() == "solu_ln": + # SoLU-LN checkpoints carry a mid-MLP LayerNorm (NativeMLP.ln); expose + # it so its params load under blocks.{i}.mlp.ln.* and its hooks fire. + submodules["ln"] = NormalizationBridge(name="ln", config=cfg) return MLPBridge( name="mlp", - submodules={ - "in": LinearBridge(name="fc_in"), - "out": LinearBridge(name="fc_out"), - }, + submodules=submodules, ) diff --git a/transformer_lens/model_bridge/supported_architectures/wav2vec2.py b/transformer_lens/model_bridge/supported_architectures/wav2vec2.py new file mode 100644 index 0000000000..630946ea75 --- /dev/null +++ b/transformer_lens/model_bridge/supported_architectures/wav2vec2.py @@ -0,0 +1,39 @@ +"""Wav2Vec2 architecture adapter. + +Wav2Vec2Model's module tree is identical to HubertModel's (zero symmetric +difference over named_modules), so the HuBERT adapter applies wholesale; only +the ForCTC nesting attribute differs (``wav2vec2.`` vs ``hubert.``). +""" + +from typing import Any + +from transformer_lens.model_bridge.generalized_components import UnembeddingBridge +from transformer_lens.model_bridge.supported_architectures.hubert import ( + HubertArchitectureAdapter, +) + + +class Wav2Vec2ArchitectureAdapter(HubertArchitectureAdapter): + """Adapter for Wav2Vec2Model (bare encoder) and Wav2Vec2ForCTC.""" + + def prepare_model(self, hf_model: Any) -> None: + """Detect nesting under 'wav2vec2.' and add the CTC head when present. + + The registered "Wav2Vec2ForPreTraining" architecture string exists so + checkpoints that DECLARE that class (facebook/wav2vec2-base/-large) + boot their encoder via AutoModel -> Wav2Vec2Model. Wrapping the + pretraining class itself is refused: its forward returns a + Wav2Vec2ForPreTrainingOutput (projected quantizer states, no + last_hidden_state), which no bridge output contract fits, and the + quantizer head has no interpretability surface. + """ + if type(hf_model).__name__ == "Wav2Vec2ForPreTraining": + raise NotImplementedError( + "Wav2Vec2ForPreTraining cannot be wrapped directly — boot the " + "checkpoint without model_class to get its encoder " + "(Wav2Vec2Model), or use Wav2Vec2ForCTC for the CTC head." + ) + if hasattr(hf_model, "wav2vec2"): + self.component_mapping = self._build_component_mapping(prefix="wav2vec2.") + if hasattr(hf_model, "lm_head"): + self.component_mapping["unembed"] = UnembeddingBridge(name="lm_head") diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index 531abb9ce8..caefeebf12 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -884,7 +884,14 @@ def encoder_output( ) resid = frames + self.conv_pos_embed(frames) - resid = self.embed_ln(resid) + # Post-LN encoders (do_stable_layer_norm=False: hubert-base, + # wav2vec2-base) normalize before the blocks; stable-LN encoders + # (wav2vec2-large and kin) apply this same module AFTER the blocks — + # HF's Wav2Vec2EncoderStableLayerNorm.forward. Mirror the real order or + # every stable-LN activation is silently wrong. + stable_ln = self._audio_encoder_is_stable_layer_norm() + if not stable_ln: + resid = self.embed_ln(resid) additive_attention_mask = None if one_zero_attention_mask is not None: @@ -898,8 +905,27 @@ def encoder_output( for block in self.blocks: output = block(resid, attention_mask=additive_attention_mask) resid = output[0] if isinstance(output, tuple) else output + if stable_ln: + resid = self.embed_ln(resid) return resid + def _audio_encoder_is_stable_layer_norm(self) -> bool: + """Whether the wrapped audio encoder is the pre-LN ("stable") variant. + + Structural, not config-driven: build_bridge_from_module never runs the + adapter's prepare_loading, so cfg.do_stable_layer_norm can be absent + on that path while the module class is authoritative on both. + """ + model = self.original_model + for attr in ("encoder", "wav2vec2", "hubert", "model"): + candidate = getattr(model, attr, None) + if candidate is None: + continue + encoder = candidate if attr == "encoder" else getattr(candidate, "encoder", None) + if encoder is not None: + return type(encoder).__name__.endswith("StableLayerNorm") + return bool(getattr(self.cfg, "do_stable_layer_norm", False)) + def _require_frame_entry_support(self) -> None: """Reject models with no conv-frame stage to re-enter.""" if not getattr(self.cfg, "is_audio_model", False): @@ -1421,6 +1447,52 @@ def W_E(self) -> torch.Tensor: """Token embedding matrix (d_vocab, d_model).""" return self.embed.W_E + @property + def W_pos(self) -> torch.Tensor: + """Positional embedding matrix (n_ctx, d_model). + + Only defined for models with learned absolute positional embeddings; + rotary/ALiBi models have no such matrix. Reflects the weights as + currently processed, like every other accessor — equal to + ``HookedTransformer.W_pos`` only under matching processing + (``from_pretrained_no_processing`` vs the bridge default, or + compatibility mode vs HT defaults). + """ + pos_embed = getattr(self, "pos_embed", None) + pos_type = getattr(self.cfg, "positional_embedding_type", "unknown") + if pos_embed is None or not hasattr(pos_embed, "W_pos"): + raise AttributeError( + "W_pos is only defined for models with a learned absolute positional " + f"embedding component; this model exposes none " + f"(positional_embedding_type={pos_type!r})." + ) + w = pos_embed.W_pos + # T5-family adapters map pos_embed to the relative attention bias — a + # [num_buckets, n_heads] table, not a positional matrix. The legacy + # accessor refused these models; keep refusing rather than hand back a + # wrong-shaped tensor. + if w.ndim != 2 or w.shape[-1] != self.cfg.d_model: + raise AttributeError( + "W_pos is only defined for learned absolute positional embeddings; " + f"this model's pos_embed holds a {tuple(w.shape)} table " + f"(positional_embedding_type={pos_type!r})." + ) + # OPT/BART-family checkpoints allocate max_position_embeddings + 2 rows + # (positions are offset by 2 in HF's forward). HookedTransformer's + # converters slice those rows off; mirror that so shapes and values agree. + if w.shape[0] == self.cfg.n_ctx + 2: + return w[2:] + return w + + @property + def W_E_pos(self) -> torch.Tensor: + """Concatenated ``[W_E; W_pos]`` (d_vocab + n_ctx, d_model). + + A full (overcomplete) basis of the input space, used for full QK/OV + circuits. Mirrors ``HookedTransformer.W_E_pos``. + """ + return torch.cat([self.W_E, self.W_pos], dim=0) + @property def QK(self): """QK circuit. On hybrids, returns attn layers only (with warning). See QK_for_attn_layers().""" diff --git a/transformer_lens/model_protocol.py b/transformer_lens/model_protocol.py index 1ebb7b8292..bdc66c3879 100644 --- a/transformer_lens/model_protocol.py +++ b/transformer_lens/model_protocol.py @@ -41,6 +41,31 @@ def to_tokens(self, *args: Any, **kwargs: Any) -> Any: ... +@runtime_checkable +class TrainableTransformerLensModel(Protocol): + """Exactly the surface the ``tools.training`` loop touches: callable with + ``return_type="loss"`` plus the standard torch parameter/mode/device + methods. Deliberately standalone (not extending TransformerLensModel): + beartype validates protocols via ``getattr_static``, and demanding the + full TL surface would spuriously reject plain ``nn.Module`` models that + train fine through this loop.""" + + def parameters(self, *args: Any, **kwargs: Any) -> Any: + ... + + def train(self, *args: Any, **kwargs: Any) -> Any: + ... + + def to(self, *args: Any, **kwargs: Any) -> Any: + ... + + def state_dict(self, *args: Any, **kwargs: Any) -> Any: + ... + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + ... + + @runtime_checkable class TransformerLensModelWithWeights(TransformerLensModel, Protocol): """Adds the weight-processing surface that ``ActivationCache``'s advanced helpers diff --git a/transformer_lens/tools/model_registry/__init__.py b/transformer_lens/tools/model_registry/__init__.py index 36b70c24e0..d0a4c3c993 100644 --- a/transformer_lens/tools/model_registry/__init__.py +++ b/transformer_lens/tools/model_registry/__init__.py @@ -121,6 +121,9 @@ "GPTNeoXForCausalLM", "HubertForCTC", "HubertModel", + "Wav2Vec2ForCTC", + "Wav2Vec2ForPreTraining", + "Wav2Vec2Model", "HrmTextForCausalLM", "HunYuanDenseV1ForCausalLM", "Idefics3ForConditionalGeneration", @@ -280,6 +283,9 @@ "GraniteMoeHybridForCausalLM": ["ibm-granite"], "HubertForCTC": ["facebook"], "HubertModel": ["facebook"], + "Wav2Vec2ForCTC": ["facebook"], + "Wav2Vec2ForPreTraining": ["facebook"], + "Wav2Vec2Model": ["facebook"], "HrmTextForCausalLM": ["sapientinc"], "HunYuanDenseV1ForCausalLM": ["tencent"], "Idefics3ForConditionalGeneration": ["HuggingFaceM4", "ibm-granite"], diff --git a/transformer_lens/tools/model_registry/checkpoints.py b/transformer_lens/tools/model_registry/checkpoints.py index fbcab44d9f..020d8a7fce 100644 --- a/transformer_lens/tools/model_registry/checkpoints.py +++ b/transformer_lens/tools/model_registry/checkpoints.py @@ -42,4 +42,19 @@ def get_checkpoint_labels(model_name: str) -> tuple[list[int], str]: "Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models." ) return PYTHIA_CHECKPOINTS, "step" + if official_name.startswith(("NeelNanda/", "ArthurConmy/", "Baidicoot/")): + # Legacy TransformerLens repos store checkpoints as files, not + # revisions: checkpoints/<name>_<step-or-tokens>.pth. + import re + + from huggingface_hub import HfApi + + labels = sorted( + int(m.group(1)) + for f in HfApi().list_repo_files(official_name) + if (m := re.match(r"checkpoints/.*_(\d+)\.pth", f)) + ) + if not labels: + raise ValueError(f"Model {official_name} is not checkpointed.") + return labels, ("token" if labels[-1] > 1e9 else "step") raise ValueError(f"Model {official_name} is not checkpointed.") diff --git a/transformer_lens/tools/training.py b/transformer_lens/tools/training.py index 8bb97f7434..4bbef82040 100644 --- a/transformer_lens/tools/training.py +++ b/transformer_lens/tools/training.py @@ -1,8 +1,9 @@ """Train loop for TransformerLens models. Utilities for training models on autoregressive language modeling tasks. -Typed against nn.Module so both HookedTransformer and TransformerBridge -work through this loop. +Typed against the ``model_protocol`` surface (``__call__`` with +``return_type="loss"`` plus standard ``nn.Module`` parameter access), so any +conforming model — ``TransformerBridge`` foremost — works through this loop. """ import dataclasses @@ -10,13 +11,13 @@ from typing import Optional, Union import torch -import torch.nn as nn import torch.optim as optim from torch.optim import Optimizer from torch.utils.data import DataLoader, Dataset from tqdm.auto import tqdm from transformer_lens import utilities as utils +from transformer_lens.model_protocol import TrainableTransformerLensModel from transformer_lens.utilities.library_utils import is_library_available @@ -62,14 +63,15 @@ class TrainConfig: def train( - model: nn.Module, + model: TrainableTransformerLensModel, config: TrainConfig, dataset: Dataset, -) -> nn.Module: +) -> TrainableTransformerLensModel: """Train a model on an autoregressive language modeling task. Args: - model: The model to train (nn.Module with __call__(tokens, return_type="loss")) + model: The model to train (TrainableTransformerLensModel: callable with + ``return_type="loss"`` and exposing torch parameters) config: The training configuration dataset: The dataset to train on - assumed set up for autoregressive language modeling. diff --git a/transformer_lens/train.py b/transformer_lens/train.py index c67653b748..46b1b16593 100644 --- a/transformer_lens/train.py +++ b/transformer_lens/train.py @@ -5,9 +5,9 @@ import warnings -import torch.nn as nn from torch.utils.data import Dataset +from transformer_lens.model_protocol import TrainableTransformerLensModel from transformer_lens.tools.training import TrainConfig as _TrainConfig from transformer_lens.tools.training import train as _train @@ -25,7 +25,9 @@ def __init__(self, *args: object, **kwargs: object) -> None: super().__init__(*args, **kwargs) # type: ignore[arg-type] -def train(model: nn.Module, config: _TrainConfig, dataset: Dataset) -> nn.Module: +def train( + model: "TrainableTransformerLensModel", config: _TrainConfig, dataset: Dataset +) -> "TrainableTransformerLensModel": """Deprecated. Use transformer_lens.tools.training.train instead.""" warnings.warn( "transformer_lens.train is deprecated; use " diff --git a/transformer_lens/utilities/architectures.py b/transformer_lens/utilities/architectures.py index 8d0f19a1fa..e36bc4c1e8 100644 --- a/transformer_lens/utilities/architectures.py +++ b/transformer_lens/utilities/architectures.py @@ -82,6 +82,11 @@ "HubertForCTC", "HubertModel", "HubertForSequenceClassification", + "Wav2Vec2ForCTC", + "Wav2Vec2Model", + # Pretraining checkpoints (facebook/wav2vec2-base/-large declare this class) + # load their encoder via AutoModel -> Wav2Vec2Model. + "Wav2Vec2ForPreTraining", } | AUDIO_CLASSIFICATION_ARCHITECTURES # Vision-only (non-multimodal, no text tower) encoder models. Split into the From ffc629caed24516a3338200cf9e2086f2ade7f5e Mon Sep 17 00:00:00 2001 From: Jonah Larson <jonahalarson@comcast.net> Date: Tue, 8 Sep 2026 20:10:48 -0500 Subject: [PATCH 67/87] Hooked Class Deprecation (#1759) * HookedTransformer Deprecation * Additional hooked deprecation prep --- .github/copilot-instructions.md | 2 +- AGENTS.md | 14 +- README.md | 2 +- debugging/hf-tl-logit-comparator.ipynb | 5 +- demos/Colab_Compatibility.ipynb | 2 +- demos/Grokking_Demo.ipynb | 13 +- demos/LIT_Integration_Demo.ipynb | 6 +- demos/Main_Demo.ipynb | 2 +- demos/No_Position_Experiment.ipynb | 13 +- demos/doc_sanitize.cfg | 8 - docs/make_docs.py | 717 +---- docs/source/conf.py | 27 +- docs/source/content/contributing.md | 4 +- .../hooked_transformer_model_properties.md | 264 ++ docs/source/content/model_tables.md | 8 +- docs/source/index.md | 2 + pyproject.toml | 15 - scripts/capture_ht_goldens.py | 361 --- scripts/capture_tl_checkpoint_fixtures.py | 66 - tests/QUARANTINES.md | 15 +- .../compatibility/test_hook_completeness.py | 2 +- tests/acceptance/test_hooked_encoder.py | 229 -- .../acceptance/test_hooked_encoder_decoder.py | 463 --- tests/goldens.py | 5 +- tests/goldens_capture_spec.py | 93 + .../test_refactor_factored_attn_matrices.py | 2 +- .../model_bridge/test_weight_processing.py | 4 +- .../integration/test_create_hooked_encoder.py | 33 - .../test_grouped_query_attention.py | 192 -- .../integration/test_head_detector_bridge.py | 31 + .../test_hooked_encoder_properties.py | 170 - .../test_loading_from_pretrained.py | 18 - .../integration/test_model_protocol_bridge.py | 41 + tests/unit/components/mlps/test_gated_mlp.py | 81 - tests/unit/components/mlps/test_mlp.py | 49 - tests/unit/components/mlps/test_moe.py | 21 - .../components/test_abstract_attention.py | 234 -- .../test_transformer_block_olmo_post_norm.py | 164 - tests/unit/factored_matrix/test_properties.py | 3 +- .../test_loading_from_pretrained_utilities.py | 54 - .../test_next_sentence_prediction.py | 213 -- .../test_gemma3_config.py | 321 -- .../test_apertus.py | 197 -- .../test_gemma.py | 358 --- .../test_hubert_weights.py | 155 - .../test_mixtral_conversion.py | 229 -- .../test_olmo3.py | 411 --- .../test_olmoe.py | 144 - .../test_openai.py | 332 -- tests/unit/test_deprecation_warnings.py | 125 +- .../test_doctest_no_hooked_transformer.py | 27 +- tests/unit/test_dropped_inference_ops.py | 347 -- tests/unit/test_goldens_infra.py | 8 +- tests/unit/test_hubert.py | 158 - tests/unit/test_hubert_hooks.py | 109 - .../test_loading_from_pretrained_utilities.py | 525 --- tests/unit/test_make_docs.py | 51 - tests/unit/test_moe_expert_fold.py | 100 + tests/unit/test_moe_fold_guard.py | 104 - tests/unit/test_next_sentence_prediction.py | 296 -- .../unit/test_post_norm_processing_guards.py | 144 +- tests/unit/test_supported_models.py | 21 +- tests/unit/test_train_config_isolation.py | 37 - tests/unit/tools/test_model_registry.py | 31 - transformer_lens/ActivationCache.py | 2 +- .../BertNextSentencePrediction.py | 280 -- transformer_lens/HookedAudioEncoder.py | 537 ---- transformer_lens/HookedEncoder.py | 572 ---- transformer_lens/HookedEncoderDecoder.py | 801 ----- transformer_lens/HookedTransformer.py | 2835 ----------------- transformer_lens/__init__.py | 109 +- transformer_lens/benchmarks/README.md | 25 +- transformer_lens/cache/key_value_cache.py | 13 +- transformer_lens/components/__init__.py | 34 - .../components/abstract_attention.py | 1050 ------ transformer_lens/components/attention.py | 59 - transformer_lens/components/bert_block.py | 78 - transformer_lens/components/bert_embed.py | 53 - transformer_lens/components/bert_mlm_head.py | 35 - transformer_lens/components/bert_nsp_head.py | 32 - transformer_lens/components/bert_pooler.py | 37 - transformer_lens/components/embed.py | 35 - .../components/grouped_query_attention.py | 204 -- transformer_lens/components/layer_norm.py | 57 - transformer_lens/components/layer_norm_pre.py | 54 - .../components/mlps/can_be_used_as_mlp.py | 82 - transformer_lens/components/mlps/gated_mlp.py | 76 - .../components/mlps/gated_mlp_4bit.py | 84 - .../components/mlps/gpt_oss_moe.py | 126 - transformer_lens/components/mlps/mlp.py | 49 - transformer_lens/components/mlps/moe.py | 114 - transformer_lens/components/pos_embed.py | 70 - transformer_lens/components/rms_norm.py | 55 - transformer_lens/components/rms_norm_pre.py | 36 - transformer_lens/components/t5_attention.py | 148 - transformer_lens/components/t5_block.py | 158 - .../components/token_typed_embed.py | 28 - .../components/transformer_block.py | 236 -- transformer_lens/components/unembed.py | 40 - transformer_lens/config/AGENTS.md | 3 +- transformer_lens/config/__init__.py | 4 +- .../config/hooked_transformer_config.py | 442 --- .../conversion_utils/hook_conversion_utils.py | 14 - transformer_lens/factories/mlp_factory.py | 25 - transformer_lens/hook_points.py | 21 +- transformer_lens/lit/README.md | 10 +- transformer_lens/lit/__init__.py | 1 - transformer_lens/loading_from_pretrained.py | 2316 -------------- .../model_bridge/remote_bridge.py | 2 +- .../sources/transformers/source.py | 2 +- transformer_lens/pretrained/__init__.py | 1 - .../pretrained/weight_conversions/__init__.py | 28 - .../pretrained/weight_conversions/apertus.py | 134 - .../pretrained/weight_conversions/bert.py | 75 - .../pretrained/weight_conversions/bloom.py | 57 - .../pretrained/weight_conversions/coder.py | 63 - .../pretrained/weight_conversions/gemma.py | 139 - .../pretrained/weight_conversions/gpt2.py | 61 - .../pretrained/weight_conversions/gptj.py | 50 - .../pretrained/weight_conversions/hubert.py | 145 - .../pretrained/weight_conversions/llama.py | 96 - .../pretrained/weight_conversions/mingpt.py | 63 - .../pretrained/weight_conversions/mistral.py | 57 - .../pretrained/weight_conversions/mixtral.py | 87 - .../pretrained/weight_conversions/nanogpt.py | 108 - .../weight_conversions/neel_solu_old.py | 38 - .../pretrained/weight_conversions/neo.py | 56 - .../pretrained/weight_conversions/neox.py | 59 - .../pretrained/weight_conversions/olmo.py | 50 - .../pretrained/weight_conversions/olmo2.py | 57 - .../pretrained/weight_conversions/olmo3.py | 91 - .../pretrained/weight_conversions/olmoe.py | 81 - .../pretrained/weight_conversions/openai.py | 155 - .../pretrained/weight_conversions/opt.py | 84 - .../pretrained/weight_conversions/phi.py | 64 - .../pretrained/weight_conversions/phi3.py | 78 - .../pretrained/weight_conversions/qwen.py | 65 - .../pretrained/weight_conversions/qwen2.py | 76 - .../pretrained/weight_conversions/qwen3.py | 69 - .../pretrained/weight_conversions/t5.py | 101 - transformer_lens/supported_models.py | 26 + transformer_lens/train.py | 38 - transformer_lens/utilities/multi_gpu.py | 15 +- transformer_lens/utils.py | 64 - uv.lock | 22 - 145 files changed, 759 insertions(+), 20616 deletions(-) create mode 100644 docs/source/content/hooked_transformer_model_properties.md delete mode 100644 scripts/capture_ht_goldens.py delete mode 100644 scripts/capture_tl_checkpoint_fixtures.py delete mode 100644 tests/acceptance/test_hooked_encoder.py delete mode 100644 tests/acceptance/test_hooked_encoder_decoder.py create mode 100644 tests/goldens_capture_spec.py delete mode 100644 tests/integration/test_create_hooked_encoder.py delete mode 100644 tests/integration/test_grouped_query_attention.py create mode 100644 tests/integration/test_head_detector_bridge.py delete mode 100644 tests/integration/test_hooked_encoder_properties.py delete mode 100644 tests/integration/test_loading_from_pretrained.py create mode 100644 tests/integration/test_model_protocol_bridge.py delete mode 100644 tests/unit/components/mlps/test_gated_mlp.py delete mode 100644 tests/unit/components/mlps/test_mlp.py delete mode 100644 tests/unit/components/mlps/test_moe.py delete mode 100644 tests/unit/components/test_abstract_attention.py delete mode 100644 tests/unit/components/test_transformer_block_olmo_post_norm.py delete mode 100644 tests/unit/model_bridge/compatibility/test_loading_from_pretrained_utilities.py delete mode 100644 tests/unit/model_bridge/compatibility/test_next_sentence_prediction.py delete mode 100644 tests/unit/model_bridge/supported_architectures/test_gemma3_config.py delete mode 100644 tests/unit/pretrained_weight_conversions/test_apertus.py delete mode 100644 tests/unit/pretrained_weight_conversions/test_gemma.py delete mode 100644 tests/unit/pretrained_weight_conversions/test_hubert_weights.py delete mode 100644 tests/unit/pretrained_weight_conversions/test_mixtral_conversion.py delete mode 100644 tests/unit/pretrained_weight_conversions/test_olmo3.py delete mode 100644 tests/unit/pretrained_weight_conversions/test_olmoe.py delete mode 100644 tests/unit/pretrained_weight_conversions/test_openai.py delete mode 100644 tests/unit/test_dropped_inference_ops.py delete mode 100644 tests/unit/test_hubert.py delete mode 100644 tests/unit/test_hubert_hooks.py delete mode 100644 tests/unit/test_loading_from_pretrained_utilities.py create mode 100644 tests/unit/test_moe_expert_fold.py delete mode 100644 tests/unit/test_moe_fold_guard.py delete mode 100644 tests/unit/test_next_sentence_prediction.py delete mode 100644 tests/unit/test_train_config_isolation.py delete mode 100644 transformer_lens/BertNextSentencePrediction.py delete mode 100644 transformer_lens/HookedAudioEncoder.py delete mode 100644 transformer_lens/HookedEncoder.py delete mode 100644 transformer_lens/HookedEncoderDecoder.py delete mode 100644 transformer_lens/HookedTransformer.py delete mode 100644 transformer_lens/components/__init__.py delete mode 100644 transformer_lens/components/abstract_attention.py delete mode 100644 transformer_lens/components/attention.py delete mode 100644 transformer_lens/components/bert_block.py delete mode 100644 transformer_lens/components/bert_embed.py delete mode 100644 transformer_lens/components/bert_mlm_head.py delete mode 100644 transformer_lens/components/bert_nsp_head.py delete mode 100644 transformer_lens/components/bert_pooler.py delete mode 100644 transformer_lens/components/embed.py delete mode 100644 transformer_lens/components/grouped_query_attention.py delete mode 100644 transformer_lens/components/layer_norm.py delete mode 100644 transformer_lens/components/layer_norm_pre.py delete mode 100644 transformer_lens/components/mlps/can_be_used_as_mlp.py delete mode 100644 transformer_lens/components/mlps/gated_mlp.py delete mode 100644 transformer_lens/components/mlps/gated_mlp_4bit.py delete mode 100644 transformer_lens/components/mlps/gpt_oss_moe.py delete mode 100644 transformer_lens/components/mlps/mlp.py delete mode 100644 transformer_lens/components/mlps/moe.py delete mode 100644 transformer_lens/components/pos_embed.py delete mode 100644 transformer_lens/components/rms_norm.py delete mode 100644 transformer_lens/components/rms_norm_pre.py delete mode 100644 transformer_lens/components/t5_attention.py delete mode 100644 transformer_lens/components/t5_block.py delete mode 100644 transformer_lens/components/token_typed_embed.py delete mode 100644 transformer_lens/components/transformer_block.py delete mode 100644 transformer_lens/components/unembed.py delete mode 100644 transformer_lens/config/hooked_transformer_config.py delete mode 100644 transformer_lens/factories/mlp_factory.py delete mode 100644 transformer_lens/loading_from_pretrained.py delete mode 100644 transformer_lens/pretrained/__init__.py delete mode 100644 transformer_lens/pretrained/weight_conversions/__init__.py delete mode 100644 transformer_lens/pretrained/weight_conversions/apertus.py delete mode 100644 transformer_lens/pretrained/weight_conversions/bert.py delete mode 100644 transformer_lens/pretrained/weight_conversions/bloom.py delete mode 100644 transformer_lens/pretrained/weight_conversions/coder.py delete mode 100644 transformer_lens/pretrained/weight_conversions/gemma.py delete mode 100644 transformer_lens/pretrained/weight_conversions/gpt2.py delete mode 100644 transformer_lens/pretrained/weight_conversions/gptj.py delete mode 100644 transformer_lens/pretrained/weight_conversions/hubert.py delete mode 100644 transformer_lens/pretrained/weight_conversions/llama.py delete mode 100644 transformer_lens/pretrained/weight_conversions/mingpt.py delete mode 100644 transformer_lens/pretrained/weight_conversions/mistral.py delete mode 100644 transformer_lens/pretrained/weight_conversions/mixtral.py delete mode 100644 transformer_lens/pretrained/weight_conversions/nanogpt.py delete mode 100644 transformer_lens/pretrained/weight_conversions/neel_solu_old.py delete mode 100644 transformer_lens/pretrained/weight_conversions/neo.py delete mode 100644 transformer_lens/pretrained/weight_conversions/neox.py delete mode 100644 transformer_lens/pretrained/weight_conversions/olmo.py delete mode 100644 transformer_lens/pretrained/weight_conversions/olmo2.py delete mode 100644 transformer_lens/pretrained/weight_conversions/olmo3.py delete mode 100644 transformer_lens/pretrained/weight_conversions/olmoe.py delete mode 100644 transformer_lens/pretrained/weight_conversions/openai.py delete mode 100644 transformer_lens/pretrained/weight_conversions/opt.py delete mode 100644 transformer_lens/pretrained/weight_conversions/phi.py delete mode 100644 transformer_lens/pretrained/weight_conversions/phi3.py delete mode 100644 transformer_lens/pretrained/weight_conversions/qwen.py delete mode 100644 transformer_lens/pretrained/weight_conversions/qwen2.py delete mode 100644 transformer_lens/pretrained/weight_conversions/qwen3.py delete mode 100644 transformer_lens/pretrained/weight_conversions/t5.py delete mode 100644 transformer_lens/train.py delete mode 100644 transformer_lens/utils.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index bcb46e53e0..adef9b5aee 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -5,7 +5,7 @@ ## Top rules to remember 1. **Use `uv`, not `pip` or `poetry`.** `uv sync` to install; `uv run <cmd>` or a `make` target to run anything. -2. **Mirror `HookedTransformer` → `TransformerBridge`** in the same PR when behaviour exists in both. The HT registry [`transformer_lens/supported_models.py`](../transformer_lens/supported_models.py) is HT-only — Bridge-only models go in the Bridge registry under [`transformer_lens/tools/model_registry/`](../transformer_lens/tools/model_registry/). +2. **The `Hooked*` model classes were removed in 4.0**; `TransformerBridge` is the one model system. [`transformer_lens/supported_models.py`](../transformer_lens/supported_models.py) is the frozen legacy name/alias ledger; Bridge models live under [`transformer_lens/tools/model_registry/`](../transformer_lens/tools/model_registry/). 3. **Base PRs against `dev`**, not `main`. PRs to `main` are maintainer-only. ## Common commands diff --git a/AGENTS.md b/AGENTS.md index b67d87d796..544697ed52 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,7 @@ This file is the single source of truth. Vendor-specific files ([CLAUDE.md](CLAU 1. **Use `uv`**, not `pip` or `poetry` (`uv sync`). 2. **Source `.env`** (`set -a; source .env; set +a`) before any HF-Hub command. 3. **Base PRs on `dev`**, not `main`. Never name a branch `main` or `dev`. -4. **Mirror HookedTransformer → TransformerBridge** when behaviour exists in both ([§2](#2-two-systems-live-in-this-repo)). +4. **The `Hooked*` model classes were removed in 4.0**; `HookedRootModule` / `HookPoint` are kept ([§2](#2-two-systems-live-in-this-repo)). 5. **`make format` + `uv run mypy .` before push** — no pre-commit hook. 6. **Never add `# type: ignore`** ([§10](#10-hard-rules)). 7. **Never dismiss a failing test as "pre-existing"** ([§10](#10-hard-rules)). @@ -29,13 +29,13 @@ Sub-folder rules: [tests/AGENTS.md](tests/AGENTS.md) · [supported_architectures | System | Status | Lives in | Numerics | Registry | |---|---|---|---|---| | **`TransformerBridge`** | v3 — default for new work | [transformer_lens/model_bridge/](transformer_lens/model_bridge/) | Raw HF weights by default; `bridge.enable_compatibility_mode()` for HT-equivalent | [transformer_lens/tools/model_registry/data/supported_models.json](transformer_lens/tools/model_registry/data/supported_models.json) | -| **`HookedTransformer`** | Legacy, maintenance mode, deprecated in 3.0 | [transformer_lens/HookedTransformer.py](transformer_lens/HookedTransformer.py) + [transformer_lens/components/](transformer_lens/components/) | Folds LayerNorm + centres weights → does NOT match HF | [transformer_lens/supported_models.py](transformer_lens/supported_models.py) (**HT-only**) | +| **`HookedTransformer`** | **Removed in 4.0** — use `TransformerBridge` + `enable_compatibility_mode()` | *(deleted; see [migrating_to_v4.md](docs/source/content/migrating_to_v4.md))* | — | — | -> ⚠ `HookedTransformer` has no acceptance suite of its own — `test_hooked_transformer.py` was removed by the reanchoring in #1603, which moved Bridge tests onto frozen goldens. Changes to `HookedTransformer` itself land untested at the acceptance level; extra manual care required. The `HookedEncoder` (BERT) and `HookedEncoderDecoder` (T5) acceptance suites do cover those two classes. See [QUARANTINES.md](tests/QUARANTINES.md) for the remaining skips. +> ⚠ `HookedRootModule` + `HookPoint` survive 4.0 as the supported way to hook an arbitrary `nn.Module`. `supported_models.py` is kept as the frozen legacy name/alias ledger (hosts the case-insensitive `get_official_model_name`). See [QUARANTINES.md](tests/QUARANTINES.md) for test skips. Bridge architecture-adapter pattern: each HF architecture has one file in [supported_architectures/](transformer_lens/model_bridge/supported_architectures/) mapping HF module paths to canonical names. Bridge hooks are architecture-native (e.g. `blocks.{i}.hook_out`); HT-style aliases live in [bridge_core.py](transformer_lens/model_bridge/bridge_core.py). -**Mirroring rule:** if you change `HookedTransformer` behaviour that has a `TransformerBridge` counterpart, update both in the same PR. [supported_models.py](transformer_lens/supported_models.py) is HT-only — Bridge-only models go in the Bridge registry data file. +There is now one model system: `TransformerBridge`. [supported_models.py](transformer_lens/supported_models.py) holds the frozen legacy name/alias ledger; Bridge models live in the Bridge registry data file. ## 3. Quickstart @@ -81,14 +81,12 @@ Python: **>=3.10, <4.0**. CI tests 3.10, 3.11, 3.12. Format/type/docstring check | Path | What's there | |---|---| | [transformer_lens/](transformer_lens/) | Core package | -| [transformer_lens/HookedTransformer.py](transformer_lens/HookedTransformer.py) | Legacy `HookedTransformer` API | | [transformer_lens/HookedEncoder.py](transformer_lens/HookedEncoder.py), [HookedEncoderDecoder.py](transformer_lens/HookedEncoderDecoder.py), [HookedAudioEncoder.py](transformer_lens/HookedAudioEncoder.py) | Encoder-only / seq2seq / audio variants | | [transformer_lens/model_bridge/](transformer_lens/model_bridge/) | `TransformerBridge` system | | [transformer_lens/model_bridge/supported_architectures/](transformer_lens/model_bridge/supported_architectures/) | One adapter file per HF architecture | | [transformer_lens/model_bridge/generalized_components/](transformer_lens/model_bridge/generalized_components/) | Bridge-side reusable components | -| [transformer_lens/components/](transformer_lens/components/) | HT-side components (attention, MLP, LN, embed) | | [transformer_lens/factories/](transformer_lens/factories/) | `architecture_adapter_factory.py`, `mlp_factory.py`, `activation_function_factory.py` | -| [transformer_lens/config/](transformer_lens/config/) | `HookedTransformerConfig` and `TransformerBridgeConfig` | +| [transformer_lens/config/](transformer_lens/config/) | `TransformerBridgeConfig` / `TransformerLensConfig` | | [transformer_lens/utilities/](transformer_lens/utilities/) | Device management, weight processing, HF utilities | | [transformer_lens/hook_points.py](transformer_lens/hook_points.py) | `HookPoint` class and `LensHandle` | | [transformer_lens/supported_models.py](transformer_lens/supported_models.py) | **HT-only** registry (`OFFICIAL_MODEL_NAMES`, `MODEL_ALIASES`) | @@ -197,7 +195,7 @@ Claude Code: `/task-complete` automates the last row. See [§15 Workflow shortcu ## 12. Pointers for further reading -- [docs/source/content/migrating_to_v3.md](docs/source/content/migrating_to_v3.md) — HT → Bridge migration recipes +- [docs/source/content/migrating_to_v4.md](docs/source/content/migrating_to_v4.md) — 4.0 removed-name → Bridge mapping ([v3 guide](docs/source/content/migrating_to_v3.md) for deeper API recipes) - [docs/source/content/adapter_development/](docs/source/content/adapter_development/) — adapter authoring deep dive - [docs/source/content/compatibility_mode.md](docs/source/content/compatibility_mode.md) — when to call `bridge.enable_compatibility_mode()`, what each flag does, four-quadrant test matrix - [docs/source/content/debugging_numerical_divergence.md](docs/source/content/debugging_numerical_divergence.md) — bisection workflow for HT-vs-Bridge / Bridge-vs-HF logit drift diff --git a/README.md b/README.md index 8cd90228f7..6708beec76 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ logits, activations = bridge.run_with_cache("Hello World") > Gated models (Llama, Mistral, Gemma, ...) require `HF_TOKEN` in your environment. See [Environment Variables](https://TransformerLensOrg.github.io/TransformerLens/content/getting_started.html#environment-variables) for the full list. -`TransformerBridge` is the recommended path and supports 15,000+ models across 140+ architecture families (see [`supported_models.json`](transformer_lens/tools/model_registry/data/supported_models.json) for the full inventory). By default it preserves raw HuggingFace weights – logits and activations match HF, *not* legacy `HookedTransformer` (which folds LayerNorm and centers weights by default). Call `bridge.enable_compatibility_mode()` after booting for HookedTransformer-equivalent numerics. The legacy `HookedTransformer.from_pretrained` API is still available but deprecated — see the [Migrating to TransformerLens 3](https://TransformerLensOrg.github.io/TransformerLens/content/migrating_to_v3.html) guide. +`TransformerBridge` is the recommended path and supports 15,000+ models across 140+ architecture families (see [`supported_models.json`](transformer_lens/tools/model_registry/data/supported_models.json) for the full inventory). By default it preserves raw HuggingFace weights – logits and activations match HF, *not* legacy `HookedTransformer` (which folds LayerNorm and centers weights by default). Call `bridge.enable_compatibility_mode()` after booting for HookedTransformer-equivalent numerics. The legacy `HookedTransformer.from_pretrained` API was removed in TransformerLens 4.0 — see the [Migrating to TransformerLens 4.0](https://TransformerLensOrg.github.io/TransformerLens/content/migrating_to_v4.html) guide. ## Key Tutorials diff --git a/debugging/hf-tl-logit-comparator.ipynb b/debugging/hf-tl-logit-comparator.ipynb index 99b2b39622..bdccee2a6f 100644 --- a/debugging/hf-tl-logit-comparator.ipynb +++ b/debugging/hf-tl-logit-comparator.ipynb @@ -15,7 +15,7 @@ "outputs": [], "source": [ "from transformers import AutoTokenizer, AutoModelForCausalLM\n", - "from transformer_lens import HookedTransformer\n", + "from transformer_lens.model_bridge import TransformerBridge\n", "import torch\n", "import torch.nn.functional as F\n", "\n", @@ -111,7 +111,8 @@ "metadata": {}, "outputs": [], "source": [ - "model = HookedTransformer.from_pretrained_no_processing(model_name, device=device)\n", + "# Unprocessed bridge load (the modern equivalent of from_pretrained_no_processing)\n", + "model = TransformerBridge.boot_transformers(model_name, device=device)\n", "tokens = model.to_tokens(sentence, prepend_bos=False)\n", "tl_logits = model(tokens)[:, -1, :]" ] diff --git a/demos/Colab_Compatibility.ipynb b/demos/Colab_Compatibility.ipynb index 9b3381cfa7..28a9edeaa8 100644 --- a/demos/Colab_Compatibility.ipynb +++ b/demos/Colab_Compatibility.ipynb @@ -72,7 +72,7 @@ "outputs": [], "source": [ "import torch\n", - "from transformer_lens import loading\n", + "from transformer_lens import supported_models as loading\n", "from transformer_lens.model_bridge import TransformerBridge\n", "from transformers import AutoTokenizer, BertForNextSentencePrediction, LlamaForCausalLM, LlamaTokenizer\n", "from typing import List\n", diff --git a/demos/Grokking_Demo.ipynb b/demos/Grokking_Demo.ipynb index bab49a9899..899436b9d8 100644 --- a/demos/Grokking_Demo.ipynb +++ b/demos/Grokking_Demo.ipynb @@ -384,18 +384,7 @@ "cell_type": "code", "execution_count": 15, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DeprecationWarning:\n", - "\n", - "HookedTransformer is deprecated and will be removed in 4.0. Use TransformerBridge.boot_transformers(...) instead, then call enable_compatibility_mode() for HookedTransformer-equivalent numerics.\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "model = TransformerBridge.boot_native(cfg, device=device)" ] diff --git a/demos/LIT_Integration_Demo.ipynb b/demos/LIT_Integration_Demo.ipynb index 40fe3df6a8..ce146d5b69 100644 --- a/demos/LIT_Integration_Demo.ipynb +++ b/demos/LIT_Integration_Demo.ipynb @@ -191,10 +191,8 @@ "model_name = \"gpt2-small\"\n", "\n", "print(f\"Loading {model_name}...\")\n", - "model = HookedTransformer.from_pretrained(\n", - " model_name,\n", - " device=device,\n", - ")\n", + "model = TransformerBridge.boot_transformers(model_name, device=device)\n", + "model.enable_compatibility_mode()\n", "print(f\"Loaded model: {model.cfg.model_name}\")\n", "print(f\" Layers: {model.cfg.n_layers}\")\n", "print(f\" Heads: {model.cfg.n_heads}\")\n", diff --git a/demos/Main_Demo.ipynb b/demos/Main_Demo.ipynb index 3a9eeb9d1d..219b159c8d 100644 --- a/demos/Main_Demo.ipynb +++ b/demos/Main_Demo.ipynb @@ -1849,7 +1849,7 @@ "metadata": {}, "outputs": [], "source": [ - "from transformer_lens.loading_from_pretrained import get_checkpoint_labels\n", + "from transformer_lens.tools.model_registry.checkpoints import get_checkpoint_labels\n", "\n", "for model_name in [\"attn-only-2l\", \"solu-12l\", \"stanford-gpt2-small-a\"]:\n", " checkpoint_labels, checkpoint_label_type = get_checkpoint_labels(model_name)\n", diff --git a/demos/No_Position_Experiment.ipynb b/demos/No_Position_Experiment.ipynb index 6fb998da7f..762c9feaef 100644 --- a/demos/No_Position_Experiment.ipynb +++ b/demos/No_Position_Experiment.ipynb @@ -145,18 +145,7 @@ "cell_type": "code", "execution_count": 35, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DeprecationWarning:\n", - "\n", - "HookedTransformer is deprecated and will be removed in 4.0. Use TransformerBridge.boot_transformers(...) instead, then call enable_compatibility_mode() for HookedTransformer-equivalent numerics.\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "cfg = TransformerBridgeConfig(\n", " n_layers=2,\n", diff --git a/demos/doc_sanitize.cfg b/demos/doc_sanitize.cfg index 19fc8a060c..6c1b235a9d 100644 --- a/demos/doc_sanitize.cfg +++ b/demos/doc_sanitize.cfg @@ -37,11 +37,3 @@ replace: \1 [regex7] regex: /(?:var|tmp|private)/\S* replace: TMP-PATH - -[regex8] -regex: [^\n]*DeprecationWarning:(?=\n\nHookedTransformer is deprecated and will be removed in 4\.0\. Use TransformerBridge\.boot_transformers\(\.\.\.\) instead, then call enable_compatibility_mode\(\) for HookedTransformer-equivalent numerics\.) -replace: DeprecationWarning: - -[regex8] -regex: /(?:var|tmp|private)/\S* -replace: TMP-PATH diff --git a/docs/make_docs.py b/docs/make_docs.py index d832d025b5..d7b25793ae 100644 --- a/docs/make_docs.py +++ b/docs/make_docs.py @@ -1,54 +1,10 @@ """Build the API Documentation.""" -import base64 -import hashlib -import json -import multiprocessing -import os import shutil import subprocess import sys -import warnings -from copy import deepcopy -from functools import lru_cache, partial from pathlib import Path -from typing import Any, Callable, Literal, Optional, Sequence, Union - -import pandas as pd # type: ignore[import-untyped] -import torch -import tqdm # type: ignore[import-untyped] -import yaml # type: ignore[import-untyped] -from muutils.dictmagic import TensorDictFormats, condense_tensor_dict -from muutils.misc import shorten_numerical_to_str -from transformers import AutoTokenizer # type: ignore[import-untyped] -from transformers import PreTrainedTokenizer - -import transformer_lens # type: ignore[import-untyped] -from transformer_lens import ( - ActivationCache, - HookedTransformer, - HookedTransformerConfig, - loading, - supported_models, -) -from transformer_lens.loading_from_pretrained import ( # type: ignore[import-untyped] - NON_HF_HOSTED_MODEL_NAMES, - get_pretrained_model_config, -) - -DEVICE: torch.device = torch.device("meta") - -# disable the symlink warning -os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1" - -try: - HF_TOKEN = os.environ.get("HF_TOKEN", "") - if not HF_TOKEN.startswith("hf_"): - raise ValueError("Invalid Hugging Face token") -except Exception as e: - warnings.warn( - f"Failed to get Hugging Face token -- info about certain models will be limited\n{e}" - ) +from typing import Any, Optional # Docs Directories CURRENT_DIR: Path = Path(__file__).parent @@ -57,597 +13,10 @@ PACKAGE_DIR: Path = CURRENT_DIR.parent DEMOS_DIR: Path = CURRENT_DIR.parent / "demos" GENERATED_DIR: Path = CURRENT_DIR.parent / "docs/source/generated" -STATIC_DIR: Path = CURRENT_DIR.parent / "docs/source/_static" - - -@lru_cache(maxsize=None) -def get_config(model_name: str): - """Retrieve the configuration of a pretrained model. - - Args: - model_name (str): Name of the pretrained model. - - Returns: - dict: Configuration of the pretrained model. - """ - return loading.get_pretrained_model_config(model_name) - - -# manually defined known model types -KNOWN_MODEL_TYPES: Sequence[str] = ( - "gpt2", - "distillgpt2", - "opt", - "gpt-neo", - "gpt-j", - "gpt-neox", - "stanford-gpt2", - "pythia", - "solu", - "gelu", - "attn-only", - "llama", - "Llama-2", - "bert", - "tiny-stories", - "stablelm", - "bloom", - "qwen", - "mistral", - "CodeLlama", - "phi", - "gemma", - "yi", - "t5", - "mixtral", - "Qwen2", -) - -MODEL_ALIASES_MAP: dict[str, str] = transformer_lens.loading.make_model_alias_map() - -# these will be copied as table columns -CONFIG_ATTRS_COPY: Sequence[str] = ( - "n_params", - "n_layers", - "n_heads", - "d_model", - "d_vocab", - "act_fn", - "positional_embedding_type", - "parallel_attn_mlp", - "original_architecture", - "normalization_type", -) - -# modify certain values when saving config -CONFIG_VALUES_PROCESS: dict[str, Callable] = { - "initializer_range": float, - "dtype": str, - "device": str, -} - -COLUMNS_ABRIDGED: Sequence[str] = ( - "name.default_alias", - "name.huggingface", - "n_params.as_str", - "n_params.as_int", - "cfg.n_params", - "cfg.n_layers", - "cfg.n_heads", - "cfg.d_model", - "cfg.d_vocab", - "cfg.act_fn", - "cfg.positional_embedding_type", - "cfg.parallel_attn_mlp", - "cfg.original_architecture", - "cfg.normalization_type", - "tokenizer.name", - "tokenizer.class", - "tokenizer.vocab_size", - "tokenizer.vocab_hash", -) - - -def get_tensor_shapes( - model: HookedTransformer, - tensor_dims_fmt: TensorDictFormats = "yaml", - except_if_forward_fails: bool = False, -) -> dict: - """get the tensor shapes from a model""" - model_info: dict = dict() - # state dict - model_info["tensor_shapes.state_dict"] = condense_tensor_dict( - model.state_dict(), fmt=tensor_dims_fmt - ) - model_info["tensor_shapes.state_dict.raw__"] = condense_tensor_dict( - model.state_dict(), fmt="dict" - ) - - try: - # input shape for activations -- "847"~="bat", subtract 7 for the context window to make it unique - input_shape: tuple[int, int] = (847, model.cfg.n_ctx - 7) - # why? to replace the batch and seq_len dims with "batch" and "seq_len" in the yaml - dims_names_map: dict[int, str] = { - input_shape[0]: "batch", - input_shape[1]: "seq_len", - } - # run with cache to activation cache - with torch.no_grad(): - cache: ActivationCache - _, cache = model.run_with_cache( - torch.empty(input_shape, dtype=torch.long, device=DEVICE) - ) - # condense using muutils and store - model_info["tensor_shapes.activation_cache"] = condense_tensor_dict( - cache.cache_dict, - fmt=tensor_dims_fmt, - dims_names_map=dims_names_map, - ) - model_info["tensor_shapes.activation_cache.raw__"] = condense_tensor_dict( - cache.cache_dict, - fmt="dict", - dims_names_map=dims_names_map, - ) - except Exception as e: - msg: str = f"Failed to get activation cache for '{model.cfg.model_name}':\n{e}" - if except_if_forward_fails: - raise ValueError(msg) from e - else: - warnings.warn(msg) - - return model_info - - -def tokenizer_vocab_hash(tokenizer: PreTrainedTokenizer) -> str: - # sort - vocab: dict[str, int] - try: - vocab = tokenizer.vocab - except Exception: - vocab = tokenizer.get_vocab() - - vocab_hashable: list[tuple[str, int]] = list( - sorted( - vocab.items(), - key=lambda x: x[1], - ) - ) - # hash it - hash_obj = hashlib.sha1(bytes(str(vocab_hashable), "UTF-8")) - # convert to base64 - return base64.b64encode( - hash_obj.digest(), - altchars=b"-_", - ).decode("UTF-8") - - -def get_tokenizer_info(model: HookedTransformer) -> dict: - tokenizer: PreTrainedTokenizer = model.tokenizer - model_info: dict = dict() - # basic info - model_info["tokenizer.name"] = tokenizer.name_or_path - model_info["tokenizer.vocab_size"] = int(tokenizer.vocab_size) - # HF uses int(1e30) as a "no max length" sentinel; clamp to None so the value round-trips through pandas JSON (ujson rejects > int64). - max_len = int(tokenizer.model_max_length) - model_info["tokenizer.max_len"] = max_len if max_len <= 2**63 - 1 else None - model_info["tokenizer.class"] = tokenizer.__class__.__name__ - - # vocab hash - model_info["tokenizer.vocab_hash"] = tokenizer_vocab_hash(tokenizer) - return model_info - - -def get_model_info( - model_name: str, - include_cfg: bool = True, - include_tensor_dims: bool = True, - include_tokenizer_info: bool = True, - tensor_dims_fmt: TensorDictFormats = "yaml", - allow_warn: bool = True, -) -> tuple[str, dict]: - """get information about the model from the default alias model name - - # Parameters: - - `model_name : str` - the default alias model name - - `include_cfg : bool` - whether to include the model config as a yaml string - (defaults to `True`) - - `include_tensor_dims : bool` - whether to include the model tensor shapes - (defaults to `True`) - - `include_tokenizer_info : bool` - whether to include the tokenizer info - (defaults to `True`) - - `tensor_dims_fmt : TensorDictFormats` - the format of the tensor shapes. one of "yaml", "json", "dict" - (defaults to `"yaml"`) - """ - - # assumes the input is a default alias - if model_name not in supported_models.DEFAULT_MODEL_ALIASES: - raise ValueError(f"Model name '{model_name}' not found in default aliases") - - # get the names and model types - official_name: Optional[str] = MODEL_ALIASES_MAP.get(model_name, None) - model_info: dict = { - "name.default_alias": model_name, - "name.huggingface": official_name, - "name.aliases": ", ".join( - list(supported_models.MODEL_ALIASES.get(official_name, [])) # type: ignore[arg-type] - ), - "model_type": None, - } - - parts: list[str] = model_name.split("-") - - # identify model type by known types - for known_type in KNOWN_MODEL_TYPES: - if known_type in model_name: - model_info["model_type"] = known_type - break - - # search for model size in name - param_count_from_name: Optional[str] = None - for part in parts: - if part[-1].lower() in ["m", "b", "k"] and part[:-1].replace(".", "", 1).isdigit(): - param_count_from_name = part - break - - # update model info from config - model_cfg: HookedTransformerConfig = get_pretrained_model_config(model_name) - model_info.update( - { - "name.from_cfg": model_cfg.model_name, - "n_params.as_str": shorten_numerical_to_str(model_cfg.n_params), # type: ignore[arg-type] - "n_params.as_int": model_cfg.n_params, - "n_params.from_name": param_count_from_name, - **{f"cfg.{attr}": getattr(model_cfg, attr) for attr in CONFIG_ATTRS_COPY}, - } - ) - - # put the whole config as yaml (for readability) - if include_cfg: - # modify certain values to make them pretty-printable - model_cfg_dict: dict = { - key: (val if key not in CONFIG_VALUES_PROCESS else CONFIG_VALUES_PROCESS[key](val)) - for key, val in model_cfg.to_dict().items() - } - - # raw config - model_info["config.raw__"] = model_cfg_dict - # dump to yaml - model_info["config"] = yaml.dump( - model_cfg_dict, - default_flow_style=False, - sort_keys=False, - width=1000, - ) - - # get tensor shapes - if include_tensor_dims or include_tokenizer_info: - # set default device to meta, so that we don't actually allocate tensors - # this can't be done at the root level because it would break other tests when we import this file - # and it has to be done inside this function due to usage of multiprocessing - with torch.device(DEVICE): - got_model: bool = False - try: - # copy the config, so we can modify it - model_cfg_copy: HookedTransformerConfig = deepcopy(model_cfg) - # set device to "meta" -- don't actually initialize the model with real tensors - model_cfg_copy.device = str(DEVICE) - if not include_tokenizer_info: - # don't need to download the tokenizer - model_cfg_copy.tokenizer_name = None - # init the fake model - model: HookedTransformer = HookedTransformer(model_cfg_copy, move_to_device=True) - # HACK: use https://huggingface.co/huggyllama to get tokenizers for original llama models - if model.cfg.tokenizer_name in NON_HF_HOSTED_MODEL_NAMES: - model.set_tokenizer( - AutoTokenizer.from_pretrained( - f"huggyllama/{model.cfg.tokenizer_name.removesuffix('-hf')}", - add_bos_token=True, - token=HF_TOKEN, - legacy=False, - ) - ) - got_model = True - except Exception as e: - msg: str = f"Failed to init model '{model_name}', can't get tensor shapes or tokenizer info" - if allow_warn: - warnings.warn(f"{msg}:\n{e}") - else: - raise ValueError(msg) from e - - if got_model: - if include_tokenizer_info: - try: - tokenizer_info: dict = get_tokenizer_info(model) - model_info.update(tokenizer_info) - except Exception as e: - msg = f"Failed to get tokenizer info for model '{model_name}'" - if allow_warn: - warnings.warn(f"{msg}:\n{e}") - else: - raise ValueError(msg) from e - - if include_tensor_dims: - try: - tensor_shapes_info: dict = get_tensor_shapes(model, tensor_dims_fmt) - model_info.update(tensor_shapes_info) - except Exception as e: - msg = f"Failed to get tensor shapes for model '{model_name}'" - if allow_warn: - warnings.warn(f"{msg}:\n{e}") - else: - raise ValueError(msg) from e - - return model_name, model_info - - -def safe_try_get_model_info( - model_name: str, kwargs: Optional[dict] = None -) -> tuple[str, Union[dict, Exception]]: - """for parallel processing, to catch exceptions and return the exception instead of raising them""" - if kwargs is None: - kwargs = {} - try: - return get_model_info(model_name, **kwargs) - except Exception as e: - warnings.warn(f"Failed to get model info for '{model_name}': {e}") - return model_name, e - - -def make_model_table( - verbose: bool, - allow_except: bool = False, - parallelize: Union[bool, int] = True, - model_names_pattern: Optional[str] = None, - **kwargs, -) -> pd.DataFrame: - """make table of all models. kwargs passed to `get_model_info()`""" - model_names: list[str] = list(supported_models.DEFAULT_MODEL_ALIASES) - model_data: list[tuple[str, Union[dict, Exception]]] = list() - - # filter by regex pattern if provided - if model_names_pattern: - model_names = [ - model_name for model_name in model_names if model_names_pattern in model_name - ] - - if parallelize: - # parallel - n_processes: int = parallelize if int(parallelize) > 1 else multiprocessing.cpu_count() - if verbose: - print(f"running in parallel with {n_processes=}") - with multiprocessing.Pool(processes=n_processes) as pool: - # Use imap for ordered results, wrapped with tqdm for progress bar - imap_results: list[tuple[str, Union[dict, Exception]]] = list( - tqdm.tqdm( - pool.imap( - partial(safe_try_get_model_info, **kwargs), - model_names, - ), - total=len(model_names), - desc="Loading model info", - disable=not verbose, - ) - ) - - model_data = imap_results - - else: - # serial - with tqdm.tqdm( - supported_models.DEFAULT_MODEL_ALIASES, - desc="Loading model info", - disable=not verbose, - ) as pbar: - for model_name in pbar: - pbar.set_postfix_str(f"model: '{model_name}'") - try: - model_data.append(get_model_info(model_name, **kwargs)) - except Exception as e: - if allow_except: - # warn and continue if we allow exceptions - warnings.warn(f"Failed to get model info for '{model_name}': {e}") - model_data.append((model_name, e)) - else: - # raise exception right away if we don't allow exceptions - # note that this differs from the parallel version, which will only except at the end - raise ValueError(f"Failed to get model info for '{model_name}'") from e - - # figure out what to do with failed models - failed_models: dict[str, Exception] = { - model_name: result for model_name, result in model_data if isinstance(result, Exception) - } - msg: str = ( - f"Failed to get model info for {len(failed_models)}/{len(model_names)} models: {failed_models}\n" - + "\n".join(f"\t'{model_name}': {expt}" for model_name, expt in failed_models.items()) - ) - if not allow_except: - if failed_models: - # raise exception if we don't allow exceptions - raise ValueError(msg + "\n\n" + "=" * 80 + "\n\n" + "NO DATA WRITTEN") - else: - if failed_models: - warnings.warn(msg + "\n\n" + "-" * 80 + "\n\n" + "WRITING PARTIAL DATA") - - # filter out failed models if we allow exceptions - model_data_filtered: list[dict] = [ - result for _, result in model_data if not isinstance(result, Exception) - ] - return pd.DataFrame(model_data_filtered) - - -OutputFormat = Literal["jsonl", "csv", "md"] - - -def huggingface_name_to_url(df: pd.DataFrame) -> pd.DataFrame: - """convert the huggingface model name to a url""" - df_new: pd.DataFrame = df.copy() - df_new["name.huggingface"] = df_new["name.huggingface"].map( - lambda x: f"[{x}](https://huggingface.co/{x})" if x else x - ) - return df_new - - -MD_TABLE_HEARDER: str = """--- -title: HookedTransformer -hide-toc: true ---- -# HookedTransformer Model Properties - -also see the <a href="../_static/model_properties_table_interactive.html" target="_blank" rel="noopener">interactive model table</a> -""" - - -def write_model_table( - model_table: pd.DataFrame, - path: Path, - format: OutputFormat = "jsonl", - include_TL_version: bool = True, - md_hf_links: bool = True, - md_header: str = MD_TABLE_HEARDER, -) -> None: - """write the model table to disk in the specified format""" - - # make sure the directory exists - path.parent.mkdir(parents=True, exist_ok=True) - - if include_TL_version: - # get `transformer_lens` version - tl_version: str = "unknown" - try: - from importlib.metadata import PackageNotFoundError, version - - tl_version = version("transformer_lens") - except PackageNotFoundError as e: - warnings.warn(f"Failed to get transformer_lens version: package not found\n{e}") - except Exception as e: - warnings.warn(f"Failed to get transformer_lens version: {e}") - - with open(path.with_suffix(".version"), "w") as f: - json.dump({"version": tl_version}, f) - - if format == "jsonl": - model_table.to_json(path.with_suffix(".jsonl"), orient="records", lines=True) - elif format == "csv": - model_table.to_csv(path.with_suffix(".csv"), index=False) - elif format == "md": - model_table_processed: pd.DataFrame = model_table - # convert huggingface name to url - if md_hf_links: - model_table_processed = huggingface_name_to_url(model_table_processed) - model_table_md_text: str = md_header + model_table_processed.to_markdown(index=False) - with open(path.with_suffix(".md"), "w") as f: - f.write(model_table_md_text) - else: - raise KeyError(f"Invalid format: {format}") - - -def abridge_model_table( - model_table: pd.DataFrame, - columns_keep: Sequence[str] = COLUMNS_ABRIDGED, - null_to_empty: bool = True, -) -> pd.DataFrame: - """keep only columns in COLUMNS_ABRIDGED - - primarily used to make the csv and md versions of the table readable - - also replaces `None` with empty string if `null_to_empty` is `True` - """ - - output: pd.DataFrame = model_table.copy() - # filter columns - output = output[list(columns_keep)] - - if null_to_empty: - output = output.fillna("") - - return output - - -def get_model_table( - model_table_path: Path, - verbose: bool = True, - force_reload: bool = True, - do_write: bool = True, - parallelize: Union[bool, int] = True, - model_names_pattern: Optional[str] = None, - **kwargs, -) -> pd.DataFrame: - """get the model table either by generating or reading from jsonl file - - # Parameters: - - `model_table_path : Path` - the path to the model table file, and the base name for the csv and md files - - `verbose : bool` - whether to show progress bar - (defaults to `True`) - - `force_reload : bool` - force creating the table from scratch, even if file exists - (defaults to `True`) - - `do_write : bool` - whether to write the table to disk, if generating - (defaults to `True`) - - `model_names_pattern : Optional[str]` - filter the model names by making them include this string. passed to `make_model_table()`. no filtering if `None` - (defaults to `None`) - - `**kwargs` - passed to `make_model_table()` - - # Returns: - - `pd.DataFrame` - the model table. rows are models, columns are model attributes - """ - - # modify the name if a pattern is provided - if model_names_pattern is not None: - model_table_path = model_table_path.with_name( - model_table_path.stem + f"-{model_names_pattern}" - ) - - model_table: pd.DataFrame - if not model_table_path.exists() or force_reload: - # generate it from scratch - model_table = make_model_table( - verbose=verbose, - parallelize=parallelize, - model_names_pattern=model_names_pattern, - **kwargs, - ) - if do_write: - # full data as jsonl - write_model_table(model_table, model_table_path, format="jsonl") - # abridged data as csv, md - abridged_table: pd.DataFrame = abridge_model_table(model_table) - write_model_table(abridged_table, model_table_path, format="csv") - write_model_table(abridged_table, model_table_path, format="md") - # The interactive table HTML (docs/source/_static/...) fetches - # `model_properties_table.{jsonl,version}` from its own dir. - # Sphinx auto-copies _static/ into the build, so co-locating the - # data there is the simplest way to expose it to the browser. - import shutil - - for ext in ("jsonl", "version"): - src = model_table_path.with_suffix(f".{ext}") - if src.exists(): - shutil.copy(src, STATIC_DIR / src.name) - else: - # read the table from jsonl - model_table = pd.read_json(model_table_path, orient="records", lines=True) - - return model_table def build_docs(): """Build the docs.""" - get_model_table( - model_table_path=GENERATED_DIR / "model_properties_table.jsonl", - force_reload=True, - allow_except=True, - ) copy_demos() generate_bridge_models_page() @@ -666,87 +35,6 @@ def build_docs(): ) -def get_property(name: str, model_name: str) -> Any: - """Retrieve a specific property of a pretrained model. - - Args: - name (str): Name of the property to retrieve. - model_name (str): Name of the pretrained model. - - Returns: - str: Value of the specified property. - """ - cfg = get_config(model_name) - - if name == "act_fn": - if cfg.attn_only: - return "attn_only" - if cfg.act_fn == "gelu_new": - return "gelu" - if cfg.act_fn == "gelu_fast": - return "gelu" - if cfg.act_fn == "solu_ln": - return "solu" - return cfg.act_fn - if name == "n_params": - n_params = cfg.n_params - if n_params < 1e4: - return f"{n_params/1e3:.1f}K" - if n_params < 1e6: - return f"{round(n_params/1e3)}K" - if n_params < 1e7: - return f"{n_params/1e6:.1f}M" - if n_params < 1e9: - return f"{round(n_params/1e6)}M" - if n_params < 1e10: - return f"{n_params/1e9:.1f}B" - if n_params < 1e12: - return f"{round(n_params/1e9)}B" - raise ValueError(f"Passed in {n_params} above 1T?") - return cfg.to_dict()[name] - - -def generate_model_table(_app: Optional[Any] = None): - """Generate a markdown table summarizing properties of pretrained models. - - This script extracts various properties of pretrained models from the `easy_transformer` - library, such as the number of parameters, layers, and heads, among others, and generates a - markdown table. - """ - - column_names = [ - "n_params", - "n_layers", - "d_model", - "n_heads", - "act_fn", - "n_ctx", - "d_vocab", - "d_head", - "d_mlp", - "n_key_value_heads", - ] - df = pd.DataFrame( - { - name: [ - get_property(name, model_name) - for model_name in supported_models.DEFAULT_MODEL_ALIASES - ] - for name in column_names - }, - index=supported_models.DEFAULT_MODEL_ALIASES, - ) - - df["n_key_value_heads"] = df["n_key_value_heads"].fillna(-1).astype(int).replace(-1, "") - markdown_string = df.to_markdown() - markdown_string = "# Model Properties Table\n\n" + markdown_string - - GENERATED_DIR.mkdir(exist_ok=True) - file_path = GENERATED_DIR / "model_properties_table.md" - with open(file_path, "w", encoding="utf-8") as file: - file.write(markdown_string) - - def copy_demos(_app: Optional[Any] = None): """Copy demo notebooks to the generated directory.""" copy_to_dir = GENERATED_DIR / "demos" @@ -1544,9 +832,6 @@ def generate_bridge_models_page(): def docs_hot_reload(): """Hot reload the docs.""" - get_model_table( - model_table_path=GENERATED_DIR / "model_properties_table.jsonl", force_reload=False - ) copy_demos() generate_bridge_models_page() diff --git a/docs/source/conf.py b/docs/source/conf.py index eea59e26a8..aa310a6851 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -74,25 +74,9 @@ # Functions to ignore as they're not interesting to the end user functions_to_ignore = [ - # functions from load_from_pretrained.py - "convert_hf_model_config", - "convert_bert_weights", - "convert_gpt2_weights", - "convert_gptj_weights", - "convert_llama_weights", - "convert_mingpt_weights", - "convert_nanogpt_weights", + # functions relocated out of the deleted loading_from_pretrained.py "convert_neel_solu_old_weights", - "convert_neo_weights", - "convert_neox_weights", - "convert_neel_model_config", - "convert_opt_weights", - "convert_gemma_weights", - "fill_missing_keys", - "get_basic_config", "get_official_model_name", - "get_pretrained_state_dict", - "make_model_alias_map", # functions from make_docs.py "get_config", "get_property", @@ -135,12 +119,7 @@ def run_apidoc(_app: Optional[Any] = None): excluded_modules = [ "ActivationCache.py", "FactoredMatrix.py", - "HookedEncoder.py", - "HookedEncoderDecoder.py", - "HookedTransformer.py", "SVDInterpreter.py", - "BertNextSentencePrediction.py", - "config/HookedTransformerConfig.py", ] args = [ "--force", # Overwrite existing files @@ -156,8 +135,8 @@ def run_apidoc(_app: Optional[Any] = None): # Add exclude-members for modules with separate docs package_excludes = { - "transformer_lens.rst": "ActivationCache, FactoredMatrix, HookedEncoder, HookedEncoderDecoder, HookedTransformer, SVDInterpreter, BertNextSentencePrediction, HookedTransformerConfig, EasyTransformerConfig", - "transformer_lens.config.rst": "HookedTransformerConfig, TransformerBridgeConfig, TransformerLensConfig", + "transformer_lens.rst": "ActivationCache, FactoredMatrix, SVDInterpreter, EasyTransformerConfig", + "transformer_lens.config.rst": "TransformerBridgeConfig, TransformerLensConfig", "transformer_lens.conversion_utils.rst": "HookConversionSet", } diff --git a/docs/source/content/contributing.md b/docs/source/content/contributing.md index b01348b17f..cc5ee84e84 100644 --- a/docs/source/content/contributing.md +++ b/docs/source/content/contributing.md @@ -1,9 +1,7 @@ # Contributing ```{warning} -`HookedTransformer` is deprecated as of TransformerLens 3.0 and will be removed in the next major version. New code should use [`TransformerBridge`](migrating_to_v3.md) instead. Existing `HookedTransformer` code continues to work through the 3.x branch via a compatibility layer. See the [migration guide](migrating_to_v3.md) for conversion recipes. - -`HookedTransformer` **has no acceptance suite of its own** — `tests/acceptance/test_hooked_transformer.py` was removed when the Bridge tests were reanchored onto frozen golden datasets. Changes that touch HookedTransformer internals therefore land essentially untested at the acceptance level — extra manual care is required. The `HookedEncoder` (BERT) and `HookedEncoderDecoder` (T5) acceptance suites are unaffected and run normally. +`HookedTransformer` and the other `Hooked*` classes were **removed in TransformerLens 4.0**. Use [`TransformerBridge`](migrating_to_v4.md) instead — call `enable_compatibility_mode()` after booting for HookedTransformer-equivalent numerics. See the [4.0 migration guide](migrating_to_v4.md). ``` ## Contributing with AI coding agents diff --git a/docs/source/content/hooked_transformer_model_properties.md b/docs/source/content/hooked_transformer_model_properties.md new file mode 100644 index 0000000000..54c813172c --- /dev/null +++ b/docs/source/content/hooked_transformer_model_properties.md @@ -0,0 +1,264 @@ +--- +title: HookedTransformer Model Properties +hide-toc: true +--- +# HookedTransformer Model Properties + +```{warning} +**Deprecated.** `HookedTransformer` was removed in TransformerLens 4.0. This +table is a frozen snapshot maintained for users still on the 2.x / 3.x branches; +it is no longer regenerated and will not gain new models. For 4.0+, see the +[TransformerBridge model table](/generated/transformer_bridge_models) and the +[migration guide](migrating_to_v3.md). +``` + +also see the <a href="../_static/model_properties_table_interactive.html" target="_blank" rel="noopener">interactive model table</a> +| name.default_alias | name.huggingface | n_params.as_str | n_params.as_int | cfg.n_params | cfg.n_layers | cfg.n_heads | cfg.d_model | cfg.d_vocab | cfg.act_fn | cfg.positional_embedding_type | cfg.parallel_attn_mlp | cfg.original_architecture | cfg.normalization_type | tokenizer.name | tokenizer.class | tokenizer.vocab_size | tokenizer.vocab_hash | +|:--------------------------------------|:----------------------------------------------------------------------------------------------------------------------------|:------------------|------------------:|---------------:|---------------:|--------------:|--------------:|--------------:|:------------------|:--------------------------------|:------------------------|:-------------------------------|:-------------------------|:--------------------------------------------|:---------------------|:-----------------------|:-----------------------------| +| yi-34b | [01-ai/Yi-34B](https://huggingface.co/01-ai/Yi-34B) | 39B | 38755368960 | 38755368960 | 60 | 56 | 7168 | 64000 | silu | rotary | False | LlamaForCausalLM | RMS | 01-ai/Yi-34B | LlamaTokenizer | 64000.0 | VBBPi7l7j0Xrv93YNq1tizlalWw= | +| yi-34b-chat | [01-ai/Yi-34B-Chat](https://huggingface.co/01-ai/Yi-34B-Chat) | 39B | 38755368960 | 38755368960 | 60 | 56 | 7168 | 64000 | silu | rotary | False | LlamaForCausalLM | RMS | 01-ai/Yi-34B-Chat | LlamaTokenizer | 63992.0 | VGXAFrTzytwGdUlX6AWH0NacncM= | +| yi-6b | [01-ai/Yi-6B](https://huggingface.co/01-ai/Yi-6B) | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 64000 | silu | rotary | False | LlamaForCausalLM | RMS | 01-ai/Yi-6B | LlamaTokenizer | 63992.0 | VGXAFrTzytwGdUlX6AWH0NacncM= | +| yi-6b-chat | [01-ai/Yi-6B-Chat](https://huggingface.co/01-ai/Yi-6B-Chat) | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 64000 | silu | rotary | False | LlamaForCausalLM | RMS | 01-ai/Yi-6B-Chat | LlamaTokenizer | 63992.0 | VGXAFrTzytwGdUlX6AWH0NacncM= | +| mGPT | | 1.2B | 1207959552 | 1207959552 | 24 | 16 | 2048 | 100000 | gelu_new | standard | False | GPT2LMHeadModel | LN | ai-forever/mGPT | GPT2Tokenizer | 100000.0 | 8j6CU_p3zgyeEBZ1Z3lu358tiy0= | +| olmo-1b | [allenai/OLMo-1B-hf](https://huggingface.co/allenai/OLMo-1B-hf) | 1.1B | 1073741824 | 1073741824 | 16 | 16 | 2048 | 50304 | silu | rotary | False | OlmoForCausalLM | LN | allenai/OLMo-1B-hf | GPTNeoXTokenizer | 50280.0 | Zon1p_mdHoNTi0EKxz2EJK3tpZg= | +| olmo-2-1b | [allenai/OLMo-2-0425-1B](https://huggingface.co/allenai/OLMo-2-0425-1B) | 1.1B | 1073741824 | 1073741824 | 16 | 16 | 2048 | 100352 | silu | rotary | False | Olmo2ForCausalLM | RMS | allenai/OLMo-2-0425-1B | TokenizersBackend | 100278.0 | VuUSAx0rE_5hHVJSa7DTIutKX9s= | +| olmo-2-7b | [allenai/OLMo-2-1124-7B](https://huggingface.co/allenai/OLMo-2-1124-7B) | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 100352 | silu | rotary | False | Olmo2ForCausalLM | RMS | allenai/OLMo-2-1124-7B | TokenizersBackend | 100278.0 | VuUSAx0rE_5hHVJSa7DTIutKX9s= | +| olmo-3-1025-7b | [allenai/Olmo-3-1025-7B](https://huggingface.co/allenai/Olmo-3-1025-7B) | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 100278 | silu | rotary | False | Olmo3ForCausalLM | RMS | allenai/Olmo-3-1025-7B | TokenizersBackend | 100278.0 | VuUSAx0rE_5hHVJSa7DTIutKX9s= | +| olmo-3-1125-32b | [allenai/Olmo-3-1125-32B](https://huggingface.co/allenai/Olmo-3-1125-32B) | 34B | 33889976320 | 33889976320 | 64 | 40 | 5120 | 100278 | silu | rotary | False | Olmo3ForCausalLM | RMS | allenai/Olmo-3-1125-32B | TokenizersBackend | 100278.0 | VuUSAx0rE_5hHVJSa7DTIutKX9s= | +| olmo-3-32b-think | [allenai/Olmo-3-32B-Think](https://huggingface.co/allenai/Olmo-3-32B-Think) | 34B | 33889976320 | 33889976320 | 64 | 40 | 5120 | 100278 | silu | rotary | False | Olmo3ForCausalLM | RMS | allenai/Olmo-3-32B-Think | TokenizersBackend | 100278.0 | VuUSAx0rE_5hHVJSa7DTIutKX9s= | +| olmo-3-7b-instruct | [allenai/Olmo-3-7B-Instruct](https://huggingface.co/allenai/Olmo-3-7B-Instruct) | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 100278 | silu | rotary | False | Olmo3ForCausalLM | RMS | allenai/Olmo-3-7B-Instruct | TokenizersBackend | 100278.0 | RNowY96fjQ_x29qA8x5WywVkiVk= | +| olmo-3-7b-think | [allenai/Olmo-3-7B-Think](https://huggingface.co/allenai/Olmo-3-7B-Think) | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 100278 | silu | rotary | False | Olmo3ForCausalLM | RMS | allenai/Olmo-3-7B-Think | TokenizersBackend | 100278.0 | VuUSAx0rE_5hHVJSa7DTIutKX9s= | +| olmo-3.1-32b-instruct | [allenai/Olmo-3.1-32B-Instruct](https://huggingface.co/allenai/Olmo-3.1-32B-Instruct) | 34B | 33889976320 | 33889976320 | 64 | 40 | 5120 | 100278 | silu | rotary | False | Olmo3ForCausalLM | RMS | allenai/Olmo-3.1-32B-Instruct | TokenizersBackend | 100278.0 | RNowY96fjQ_x29qA8x5WywVkiVk= | +| olmo-3.1-32b-think | [allenai/Olmo-3.1-32B-Think](https://huggingface.co/allenai/Olmo-3.1-32B-Think) | 34B | 33889976320 | 33889976320 | 64 | 40 | 5120 | 100278 | silu | rotary | False | Olmo3ForCausalLM | RMS | allenai/Olmo-3.1-32B-Think | TokenizersBackend | 100278.0 | VuUSAx0rE_5hHVJSa7DTIutKX9s= | +| olmo-7b | [allenai/OLMo-7B-hf](https://huggingface.co/allenai/OLMo-7B-hf) | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 50304 | silu | rotary | False | OlmoForCausalLM | LN | allenai/OLMo-7B-hf | GPTNeoXTokenizer | 50280.0 | Zon1p_mdHoNTi0EKxz2EJK3tpZg= | +| olmoe | [allenai/OLMoE-1B-7B-0924](https://huggingface.co/allenai/OLMoE-1B-7B-0924) | 6.7B | 6712983552 | 6712983552 | 16 | 16 | 2048 | 50304 | silu | rotary | False | OlmoeForCausalLM | RMS | allenai/OLMoE-1B-7B-0924 | GPTNeoXTokenizer | 50280.0 | Zon1p_mdHoNTi0EKxz2EJK3tpZg= | +| redwood_attn_2l | [ArthurConmy/redwood_attn_2l](https://huggingface.co/ArthurConmy/redwood_attn_2l) | 524K | 524288 | 524288 | 2 | 8 | 256 | 50259 | gelu_new | shortformer | False | neel | LN | | | | | +| othello-gpt | [Baidicoot/Othello-GPT-Transformer-Lens](https://huggingface.co/Baidicoot/Othello-GPT-Transformer-Lens) | 25M | 25165824 | 25165824 | 8 | 8 | 512 | 61 | gelu | standard | False | mingpt | LN | | | | | +| bloom-1b1 | [bigscience/bloom-1b1](https://huggingface.co/bigscience/bloom-1b1) | 679M | 679477248 | 679477248 | 24 | 16 | 1536 | 250880 | gelu_fast | alibi | False | BloomForCausalLM | LN | bigscience/bloom-1b1 | TokenizersBackend | 250680.0 | OO9NZoesMCpWsijo1O2DAbq9GqI= | +| bloom-1b7 | [bigscience/bloom-1b7](https://huggingface.co/bigscience/bloom-1b7) | 1.2B | 1207959552 | 1207959552 | 24 | 16 | 2048 | 250880 | gelu_fast | alibi | False | BloomForCausalLM | LN | bigscience/bloom-1b7 | TokenizersBackend | 250680.0 | OO9NZoesMCpWsijo1O2DAbq9GqI= | +| bloom-3b | [bigscience/bloom-3b](https://huggingface.co/bigscience/bloom-3b) | 2.4B | 2359296000 | 2359296000 | 30 | 32 | 2560 | 250880 | gelu_fast | alibi | False | BloomForCausalLM | LN | bigscience/bloom-3b | TokenizersBackend | 250680.0 | OO9NZoesMCpWsijo1O2DAbq9GqI= | +| bloom-560m | [bigscience/bloom-560m](https://huggingface.co/bigscience/bloom-560m) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | 250880 | gelu_fast | alibi | False | BloomForCausalLM | LN | bigscience/bloom-560m | TokenizersBackend | 250680.0 | OO9NZoesMCpWsijo1O2DAbq9GqI= | +| bloom-7b1 | [bigscience/bloom-7b1](https://huggingface.co/bigscience/bloom-7b1) | 6.0B | 6039797760 | 6039797760 | 30 | 32 | 4096 | 250880 | gelu_fast | alibi | False | BloomForCausalLM | LN | bigscience/bloom-7b1 | TokenizersBackend | 250680.0 | OO9NZoesMCpWsijo1O2DAbq9GqI= | +| CodeLlamallama-2-7b | | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 32016 | silu | rotary | False | LlamaForCausalLM | RMS | codellama/CodeLlama-7b-hf | CodeLlamaTokenizer | 32016.0 | TKLN0vmDArFwHMvDHh9FaoGA3_k= | +| CodeLlama-7b-instruct | | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 32016 | silu | rotary | False | LlamaForCausalLM | RMS | codellama/CodeLlama-7b-Instruct-hf | CodeLlamaTokenizer | 32016.0 | TKLN0vmDArFwHMvDHh9FaoGA3_k= | +| CodeLlama-7b-python | | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 32000 | silu | rotary | False | LlamaForCausalLM | RMS | codellama/CodeLlama-7b-Python-hf | CodeLlamaTokenizer | 32000.0 | demd-4_2c5nH6yyymVgGMcZU9lU= | +| distillgpt2 | [distilgpt2](https://huggingface.co/distilgpt2) | 42M | 42467328 | 42467328 | 6 | 12 | 768 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | distilgpt2 | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| gpt-j-6B | | 5.6B | 5637144576 | 5637144576 | 28 | 16 | 4096 | 50400 | gelu_new | rotary | True | GPTJForCausalLM | LN | EleutherAI/gpt-j-6B | GPT2Tokenizer | 50257.0 | aKfp-BCA9d3W27qknxFiS0DGC5s= | +| gpt-neo-1.3B | | 1.2B | 1207959552 | 1207959552 | 24 | 16 | 2048 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | EleutherAI/gpt-neo-1.3B | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| gpt-neo-125M | | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | EleutherAI/gpt-neo-125M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| gpt-neo-2.7B | | 2.5B | 2516582400 | 2516582400 | 32 | 20 | 2560 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | EleutherAI/gpt-neo-2.7B | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| gpt-neox-20b | [EleutherAI/gpt-neox-20b](https://huggingface.co/EleutherAI/gpt-neox-20b) | 20B | 19931332608 | 19931332608 | 44 | 64 | 6144 | 50432 | gelu_fast | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/gpt-neox-20b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-1.4b | [EleutherAI/pythia-1.4b](https://huggingface.co/EleutherAI/pythia-1.4b) | 1.2B | 1207959552 | 1207959552 | 24 | 16 | 2048 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-1.4b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-1.4b-deduped | [EleutherAI/pythia-1.4b-deduped](https://huggingface.co/EleutherAI/pythia-1.4b-deduped) | 1.2B | 1207959552 | 1207959552 | 24 | 16 | 2048 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-1.4b-deduped | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-1.4b-deduped-v0 | [EleutherAI/pythia-1.4b-deduped-v0](https://huggingface.co/EleutherAI/pythia-1.4b-deduped-v0) | 1.2B | 1207959552 | 1207959552 | 24 | 16 | 2048 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-1.4b-deduped-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-1.4b-v0 | [EleutherAI/pythia-1.4b-v0](https://huggingface.co/EleutherAI/pythia-1.4b-v0) | 1.2B | 1207959552 | 1207959552 | 24 | 16 | 2048 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-1.4b-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-12b | [EleutherAI/pythia-12b](https://huggingface.co/EleutherAI/pythia-12b) | 11B | 11324620800 | 11324620800 | 36 | 40 | 5120 | 50688 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-12b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-12b-deduped | [EleutherAI/pythia-12b-deduped](https://huggingface.co/EleutherAI/pythia-12b-deduped) | 11B | 11324620800 | 11324620800 | 36 | 40 | 5120 | 50688 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-12b-deduped | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-12b-deduped-v0 | [EleutherAI/pythia-12b-deduped-v0](https://huggingface.co/EleutherAI/pythia-12b-deduped-v0) | 11B | 11324620800 | 11324620800 | 36 | 40 | 5120 | 50688 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-12b-deduped-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-12b-v0 | [EleutherAI/pythia-12b-v0](https://huggingface.co/EleutherAI/pythia-12b-v0) | 11B | 11324620800 | 11324620800 | 36 | 40 | 5120 | 50688 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-12b-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-14m | [EleutherAI/pythia-14m](https://huggingface.co/EleutherAI/pythia-14m) | 1.2M | 1179648 | 1179648 | 6 | 4 | 128 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-14m | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-160m | [EleutherAI/pythia-160m](https://huggingface.co/EleutherAI/pythia-160m) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-160m | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-160m-deduped | [EleutherAI/pythia-160m-deduped](https://huggingface.co/EleutherAI/pythia-160m-deduped) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-160m-deduped | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-160m-deduped-v0 | [EleutherAI/pythia-160m-deduped-v0](https://huggingface.co/EleutherAI/pythia-160m-deduped-v0) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-160m-deduped-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-160m-seed1 | [EleutherAI/pythia-160m-seed1](https://huggingface.co/EleutherAI/pythia-160m-seed1) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-160m-seed1 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-160m-seed2 | [EleutherAI/pythia-160m-seed2](https://huggingface.co/EleutherAI/pythia-160m-seed2) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-160m-seed2 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-160m-seed3 | [EleutherAI/pythia-160m-seed3](https://huggingface.co/EleutherAI/pythia-160m-seed3) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-160m-seed3 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-160m-v0 | [EleutherAI/pythia-160m-v0](https://huggingface.co/EleutherAI/pythia-160m-v0) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-160m-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-1b | [EleutherAI/pythia-1b](https://huggingface.co/EleutherAI/pythia-1b) | 805M | 805306368 | 805306368 | 16 | 8 | 2048 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-1b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-1b-deduped | [EleutherAI/pythia-1b-deduped](https://huggingface.co/EleutherAI/pythia-1b-deduped) | 805M | 805306368 | 805306368 | 16 | 8 | 2048 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-1b-deduped | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-1b-deduped-v0 | [EleutherAI/pythia-1b-deduped-v0](https://huggingface.co/EleutherAI/pythia-1b-deduped-v0) | 805M | 805306368 | 805306368 | 16 | 8 | 2048 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-1b-deduped-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-1b-v0 | [EleutherAI/pythia-1b-v0](https://huggingface.co/EleutherAI/pythia-1b-v0) | 805M | 805306368 | 805306368 | 16 | 8 | 2048 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-1b-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-2.8b | [EleutherAI/pythia-2.8b](https://huggingface.co/EleutherAI/pythia-2.8b) | 2.5B | 2516582400 | 2516582400 | 32 | 32 | 2560 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-2.8b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-2.8b-deduped | [EleutherAI/pythia-2.8b-deduped](https://huggingface.co/EleutherAI/pythia-2.8b-deduped) | 2.5B | 2516582400 | 2516582400 | 32 | 32 | 2560 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-2.8b-deduped | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-2.8b-deduped-v0 | [EleutherAI/pythia-2.8b-deduped-v0](https://huggingface.co/EleutherAI/pythia-2.8b-deduped-v0) | 2.5B | 2516582400 | 2516582400 | 32 | 32 | 2560 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-2.8b-deduped-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-2.8b-v0 | [EleutherAI/pythia-2.8b-v0](https://huggingface.co/EleutherAI/pythia-2.8b-v0) | 2.5B | 2516582400 | 2516582400 | 32 | 32 | 2560 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-2.8b-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-31m | [EleutherAI/pythia-31m](https://huggingface.co/EleutherAI/pythia-31m) | 4.7M | 4718592 | 4718592 | 6 | 8 | 256 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-31m | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-410m | [EleutherAI/pythia-410m](https://huggingface.co/EleutherAI/pythia-410m) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-410m | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-410m-deduped | [EleutherAI/pythia-410m-deduped](https://huggingface.co/EleutherAI/pythia-410m-deduped) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-410m-deduped | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-410m-deduped-v0 | [EleutherAI/pythia-410m-deduped-v0](https://huggingface.co/EleutherAI/pythia-410m-deduped-v0) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-410m-deduped-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-410m-v0 | [EleutherAI/pythia-410m-v0](https://huggingface.co/EleutherAI/pythia-410m-v0) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-410m-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-6.9b | [EleutherAI/pythia-6.9b](https://huggingface.co/EleutherAI/pythia-6.9b) | 6.4B | 6442450944 | 6442450944 | 32 | 32 | 4096 | 50432 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-6.9b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-6.9b-deduped | [EleutherAI/pythia-6.9b-deduped](https://huggingface.co/EleutherAI/pythia-6.9b-deduped) | 6.4B | 6442450944 | 6442450944 | 32 | 32 | 4096 | 50432 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-6.9b-deduped | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-6.9b-deduped-v0 | [EleutherAI/pythia-6.9b-deduped-v0](https://huggingface.co/EleutherAI/pythia-6.9b-deduped-v0) | 6.4B | 6442450944 | 6442450944 | 32 | 32 | 4096 | 50432 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-6.9b-deduped-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-6.9b-v0 | [EleutherAI/pythia-6.9b-v0](https://huggingface.co/EleutherAI/pythia-6.9b-v0) | 6.4B | 6442450944 | 6442450944 | 32 | 32 | 4096 | 50432 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-6.9b-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-70m | [EleutherAI/pythia-70m](https://huggingface.co/EleutherAI/pythia-70m) | 19M | 18874368 | 18874368 | 6 | 8 | 512 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-70m | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-70m-deduped | [EleutherAI/pythia-70m-deduped](https://huggingface.co/EleutherAI/pythia-70m-deduped) | 19M | 18874368 | 18874368 | 6 | 8 | 512 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-70m-deduped | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-70m-deduped-v0 | [EleutherAI/pythia-70m-deduped-v0](https://huggingface.co/EleutherAI/pythia-70m-deduped-v0) | 19M | 18874368 | 18874368 | 6 | 8 | 512 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-70m-deduped-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| pythia-70m-v0 | [EleutherAI/pythia-70m-v0](https://huggingface.co/EleutherAI/pythia-70m-v0) | 19M | 18874368 | 18874368 | 6 | 8 | 512 | 50304 | gelu | rotary | True | GPTNeoXForCausalLM | LN | EleutherAI/pythia-70m-v0 | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| hubert-base-ls960 | [facebook/hubert-base-ls960](https://huggingface.co/facebook/hubert-base-ls960) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | -1 | gelu | standard | False | HubertModel | LN | | | | | +| opt-1.3b | [facebook/opt-1.3b](https://huggingface.co/facebook/opt-1.3b) | 1.2B | 1207959552 | 1207959552 | 24 | 32 | 2048 | 50272 | relu | standard | False | OPTForCausalLM | LN | facebook/opt-1.3b | GPT2Tokenizer | 50265.0 | f1FIzqnRiMYzke1CU0hp8TDxq7k= | +| opt-125m | [facebook/opt-125m](https://huggingface.co/facebook/opt-125m) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50272 | relu | standard | False | OPTForCausalLM | LN | facebook/opt-125m | GPT2Tokenizer | 50265.0 | f1FIzqnRiMYzke1CU0hp8TDxq7k= | +| opt-13b | [facebook/opt-13b](https://huggingface.co/facebook/opt-13b) | 13B | 12582912000 | 12582912000 | 40 | 40 | 5120 | 50272 | relu | standard | False | OPTForCausalLM | LN | facebook/opt-13b | GPT2Tokenizer | 50265.0 | f1FIzqnRiMYzke1CU0hp8TDxq7k= | +| opt-2.7b | [facebook/opt-2.7b](https://huggingface.co/facebook/opt-2.7b) | 2.5B | 2516582400 | 2516582400 | 32 | 32 | 2560 | 50272 | relu | standard | False | OPTForCausalLM | LN | facebook/opt-2.7b | GPT2Tokenizer | 50265.0 | f1FIzqnRiMYzke1CU0hp8TDxq7k= | +| opt-30b | [facebook/opt-30b](https://huggingface.co/facebook/opt-30b) | 30B | 29595009024 | 29595009024 | 48 | 56 | 7168 | 50272 | relu | standard | False | OPTForCausalLM | LN | facebook/opt-30b | GPT2Tokenizer | 50265.0 | f1FIzqnRiMYzke1CU0hp8TDxq7k= | +| opt-6.7b | [facebook/opt-6.7b](https://huggingface.co/facebook/opt-6.7b) | 6.4B | 6442450944 | 6442450944 | 32 | 32 | 4096 | 50272 | relu | standard | False | OPTForCausalLM | LN | facebook/opt-6.7b | GPT2Tokenizer | 50265.0 | f1FIzqnRiMYzke1CU0hp8TDxq7k= | +| opt-66b | [facebook/opt-66b](https://huggingface.co/facebook/opt-66b) | 65B | 65229815808 | 65229815808 | 64 | 72 | 9216 | 50272 | relu | standard | False | OPTForCausalLM | LN | facebook/opt-66b | GPT2Tokenizer | 50265.0 | f1FIzqnRiMYzke1CU0hp8TDxq7k= | +| wav2vec2-base | [facebook/wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | -1 | gelu | standard | False | Wav2Vec2ForPreTraining | LN | facebook/wav2vec2-base | Wav2Vec2CTCTokenizer | 32.0 | tJ24E7IRYtfON5KH_y7lgJl6WV4= | +| wav2vec2-large | [facebook/wav2vec2-large](https://huggingface.co/facebook/wav2vec2-large) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | -1 | gelu | standard | False | Wav2Vec2ForPreTraining | LN | | | | | +| bert-base-cased | [google-bert/bert-base-cased](https://huggingface.co/google-bert/bert-base-cased) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 28996 | gelu | standard | False | BertForMaskedLM | LN | google-bert/bert-base-cased | BertTokenizer | 28996.0 | SSKvHuFYtPbvgwMSLSIhfFE_kF8= | +| bert-base-uncased | [google-bert/bert-base-uncased](https://huggingface.co/google-bert/bert-base-uncased) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 30522 | gelu | standard | False | BertForMaskedLM | LN | google-bert/bert-base-uncased | BertTokenizer | 30522.0 | G9iEWgpI_JY73i8Lym9gBVhq4BI= | +| bert-large-cased | [google-bert/bert-large-cased](https://huggingface.co/google-bert/bert-large-cased) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | 28996 | gelu | standard | False | BertForMaskedLM | LN | google-bert/bert-large-cased | BertTokenizer | 28996.0 | SSKvHuFYtPbvgwMSLSIhfFE_kF8= | +| bert-large-uncased | [google-bert/bert-large-uncased](https://huggingface.co/google-bert/bert-large-uncased) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | 30522 | gelu | standard | False | BertForMaskedLM | LN | google-bert/bert-large-uncased | BertTokenizer | 30522.0 | G9iEWgpI_JY73i8Lym9gBVhq4BI= | +| t5-base | [google-t5/t5-base](https://huggingface.co/google-t5/t5-base) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 32128 | relu | relative_positional_bias | False | T5ForConditionalGeneration | LN | google-t5/t5-base | T5Tokenizer | 32100.0 | jQeywCyCMVL_vza2wKfpuwjNVys= | +| t5-large | [google-t5/t5-large](https://huggingface.co/google-t5/t5-large) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | 32128 | relu | relative_positional_bias | False | T5ForConditionalGeneration | LN | google-t5/t5-large | T5Tokenizer | 32100.0 | jQeywCyCMVL_vza2wKfpuwjNVys= | +| t5-small | [google-t5/t5-small](https://huggingface.co/google-t5/t5-small) | 19M | 18874368 | 18874368 | 6 | 8 | 512 | 32128 | relu | relative_positional_bias | False | T5ForConditionalGeneration | LN | google-t5/t5-small | T5Tokenizer | 32100.0 | jQeywCyCMVL_vza2wKfpuwjNVys= | +| gemma-2-27b | [google/gemma-2-27b](https://huggingface.co/google/gemma-2-27b) | 27B | 26914848768 | 26914848768 | 46 | 32 | 4608 | 256000 | gelu_pytorch_tanh | rotary | False | Gemma2ForCausalLM | RMS | google/gemma-2-27b | GemmaTokenizer | 256000.0 | 9GdVxqTUEa0Qt2kBBEHf3ebUisw= | +| gemma-2-27b-it | [google/gemma-2-27b-it](https://huggingface.co/google/gemma-2-27b-it) | 27B | 26914848768 | 26914848768 | 46 | 32 | 4608 | 256000 | gelu_pytorch_tanh | rotary | False | Gemma2ForCausalLM | RMS | google/gemma-2-27b-it | GemmaTokenizer | 256000.0 | 9GdVxqTUEa0Qt2kBBEHf3ebUisw= | +| gemma-2-2b | [google/gemma-2-2b](https://huggingface.co/google/gemma-2-2b) | 2.1B | 2146959360 | 2146959360 | 26 | 8 | 2304 | 256000 | gelu_pytorch_tanh | rotary | False | Gemma2ForCausalLM | RMS | google/gemma-2-2b | GemmaTokenizer | 256000.0 | 9GdVxqTUEa0Qt2kBBEHf3ebUisw= | +| gemma-2-2b-it | [google/gemma-2-2b-it](https://huggingface.co/google/gemma-2-2b-it) | 2.1B | 2146959360 | 2146959360 | 26 | 8 | 2304 | 256000 | gelu_pytorch_tanh | rotary | False | Gemma2ForCausalLM | RMS | google/gemma-2-2b-it | GemmaTokenizer | 256000.0 | 9GdVxqTUEa0Qt2kBBEHf3ebUisw= | +| gemma-2-9b | [google/gemma-2-9b](https://huggingface.co/google/gemma-2-9b) | 8.9B | 8940158976 | 8940158976 | 42 | 16 | 3584 | 256000 | gelu_pytorch_tanh | rotary | False | Gemma2ForCausalLM | RMS | google/gemma-2-9b | GemmaTokenizer | 256000.0 | 9GdVxqTUEa0Qt2kBBEHf3ebUisw= | +| gemma-2-9b-it | [google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) | 8.9B | 8940158976 | 8940158976 | 42 | 16 | 3584 | 256000 | gelu_pytorch_tanh | rotary | False | Gemma2ForCausalLM | RMS | google/gemma-2-9b-it | GemmaTokenizer | 256000.0 | 9GdVxqTUEa0Qt2kBBEHf3ebUisw= | +| gemma-2b | [google/gemma-2b](https://huggingface.co/google/gemma-2b) | 2.1B | 2113929216 | 2113929216 | 18 | 8 | 2048 | 256000 | gelu | rotary | False | GemmaForCausalLM | RMS | google/gemma-2b | GemmaTokenizer | 256000.0 | 9GdVxqTUEa0Qt2kBBEHf3ebUisw= | +| gemma-2b-it | [google/gemma-2b-it](https://huggingface.co/google/gemma-2b-it) | 2.1B | 2113929216 | 2113929216 | 18 | 8 | 2048 | 256000 | gelu | rotary | False | GemmaForCausalLM | RMS | google/gemma-2b-it | GemmaTokenizer | 256000.0 | 9GdVxqTUEa0Qt2kBBEHf3ebUisw= | +| gemma-3-12b-it | [google/gemma-3-12b-it](https://huggingface.co/google/gemma-3-12b-it) | 12B | 11513364480 | 11513364480 | 48 | 16 | 3840 | 262208 | gelu_pytorch_tanh | rotary | False | Gemma3ForConditionalGeneration | RMS | google/gemma-3-12b-it | GemmaTokenizer | 262144.0 | cTyOT2pw07eF2vN0sn4RYJBX-mY= | +| gemma-3-12b-pt | [google/gemma-3-12b-pt](https://huggingface.co/google/gemma-3-12b-pt) | 12B | 11513364480 | 11513364480 | 48 | 16 | 3840 | 262208 | gelu_pytorch_tanh | rotary | False | Gemma3ForConditionalGeneration | RMS | google/gemma-3-12b-pt | GemmaTokenizer | 262144.0 | cTyOT2pw07eF2vN0sn4RYJBX-mY= | +| gemma-3-1b-it | [google/gemma-3-1b-it](https://huggingface.co/google/gemma-3-1b-it) | 744M | 743768064 | 743768064 | 26 | 4 | 1152 | 262144 | gelu_pytorch_tanh | rotary | False | Gemma3ForCausalLM | RMS | google/gemma-3-1b-it | GemmaTokenizer | 262144.0 | cTyOT2pw07eF2vN0sn4RYJBX-mY= | +| gemma-3-1b-pt | [google/gemma-3-1b-pt](https://huggingface.co/google/gemma-3-1b-pt) | 744M | 743768064 | 743768064 | 26 | 4 | 1152 | 262144 | gelu_pytorch_tanh | rotary | False | Gemma3ForCausalLM | RMS | google/gemma-3-1b-pt | GemmaTokenizer | 262144.0 | cTyOT2pw07eF2vN0sn4RYJBX-mY= | +| gemma-3-270m | [google/gemma-3-270m](https://huggingface.co/google/gemma-3-270m) | 118M | 117964800 | 117964800 | 18 | 4 | 640 | 262144 | gelu_pytorch_tanh | rotary | False | Gemma3ForCausalLM | RMS | google/gemma-3-270m | GemmaTokenizer | 262144.0 | cTyOT2pw07eF2vN0sn4RYJBX-mY= | +| gemma-3-270m-it | [google/gemma-3-270m-it](https://huggingface.co/google/gemma-3-270m-it) | 118M | 117964800 | 117964800 | 18 | 4 | 640 | 262144 | gelu_pytorch_tanh | rotary | False | Gemma3ForCausalLM | RMS | google/gemma-3-270m-it | GemmaTokenizer | 262144.0 | cTyOT2pw07eF2vN0sn4RYJBX-mY= | +| gemma-3-27b-it | [google/gemma-3-27b-it](https://huggingface.co/google/gemma-3-27b-it) | 27B | 26963607552 | 26963607552 | 62 | 32 | 5376 | 262208 | gelu_pytorch_tanh | rotary | False | Gemma3ForConditionalGeneration | RMS | google/gemma-3-27b-it | GemmaTokenizer | 262144.0 | cTyOT2pw07eF2vN0sn4RYJBX-mY= | +| gemma-3-27b-pt | [google/gemma-3-27b-pt](https://huggingface.co/google/gemma-3-27b-pt) | 27B | 26963607552 | 26963607552 | 62 | 32 | 5376 | 262208 | gelu_pytorch_tanh | rotary | False | Gemma3ForConditionalGeneration | RMS | google/gemma-3-27b-pt | GemmaTokenizer | 262144.0 | cTyOT2pw07eF2vN0sn4RYJBX-mY= | +| gemma-3-4b-it | [google/gemma-3-4b-it](https://huggingface.co/google/gemma-3-4b-it) | 3.4B | 3386900480 | 3386900480 | 34 | 8 | 2560 | 262208 | gelu_pytorch_tanh | rotary | False | Gemma3ForConditionalGeneration | RMS | google/gemma-3-4b-it | GemmaTokenizer | 262144.0 | cTyOT2pw07eF2vN0sn4RYJBX-mY= | +| gemma-3-4b-pt | [google/gemma-3-4b-pt](https://huggingface.co/google/gemma-3-4b-pt) | 3.4B | 3386900480 | 3386900480 | 34 | 8 | 2560 | 262208 | gelu_pytorch_tanh | rotary | False | Gemma3ForConditionalGeneration | RMS | google/gemma-3-4b-pt | GemmaTokenizer | 262144.0 | cTyOT2pw07eF2vN0sn4RYJBX-mY= | +| gemma-7b | [google/gemma-7b](https://huggingface.co/google/gemma-7b) | 7.8B | 7751073792 | 7751073792 | 28 | 16 | 3072 | 256000 | gelu | rotary | False | GemmaForCausalLM | RMS | google/gemma-7b | GemmaTokenizer | 256000.0 | 9GdVxqTUEa0Qt2kBBEHf3ebUisw= | +| gemma-7b-it | [google/gemma-7b-it](https://huggingface.co/google/gemma-7b-it) | 7.8B | 7751073792 | 7751073792 | 28 | 16 | 3072 | 256000 | gelu | rotary | False | GemmaForCausalLM | RMS | google/gemma-7b-it | GemmaTokenizer | 256000.0 | 9GdVxqTUEa0Qt2kBBEHf3ebUisw= | +| medgemma-27b-it | [google/medgemma-27b-it](https://huggingface.co/google/medgemma-27b-it) | 27B | 26963607552 | 26963607552 | 62 | 32 | 5376 | 262208 | gelu_pytorch_tanh | rotary | False | Gemma3ForConditionalGeneration | RMS | | | | | +| medgemma-27b-text-it | [google/medgemma-27b-text-it](https://huggingface.co/google/medgemma-27b-text-it) | 27B | 26963607552 | 26963607552 | 62 | 32 | 5376 | 262144 | gelu_pytorch_tanh | rotary | False | Gemma3ForCausalLM | RMS | | | | | +| medgemma-4b-it | [google/medgemma-4b-it](https://huggingface.co/google/medgemma-4b-it) | 3.4B | 3386900480 | 3386900480 | 34 | 8 | 2560 | 262208 | gelu_pytorch_tanh | rotary | False | Gemma3ForConditionalGeneration | RMS | | | | | +| medgemma-4b-pt | [google/medgemma-4b-pt](https://huggingface.co/google/medgemma-4b-pt) | 3.4B | 3386900480 | 3386900480 | 34 | 8 | 2560 | 262208 | gelu_pytorch_tanh | rotary | False | Gemma3ForConditionalGeneration | RMS | google/medgemma-4b-pt | GemmaTokenizer | 262144.0 | cTyOT2pw07eF2vN0sn4RYJBX-mY= | +| gpt2-small | [gpt2](https://huggingface.co/gpt2) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | gpt2 | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| gpt2-large | [gpt2-large](https://huggingface.co/gpt2-large) | 708M | 707788800 | 707788800 | 36 | 20 | 1280 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | gpt2-large | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| gpt2-medium | [gpt2-medium](https://huggingface.co/gpt2-medium) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | gpt2-medium | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| gpt2-xl | [gpt2-xl](https://huggingface.co/gpt2-xl) | 1.5B | 1474560000 | 1474560000 | 48 | 25 | 1600 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | gpt2-xl | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| llama-13b | [llama-13b-hf](https://huggingface.co/llama-13b-hf) | 13B | 12687769600 | 12687769600 | 40 | 40 | 5120 | 32000 | silu | rotary | False | LlamaForCausalLM | RMS | huggyllama/llama-13b | LlamaTokenizer | 32000.0 | e3A7wYziNQPAWcJ15GMAQY8qZqw= | +| llama-30b | [llama-30b-hf](https://huggingface.co/llama-30b-hf) | 32B | 32102154240 | 32102154240 | 60 | 52 | 6656 | 32000 | silu | rotary | False | LlamaForCausalLM | RMS | huggyllama/llama-30b | LlamaTokenizer | 32000.0 | e3A7wYziNQPAWcJ15GMAQY8qZqw= | +| llama-65b | [llama-65b-hf](https://huggingface.co/llama-65b-hf) | 65B | 64760053760 | 64760053760 | 80 | 64 | 8192 | 32000 | silu | rotary | False | LlamaForCausalLM | RMS | huggyllama/llama-65b | LlamaTokenizer | 32000.0 | e3A7wYziNQPAWcJ15GMAQY8qZqw= | +| llama-7b | [llama-7b-hf](https://huggingface.co/llama-7b-hf) | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 32000 | silu | rotary | False | LlamaForCausalLM | RMS | huggyllama/llama-7b | LlamaTokenizer | 32000.0 | e3A7wYziNQPAWcJ15GMAQY8qZqw= | +| Llama-2-13b-chat | | 13B | 12687769600 | 12687769600 | 40 | 40 | 5120 | 32000 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Llama-2-13b-chat-hf | LlamaTokenizer | 32000.0 | e3A7wYziNQPAWcJ15GMAQY8qZqw= | +| Llama-2-13b | | 13B | 12687769600 | 12687769600 | 40 | 40 | 5120 | 32000 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Llama-2-13b-hf | LlamaTokenizer | 32000.0 | e3A7wYziNQPAWcJ15GMAQY8qZqw= | +| Llama-2-70b-chat | | 78B | 77846282240 | 77846282240 | 80 | 64 | 8192 | 32000 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Llama-2-70b-chat-hf | LlamaTokenizer | 32000.0 | e3A7wYziNQPAWcJ15GMAQY8qZqw= | +| Llama-2-7b-chat | | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 32000 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Llama-2-7b-chat-hf | LlamaTokenizer | 32000.0 | e3A7wYziNQPAWcJ15GMAQY8qZqw= | +| Llama-2-7b | | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 32000 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Llama-2-7b-hf | LlamaTokenizer | 32000.0 | e3A7wYziNQPAWcJ15GMAQY8qZqw= | +| meta-llama/Llama-3.1-70B | | 78B | 77846282240 | 77846282240 | 80 | 64 | 8192 | 128256 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Llama-3.1-70B | TokenizersBackend | 128000.0 | j9N50ddC7mjCgS4GseU9LmKZDKk= | +| meta-llama/Llama-3.1-70B-Instruct | | 78B | 77846282240 | 77846282240 | 80 | 64 | 8192 | 128256 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Llama-3.1-70B-Instruct | TokenizersBackend | 128000.0 | j9N50ddC7mjCgS4GseU9LmKZDKk= | +| meta-llama/Llama-3.1-8B | | 7.8B | 7784628224 | 7784628224 | 32 | 32 | 4096 | 128256 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Llama-3.1-8B | TokenizersBackend | 128000.0 | j9N50ddC7mjCgS4GseU9LmKZDKk= | +| meta-llama/Llama-3.1-8B-Instruct | | 7.8B | 7784628224 | 7784628224 | 32 | 32 | 4096 | 128256 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Llama-3.1-8B-Instruct | TokenizersBackend | 128000.0 | j9N50ddC7mjCgS4GseU9LmKZDKk= | +| meta-llama/Llama-3.2-1B | | 1.1B | 1073741824 | 1073741824 | 16 | 32 | 2048 | 128256 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Llama-3.2-1B | TokenizersBackend | 128000.0 | j9N50ddC7mjCgS4GseU9LmKZDKk= | +| meta-llama/Llama-3.2-1B-Instruct | | 1.1B | 1073741824 | 1073741824 | 16 | 32 | 2048 | 128256 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Llama-3.2-1B-Instruct | TokenizersBackend | 128000.0 | j9N50ddC7mjCgS4GseU9LmKZDKk= | +| meta-llama/Llama-3.2-3B | | 3.2B | 3170893824 | 3170893824 | 28 | 24 | 3072 | 128256 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Llama-3.2-3B | TokenizersBackend | 128000.0 | j9N50ddC7mjCgS4GseU9LmKZDKk= | +| meta-llama/Llama-3.2-3B-Instruct | | 3.2B | 3170893824 | 3170893824 | 28 | 24 | 3072 | 128256 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Llama-3.2-3B-Instruct | TokenizersBackend | 128000.0 | j9N50ddC7mjCgS4GseU9LmKZDKk= | +| meta-llama/Llama-3.3-70B-Instruct | | 78B | 77846282240 | 77846282240 | 80 | 64 | 8192 | 128256 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Llama-3.3-70B-Instruct | TokenizersBackend | 128000.0 | j9N50ddC7mjCgS4GseU9LmKZDKk= | +| meta-llama/Meta-Llama-3-70B | | 78B | 77846282240 | 77846282240 | 80 | 64 | 8192 | 128256 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Meta-Llama-3-70B | TokenizersBackend | 128000.0 | RnzNv9w_ITBp6b2dcibKR7_l85I= | +| meta-llama/Meta-Llama-3-70B-Instruct | | 78B | 77846282240 | 77846282240 | 80 | 64 | 8192 | 128256 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Meta-Llama-3-70B-Instruct | TokenizersBackend | 128000.0 | RnzNv9w_ITBp6b2dcibKR7_l85I= | +| meta-llama/Meta-Llama-3-8B | | 7.8B | 7784628224 | 7784628224 | 32 | 32 | 4096 | 128256 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Meta-Llama-3-8B | TokenizersBackend | 128000.0 | RnzNv9w_ITBp6b2dcibKR7_l85I= | +| meta-llama/Meta-Llama-3-8B-Instruct | | 7.8B | 7784628224 | 7784628224 | 32 | 32 | 4096 | 128256 | silu | rotary | False | LlamaForCausalLM | RMS | meta-llama/Meta-Llama-3-8B-Instruct | TokenizersBackend | 128000.0 | RnzNv9w_ITBp6b2dcibKR7_l85I= | +| phi-1 | [microsoft/phi-1](https://huggingface.co/microsoft/phi-1) | 1.2B | 1207959552 | 1207959552 | 24 | 32 | 2048 | 51200 | gelu_new | rotary | True | PhiForCausalLM | LN | microsoft/phi-1 | CodeGenTokenizer | 50257.0 | TYk6J3OrqdU2F7JYiSfFXtd-vB4= | +| phi-1_5 | [microsoft/phi-1_5](https://huggingface.co/microsoft/phi-1_5) | 1.2B | 1207959552 | 1207959552 | 24 | 32 | 2048 | 51200 | gelu_new | rotary | True | PhiForCausalLM | LN | microsoft/phi-1_5 | CodeGenTokenizer | 50257.0 | TYk6J3OrqdU2F7JYiSfFXtd-vB4= | +| phi-2 | [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) | 2.5B | 2516582400 | 2516582400 | 32 | 32 | 2560 | 51200 | gelu_new | rotary | True | PhiForCausalLM | LN | microsoft/phi-2 | CodeGenTokenizer | 50257.0 | TYk6J3OrqdU2F7JYiSfFXtd-vB4= | +| phi-3 | [microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct) | 3.6B | 3623878656 | 3623878656 | 32 | 32 | 3072 | 32064 | silu | rotary | False | Phi3ForCausalLM | RMS | microsoft/Phi-3-mini-4k-instruct | TokenizersBackend | 32000.0 | 2BcGXsWoZjuOkMtb6uTbGL68fbc= | +| phi-4 | [microsoft/phi-4](https://huggingface.co/microsoft/phi-4) | 15B | 15204352000 | 15204352000 | 40 | 40 | 5120 | 100352 | silu | rotary | False | Phi3ForCausalLM | RMS | microsoft/phi-4 | TokenizersBackend | 100352.0 | uJZqWk6gqn6tO_nlSJEZsP9MITQ= | +| mistral-7b-instruct | [mistralai/Mistral-7B-Instruct-v0.1](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1) | 7.8B | 7784628224 | 7784628224 | 32 | 32 | 4096 | 32000 | silu | rotary | False | MistralForCausalLM | RMS | mistralai/Mistral-7B-Instruct-v0.1 | TokenizersBackend | 32000.0 | kkCQxUk-PF9Ay_ZKDdKCh02YaGQ= | +| mistral-7b | [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) | 7.8B | 7784628224 | 7784628224 | 32 | 32 | 4096 | 32000 | silu | rotary | False | MistralForCausalLM | RMS | mistralai/Mistral-7B-v0.1 | TokenizersBackend | 32000.0 | kkCQxUk-PF9Ay_ZKDdKCh02YaGQ= | +| mistral-nemo-base-2407 | [mistralai/Mistral-Nemo-Base-2407](https://huggingface.co/mistralai/Mistral-Nemo-Base-2407) | 12B | 12163481600 | 12163481600 | 40 | 32 | 5120 | 131072 | silu | rotary | False | MistralForCausalLM | RMS | mistralai/Mistral-Nemo-Base-2407 | TokenizersBackend | 131072.0 | 0xs_eSvVgsyZGbcLSIpGUn4Gdms= | +| mistralai/Mistral-Small-24B-Base-2501 | | 23B | 23488102400 | 23488102400 | 40 | 32 | 5120 | 131072 | silu | rotary | False | MistralForCausalLM | RMS | mistralai/Mistral-Small-24B-Base-2501 | TokenizersBackend | 131072.0 | GEwgZayWxpmhQxtMq-WrVFJEfqM= | +| mixtral-instruct | [mistralai/Mixtral-8x7B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1) | 47B | 47245688832 | 47245688832 | 32 | 32 | 4096 | 32000 | silu | rotary | False | MixtralForCausalLM | RMS | mistralai/Mixtral-8x7B-Instruct-v0.1 | TokenizersBackend | 32000.0 | kkCQxUk-PF9Ay_ZKDdKCh02YaGQ= | +| mixtral | [mistralai/Mixtral-8x7B-v0.1](https://huggingface.co/mistralai/Mixtral-8x7B-v0.1) | 47B | 47245688832 | 47245688832 | 32 | 32 | 4096 | 32000 | silu | rotary | False | MixtralForCausalLM | RMS | mistralai/Mixtral-8x7B-v0.1 | TokenizersBackend | 32000.0 | kkCQxUk-PF9Ay_ZKDdKCh02YaGQ= | +| attn-only-2l-demo | [NeelNanda/Attn-Only-2L512W-Shortformer-6B-big-lr](https://huggingface.co/NeelNanda/Attn-Only-2L512W-Shortformer-6B-big-lr) | 2.1M | 2097152 | 2097152 | 2 | 8 | 512 | 50277 | solu_ln | shortformer | False | neel | | EleutherAI/gpt-neox-20b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| attn-only-1l | [NeelNanda/Attn_Only_1L512W_C4_Code](https://huggingface.co/NeelNanda/Attn_Only_1L512W_C4_Code) | 1.0M | 1048576 | 1048576 | 1 | 8 | 512 | 48262 | solu_ln | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| attn-only-2l | [NeelNanda/Attn_Only_2L512W_C4_Code](https://huggingface.co/NeelNanda/Attn_Only_2L512W_C4_Code) | 2.1M | 2097152 | 2097152 | 2 | 8 | 512 | 48262 | solu_ln | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| attn-only-3l | [NeelNanda/Attn_Only_3L512W_C4_Code](https://huggingface.co/NeelNanda/Attn_Only_3L512W_C4_Code) | 3.1M | 3145728 | 3145728 | 3 | 8 | 512 | 48262 | solu_ln | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| attn-only-4l | [NeelNanda/Attn_Only_4L512W_C4_Code](https://huggingface.co/NeelNanda/Attn_Only_4L512W_C4_Code) | 4.2M | 4194304 | 4194304 | 4 | 8 | 512 | 48262 | solu_ln | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| gelu-1l | [NeelNanda/GELU_1L512W_C4_Code](https://huggingface.co/NeelNanda/GELU_1L512W_C4_Code) | 3.1M | 3145728 | 3145728 | 1 | 8 | 512 | 48262 | gelu | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| gelu-2l | [NeelNanda/GELU_2L512W_C4_Code](https://huggingface.co/NeelNanda/GELU_2L512W_C4_Code) | 6.3M | 6291456 | 6291456 | 2 | 8 | 512 | 48262 | gelu | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| gelu-3l | [NeelNanda/GELU_3L512W_C4_Code](https://huggingface.co/NeelNanda/GELU_3L512W_C4_Code) | 9.4M | 9437184 | 9437184 | 3 | 8 | 512 | 48262 | gelu | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| gelu-4l | [NeelNanda/GELU_4L512W_C4_Code](https://huggingface.co/NeelNanda/GELU_4L512W_C4_Code) | 13M | 12582912 | 12582912 | 4 | 8 | 512 | 48262 | gelu | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| solu-10l | [NeelNanda/SoLU_10L1280W_C4_Code](https://huggingface.co/NeelNanda/SoLU_10L1280W_C4_Code) | 197M | 196608000 | 196608000 | 10 | 20 | 1280 | 48262 | solu_ln | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| solu-10l-pile | [NeelNanda/SoLU_10L_v22_old](https://huggingface.co/NeelNanda/SoLU_10L_v22_old) | 197M | 196608000 | 196608000 | 10 | 20 | 1280 | 50278 | solu_ln | standard | False | neel-solu-old | LNPre | EleutherAI/gpt-neox-20b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| solu-12l | [NeelNanda/SoLU_12L1536W_C4_Code](https://huggingface.co/NeelNanda/SoLU_12L1536W_C4_Code) | 340M | 339738624 | 339738624 | 12 | 24 | 1536 | 48262 | solu_ln | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| solu-12l-pile | [NeelNanda/SoLU_12L_v23_old](https://huggingface.co/NeelNanda/SoLU_12L_v23_old) | 340M | 339738624 | 339738624 | 12 | 24 | 1536 | 50278 | solu_ln | standard | False | neel-solu-old | LN | EleutherAI/gpt-neox-20b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| solu-1l | [NeelNanda/SoLU_1L512W_C4_Code](https://huggingface.co/NeelNanda/SoLU_1L512W_C4_Code) | 3.1M | 3145728 | 3145728 | 1 | 8 | 512 | 48262 | solu_ln | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| solu-1l-wiki | [NeelNanda/SoLU_1L512W_Wiki_Finetune](https://huggingface.co/NeelNanda/SoLU_1L512W_Wiki_Finetune) | 3.1M | 3145728 | 3145728 | 1 | 8 | 512 | 48262 | solu_ln | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| solu-1l-pile | [NeelNanda/SoLU_1L_v9_old](https://huggingface.co/NeelNanda/SoLU_1L_v9_old) | 13M | 12582912 | 12582912 | 1 | 16 | 1024 | 50278 | solu_ln | standard | False | neel-solu-old | LN | EleutherAI/gpt-neox-20b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| solu-2l | [NeelNanda/SoLU_2L512W_C4_Code](https://huggingface.co/NeelNanda/SoLU_2L512W_C4_Code) | 6.3M | 6291456 | 6291456 | 2 | 8 | 512 | 48262 | solu_ln | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| solu-2l-pile | [NeelNanda/SoLU_2L_v10_old](https://huggingface.co/NeelNanda/SoLU_2L_v10_old) | 13M | 12812288 | 12812288 | 2 | 11 | 736 | 50278 | solu_ln | standard | False | neel-solu-old | LNPre | EleutherAI/gpt-neox-20b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| solu-3l | [NeelNanda/SoLU_3L512W_C4_Code](https://huggingface.co/NeelNanda/SoLU_3L512W_C4_Code) | 9.4M | 9437184 | 9437184 | 3 | 8 | 512 | 48262 | solu_ln | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| solu-4l | [NeelNanda/SoLU_4L512W_C4_Code](https://huggingface.co/NeelNanda/SoLU_4L512W_C4_Code) | 13M | 12582912 | 12582912 | 4 | 8 | 512 | 48262 | solu_ln | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| solu-4l-wiki | [NeelNanda/SoLU_4L512W_Wiki_Finetune](https://huggingface.co/NeelNanda/SoLU_4L512W_Wiki_Finetune) | 13M | 12582912 | 12582912 | 4 | 8 | 512 | 48262 | solu_ln | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| solu-4l-pile | [NeelNanda/SoLU_4L_v11_old](https://huggingface.co/NeelNanda/SoLU_4L_v11_old) | 13M | 12582912 | 12582912 | 4 | 8 | 512 | 50278 | solu_ln | standard | False | neel-solu-old | LNPre | EleutherAI/gpt-neox-20b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| solu-6l | [NeelNanda/SoLU_6L768W_C4_Code](https://huggingface.co/NeelNanda/SoLU_6L768W_C4_Code) | 42M | 42467328 | 42467328 | 6 | 12 | 768 | 48262 | solu_ln | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| solu-6l-pile | [NeelNanda/SoLU_6L_v13_old](https://huggingface.co/NeelNanda/SoLU_6L_v13_old) | 42M | 42467328 | 42467328 | 6 | 12 | 768 | 50278 | solu_ln | standard | False | neel-solu-old | LNPre | EleutherAI/gpt-neox-20b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| solu-8l | [NeelNanda/SoLU_8L1024W_C4_Code](https://huggingface.co/NeelNanda/SoLU_8L1024W_C4_Code) | 101M | 100663296 | 100663296 | 8 | 16 | 1024 | 48262 | solu_ln | standard | False | neel | LN | NeelNanda/gpt-neox-tokenizer-digits | TokenizersBackend | 48262.0 | AsGo9tS8Sq4-rlVHM2o3-GyDkJU= | +| solu-8l-pile | [NeelNanda/SoLU_8L_v21_old](https://huggingface.co/NeelNanda/SoLU_8L_v21_old) | 101M | 100663296 | 100663296 | 8 | 16 | 1024 | 50278 | solu_ln | standard | False | neel-solu-old | LNPre | EleutherAI/gpt-neox-20b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| gpt-oss-20b | [openai/gpt-oss-20b](https://huggingface.co/openai/gpt-oss-20b) | 20B | 20244971520 | 20244971520 | 24 | 64 | 2880 | 201088 | silu | rotary | False | GptOssForCausalLM | RMS | openai/gpt-oss-20b | TokenizersBackend | 199998.0 | c9AMJHU0SnHAXwla61AkYA78n-w= | +| qwen-14b | [Qwen/Qwen-14B](https://huggingface.co/Qwen/Qwen-14B) | 13B | 12609126400 | 12609126400 | 40 | 40 | 5120 | 152064 | silu | rotary | False | QWenLMHeadModel | RMS | | | | | +| qwen-14b-chat | [Qwen/Qwen-14B-Chat](https://huggingface.co/Qwen/Qwen-14B-Chat) | 13B | 12609126400 | 12609126400 | 40 | 40 | 5120 | 152064 | silu | rotary | False | QWenLMHeadModel | RMS | | | | | +| qwen-1.8b | [Qwen/Qwen-1_8B](https://huggingface.co/Qwen/Qwen-1_8B) | 1.2B | 1214251008 | 1214251008 | 24 | 16 | 2048 | 151936 | silu | rotary | False | QWenLMHeadModel | RMS | | | | | +| qwen-7b | [Qwen/Qwen-7B](https://huggingface.co/Qwen/Qwen-7B) | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 151936 | silu | rotary | False | QWenLMHeadModel | RMS | | | | | +| qwen-7b-chat | [Qwen/Qwen-7B-Chat](https://huggingface.co/Qwen/Qwen-7B-Chat) | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 151936 | silu | rotary | False | QWenLMHeadModel | RMS | | | | | +| qwen1.5-0.5b | [Qwen/Qwen1.5-0.5B](https://huggingface.co/Qwen/Qwen1.5-0.5B) | 308M | 308281344 | 308281344 | 24 | 16 | 1024 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen1.5-0.5B | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen1.5-0.5b-chat | [Qwen/Qwen1.5-0.5B-Chat](https://huggingface.co/Qwen/Qwen1.5-0.5B-Chat) | 308M | 308281344 | 308281344 | 24 | 16 | 1024 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen1.5-0.5B-Chat | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen1.5-1.8b | [Qwen/Qwen1.5-1.8B](https://huggingface.co/Qwen/Qwen1.5-1.8B) | 1.2B | 1214251008 | 1214251008 | 24 | 16 | 2048 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen1.5-1.8B | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen1.5-1.8b-chat | [Qwen/Qwen1.5-1.8B-Chat](https://huggingface.co/Qwen/Qwen1.5-1.8B-Chat) | 1.2B | 1214251008 | 1214251008 | 24 | 16 | 2048 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen1.5-1.8B-Chat | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen1.5-14b | [Qwen/Qwen1.5-14B](https://huggingface.co/Qwen/Qwen1.5-14B) | 13B | 12609126400 | 12609126400 | 40 | 40 | 5120 | 152064 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen1.5-14B | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen1.5-14b-chat | [Qwen/Qwen1.5-14B-Chat](https://huggingface.co/Qwen/Qwen1.5-14B-Chat) | 13B | 12609126400 | 12609126400 | 40 | 40 | 5120 | 152064 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen1.5-14B-Chat | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen1.5-4b | [Qwen/Qwen1.5-4B](https://huggingface.co/Qwen/Qwen1.5-4B) | 3.2B | 3171942400 | 3171942400 | 40 | 20 | 2560 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen1.5-4B | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen1.5-4b-chat | [Qwen/Qwen1.5-4B-Chat](https://huggingface.co/Qwen/Qwen1.5-4B-Chat) | 3.2B | 3171942400 | 3171942400 | 40 | 20 | 2560 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen1.5-4B-Chat | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen1.5-7b | [Qwen/Qwen1.5-7B](https://huggingface.co/Qwen/Qwen1.5-7B) | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen1.5-7B | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen1.5-7b-chat | [Qwen/Qwen1.5-7B-Chat](https://huggingface.co/Qwen/Qwen1.5-7B-Chat) | 6.5B | 6476005376 | 6476005376 | 32 | 32 | 4096 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen1.5-7B-Chat | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen2-0.5b | [Qwen/Qwen2-0.5B](https://huggingface.co/Qwen/Qwen2-0.5B) | 391M | 390856704 | 390856704 | 24 | 14 | 896 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2-0.5B | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen2-0.5b-instruct | [Qwen/Qwen2-0.5B-Instruct](https://huggingface.co/Qwen/Qwen2-0.5B-Instruct) | 391M | 390856704 | 390856704 | 24 | 14 | 896 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2-0.5B-Instruct | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen2-1.5b | [Qwen/Qwen2-1.5B](https://huggingface.co/Qwen/Qwen2-1.5B) | 1.4B | 1420296192 | 1420296192 | 28 | 12 | 1536 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2-1.5B | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen2-1.5b-instruct | [Qwen/Qwen2-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2-1.5B-Instruct) | 1.4B | 1420296192 | 1420296192 | 28 | 12 | 1536 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2-1.5B-Instruct | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen2-7b | [Qwen/Qwen2-7B](https://huggingface.co/Qwen/Qwen2-7B) | 7.1B | 7141851136 | 7141851136 | 28 | 28 | 3584 | 152064 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2-7B | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen2-7b-instruct | [Qwen/Qwen2-7B-Instruct](https://huggingface.co/Qwen/Qwen2-7B-Instruct) | 7.1B | 7141851136 | 7141851136 | 28 | 28 | 3584 | 152064 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2-7B-Instruct | Qwen2Tokenizer | 151643.0 | vakQOjPaHpZ23kxcqX0tTXi2EzQ= | +| qwen2.5-0.5b | [Qwen/Qwen2.5-0.5B](https://huggingface.co/Qwen/Qwen2.5-0.5B) | 391M | 390856704 | 390856704 | 24 | 14 | 896 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2.5-0.5B | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| qwen2.5-0.5b-instruct | [Qwen/Qwen2.5-0.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct) | 391M | 390856704 | 390856704 | 24 | 14 | 896 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2.5-0.5B-Instruct | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| qwen2.5-1.5b | [Qwen/Qwen2.5-1.5B](https://huggingface.co/Qwen/Qwen2.5-1.5B) | 1.4B | 1420296192 | 1420296192 | 28 | 12 | 1536 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2.5-1.5B | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| qwen2.5-1.5b-instruct | [Qwen/Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct) | 1.4B | 1420296192 | 1420296192 | 28 | 12 | 1536 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2.5-1.5B-Instruct | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| qwen2.5-14b | [Qwen/Qwen2.5-14B](https://huggingface.co/Qwen/Qwen2.5-14B) | 15B | 15225323520 | 15225323520 | 48 | 40 | 5120 | 152064 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2.5-14B | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| qwen2.5-14b-instruct | [Qwen/Qwen2.5-14B-Instruct](https://huggingface.co/Qwen/Qwen2.5-14B-Instruct) | 15B | 15225323520 | 15225323520 | 48 | 40 | 5120 | 152064 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2.5-14B-Instruct | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| qwen2.5-32b | [Qwen/Qwen2.5-32B](https://huggingface.co/Qwen/Qwen2.5-32B) | 34B | 33889976320 | 33889976320 | 64 | 40 | 5120 | 152064 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2.5-32B | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| qwen2.5-32b-instruct | [Qwen/Qwen2.5-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-32B-Instruct) | 34B | 33889976320 | 33889976320 | 64 | 40 | 5120 | 152064 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2.5-32B-Instruct | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| qwen2.5-3b | [Qwen/Qwen2.5-3B](https://huggingface.co/Qwen/Qwen2.5-3B) | 3.0B | 3038773248 | 3038773248 | 36 | 16 | 2048 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2.5-3B | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| qwen2.5-3b-instruct | [Qwen/Qwen2.5-3B-Instruct](https://huggingface.co/Qwen/Qwen2.5-3B-Instruct) | 3.0B | 3038773248 | 3038773248 | 36 | 16 | 2048 | 151936 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2.5-3B-Instruct | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| qwen2.5-72b | [Qwen/Qwen2.5-72B](https://huggingface.co/Qwen/Qwen2.5-72B) | 80B | 79607889920 | 79607889920 | 80 | 64 | 8192 | 152064 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2.5-72B | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| qwen2.5-72b-instruct | [Qwen/Qwen2.5-72B-Instruct](https://huggingface.co/Qwen/Qwen2.5-72B-Instruct) | 80B | 79607889920 | 79607889920 | 80 | 64 | 8192 | 152064 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2.5-72B-Instruct | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| qwen2.5-7b | [Qwen/Qwen2.5-7B](https://huggingface.co/Qwen/Qwen2.5-7B) | 7.1B | 7141851136 | 7141851136 | 28 | 28 | 3584 | 152064 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2.5-7B | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| qwen2.5-7b-instruct | [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) | 7.1B | 7141851136 | 7141851136 | 28 | 28 | 3584 | 152064 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/Qwen2.5-7B-Instruct | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| qwen3-0.6b | [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) | 499M | 499122176 | 499122176 | 28 | 16 | 1024 | 151936 | silu | rotary | False | Qwen3ForCausalLM | RMS | Qwen/Qwen3-0.6B | Qwen2Tokenizer | 151643.0 | OfOK7SjmrN4KFUqlDylX-Up77SM= | +| qwen3-0.6b-base | [Qwen/Qwen3-0.6B-Base](https://huggingface.co/Qwen/Qwen3-0.6B-Base) | 499M | 499122176 | 499122176 | 28 | 16 | 1024 | 151936 | silu | rotary | False | Qwen3ForCausalLM | RMS | Qwen/Qwen3-0.6B-Base | Qwen2Tokenizer | 151643.0 | OfOK7SjmrN4KFUqlDylX-Up77SM= | +| qwen3-1.7b | [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B) | 1.5B | 1526726656 | 1526726656 | 28 | 16 | 2048 | 151936 | silu | rotary | False | Qwen3ForCausalLM | RMS | Qwen/Qwen3-1.7B | Qwen2Tokenizer | 151643.0 | OfOK7SjmrN4KFUqlDylX-Up77SM= | +| qwen3-14b | [Qwen/Qwen3-14B](https://huggingface.co/Qwen/Qwen3-14B) | 15B | 14889779200 | 14889779200 | 40 | 40 | 5120 | 151936 | silu | rotary | False | Qwen3ForCausalLM | RMS | Qwen/Qwen3-14B | Qwen2Tokenizer | 151643.0 | OfOK7SjmrN4KFUqlDylX-Up77SM= | +| qwen3-4b | [Qwen/Qwen3-4B](https://huggingface.co/Qwen/Qwen3-4B) | 4.2B | 4199546880 | 4199546880 | 36 | 32 | 2560 | 151936 | silu | rotary | False | Qwen3ForCausalLM | RMS | Qwen/Qwen3-4B | Qwen2Tokenizer | 151643.0 | OfOK7SjmrN4KFUqlDylX-Up77SM= | +| qwen3-8b | [Qwen/Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B) | 7.9B | 7851737088 | 7851737088 | 36 | 32 | 4096 | 151936 | silu | rotary | False | Qwen3ForCausalLM | RMS | Qwen/Qwen3-8B | Qwen2Tokenizer | 151643.0 | OfOK7SjmrN4KFUqlDylX-Up77SM= | +| qwen-32b-preview | [Qwen/QwQ-32B-Preview](https://huggingface.co/Qwen/QwQ-32B-Preview) | 34B | 33889976320 | 33889976320 | 64 | 40 | 5120 | 152064 | silu | rotary | False | Qwen2ForCausalLM | RMS | Qwen/QwQ-32B-Preview | Qwen2Tokenizer | 151643.0 | NI384GYDfJidzgXg_-9habj8lOk= | +| tiny-stories-1L-21M | | 13M | 12582912 | 12582912 | 1 | 16 | 1024 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | roneneldan/TinyStories-1Layer-21M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| tiny-stories-1M | | 393K | 393216 | 393216 | 8 | 16 | 64 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | roneneldan/TinyStories-1M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| tiny-stories-28M | | 25M | 25165824 | 25165824 | 8 | 16 | 512 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | roneneldan/TinyStories-28M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| tiny-stories-2L-33M | | 25M | 25165824 | 25165824 | 2 | 16 | 1024 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | roneneldan/TinyStories-2Layers-33M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| tiny-stories-33M | | 28M | 28311552 | 28311552 | 4 | 16 | 768 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | roneneldan/TinyStories-33M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| tiny-stories-3M | | 1.6M | 1572864 | 1572864 | 8 | 16 | 128 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | roneneldan/TinyStories-3M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| tiny-stories-8M | | 6.3M | 6291456 | 6291456 | 8 | 16 | 256 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | roneneldan/TinyStories-8M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| tiny-stories-instruct-1M | | 393K | 393216 | 393216 | 8 | 16 | 64 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | roneneldan/TinyStories-Instruct-1M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| tiny-stories-instruct-28M | | 25M | 25165824 | 25165824 | 8 | 16 | 512 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | roneneldan/TinyStories-Instruct-28M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| tiny-stories-instruct-2L-33M | | 25M | 25165824 | 25165824 | 2 | 16 | 1024 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | roneneldan/TinyStories-Instruct-2Layers-33M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| tiny-stories-instruct-33M | | 28M | 28311552 | 28311552 | 4 | 16 | 768 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | roneneldan/TinyStories-Instruct-33M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| tiny-stories-instruct-3M | | 1.6M | 1572864 | 1572864 | 8 | 16 | 128 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | roneneldan/TinyStories-Instruct-3M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| tiny-stories-instruct-8M | | 6.3M | 6291456 | 6291456 | 8 | 16 | 256 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | roneneldan/TinyStories-Instruct-8M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| tiny-stories-instruct-1L-21M | | 13M | 12582912 | 12582912 | 1 | 16 | 1024 | 50257 | gelu_new | standard | False | GPTNeoForCausalLM | LN | roneneldan/TinyStories-Instuct-1Layer-21M | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| stablelm-base-alpha-3b | [stabilityai/stablelm-base-alpha-3b](https://huggingface.co/stabilityai/stablelm-base-alpha-3b) | 3.2B | 3221225472 | 3221225472 | 16 | 32 | 4096 | 50688 | gelu | rotary | True | GPTNeoXForCausalLM | LN | stabilityai/stablelm-base-alpha-3b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| stablelm-base-alpha-7b | [stabilityai/stablelm-base-alpha-7b](https://huggingface.co/stabilityai/stablelm-base-alpha-7b) | 7.2B | 7247757312 | 7247757312 | 16 | 48 | 6144 | 50432 | gelu | rotary | True | GPTNeoXForCausalLM | LN | stabilityai/stablelm-base-alpha-7b | GPTNeoXTokenizer | 50254.0 | 96EawM8Lij99W7OBTk0KW2ELUrQ= | +| stablelm-tuned-alpha-3b | [stabilityai/stablelm-tuned-alpha-3b](https://huggingface.co/stabilityai/stablelm-tuned-alpha-3b) | 3.2B | 3221225472 | 3221225472 | 16 | 32 | 4096 | 50688 | gelu | rotary | True | GPTNeoXForCausalLM | LN | stabilityai/stablelm-tuned-alpha-3b | GPTNeoXTokenizer | 50254.0 | RD3vcWSd_TiTpqo5dHyICzaXtGQ= | +| stablelm-tuned-alpha-7b | [stabilityai/stablelm-tuned-alpha-7b](https://huggingface.co/stabilityai/stablelm-tuned-alpha-7b) | 7.2B | 7247757312 | 7247757312 | 16 | 48 | 6144 | 50432 | gelu | rotary | True | GPTNeoXForCausalLM | LN | stabilityai/stablelm-tuned-alpha-7b | GPTNeoXTokenizer | 50254.0 | RD3vcWSd_TiTpqo5dHyICzaXtGQ= | +| stanford-gpt2-small-a | [stanford-crfm/alias-gpt2-small-x21](https://huggingface.co/stanford-crfm/alias-gpt2-small-x21) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | stanford-crfm/alias-gpt2-small-x21 | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| stanford-gpt2-medium-a | [stanford-crfm/arwen-gpt2-medium-x21](https://huggingface.co/stanford-crfm/arwen-gpt2-medium-x21) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | stanford-crfm/arwen-gpt2-medium-x21 | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| stanford-gpt2-small-b | [stanford-crfm/battlestar-gpt2-small-x49](https://huggingface.co/stanford-crfm/battlestar-gpt2-small-x49) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | stanford-crfm/battlestar-gpt2-small-x49 | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| stanford-gpt2-medium-b | [stanford-crfm/beren-gpt2-medium-x49](https://huggingface.co/stanford-crfm/beren-gpt2-medium-x49) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | stanford-crfm/beren-gpt2-medium-x49 | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| stanford-gpt2-small-c | [stanford-crfm/caprica-gpt2-small-x81](https://huggingface.co/stanford-crfm/caprica-gpt2-small-x81) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | stanford-crfm/caprica-gpt2-small-x81 | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| stanford-gpt2-medium-c | [stanford-crfm/celebrimbor-gpt2-medium-x81](https://huggingface.co/stanford-crfm/celebrimbor-gpt2-medium-x81) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | stanford-crfm/celebrimbor-gpt2-medium-x81 | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| stanford-gpt2-small-d | [stanford-crfm/darkmatter-gpt2-small-x343](https://huggingface.co/stanford-crfm/darkmatter-gpt2-small-x343) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | stanford-crfm/darkmatter-gpt2-small-x343 | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| stanford-gpt2-medium-d | [stanford-crfm/durin-gpt2-medium-x343](https://huggingface.co/stanford-crfm/durin-gpt2-medium-x343) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | stanford-crfm/durin-gpt2-medium-x343 | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| stanford-gpt2-medium-e | [stanford-crfm/eowyn-gpt2-medium-x777](https://huggingface.co/stanford-crfm/eowyn-gpt2-medium-x777) | 302M | 301989888 | 301989888 | 24 | 16 | 1024 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | stanford-crfm/eowyn-gpt2-medium-x777 | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| stanford-gpt2-small-e | [stanford-crfm/expanse-gpt2-small-x777](https://huggingface.co/stanford-crfm/expanse-gpt2-small-x777) | 85M | 84934656 | 84934656 | 12 | 12 | 768 | 50257 | gelu_new | standard | False | GPT2LMHeadModel | LN | stanford-crfm/expanse-gpt2-small-x777 | GPT2Tokenizer | 50257.0 | v8xfIj5kwZX5RwgLU66lZNZUlE4= | +| apertus-8b | [swiss-ai/Apertus-8B-2509](https://huggingface.co/swiss-ai/Apertus-8B-2509) | 7.8B | 7784628224 | 7784628224 | 32 | 32 | 4096 | 131072 | xielu | rotary | False | ApertusForCausalLM | RMS | swiss-ai/Apertus-8B-2509 | TokenizersBackend | 131072.0 | C46N1vCQdy2ADcCzoHO1wqva-9w= | +| apertus-8b-instruct | [swiss-ai/Apertus-8B-Instruct-2509](https://huggingface.co/swiss-ai/Apertus-8B-Instruct-2509) | 7.8B | 7784628224 | 7784628224 | 32 | 32 | 4096 | 131072 | xielu | rotary | False | ApertusForCausalLM | RMS | swiss-ai/Apertus-8B-Instruct-2509 | TokenizersBackend | 131072.0 | C46N1vCQdy2ADcCzoHO1wqva-9w= | diff --git a/docs/source/content/model_tables.md b/docs/source/content/model_tables.md index 5b01d64a31..64d30ab128 100644 --- a/docs/source/content/model_tables.md +++ b/docs/source/content/model_tables.md @@ -4,15 +4,15 @@ title: Model Tables # Model Tables ```{warning} -`HookedTransformer` is deprecated as of TransformerLens 3.0 and will be removed in the next major version. New code should use [`TransformerBridge`](migrating_to_v3.md) instead. Existing `HookedTransformer` code continues to work through the 3.x branch via a compatibility layer. See the [migration guide](migrating_to_v3.md) for conversion recipes. +`HookedTransformer` was removed in TransformerLens 4.0. New code should use [`TransformerBridge`](migrating_to_v3.md), which reproduces HookedTransformer numerics via `enable_compatibility_mode()`. The HookedTransformer model table below is a frozen snapshot kept for users still on the 2.x / 3.x branches. See the [migration guide](migrating_to_v3.md) for conversion recipes. ``` -TransformerLens 3.0 provides two model loading paths, each with its own set of supported models. +TransformerLens documents two model tables: -- **HookedTransformer** -- The original TransformerLens models with full hook-point access and mechanistic interpretability support. +- **HookedTransformer** (removed in 4.0) -- the original TransformerLens models; a frozen table kept for 2.x / 3.x users. - **TransformerBridge Models** -- Automatic compatibility layer for thousands of HuggingFace models across supported architectures. ```{toctree} -/generated/model_properties_table +hooked_transformer_model_properties /generated/transformer_bridge_models ``` diff --git a/docs/source/index.md b/docs/source/index.md index e5d105e8dc..c08d031833 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -33,6 +33,7 @@ content/gallery :hidden: :caption: News +content/news/release-4.0 content/news/release-3.0 content/news/release-2.0 ``` @@ -50,6 +51,7 @@ content/model_structure :hidden: :caption: Resources +content/migrating_to_v4 content/migrating_to_v3 content/tutorials content/citation diff --git a/pyproject.toml b/pyproject.toml index cf3924c0b4..fc7142c950 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,6 @@ dependencies=[ "accelerate>=1.1.0", # align_module_device (device_map disk offload) "beartype>=0.14.1", - "better-abc>=0.0.3", "datasets>=2.7.1", "einops>=0.6.0", "fancy-einsum>=0.0.3", @@ -87,7 +86,6 @@ ] docs=[ "furo>=2023.3.27,<2024.0.0", - "muutils>=0.6.13", "myst-parser>=2.0.0,<3.0.0", "nbconvert>=7.9.2", "nbsphinx>=0.9.3", @@ -139,19 +137,6 @@ filterwarnings=[ "ignore:distutils Version classes are deprecated:DeprecationWarning", "ignore:pkg_resources is deprecated as an API:DeprecationWarning", - # Tests still exercise the deprecated Hooked* stack on purpose, so the - # notice is noise here while staying visible to users. Delete these - # when the classes go in 4.0. - "ignore:HookedTransformer is deprecated:DeprecationWarning", - "ignore:HookedTransformer.from_pretrained is deprecated:DeprecationWarning", - "ignore:HookedEncoder is deprecated:DeprecationWarning", - "ignore:HookedEncoder.from_pretrained is deprecated:DeprecationWarning", - "ignore:HookedEncoderDecoder is deprecated:DeprecationWarning", - "ignore:HookedEncoderDecoder.from_pretrained is deprecated:DeprecationWarning", - "ignore:HookedAudioEncoder is deprecated:DeprecationWarning", - "ignore:HookedAudioEncoder.from_pretrained is deprecated:DeprecationWarning", - "ignore:HookedTransformer.from_pretrained_no_processing is deprecated:DeprecationWarning", - "ignore:BertNextSentencePrediction is deprecated:DeprecationWarning", ] markers=[ "slow: marks tests as slow (deselect with '-m \"not slow\"')", diff --git a/scripts/capture_ht_goldens.py b/scripts/capture_ht_goldens.py deleted file mode 100644 index fdc0e8a3eb..0000000000 --- a/scripts/capture_ht_goldens.py +++ /dev/null @@ -1,361 +0,0 @@ -"""Capture golden fixtures from HookedTransformer before its removal. - -The bridge's ``enable_compatibility_mode()`` promises HookedTransformer-equivalent -numerics. Once ``HookedTransformer`` is deleted, frozen goldens captured by this -script are the only way to certify that promise. Run it while HT still exists; -upload the output directory to the goldens dataset repo (see -``tests/goldens.py`` for how tests consume it). - -Usage (run serially — never load two models at once): - uv run python scripts/capture_ht_goldens.py --output-dir tl-compat-goldens - uv run python scripts/capture_ht_goldens.py --models gpt2 --configs no_processing - uv run python scripts/capture_ht_goldens.py --include-olmo2 - -Captured per (model, processing-config): - state_dict.safetensors full processed TL-key state dict (2 of 5 configs) - state_dict.checksums.json per-tensor sha256 + shape/dtype (all configs) - state_dict.samples.safetensors seeded 1024-element samples per tensor (all configs) - views.safetensors W_E / W_U / b_U (+ QK/OV factors on GQA models) - activations.safetensors full run_with_cache snapshot for the short prompt - logits_short.safetensors full logits for the short prompt - logits_long_final.safetensors final-position logits for the Main-Demo text - hook_manifest.json every hook_dict name + fired shape (null if unfired) - scalars.json CE losses + the L0/H8 hook_v ablation anchors - provenance.json versions, commit, platform, exact load kwargs -""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import platform -import subprocess -import sys -from pathlib import Path -from typing import Any - -import torch - -SCHEMA_VERSION = 1 - -# Short prompt: used across today's HT-vs-bridge tests. Long text: the Main Demo -# anchor pinned by tests/integration/model_bridge/test_weight_processing.py. -SHORT_PROMPT = "Natural language processing" -MAIN_DEMO_TEXT = ( - "Natural language processing tasks, such as question answering, machine " - "translation, reading comprehension, and summarization, are typically " - "approached with supervised learning on taskspecific datasets." -) -MAIN_DEMO_LAYER = 0 -MAIN_DEMO_HEAD = 8 - -SAMPLE_COUNT = 1024 -SAMPLE_SEED = 20260803 - -# Models chosen to cover the processing branches (see ws9-oracle-reanchoring-plan.md): -# canonical GPT-2, the flag-matrix workhorse, parallel-block/rotary, GQA, SoLU. -DEFAULT_MODELS = [ - "gpt2", - "distilgpt2", - "EleutherAI/pythia-14m", - "Qwen/Qwen2-0.5B", - "solu-1l", -] -# Post-norm carve-out model (fold_ln refused); opt-in via --include-olmo2. -OLMO2_MODEL = "allenai/OLMo-2-0425-1B" - -# GQA models where QK/OV factor expansion is the contract under test. -GQA_MODELS = {"Qwen/Qwen2-0.5B"} - -# Processing configs. `full_state_dict` marks the two configs whose complete -# processed state dict is stored; the rest store checksums + samples only. -CONFIGS: dict[str, dict[str, Any]] = { - "no_processing": { - "kwargs": {}, - "no_processing": True, - "full_state_dict": True, - }, - "full_defaults": { - "kwargs": { - "fold_ln": True, - "center_writing_weights": True, - "center_unembed": True, - "fold_value_biases": True, - }, - "no_processing": False, - "full_state_dict": True, - }, - "fold_ln_only": { - "kwargs": { - "fold_ln": True, - "center_writing_weights": False, - "center_unembed": False, - "fold_value_biases": False, - }, - "no_processing": False, - "full_state_dict": False, - }, - "fold_ln_center_writing": { - "kwargs": { - "fold_ln": True, - "center_writing_weights": True, - "center_unembed": False, - "fold_value_biases": False, - }, - "no_processing": False, - "full_state_dict": False, - }, - # gpt2 only: the sole model whose refactor test exists today. - "refactor_factored": { - "kwargs": { - "fold_ln": True, - "center_writing_weights": True, - "center_unembed": True, - "fold_value_biases": True, - "refactor_factored_attn_matrices": True, - }, - "no_processing": False, - "full_state_dict": False, - "models": ["gpt2"], - }, -} - - -def _model_dir_name(model_name: str) -> str: - return model_name.replace("/", "__") - - -def _git_commit() -> str | None: - try: - return subprocess.check_output( - ["git", "rev-parse", "HEAD"], cwd=Path(__file__).parent, text=True - ).strip() - except Exception: - return None - - -def _tensor_checksum(t: torch.Tensor) -> str: - return hashlib.sha256(t.detach().cpu().contiguous().float().numpy().tobytes()).hexdigest() - - -def _seeded_sample(t: torch.Tensor, seed: int) -> torch.Tensor: - """Fixed random-index sample of a tensor, deterministic across runs.""" - flat = t.detach().cpu().contiguous().float().flatten() - if flat.numel() <= SAMPLE_COUNT: - return flat.clone() - gen = torch.Generator().manual_seed(seed) - idx = torch.randperm(flat.numel(), generator=gen)[:SAMPLE_COUNT] - return flat[idx.sort().values] - - -def _save_safetensors(path: Path, tensors: dict[str, torch.Tensor]) -> None: - from safetensors.torch import save_file - - # .clone() breaks storage sharing: HT's cache aliases tensors (e.g. - # blocks.N.hook_resid_post IS blocks.N+1.hook_resid_pre), which safetensors - # rejects. Each entry must own its memory. - save_file({k: v.detach().cpu().contiguous().clone() for k, v in tensors.items()}, str(path)) - - -def _run_ablation(model: Any, text: str, layer: int, head: int) -> tuple[float, float]: - """Mirror of tests/integration/model_bridge/test_weight_processing.py::_run_ablation.""" - from transformer_lens import utilities as utils - - tokens = model.to_tokens(text) - - def ablation_hook(value: torch.Tensor, hook: Any) -> torch.Tensor: - value[:, :, head, :] = 0.0 - return value - - hook_name = utils.get_act_name("v", layer) - orig = model(tokens, return_type="loss").item() - ablated = model.run_with_hooks( - tokens, return_type="loss", fwd_hooks=[(hook_name, ablation_hook)] - ).item() - return orig, ablated - - -def capture_one(model_name: str, config_name: str, out_root: Path, skip_existing: bool) -> None: - """Capture the full golden set for one (model, processing-config) cell.""" - from transformer_lens import HookedTransformer - - cfg = CONFIGS[config_name] - out_dir = out_root / _model_dir_name(model_name) / config_name - if skip_existing and (out_dir / "provenance.json").exists(): - print(f"[skip] {model_name} / {config_name} (exists)") - return - out_dir.mkdir(parents=True, exist_ok=True) - print(f"[capture] {model_name} / {config_name}") - - if cfg["no_processing"]: - model = HookedTransformer.from_pretrained_no_processing( - model_name, device="cpu", dtype=torch.float32 - ) - load_kwargs: dict[str, Any] = {"no_processing": True} - else: - load_kwargs = dict(cfg["kwargs"]) - model = HookedTransformer.from_pretrained( - model_name, device="cpu", dtype=torch.float32, **load_kwargs - ) - model.eval() - - with torch.no_grad(): - state = model.state_dict() - - # --- state dict: checksums + samples always; full dict for the 2 pinned configs - checksums = { - k: { - "sha256": _tensor_checksum(v), - "shape": list(v.shape), - "dtype": str(v.dtype), - } - for k, v in state.items() - } - (out_dir / "state_dict.checksums.json").write_text( - json.dumps(checksums, indent=1, sort_keys=True) - ) - _save_safetensors( - out_dir / "state_dict.samples.safetensors", - {k: _seeded_sample(v, SAMPLE_SEED) for k, v in state.items()}, - ) - if cfg["full_state_dict"]: - _save_safetensors(out_dir / "state_dict.safetensors", dict(state)) - - # --- derived weight views asserted by today's tests - views: dict[str, torch.Tensor] = { - "W_E": model.W_E, - "W_U": model.W_U, - "b_U": model.b_U, - } - if model_name in GQA_MODELS: - views.update( - { - "QK.A": model.QK.A, - "QK.B": model.QK.B, - "OV.A": model.OV.A, - "OV.B": model.OV.B, - } - ) - _save_safetensors(out_dir / "views.safetensors", views) - - # --- short-prompt logits + full activation snapshot + hook manifest - logits, cache = model.run_with_cache(SHORT_PROMPT) - _save_safetensors(out_dir / "logits_short.safetensors", {"logits": logits}) - _save_safetensors( - out_dir / "activations.safetensors", - {k: v for k, v in cache.items() if isinstance(v, torch.Tensor)}, - ) - manifest = { - name: (list(cache[name].shape) if name in cache else None) - for name in sorted(model.hook_dict.keys()) - } - (out_dir / "hook_manifest.json").write_text(json.dumps(manifest, indent=1)) - - # --- short-prompt loss gradients (incl_bwd): the HT-free anchor for - # run_with_cache(incl_bwd=True) value parity after HT is deleted. - # enable_grad: the surrounding capture runs under no_grad. - with torch.enable_grad(): - _, grad_cache = model.run_with_cache(SHORT_PROMPT, return_type="loss", incl_bwd=True) - model.zero_grad(set_to_none=True) - _save_safetensors( - out_dir / "gradients.safetensors", - { - k: v - for k, v in grad_cache.items() - if k.endswith("_grad") and isinstance(v, torch.Tensor) - }, - ) - - # --- long-text loss + final-position logits; Main-Demo ablation anchors - long_tokens = model.to_tokens(MAIN_DEMO_TEXT) - long_logits = model(long_tokens, return_type="logits") - long_loss = model(long_tokens, return_type="loss").item() - _save_safetensors( - out_dir / "logits_long_final.safetensors", - {"final_logits": long_logits[:, -1, :]}, - ) - - # Clamp the Main-Demo anchor (gpt2's L0H8) to this model's dims — smaller - # models (pythia-14m has 4 heads) just need *a* fixed causal intervention. - # hook_v is indexed by KV heads under GQA, so clamp against those when set. - n_v_heads = getattr(model.cfg, "n_key_value_heads", None) or model.cfg.n_heads - ablation_layer = min(MAIN_DEMO_LAYER, model.cfg.n_layers - 1) - ablation_head = min(MAIN_DEMO_HEAD, n_v_heads - 1) - orig_loss, ablated_loss = _run_ablation(model, MAIN_DEMO_TEXT, ablation_layer, ablation_head) - - scalars = { - "short_prompt": SHORT_PROMPT, - "long_text_ce_loss": long_loss, - "ablation": { - "text": MAIN_DEMO_TEXT, - "layer": ablation_layer, - "head": ablation_head, - "hook": f"blocks.{ablation_layer}.attn.hook_v", - "orig_loss": orig_loss, - "ablated_loss": ablated_loss, - }, - } - (out_dir / "scalars.json").write_text(json.dumps(scalars, indent=1)) - - import transformers - - import transformer_lens - - provenance = { - "schema_version": SCHEMA_VERSION, - "model": model_name, - "config": config_name, - "load_kwargs": load_kwargs, - "device": "cpu", - "dtype": "float32", - "transformer_lens_version": getattr(transformer_lens, "__version__", None), - "transformer_lens_commit": _git_commit(), - "transformers_version": transformers.__version__, - "torch_version": torch.__version__, - "platform": platform.platform(), - "sample_seed": SAMPLE_SEED, - "sample_count": SAMPLE_COUNT, - } - (out_dir / "provenance.json").write_text(json.dumps(provenance, indent=1)) - - del model, state, cache, logits, long_logits - print(f"[done] {model_name} / {config_name}") - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) - parser.add_argument("--output-dir", default="tl-compat-goldens", type=Path) - parser.add_argument("--models", nargs="*", default=None, help="subset of models") - parser.add_argument("--configs", nargs="*", default=None, help="subset of config names") - parser.add_argument( - "--include-olmo2", action="store_true", help="also capture the post-norm carve-out model" - ) - parser.add_argument("--skip-existing", action="store_true") - parser.add_argument("--dry-run", action="store_true", help="print the capture matrix and exit") - args = parser.parse_args() - - models = list(args.models) if args.models else list(DEFAULT_MODELS) - if args.include_olmo2 and OLMO2_MODEL not in models: - models.append(OLMO2_MODEL) - config_names = list(args.configs) if args.configs else list(CONFIGS.keys()) - unknown = [c for c in config_names if c not in CONFIGS] - if unknown: - parser.error(f"unknown configs: {unknown}; valid: {list(CONFIGS)}") - - cells = [(m, c) for m in models for c in config_names if m in CONFIGS[c].get("models", models)] - if args.dry_run: - for m, c in cells: - print(f"{m} / {c}") - print(f"{len(cells)} cells -> {args.output_dir}") - return 0 - - # Serial on purpose: never hold two models in memory. - for m, c in cells: - capture_one(m, c, args.output_dir, args.skip_existing) - print(f"All {len(cells)} cells captured to {args.output_dir}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/capture_tl_checkpoint_fixtures.py b/scripts/capture_tl_checkpoint_fixtures.py deleted file mode 100644 index 99fe1bd1f6..0000000000 --- a/scripts/capture_tl_checkpoint_fixtures.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Freeze legacy HookedTransformer checkpoints for the converter tests. - -The legacy TL property-format is frozen by definition — historical checkpoints -(OthelloGPT, grokking, ARENA) never change — so the converter's test inputs are -captured once from a live HookedTransformer and committed, letting the tests -survive HookedTransformer's 4.0 deletion. Rerun only to ADD variants. - - uv run python scripts/capture_tl_checkpoint_fixtures.py -""" -from __future__ import annotations - -import json -from pathlib import Path - -import torch - -from transformer_lens import HookedTransformer -from transformer_lens.config import HookedTransformerConfig - -OUT = Path(__file__).parents[1] / "tests" / "fixtures" / "tl_checkpoints" - -BASE = dict( - d_model=32, - d_head=16, - n_heads=2, - n_layers=2, - n_ctx=8, - d_vocab=16, - d_mlp=64, - act_fn="gelu", - normalization_type="LN", - seed=0, -) - -VARIANTS: dict[str, dict] = { - "default": {}, - "gqa": dict(n_heads=4, d_head=8, n_key_value_heads=2), - "lnpre": dict(normalization_type="LNPre"), - "attn_only": dict(attn_only=True), - "gated_rms": dict(gated_mlp=True, normalization_type="RMS", act_fn="silu"), -} - - -def main() -> None: - torch.manual_seed(0) - for name, overrides in VARIANTS.items(): - kwargs = {**BASE, **overrides} - ht = HookedTransformer(HookedTransformerConfig(**kwargs)) - tokens = torch.randint(0, kwargs["d_vocab"], (1, 4)) - with torch.no_grad(): - logits = ht(tokens) - out_dir = OUT / name - out_dir.mkdir(parents=True, exist_ok=True) - torch.save(ht.state_dict(), out_dir / "checkpoint.pt") - reference = {"tokens": tokens, "logits": logits} - if name == "default": - # stacked per-head views for the head-slot placement test - for attr in ("W_Q", "W_K", "W_V", "W_O", "b_Q", "b_K", "b_V", "b_O"): - reference[attr] = getattr(ht, attr) - torch.save(reference, out_dir / "reference.pt") - (out_dir / "meta.json").write_text(json.dumps(kwargs, indent=1) + "\n") - print(f"[done] {name}: {sum(v.numel() for v in ht.state_dict().values())} params") - - -if __name__ == "__main__": - main() diff --git a/tests/QUARANTINES.md b/tests/QUARANTINES.md index 6fa7eb7f67..359eda3a2f 100644 --- a/tests/QUARANTINES.md +++ b/tests/QUARANTINES.md @@ -56,14 +56,9 @@ A `[vllm]` extra exists (Linux-only marker; declared conflicting with `[lit]` in | Path | Marker | Required | |---|---|---| -| [`unit/test_next_sentence_prediction.py`:131](unit/test_next_sentence_prediction.py) | `skipif(not cuda)` | Any CUDA | -| [`unit/model_bridge/compatibility/test_next_sentence_prediction.py`:88](unit/model_bridge/compatibility/test_next_sentence_prediction.py) | `skipif(not cuda)` | Any CUDA | | [`unit/test_generate_no_tokenizer.py`:112](unit/test_generate_no_tokenizer.py) | `skipif(not cuda)` | Any CUDA | | [`unit/model_bridge/test_driver_protocol.py`:103](unit/model_bridge/test_driver_protocol.py) | `skipif(not cuda)` | Any CUDA | | [`unit/test_weight_processing.py`:475](unit/test_weight_processing.py) | `skipif(not cuda and not mps)` (cross-device fold) | Any non-CPU accelerator | -| [`acceptance/test_hooked_encoder.py`:171](acceptance/test_hooked_encoder.py) | `skipif(mps or not cuda)` (bf16/fp16) | CUDA, non-MPS | -| [`acceptance/test_hooked_encoder.py`:226](acceptance/test_hooked_encoder.py) | `skipif(not cuda)` | Any CUDA | -| [`acceptance/test_hooked_encoder_decoder.py`:460](acceptance/test_hooked_encoder_decoder.py) | `skipif(not cuda)` | Any CUDA | | [`acceptance/model_bridge/test_bridge_multigpu.py`](acceptance/model_bridge/test_bridge_multigpu.py) module-level | `multigpu` marker + `skipif(device_count < 2)` | 2+ CUDA | | [`acceptance/model_bridge/test_bridge_multigpu_device_map.py`](acceptance/model_bridge/test_bridge_multigpu_device_map.py) module-level | `multigpu` marker + `skipif(device_count < 2)` | 2+ CUDA | | [`mps/test_mps_basic.py`](mps/test_mps_basic.py) module-level | `skipif(not mps)` | Apple Silicon | @@ -146,11 +141,10 @@ and the acceptance tier is green. What the skips were hiding was four genuine fa Two silent TransformerLens bugs also lived in this blind spot the whole time: T5's decoder self-attention was never causally masked, and its relative-position bias used the encoder's -bucketing. Both are fixed, and bound by -[`acceptance/test_hooked_encoder_decoder.py`](acceptance/test_hooked_encoder_decoder.py)'s -`test_full_model_multi_token_decoder` plus -[`unit/model_bridge/test_t5_block_parity.py`](unit/model_bridge/test_t5_block_parity.py). Keep -the encoder modules enabled. +bucketing. Both are fixed. **Update (4.0):** the acceptance suites named above were deleted +with the `Hooked*` classes in the 4.0 removal (the bugs lived in deleted code); the T5 fix is +now bound solely by the surviving +[`unit/model_bridge/test_t5_block_parity.py`](unit/model_bridge/test_t5_block_parity.py). --- @@ -165,7 +159,6 @@ the encoder modules enabled. | [`unit/model_bridge/supported_architectures/test_qwen3_5_adapter.py`:448,464,494,514,605,700,805,947,1133](unit/model_bridge/supported_architectures/test_qwen3_5_adapter.py) | `skipif` ×9 | Qwen3_5 classes absent from installed transformers | | [`unit/model_bridge/supported_architectures/test_qwen3_next_adapter.py`:397](unit/model_bridge/supported_architectures/test_qwen3_next_adapter.py) | `skipif` | Qwen3NextForCausalLM absent from installed transformers | | [`integration/test_weight_processing_integration.py`:279](integration/test_weight_processing_integration.py) | `skip` | Weight-processing edge case | -| [`integration/test_hooked_encoder_properties.py`:71](integration/test_hooked_encoder_properties.py) | `xfail` | HookedEncoder properties | | [`acceptance/model_bridge/compatibility/test_backward_hooks.py`:11](acceptance/model_bridge/compatibility/test_backward_hooks.py) | `skip` | Backward-hook compatibility | **Un-skip:** debug the underlying issue and remove the marker. Each removal lands in a focused PR with a regression test. diff --git a/tests/acceptance/model_bridge/compatibility/test_hook_completeness.py b/tests/acceptance/model_bridge/compatibility/test_hook_completeness.py index d8d0f226b5..258436d66c 100644 --- a/tests/acceptance/model_bridge/compatibility/test_hook_completeness.py +++ b/tests/acceptance/model_bridge/compatibility/test_hook_completeness.py @@ -19,7 +19,7 @@ # Diverse architectures for hook completeness testing. # Constraint: the reference-anchored tests need a golden cell for the model -# (see scripts/capture_ht_goldens.py). Tiny Llama/Qwen/Gemma families have no +# (see tests/goldens_capture_spec.py). Tiny Llama/Qwen/Gemma families have no # goldens; for those, tests/unit/model_bridge/test_component_hooks_fire.py # (Tier 2) provides direct per-adapter hook-firing coverage. MODELS_TO_TEST = [ diff --git a/tests/acceptance/test_hooked_encoder.py b/tests/acceptance/test_hooked_encoder.py deleted file mode 100644 index 77f65f6107..0000000000 --- a/tests/acceptance/test_hooked_encoder.py +++ /dev/null @@ -1,229 +0,0 @@ -from typing import List - -import pytest -import torch -import torch.nn.functional as F -from jaxtyping import Float -from torch.testing import assert_close -from transformers import AutoTokenizer, BertForPreTraining - -from transformer_lens import HookedEncoder - -MODEL_NAME = "bert-base-cased" - - -@pytest.fixture(scope="module") -def our_bert(): - return HookedEncoder.from_pretrained(MODEL_NAME, device="cpu") - - -@pytest.fixture(scope="module") -def huggingface_bert(): - return BertForPreTraining.from_pretrained(MODEL_NAME) - - -@pytest.fixture(scope="module") -def tokenizer(): - return AutoTokenizer.from_pretrained(MODEL_NAME) - - -@pytest.fixture -def tokens(tokenizer): - return tokenizer("The [MASK] sat on the mat", return_tensors="pt")["input_ids"] - - -def test_full_model(our_bert, huggingface_bert, tokenizer): - sequences = [ - "Hello, my [MASK] is Bert.", - "I went to the [MASK] to buy some groceries.", - ] - tokenized = tokenizer(sequences, return_tensors="pt", padding=True) - input_ids = tokenized["input_ids"] - attention_mask = tokenized["attention_mask"] - - huggingface_bert_logits = huggingface_bert( - input_ids, attention_mask=attention_mask - ).prediction_logits - our_bert_logits = our_bert(input_ids, one_zero_attention_mask=attention_mask) - assert_close(huggingface_bert_logits, our_bert_logits, rtol=1.3e-6, atol=4e-5) - - -def test_embed_one_prediction(our_bert, huggingface_bert, tokens): - huggingface_embed = huggingface_bert.bert.embeddings - our_embed = our_bert.embed - - huggingface_embed_out = huggingface_embed(tokens)[0] - our_embed_out = our_embed(tokens).squeeze(0) - assert_close(huggingface_embed_out, our_embed_out) - - -def test_embed_two_predictions(our_bert, huggingface_bert, tokenizer): - encoding = tokenizer( - "Hello, my [MASK] is Bert.", - "I went to the [MASK] to buy some groceries.", - return_tensors="pt", - ) - input_ids = encoding["input_ids"] - token_type_ids = encoding["token_type_ids"] - - huggingface_embed_out = huggingface_bert.bert.embeddings( - input_ids, token_type_ids=token_type_ids - )[0] - our_embed_out = our_bert.embed(input_ids, token_type_ids=token_type_ids).squeeze(0) - assert_close(huggingface_embed_out, our_embed_out) - - -def test_attention(our_bert, huggingface_bert, tokens): - huggingface_embed = huggingface_bert.bert.embeddings - huggingface_attn = huggingface_bert.bert.encoder.layer[0].attention - - embed_out = huggingface_embed(tokens) - - our_attn = our_bert.blocks[0].attn - - our_attn_out = our_attn(embed_out, embed_out, embed_out) - huggingface_self_attn_out = huggingface_attn.self(embed_out)[0] - huggingface_attn_out = huggingface_attn.output.dense(huggingface_self_attn_out) - assert_close(our_attn_out, huggingface_attn_out) - - -def test_bert_block(our_bert, huggingface_bert, tokens): - huggingface_embed = huggingface_bert.bert.embeddings - huggingface_block = huggingface_bert.bert.encoder.layer[0] - - embed_out = huggingface_embed(tokens) - - our_block = our_bert.blocks[0] - - our_block_out = our_block(embed_out) - # transformers v5 BertLayer.forward returns the hidden states directly; v4 - # returned a (hidden_states, *optional) tuple, so this used to index [0]. - huggingface_block_out = huggingface_block(embed_out) - assert_close(our_block_out, huggingface_block_out) - - -def test_bert_pooler(our_bert, huggingface_bert, tokens): - huggingface_embed_out = huggingface_bert.bert.embeddings(tokens) - huggingface_encoder_out = huggingface_bert.bert.encoder(huggingface_embed_out) - cls_token_representation = huggingface_encoder_out[0] - - our_pooler_out = our_bert.pooler(cls_token_representation) - huggingface_pooler_out = huggingface_bert.bert.pooler(cls_token_representation) - assert_close(our_pooler_out, huggingface_pooler_out) - - -def test_nsp_head(our_bert, huggingface_bert, tokens): - huggingface_bert_pooler_output = huggingface_bert.bert(tokens).pooler_output - our_nsp_head_out = our_bert.nsp_head(huggingface_bert_pooler_output) - huggingface_nsp_head_out = huggingface_bert.cls.seq_relationship(huggingface_bert_pooler_output) - - assert_close(our_nsp_head_out, huggingface_nsp_head_out) - - -def test_mlm_head(our_bert, huggingface_bert, tokens): - huggingface_bert_core_outputs = huggingface_bert.bert(tokens).last_hidden_state - - our_mlm_head_out = our_bert.mlm_head(huggingface_bert_core_outputs) - huggingface_predictions_out = huggingface_bert.cls.predictions.transform( - huggingface_bert_core_outputs - ) - - print((our_mlm_head_out - huggingface_predictions_out).abs().max()) - assert_close(our_mlm_head_out, huggingface_predictions_out, rtol=1.3e-3, atol=1e-5) - - -def test_unembed(our_bert, huggingface_bert, tokens): - huggingface_bert_core_outputs = huggingface_bert.bert(tokens).last_hidden_state - - our_mlm_head_out = our_bert.mlm_head(huggingface_bert_core_outputs) - our_unembed_out = our_bert.unembed(our_mlm_head_out) - huggingface_predictions_out = huggingface_bert.cls.predictions(huggingface_bert_core_outputs) - - assert_close(our_unembed_out, huggingface_predictions_out, rtol=1.3e-6, atol=4e-5) - - -def test_run_with_cache(our_bert, tokens): - _, cache = our_bert.run_with_cache(tokens) - - # check that an arbitrary subset of the keys exist - assert "embed.hook_embed" in cache - assert "blocks.0.attn.hook_q" in cache - assert "blocks.3.attn.hook_attn_scores" in cache - assert "blocks.7.hook_resid_post" in cache - assert "mlm_head.ln.hook_normalized" in cache - - -def test_from_pretrained_revision(): - """ - Check that the from_pretrained parameter `revision` (= git version) works - """ - - _ = HookedEncoder.from_pretrained(MODEL_NAME, revision="main") - - try: - _ = HookedEncoder.from_pretrained(MODEL_NAME, revision="inexistent_branch_name") - except: - pass - else: - raise AssertionError("Should have raised an error") - - -@pytest.mark.skipif( - torch.backends.mps.is_available() or not torch.cuda.is_available(), - reason="bfloat16 unsupported by MPS: https://github.com/pytorch/pytorch/issues/78168 or no GPU", -) -@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) -def test_half_precision(dtype): - """Check the 16 bits loading and inferences.""" - model = HookedEncoder.from_pretrained(MODEL_NAME, torch_dtype=dtype) - assert model.W_K.dtype == dtype - - _ = model(model.tokenizer("Hello, world", return_tensors="pt")["input_ids"]) - - -def _get_predictions( - logits: Float[torch.Tensor, "batch pos d_vocab"], positions: List[int], tokenizer -): - logits_at_position = logits.squeeze(0)[positions] - predicted_tokens = F.softmax(logits_at_position, dim=-1).argmax(dim=-1) - return tokenizer.batch_decode(predicted_tokens) - - -def test_predictions_mlm(our_bert, huggingface_bert, tokenizer): - input_ids = tokenizer("The [MASK] sat on the mat", return_tensors="pt")["input_ids"] - - our_bert_logits = our_bert(input_ids) - our_prediction = _get_predictions(our_bert_logits, [2], tokenizer) - - huggingface_bert_out = huggingface_bert(input_ids).prediction_logits - huggingface_prediction = _get_predictions(huggingface_bert_out, [2], tokenizer) - - assert our_prediction == huggingface_prediction - - -def test_predictions_from_forward_function_mlm(our_bert, huggingface_bert, tokenizer): - input_ids = tokenizer("The [MASK] sat on the mat", return_tensors="pt")["input_ids"] - our_prediction = our_bert(input_ids, return_type="predictions") - - huggingface_bert_out = huggingface_bert(input_ids).prediction_logits - huggingface_prediction = _get_predictions(huggingface_bert_out, [2], tokenizer)[ - 0 - ] # prediction is returned as a list - - assert our_prediction == huggingface_prediction - - -def test_input_list_of_strings_mlm(our_bert, huggingface_bert, tokenizer): - prompts = ["The [MASK] sat on the mat", "She [MASK] to the store", "The dog [MASK] the ball"] - encodings = tokenizer(prompts, return_tensors="pt", truncation=True, padding=True) - our_bert_logits = our_bert(prompts) - - huggingface_bert_logits = huggingface_bert(**encodings).prediction_logits - - assert_close(our_bert_logits, huggingface_bert_logits, rtol=1.3e-6, atol=4e-5) - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires a CUDA device") -def test_cuda(tokens): - model = HookedEncoder.from_pretrained(MODEL_NAME) - model(tokens) diff --git a/tests/acceptance/test_hooked_encoder_decoder.py b/tests/acceptance/test_hooked_encoder_decoder.py deleted file mode 100644 index 579a12f89a..0000000000 --- a/tests/acceptance/test_hooked_encoder_decoder.py +++ /dev/null @@ -1,463 +0,0 @@ -import pytest -import torch -from jaxtyping import Float -from torch.testing import assert_close -from transformers import AutoTokenizer, T5ForConditionalGeneration - -from transformer_lens import HookedEncoderDecoder - -MODEL_NAME = "t5-small" - - -@pytest.fixture(scope="module") -def our_model(): - return HookedEncoderDecoder.from_pretrained(MODEL_NAME, device="cpu") - - -@pytest.fixture(scope="module") -def huggingface_model(): - return T5ForConditionalGeneration.from_pretrained(MODEL_NAME).eval() - - -@pytest.fixture(scope="module") -def tokenizer(): - return AutoTokenizer.from_pretrained(MODEL_NAME) - - -@pytest.fixture -def hello_world_tokens(tokenizer): - return tokenizer("Hello, world!", return_tensors="pt")["input_ids"] - - -@pytest.fixture -def decoder_input_ids(tokenizer): - return torch.LongTensor([[tokenizer.pad_token_id]]) - - -def test_full_model(our_model, huggingface_model, tokenizer, decoder_input_ids): - sequences = ["Hello, world!", "this is another sequence of tokens"] - - tokenized = tokenizer(sequences, return_tensors="pt", padding=True) - decoder_ids = torch.stack([decoder_input_ids[0]] * len(sequences), dim=0) - input_ids = tokenized["input_ids"] - - attention_mask = tokenized["attention_mask"] - - huggingface_model_out = huggingface_model( - input_ids=input_ids, - attention_mask=attention_mask, - decoder_input_ids=decoder_ids, - ).logits - our_model_out = our_model( - input_ids, - decoder_input=decoder_ids, - one_zero_attention_mask=attention_mask, - ) - assert_close(huggingface_model_out, our_model_out, rtol=1.3e-6, atol=4e-5) - - -def test_full_model_multi_token_decoder(our_model, huggingface_model, tokenizer): - """Every other full-model test decodes a single pad token, where neither - causal masking nor relative-position bucketing can differ. Two bugs hid in - that gap: unmasked decoder self-attention (wrong from length 2) and - encoder-style bias buckets (wrong from offset 9); this binds both. - """ - tokenized = tokenizer( - ["Hello, world!", "this is another sequence of tokens"], return_tensors="pt", padding=True - ) - target = tokenizer( - "Das Haus ist wunderbar und sehr gross heute Abend am Fluss neben dem alten Turm", - return_tensors="pt", - ) - decoder_ids = torch.cat( - [torch.LongTensor([[tokenizer.pad_token_id]]), target["input_ids"][:, :15]], dim=1 - ).repeat(2, 1) - assert decoder_ids.shape[1] >= 12, "shorter than this and the bias buckets coincide" - - huggingface_model_out = huggingface_model( - input_ids=tokenized["input_ids"], - attention_mask=tokenized["attention_mask"], - decoder_input_ids=decoder_ids, - ).logits - our_model_out = our_model( - tokenized["input_ids"], - decoder_input=decoder_ids, - one_zero_attention_mask=tokenized["attention_mask"], - ) - assert_close(huggingface_model_out, our_model_out, rtol=1.3e-6, atol=4e-5) - - -def test_encoder(our_model, huggingface_model, hello_world_tokens): - our_embeds = our_model.embed(hello_world_tokens) - pos_bias = our_model.encoder[0].attn.compute_relative_attention_bias( - hello_world_tokens.shape[1], hello_world_tokens.shape[1] - ) - - for our_layer in our_model.encoder: - our_embeds = our_layer(resid_pre=our_embeds, position_bias=pos_bias) - - our_encoder_out = our_model.encoder_final_ln(our_embeds) - - huggingface_encoder_out = huggingface_model.encoder(hello_world_tokens).last_hidden_state - - assert_close(our_encoder_out, huggingface_encoder_out, rtol=1.3e-6, atol=4e-5) - - -def test_decoder(our_model, huggingface_model, hello_world_tokens, decoder_input_ids): - encoder_hidden = huggingface_model.encoder(hello_world_tokens)[0] - - embeds = our_model.embed(decoder_input_ids) - pos_bias = our_model.decoder[0].attn.compute_relative_attention_bias( - decoder_input_ids.shape[1], decoder_input_ids.shape[1] - ) - for layer in our_model.decoder: - embeds = layer(embeds, encoder_hidden_states=encoder_hidden, position_bias=pos_bias) - - our_decoder_out = our_model.decoder_final_ln(embeds) - hf_decoder_out = huggingface_model.decoder( - decoder_input_ids, encoder_hidden_states=encoder_hidden - )[0] - - assert_close(our_decoder_out, hf_decoder_out, rtol=1.3e-6, atol=4e-5) - - -def test_embed_one_sentence(our_model, huggingface_model, hello_world_tokens): - huggingface_embed = huggingface_model.encoder.embed_tokens - our_embed = our_model.embed - - huggingface_embed_out = huggingface_embed(hello_world_tokens)[0] - our_embed_out = our_embed(hello_world_tokens).squeeze(0) - assert_close(huggingface_embed_out, our_embed_out) - - -def test_relative_attention_bias(our_model, huggingface_model, hello_world_tokens): - # it is used only in self attention of first layer of encoder - huggingface_embed = huggingface_model.encoder.embed_tokens - huggingface_attn = huggingface_model.encoder.block[0].layer[0].SelfAttention - our_attn = our_model.encoder[0].attn - - assert huggingface_attn.has_relative_attention_bias - assert our_attn.has_relative_attention_bias - assert ( - our_attn.relative_attention_num_buckets == huggingface_attn.relative_attention_num_buckets - ) - assert ( - our_attn.relative_attention_max_distance == huggingface_attn.relative_attention_max_distance - ) - assert_close(our_attn.rel_pos_bias.weight, huggingface_attn.relative_attention_bias.weight) - - input_len = hello_world_tokens.shape[1] - our_bias = our_attn.compute_relative_attention_bias(input_len, input_len) - hf_bias = huggingface_attn.compute_bias(input_len, input_len) - assert_close(our_bias, hf_bias, rtol=1e-5, atol=1e-5) - - embed_out = huggingface_embed(hello_world_tokens) - - cache_position = torch.arange(input_len) - huggingface_attn_out = huggingface_attn(embed_out, cache_position=cache_position)[0] - our_attn_out = our_attn(embed_out, embed_out, embed_out, position_bias=our_bias) - - assert_close(our_attn_out, huggingface_attn_out, rtol=7.4e-4, atol=1e-5) - - -def test_relative_attention_layer(our_model, huggingface_model, hello_world_tokens): - # it is used only in self attention of first layer of encoder - hf_block = huggingface_model.encoder.block[0].layer[0] - our_block = our_model.encoder[0] - resid = huggingface_model.encoder.embed_tokens(hello_world_tokens) - - input_len = hello_world_tokens.shape[1] - our_bias = our_block.attn.compute_relative_attention_bias(input_len, input_len) - resid_norm = our_block.ln1(resid) - our_out = resid + our_block.attn(resid_norm, resid_norm, resid_norm, position_bias=our_bias) - - cache_position = torch.arange(input_len) - hf_out = hf_block(resid, cache_position=cache_position)[0] - assert_close(our_out, hf_out, rtol=1.3e-6, atol=4e-5) - - -def test_attention(our_model, huggingface_model, hello_world_tokens): - huggingface_embed = huggingface_model.encoder.embed_tokens - huggingface_attn = huggingface_model.encoder.block[1].layer[0].SelfAttention - - embed_out = huggingface_embed(hello_world_tokens) - our_attn = our_model.encoder[1].attn - - our_attn_out = our_attn(embed_out, embed_out, embed_out) - - input_len = hello_world_tokens.shape[1] - cache_position = torch.arange(input_len) - huggingface_attn_out = huggingface_attn(embed_out, cache_position=cache_position)[0] - - assert_close(our_attn_out, huggingface_attn_out, rtol=5e-4, atol=1e-5) - - -def _causal_float_mask(seq_len): - """The additive causal mask T5Stack builds for the decoder; passing HF's - attention module a bare call leaves it unmasked, which is not how the - decoder ever runs.""" - allowed = torch.tril(torch.ones(seq_len, seq_len, dtype=torch.bool)) - return torch.where(allowed, 0.0, torch.finfo(torch.float32).min)[None, None] - - -def test_decoder_attention(our_model, huggingface_model, hello_world_tokens): - huggingface_embed = huggingface_model.decoder.embed_tokens - huggingface_attn = huggingface_model.decoder.block[1].layer[0].SelfAttention - - embed_out = huggingface_embed(hello_world_tokens) - our_attn = our_model.decoder[1].attn - - our_attn_out = our_attn(embed_out, embed_out, embed_out) - - input_len = hello_world_tokens.shape[1] - cache_position = torch.arange(input_len) - huggingface_attn_out = huggingface_attn( - embed_out, mask=_causal_float_mask(input_len), cache_position=cache_position - )[0] - assert_close(our_attn_out, huggingface_attn_out, rtol=5e-4, atol=1e-5) - - -def test_attention_layer(our_model, huggingface_model, hello_world_tokens): - huggingface_embed = huggingface_model.encoder.embed_tokens - huggingface_attn = huggingface_model.encoder.block[1].layer[0] - - embed_out = huggingface_embed(hello_world_tokens) - our_attn = our_model.encoder[1].attn - norm_embed = our_model.encoder[1].ln1(embed_out) - our_attn_out = our_attn(norm_embed, norm_embed, norm_embed) + embed_out - - input_len = hello_world_tokens.shape[1] - cache_position = torch.arange(input_len) - huggingface_attn_out = huggingface_attn(embed_out, cache_position=cache_position)[0] - assert_close(our_attn_out, huggingface_attn_out, rtol=2e-4, atol=1e-5) - - -def test_decoder_attention_layer(our_model, huggingface_model, hello_world_tokens): - huggingface_embed = huggingface_model.decoder.embed_tokens - huggingface_attn = huggingface_model.decoder.block[1].layer[0] - - embed_out = huggingface_embed(hello_world_tokens) - our_attn = our_model.decoder[1].attn - norm_embed = our_model.decoder[1].ln1(embed_out) - our_attn_out = our_attn(norm_embed, norm_embed, norm_embed) + embed_out - - input_len = hello_world_tokens.shape[1] - cache_position = torch.arange(input_len) - huggingface_attn_out = huggingface_attn( - embed_out, attention_mask=_causal_float_mask(input_len), cache_position=cache_position - )[0] - assert_close(our_attn_out, huggingface_attn_out, rtol=3e-4, atol=4e-5) - - -def test_cross_attention(our_model, huggingface_model, hello_world_tokens, decoder_input_ids): - encoder_hidden = huggingface_model.encoder(hello_world_tokens).last_hidden_state - decoder_hidden = huggingface_model.decoder.embed_tokens(decoder_input_ids) - - huggingface_cross_attn = huggingface_model.decoder.block[0].layer[1].EncDecAttention - our_cross_attn = our_model.decoder[0].cross_attn - - our_cross_attn_out = our_cross_attn(decoder_hidden, encoder_hidden, encoder_hidden) - huggingface_cross_attn_out = huggingface_cross_attn( - decoder_hidden, key_value_states=encoder_hidden, cache_position=encoder_hidden - )[0] - assert_close(our_cross_attn_out, huggingface_cross_attn_out, rtol=2e-3, atol=1e-4) - - -def test_cross_attention_layer(our_model, huggingface_model, hello_world_tokens, decoder_input_ids): - encoder_hidden = huggingface_model.encoder(hello_world_tokens).last_hidden_state - decoder_hidden = huggingface_model.decoder.embed_tokens(decoder_input_ids) - - hf_layer = huggingface_model.decoder.block[0].layer[1] - our_layer = our_model.decoder[0] - assert_close(hf_layer.layer_norm.weight, our_layer.ln2.w) - - our_cross_attn_out = ( - our_layer.cross_attn(our_layer.ln2(decoder_hidden), encoder_hidden, encoder_hidden) - + decoder_hidden - ) - huggingface_cross_attn_out = hf_layer( - decoder_hidden, key_value_states=encoder_hidden, cache_position=encoder_hidden - )[0] - assert_close(our_cross_attn_out, huggingface_cross_attn_out, rtol=2e-4, atol=1e-5) - - -def test_encoder_block(our_model, huggingface_model, hello_world_tokens): - huggingface_embed = huggingface_model.encoder.embed_tokens - huggingface_block = huggingface_model.encoder.block[1] - our_block = our_model.encoder[1] - - embed_out = huggingface_embed(hello_world_tokens) - - input_len = hello_world_tokens.shape[1] - cache_position = torch.arange(input_len) - hf_out = huggingface_block(embed_out, cache_position=cache_position)[0] - our_out = our_block(embed_out) - - assert_close(our_out, hf_out, rtol=2e-4, atol=2e-5) - - -def test_decoder_block(our_model, huggingface_model, hello_world_tokens, decoder_input_ids): - huggingface_embed = huggingface_model.decoder.embed_tokens - huggingface_block = huggingface_model.decoder.block[1] - our_block = our_model.decoder[1] - - encoder_hidden = huggingface_model.encoder(hello_world_tokens)[0] - - input_len = decoder_input_ids.shape[1] - cache_position = torch.arange(input_len) - decoder_hidden = huggingface_model.decoder.block[0]( - huggingface_embed(decoder_input_ids), cache_position=cache_position - )[0] - - our_out = our_block(decoder_hidden, encoder_hidden_states=encoder_hidden) - hf_out = huggingface_block( - decoder_hidden, encoder_hidden_states=encoder_hidden, cache_position=encoder_hidden - )[0] - - assert_close(hf_out, our_out, rtol=2e-4, atol=2e-5) - - -def test_layernorm(our_model, huggingface_model, hello_world_tokens): - huggingface_embed = huggingface_model.encoder.embed_tokens - huggingface_layernorm = huggingface_model.encoder.block[0].layer[0].layer_norm - our_layernorm = our_model.encoder[0].ln1 - - embed_out = huggingface_embed(hello_world_tokens) - - our_layernorm_out = our_layernorm(embed_out) - huggingface_layernorm_out = huggingface_layernorm(embed_out) - assert_close(our_layernorm_out, huggingface_layernorm_out) - - -def test_unembed(our_model, huggingface_model, hello_world_tokens): - huggingface_model_hidden = huggingface_model.decoder(hello_world_tokens).last_hidden_state - - our_model_logits = our_model.unembed(huggingface_model_hidden) - huggingface_model_logits = huggingface_model.lm_head(huggingface_model_hidden) - - assert_close(our_model_logits, huggingface_model_logits, rtol=1.3e-3, atol=1e-5) - - -def test_run_with_cache(our_model, hello_world_tokens, decoder_input_ids): - logits, cache = our_model.run_with_cache(hello_world_tokens, decoder_input=decoder_input_ids) - - # check that an arbitrary subset of the keys exist and have the right shape - seq_len = 5 - generated_len = 1 - assert "hook_embed" in cache - assert cache["hook_embed"].shape == (1, seq_len, 512) - assert "encoder.1.attn.hook_v" in cache - assert cache["encoder.1.attn.hook_v"].shape == (1, seq_len, 8, 64) - assert "encoder.3.attn.hook_attn_scores" in cache - assert cache["encoder.3.attn.hook_attn_scores"].shape == (1, 8, seq_len, seq_len) - assert "decoder.0.cross_attn.hook_k" in cache - assert cache["decoder.0.cross_attn.hook_attn_scores"].shape == ( - 1, - 8, - generated_len, - seq_len, - ) - assert "decoder.3.hook_resid_post" in cache - assert cache["decoder.3.hook_resid_post"].shape == (1, generated_len, 512) - - -def test_from_pretrained_revision(): - """ - Check that the from_pretrained parameter `revision` (= git version) works - """ - - _ = HookedEncoderDecoder.from_pretrained(MODEL_NAME, revision="main") - - try: - _ = HookedEncoderDecoder.from_pretrained(MODEL_NAME, revision="inexistent_branch_name") - except: - pass - else: - raise AssertionError("Should have raised an error") - - -def test_predictions(our_model, huggingface_model, tokenizer, decoder_input_ids): - input_ids = tokenizer("My name is Wolfgang and I live in Berlin", return_tensors="pt")[ - "input_ids" - ] - - def get_predictions(logits: Float[torch.Tensor, "batch pos d_vocab"]): - predicted_tokens = logits[0].argmax(dim=-1) - return tokenizer.batch_decode(predicted_tokens) - - our_model_logits = our_model(input_ids, decoder_input=decoder_input_ids) - our_prediction = get_predictions(our_model_logits) - - huggingface_model_logits = huggingface_model( - input_ids, decoder_input_ids=decoder_input_ids - ).logits - huggingface_prediction = get_predictions(huggingface_model_logits) - - assert our_prediction == huggingface_prediction - - -def test_predictions_string_input(our_model, huggingface_model, tokenizer): - prompt = "translate English to German: Hello, do you like bananas?" - - encodings = tokenizer(prompt, return_tensors="pt") - tokens = encodings.input_ids - batch_size, seq_len = tokens.shape - decoder_input_ids = torch.full((batch_size, 1), tokenizer.pad_token_id) - - our_model_logits = our_model(prompt) - - huggingface_model_logits = huggingface_model( - input_ids=tokens, - attention_mask=encodings.attention_mask, - decoder_input_ids=decoder_input_ids, - ).logits - - assert_close(our_model_logits, huggingface_model_logits, rtol=1e-5, atol=1e-5) - - -def test_predictions_string_list_input(our_model, huggingface_model, tokenizer): - prompt = [ - "translate English to German: Hello, do you like bananas?", - "translate English to French: Hello, do you like bananas?", - "translate English to Spanish: Hello, do you like bananas?", - ] - - encodings = tokenizer(prompt, return_tensors="pt") - tokens = encodings.input_ids - batch_size, seq_len = tokens.shape - decoder_input_ids = torch.full((batch_size, 1), tokenizer.pad_token_id) - - our_model_logits = our_model(prompt) - - huggingface_model_logits = huggingface_model( - input_ids=tokens, - attention_mask=encodings.attention_mask, - decoder_input_ids=decoder_input_ids, - ).logits - - assert_close(our_model_logits, huggingface_model_logits, rtol=1e-5, atol=1e-5) - - -def test_generate(our_model, huggingface_model, tokenizer): - prompt = "translate English to German: Hello, do you like bananas?" - - encodings = tokenizer(prompt, return_tensors="pt") - - our_generation = our_model.generate(prompt, do_sample=False, max_new_tokens=20) - huggingface_generated_tokens = huggingface_model.generate( - input_ids=encodings.input_ids, - attention_mask=encodings.attention_mask, - do_sample=False, - )[0] - - huggingface_generation = tokenizer.decode( - huggingface_generated_tokens, skip_special_tokens=True - ) - - assert our_generation.lower() == huggingface_generation.lower() - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires a CUDA device") -def test_cuda(hello_world_tokens, decoder_input_ids): - model = HookedEncoderDecoder.from_pretrained(MODEL_NAME) - model(hello_world_tokens, decoder_input=decoder_input_ids.cuda()) diff --git a/tests/goldens.py b/tests/goldens.py index 332a327144..055738e5a0 100644 --- a/tests/goldens.py +++ b/tests/goldens.py @@ -1,7 +1,8 @@ """Loader for the HookedTransformer golden fixtures (the post-deletion oracle). -Goldens are captured by ``scripts/capture_ht_goldens.py`` while HookedTransformer -still exists, and hosted on a HF Hub dataset repo pinned by revision. Tests that +Goldens were captured from HookedTransformer before its 4.0 removal and are +hosted on a HF Hub dataset repo pinned by revision; the capture script was +removed with HookedTransformer, so the fixtures are now frozen. Tests that certify the compatibility-mode contract load their reference tensors from here instead of constructing a live HookedTransformer. diff --git a/tests/goldens_capture_spec.py b/tests/goldens_capture_spec.py new file mode 100644 index 0000000000..68d3e32de0 --- /dev/null +++ b/tests/goldens_capture_spec.py @@ -0,0 +1,93 @@ +"""Pure, model-free spec for the frozen golden fixtures. + +The runnable capture script (``scripts/capture_ht_goldens.py``) was removed with +HookedTransformer at 4.0 — it booted the class to produce the goldens. Its +deterministic, model-free helpers (checksum, seeded sampling, the config matrix, +directory naming) live on here: the committed goldens were built with these, and +``tests/goldens.py`` plus the golden round-trip tests verify against them. +""" + +from __future__ import annotations + +import hashlib +from typing import Any + +import torch + +SCHEMA_VERSION = 1 +SAMPLE_COUNT = 1024 +SAMPLE_SEED = 20260803 + +SHORT_PROMPT = "Natural language processing" + +# Processing configs the goldens are captured under (data-only; the removed +# script mapped these to HookedTransformer load flags). Preserved verbatim so +# the golden cell names and the fixtures stay consistent. +CONFIGS: dict[str, dict[str, Any]] = { + "no_processing": { + "kwargs": {}, + "no_processing": True, + "full_state_dict": True, + }, + "full_defaults": { + "kwargs": { + "fold_ln": True, + "center_writing_weights": True, + "center_unembed": True, + "fold_value_biases": True, + }, + "no_processing": False, + "full_state_dict": True, + }, + "fold_ln_only": { + "kwargs": { + "fold_ln": True, + "center_writing_weights": False, + "center_unembed": False, + "fold_value_biases": False, + }, + "no_processing": False, + "full_state_dict": False, + }, + "fold_ln_center_writing": { + "kwargs": { + "fold_ln": True, + "center_writing_weights": True, + "center_unembed": False, + "fold_value_biases": False, + }, + "no_processing": False, + "full_state_dict": False, + }, + # gpt2 only: the sole model whose refactor test exists today. + "refactor_factored": { + "kwargs": { + "fold_ln": True, + "center_writing_weights": True, + "center_unembed": True, + "fold_value_biases": True, + "refactor_factored_attn_matrices": True, + }, + "no_processing": False, + "full_state_dict": False, + "models": ["gpt2"], + }, +} + + +def _model_dir_name(model_name: str) -> str: + return model_name.replace("/", "__") + + +def _tensor_checksum(t: torch.Tensor) -> str: + return hashlib.sha256(t.detach().cpu().contiguous().float().numpy().tobytes()).hexdigest() + + +def _seeded_sample(t: torch.Tensor, seed: int) -> torch.Tensor: + """Fixed random-index sample of a tensor, deterministic across runs.""" + flat = t.detach().cpu().contiguous().float().flatten() + if flat.numel() <= SAMPLE_COUNT: + return flat.clone() + gen = torch.Generator().manual_seed(seed) + idx = torch.randperm(flat.numel(), generator=gen)[:SAMPLE_COUNT] + return flat[idx.sort().values] diff --git a/tests/integration/model_bridge/test_refactor_factored_attn_matrices.py b/tests/integration/model_bridge/test_refactor_factored_attn_matrices.py index 446384bd7c..a9293b97fd 100644 --- a/tests/integration/model_bridge/test_refactor_factored_attn_matrices.py +++ b/tests/integration/model_bridge/test_refactor_factored_attn_matrices.py @@ -2,7 +2,7 @@ Verifies that the refactored attention matrices produce correct results when used via TransformerBridge, matching the frozen HookedTransformer goldens -(gpt2 is the golden refactor model — see scripts/capture_ht_goldens.py). +(gpt2 is the golden refactor model — see tests/goldens_capture_spec.py). """ import pytest diff --git a/tests/integration/model_bridge/test_weight_processing.py b/tests/integration/model_bridge/test_weight_processing.py index 2ee7198701..6c5557f902 100644 --- a/tests/integration/model_bridge/test_weight_processing.py +++ b/tests/integration/model_bridge/test_weight_processing.py @@ -14,7 +14,7 @@ from jaxtyping import Float from tests import goldens -from transformer_lens import utils +from transformer_lens import utilities as utils from transformer_lens.model_bridge import TransformerBridge # --------------------------------------------------------------------------- @@ -22,7 +22,7 @@ # --------------------------------------------------------------------------- -# The four golden processing configs (see scripts/capture_ht_goldens.py). The +# The four golden processing configs (see tests/goldens_capture_spec.py, CONFIGS). The # former @slow flag combinations are covered by the per-function math invariant # tests instead — they had no independent reference once HT is frozen. GOLDEN_FLAG_CONFIGS = { diff --git a/tests/integration/test_create_hooked_encoder.py b/tests/integration/test_create_hooked_encoder.py deleted file mode 100644 index 45e549aaad..0000000000 --- a/tests/integration/test_create_hooked_encoder.py +++ /dev/null @@ -1,33 +0,0 @@ -import pytest -from transformers import AutoTokenizer, BertTokenizerFast - -from transformer_lens import HookedEncoder, HookedTransformerConfig - - -@pytest.fixture -def cfg(): - return HookedTransformerConfig(d_head=4, d_model=12, n_ctx=5, n_layers=3, act_fn="gelu") - - -def test_pass_tokenizer(cfg): - tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") - model = HookedEncoder(cfg, tokenizer=tokenizer) - assert model.tokenizer == tokenizer - - -def test_load_tokenizer_from_config(cfg): - cfg.tokenizer_name = "bert-base-cased" - model = HookedEncoder(cfg) - assert isinstance(model.tokenizer, BertTokenizerFast) - - -def test_load_without_tokenizer(cfg): - cfg.d_vocab = 22 - model = HookedEncoder(cfg) - assert model.tokenizer is None - - -def test_cannot_load_without_tokenizer_or_d_vocab(cfg): - with pytest.raises(AssertionError) as e: - HookedEncoder(cfg) - assert "Must provide a tokenizer if d_vocab is not provided" in str(e.value) diff --git a/tests/integration/test_grouped_query_attention.py b/tests/integration/test_grouped_query_attention.py deleted file mode 100644 index 98ec98aa42..0000000000 --- a/tests/integration/test_grouped_query_attention.py +++ /dev/null @@ -1,192 +0,0 @@ -import einops -import torch - -from transformer_lens.components import Attention, GroupedQueryAttention -from transformer_lens.config import HookedTransformerConfig - - -def test_grouped_query_attention_output_is_correct(): - """Verifies that grouped query attention (GPA) block behaves correctly - see https://arxiv.org/abs/2305.13245v2 for details on GPA. - A GPA block with h query heads, n key-value heads, key parameters _K and value parameters _V should have the same output as a regular attention block - with h heads, whose parameters K and V are _K and _V repeated h/n times respectively. This test uses torch.repeat_interleave, which is also used by - the GPA block internally, to generate K and V from _K and _V""" - d_model = 512 - d_head = 32 - n_heads = 16 - n_ctx = 128 - n_key_value_heads = 4 - n_layers = 1 - - cfg = HookedTransformerConfig( - d_model=d_model, - d_head=d_head, - n_heads=n_heads, - n_ctx=n_ctx, - n_key_value_heads=n_key_value_heads, - n_layers=n_layers, - act_fn="silu", - ) - - regular_attention = Attention(cfg) - grouped_query_attention = GroupedQueryAttention(cfg) - - W_Q = torch.rand((n_heads, d_model, d_head)) - b_Q = torch.rand((n_heads, d_head)) - _W_K = torch.rand((n_key_value_heads, d_model, d_head)) - W_K = torch.repeat_interleave(_W_K, dim=0, repeats=n_heads // n_key_value_heads) - _b_K = torch.rand((n_key_value_heads, d_head)) - b_K = torch.repeat_interleave(_b_K, dim=0, repeats=n_heads // n_key_value_heads) - _W_V = torch.rand((n_key_value_heads, d_model, d_head)) - W_V = torch.repeat_interleave(_W_V, dim=0, repeats=n_heads // n_key_value_heads) - _b_V = torch.rand((n_key_value_heads, d_head)) - b_V = torch.repeat_interleave(_b_V, dim=0, repeats=n_heads // n_key_value_heads) - W_O = torch.rand((n_heads, d_head, d_model)) - b_O = torch.rand(d_model) - - regular_attention_state_dict = { - "W_Q": W_Q, - "b_Q": b_Q, - "W_O": W_O, - "b_O": b_O, - "W_K": W_K, - "b_K": b_K, - "W_V": W_V, - "b_V": b_V, - "mask": regular_attention.state_dict()["mask"], - "IGNORE": regular_attention.state_dict()["IGNORE"], - } - grouped_query_attention_state_dict = { - "W_Q": W_Q, - "b_Q": b_Q, - "W_O": W_O, - "b_O": b_O, - "_W_K": _W_K, - "_b_K": _b_K, - "_W_V": _W_V, - "_b_V": _b_V, - "mask": grouped_query_attention.state_dict()["mask"], - "IGNORE": grouped_query_attention.state_dict()["IGNORE"], - } - - regular_attention.load_state_dict(regular_attention_state_dict) - grouped_query_attention.load_state_dict(grouped_query_attention_state_dict) - - query_input = torch.rand((1, 5, d_model)) - key_input = torch.rand((1, 5, d_model)) - value_input = torch.rand((1, 5, d_model)) - - regular_attn_output = regular_attention(query_input, key_input, value_input) - grouped_query_attn_output = grouped_query_attention(query_input, key_input, value_input) - - assert torch.allclose(regular_attn_output, grouped_query_attn_output, atol=1e-4, rtol=1e-4) - - # Test GQA behaves correctly when use_split_qkv_input is True - grouped_query_attention.cfg.use_split_qkv_input = True - - split_query_input = einops.repeat(query_input, "b n d -> b n h d", h=n_heads).clone() - split_key_input = einops.repeat(key_input, "b n d -> b n h d", h=n_key_value_heads).clone() - split_value_input = einops.repeat(value_input, "b n d -> b n h d", h=n_key_value_heads).clone() - - split_grouped_query_attn_output = grouped_query_attention( - split_query_input, split_key_input, split_value_input - ) - - # split vs non-split paths differ in fp; loose atol/rtol needed across hardware - - # Calculate differences for debugging - abs_diff = torch.abs(regular_attn_output - split_grouped_query_attn_output) - max_abs_diff = torch.max(abs_diff).item() - mean_abs_diff = torch.mean(abs_diff).item() - - # Calculate relative differences (avoid division by zero) - regular_abs = torch.abs(regular_attn_output) - rel_diff = abs_diff / torch.where(regular_abs > 1e-8, regular_abs, torch.ones_like(regular_abs)) - max_rel_diff = torch.max(rel_diff).item() - mean_rel_diff = torch.mean(rel_diff).item() - - # Print diagnostic information - print(f"\n=== Split vs Non-Split Attention Output Comparison ===") - print(f"Max absolute difference: {max_abs_diff:.6e}") - print(f"Mean absolute difference: {mean_abs_diff:.6e}") - print(f"Max relative difference: {max_rel_diff:.6e}") - print(f"Mean relative difference: {mean_rel_diff:.6e}") - print( - f"Output value range: [{torch.min(regular_attn_output).item():.2f}, {torch.max(regular_attn_output).item():.2f}]" - ) - - assert torch.allclose( - regular_attn_output, split_grouped_query_attn_output, rtol=5e-5, atol=0.5 - ), f"Outputs differ: max_abs_diff={max_abs_diff:.6e}, max_rel_diff={max_rel_diff:.6e}" - - -def test_ungroup_grouped_query_attention_flag_produces_same_result(): - d_model = 512 - d_head = 32 - n_heads = 16 - n_ctx = 128 - n_key_value_heads = 4 - n_layers = 1 - - cfg_flag_off = HookedTransformerConfig( - d_model=d_model, - d_head=d_head, - n_heads=n_heads, - n_ctx=n_ctx, - n_key_value_heads=n_key_value_heads, - n_layers=n_layers, - act_fn="silu", - ungroup_grouped_query_attention=False, - ) - grouped_query_attention_flag_off = GroupedQueryAttention(cfg_flag_off) - - cfg_flag_on = HookedTransformerConfig( - d_model=d_model, - d_head=d_head, - n_heads=n_heads, - n_ctx=n_ctx, - n_key_value_heads=n_key_value_heads, - n_layers=n_layers, - act_fn="silu", - ungroup_grouped_query_attention=True, - ) - grouped_query_attention_flag_on = GroupedQueryAttention(cfg_flag_on) - - W_Q = torch.rand((n_heads, d_model, d_head)) - b_Q = torch.rand((n_heads, d_head)) - _W_K = torch.rand((n_key_value_heads, d_model, d_head)) - _b_K = torch.rand((n_key_value_heads, d_head)) - _W_V = torch.rand((n_key_value_heads, d_model, d_head)) - _b_V = torch.rand((n_key_value_heads, d_head)) - W_O = torch.rand((n_heads, d_head, d_model)) - b_O = torch.rand(d_model) - - grouped_query_attention_state_dict = { - "W_Q": W_Q, - "b_Q": b_Q, - "W_O": W_O, - "b_O": b_O, - "_W_K": _W_K, - "_b_K": _b_K, - "_W_V": _W_V, - "_b_V": _b_V, - "mask": grouped_query_attention_flag_off.state_dict()["mask"], - "IGNORE": grouped_query_attention_flag_off.state_dict()["IGNORE"], - } - - grouped_query_attention_flag_off.load_state_dict(grouped_query_attention_state_dict) - grouped_query_attention_flag_on.load_state_dict(grouped_query_attention_state_dict) - - query_input = torch.rand((1, 5, d_model)) - key_input = torch.rand((1, 5, d_model)) - value_input = torch.rand((1, 5, d_model)) - - grouped_query_attn_flag_off_output = grouped_query_attention_flag_off( - query_input, key_input, value_input - ) - grouped_query_attn_flag_on_output = grouped_query_attention_flag_on( - query_input, key_input, value_input - ) - - assert torch.allclose( - grouped_query_attn_flag_off_output, grouped_query_attn_flag_on_output, atol=1e-4, rtol=1e-4 - ) diff --git a/tests/integration/test_head_detector_bridge.py b/tests/integration/test_head_detector_bridge.py new file mode 100644 index 0000000000..85327ba58d --- /dev/null +++ b/tests/integration/test_head_detector_bridge.py @@ -0,0 +1,31 @@ +"""Smoke coverage for head_detector on a TransformerBridge. + +head_detector.py is kept post-4.0 and typed against the model_protocol surface, +but shipped without a test. This pins that the detectors run end-to-end on a +real bridge (the only model system after the Hooked* removal). +""" + +from __future__ import annotations + +import pytest +import torch + +from transformer_lens.head_detector import HEAD_NAMES, detect_head +from transformer_lens.model_bridge.bridge import TransformerBridge + + +@pytest.fixture(scope="module") +def bridge() -> TransformerBridge: + b = TransformerBridge.boot_transformers("gpt2", device="cpu") + b.enable_compatibility_mode() + return b + + +@pytest.mark.parametrize( + "head_name", ["previous_token_head", "duplicate_token_head", "induction_head"] +) +def test_detect_head_runs_on_bridge(bridge, head_name): + assert head_name in HEAD_NAMES + scores = detect_head(bridge, "The cat sat on the cat sat on the mat", head_name) + assert scores.shape == (bridge.cfg.n_layers, bridge.cfg.n_heads) + assert torch.isfinite(scores).all() diff --git a/tests/integration/test_hooked_encoder_properties.py b/tests/integration/test_hooked_encoder_properties.py deleted file mode 100644 index 3c14445735..0000000000 --- a/tests/integration/test_hooked_encoder_properties.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Convenience-property tests for ``HookedEncoder``. - -Closes the last open ask in #277 — verify each ``W_*`` / ``b_*`` / circuit -property has the right shape AND aliases the right underlying parameter, so -property-level mech-interp work doesn't silently read the wrong tensor. - -Uses a randomly-initialized small encoder (no HF download) so the tests run -fast and deterministically. -""" - -from __future__ import annotations - -import pytest -import torch - -from transformer_lens import FactoredMatrix, HookedEncoder, HookedTransformerConfig - -D_MODEL = 12 -D_HEAD = 4 -N_HEADS = D_MODEL // D_HEAD -D_MLP = 4 * D_MODEL -N_CTX = 5 -N_LAYERS = 3 -D_VOCAB = 22 - - -@pytest.fixture -def model() -> HookedEncoder: - cfg = HookedTransformerConfig( - d_head=D_HEAD, - d_model=D_MODEL, - n_ctx=N_CTX, - n_layers=N_LAYERS, - act_fn="gelu", - d_vocab=D_VOCAB, - ) - encoder = HookedEncoder(cfg) - # HookedEncoder uses torch.empty() for params and does no init pass; the - # uninitialized memory contains NaNs which break torch.equal comparisons. - torch.manual_seed(0) - for p in encoder.parameters(): - torch.nn.init.normal_(p, std=0.02) - return encoder - - -# --------------------------------------------------------------------------- -# Embed / unembed -# --------------------------------------------------------------------------- - - -def test_W_U(model: HookedEncoder): - assert model.W_U.shape == (D_MODEL, D_VOCAB) - assert model.W_U is model.unembed.W_U - - -def test_b_U(model: HookedEncoder): - assert model.b_U.shape == (D_VOCAB,) - assert model.b_U is model.unembed.b_U - - -def test_W_E(model: HookedEncoder): - assert model.W_E.shape == (D_VOCAB, D_MODEL) - assert model.W_E is model.embed.embed.W_E - - -def test_W_pos(model: HookedEncoder): - assert model.W_pos.shape == (N_CTX, D_MODEL) - assert model.W_pos is model.embed.pos_embed.W_pos - - -@pytest.mark.xfail( - reason=( - "HookedEncoder.W_E_pos return annotation 'd_vocab+n_ctx d_model' references " - "unbound dimension names (no input args supply them), so the jaxtyping import-hook " - "can't resolve the sum at runtime. Fixing it is a separate API-touch." - ), - strict=True, -) -def test_W_E_pos(model: HookedEncoder): - assert model.W_E_pos.shape == (D_VOCAB + N_CTX, D_MODEL) - # Concatenation, so identity doesn't apply — verify the slices match. - assert torch.equal(model.W_E_pos[:D_VOCAB], model.W_E) - assert torch.equal(model.W_E_pos[D_VOCAB:], model.W_pos) - - -# --------------------------------------------------------------------------- -# Per-layer attention weights/biases — stacked across blocks -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize("attr", ["W_Q", "W_K", "W_V"]) -def test_attn_qkv_weight(model: HookedEncoder, attr: str): - stacked = getattr(model, attr) - assert stacked.shape == (N_LAYERS, N_HEADS, D_MODEL, D_HEAD) - for layer_idx, block in enumerate(model.blocks): - assert torch.equal(stacked[layer_idx], getattr(block.attn, attr)) - - -def test_W_O(model: HookedEncoder): - assert model.W_O.shape == (N_LAYERS, N_HEADS, D_HEAD, D_MODEL) - for layer_idx, block in enumerate(model.blocks): - assert torch.equal(model.W_O[layer_idx], block.attn.W_O) - - -@pytest.mark.parametrize("attr", ["b_Q", "b_K", "b_V"]) -def test_attn_qkv_bias(model: HookedEncoder, attr: str): - stacked = getattr(model, attr) - assert stacked.shape == (N_LAYERS, N_HEADS, D_HEAD) - for layer_idx, block in enumerate(model.blocks): - assert torch.equal(stacked[layer_idx], getattr(block.attn, attr)) - - -def test_b_O(model: HookedEncoder): - assert model.b_O.shape == (N_LAYERS, D_MODEL) - for layer_idx, block in enumerate(model.blocks): - assert torch.equal(model.b_O[layer_idx], block.attn.b_O) - - -# --------------------------------------------------------------------------- -# Per-layer MLP weights/biases — stacked across blocks -# --------------------------------------------------------------------------- - - -def test_W_in(model: HookedEncoder): - assert model.W_in.shape == (N_LAYERS, D_MODEL, D_MLP) - for layer_idx, block in enumerate(model.blocks): - assert torch.equal(model.W_in[layer_idx], block.mlp.W_in) - - -def test_W_out(model: HookedEncoder): - assert model.W_out.shape == (N_LAYERS, D_MLP, D_MODEL) - for layer_idx, block in enumerate(model.blocks): - assert torch.equal(model.W_out[layer_idx], block.mlp.W_out) - - -def test_b_in(model: HookedEncoder): - assert model.b_in.shape == (N_LAYERS, D_MLP) - for layer_idx, block in enumerate(model.blocks): - assert torch.equal(model.b_in[layer_idx], block.mlp.b_in) - - -def test_b_out(model: HookedEncoder): - assert model.b_out.shape == (N_LAYERS, D_MODEL) - for layer_idx, block in enumerate(model.blocks): - assert torch.equal(model.b_out[layer_idx], block.mlp.b_out) - - -# --------------------------------------------------------------------------- -# Factored circuits -# --------------------------------------------------------------------------- - - -def test_QK_circuit(model: HookedEncoder): - qk = model.QK - assert isinstance(qk, FactoredMatrix) - # Left factor is W_Q [..., d_model, d_head]; right factor is W_K transposed - # to [..., d_head, d_model]. Their product would be [..., d_model, d_model]. - assert qk.A.shape == (N_LAYERS, N_HEADS, D_MODEL, D_HEAD) - assert qk.B.shape == (N_LAYERS, N_HEADS, D_HEAD, D_MODEL) - assert torch.equal(qk.A, model.W_Q) - assert torch.equal(qk.B, model.W_K.transpose(-2, -1)) - - -def test_OV_circuit(model: HookedEncoder): - ov = model.OV - assert isinstance(ov, FactoredMatrix) - assert ov.A.shape == (N_LAYERS, N_HEADS, D_MODEL, D_HEAD) - assert ov.B.shape == (N_LAYERS, N_HEADS, D_HEAD, D_MODEL) - assert torch.equal(ov.A, model.W_V) - assert torch.equal(ov.B, model.W_O) diff --git a/tests/integration/test_loading_from_pretrained.py b/tests/integration/test_loading_from_pretrained.py deleted file mode 100644 index ee20ddf951..0000000000 --- a/tests/integration/test_loading_from_pretrained.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -Tests for the ``loading_from_pretrained`` helpers (basic config lookup). -""" - -from transformer_lens import loading_from_pretrained as loading - - -def test_get_basic_config(): - cfg = loading.get_basic_config("gpt2-small") - assert cfg.d_model - assert cfg.layer_norm_eps - assert cfg.d_vocab - assert cfg.init_range - assert cfg.n_ctx - assert cfg.d_head - assert cfg.d_mlp - assert cfg.n_heads - assert cfg.n_layers diff --git a/tests/integration/test_model_protocol_bridge.py b/tests/integration/test_model_protocol_bridge.py new file mode 100644 index 0000000000..d7a4b9da8d --- /dev/null +++ b/tests/integration/test_model_protocol_bridge.py @@ -0,0 +1,41 @@ +"""TransformerBridge satisfies the model_protocol structural types. + +model_protocol.py was introduced by the deprecation program and is what +head_detector, SVDInterpreter, tools.training and the ActivationCache helpers +type against after the Hooked* removal — but it shipped without a test. This +pins that a live bridge is a structural instance of each protocol. +""" + +from __future__ import annotations + +import pytest + +from transformer_lens.model_bridge.bridge import TransformerBridge +from transformer_lens.model_protocol import ( + TrainableTransformerLensModel, + TransformerLensModel, + TransformerLensModelWithWeights, +) + + +@pytest.fixture(scope="module") +def bridge() -> TransformerBridge: + return TransformerBridge.boot_transformers("gpt2", device="cpu") + + +def test_bridge_is_transformer_lens_model(bridge): + assert isinstance(bridge, TransformerLensModel) + + +def test_bridge_is_trainable_model(bridge): + assert isinstance(bridge, TrainableTransformerLensModel) + + +def test_bridge_has_weight_surface(bridge): + # WithWeights checks getattr-static-visible members; the bridge exposes them + # as properties, so at minimum the runtime attributes must resolve. + assert isinstance(bridge, TransformerLensModel) + for attr in ("blocks", "ln_final", "W_U", "W_E"): + assert hasattr(bridge, attr), attr + # The protocol type itself is importable and usable as an annotation target. + assert TransformerLensModelWithWeights is not None diff --git a/tests/unit/components/mlps/test_gated_mlp.py b/tests/unit/components/mlps/test_gated_mlp.py deleted file mode 100644 index 01175d7992..0000000000 --- a/tests/unit/components/mlps/test_gated_mlp.py +++ /dev/null @@ -1,81 +0,0 @@ -from typing import Any, Dict - -import pytest -import torch -import torch.nn as nn -import torch.nn.functional as F - -from transformer_lens.components import GatedMLP, LayerNorm -from transformer_lens.utilities import solu - - -@pytest.fixture -def cfg() -> Dict[str, Any]: - return { - "n_layers": 12, - "n_ctx": 1024, - "d_head": 64, - "d_model": 128, - "d_mlp": 256, - "dtype": torch.float32, - "act_fn": "solu_ln", - "normalization_type": "LN", - "load_in_4bit": False, - } - - -def test_initialization(cfg: Dict[str, Any]): - model = GatedMLP(cfg) - assert isinstance(model.W_in, nn.Parameter) - assert isinstance(model.W_gate, nn.Parameter) - assert isinstance(model.W_out, nn.Parameter) - assert isinstance(model.b_in, nn.Parameter) - assert isinstance(model.b_out, nn.Parameter) - assert model.act_fn == solu - assert isinstance(model.ln, LayerNorm) - - -def test_forward(cfg: Dict[str, Any]): - model = GatedMLP(cfg) - x = torch.randn(2, 10, cfg["d_model"]) - output = model(x) - assert output.shape == (2, 10, cfg["d_model"]) - - -def test_forward_matches_reference_equation(): - """Numeric equivalence vs a hand-rolled gated-MLP reference (issue #264). - - Closes the original ask in the thread: build an "equivalent gated MLP in - pytorch" and confirm the component matches it under ``torch.allclose``. - Uses ``silu`` so the LN-activation branch is not exercised — that keeps the - reference equation to the documented form. - """ - cfg: Dict[str, Any] = { - "n_layers": 1, - "n_ctx": 16, - "d_head": 32, - "d_model": 64, - "d_mlp": 128, - "dtype": torch.float32, - "act_fn": "silu", - "normalization_type": None, - "load_in_4bit": False, - } - torch.manual_seed(0) - model = GatedMLP(cfg).eval() - # Randomize the params so the test isn't run against zero-bias defaults. - for p in model.parameters(): - torch.nn.init.normal_(p, std=0.02) - - x = torch.randn(2, 5, cfg["d_model"]) - actual = model(x) - - # Reference: mlp_out = (silu(x @ W_gate) * (x @ W_in) + b_in) @ W_out + b_out. - # GatedMLP uses F.linear with .T.contiguous() to match HF accumulation order; - # mirror that here so the two compute graphs are bitwise comparable in fp32. - pre_act = F.linear(x, model.W_gate.T.contiguous()) - pre_linear = F.linear(x, model.W_in.T.contiguous()) - post_act = F.silu(pre_act) * pre_linear + model.b_in - expected = F.linear(post_act, model.W_out.T.contiguous(), model.b_out) - - assert torch.allclose(actual, expected, atol=1e-6) diff --git a/tests/unit/components/mlps/test_mlp.py b/tests/unit/components/mlps/test_mlp.py deleted file mode 100644 index feb8be7c61..0000000000 --- a/tests/unit/components/mlps/test_mlp.py +++ /dev/null @@ -1,49 +0,0 @@ -from typing import Any, Dict - -import pytest -import torch - -from transformer_lens.components import LayerNorm -from transformer_lens.components.mlps.mlp import MLP -from transformer_lens.hook_points import HookPoint - - -@pytest.fixture -def cfg() -> Dict[str, Any]: - return { - "n_layers": 12, - "n_ctx": 1024, - "d_head": 64, - "d_model": 128, - "d_mlp": 256, - "dtype": torch.float32, - "act_fn": "solu_ln", - "normalization_type": "LN", - "load_in_4bit": False, - } - - -def test_initialization(cfg: Dict[str, Any]): - MLP(cfg) - - -def test_forward_without_layer_norm(cfg: Dict[str, Any]): - cfg["act_fn"] = "solu" - - model = MLP(cfg) - - input = torch.full((1, 1, 128), 0.085) - - result = model(input) - - assert result.shape == (1, 1, 128) - - -def test_forward_with_layer_norm(cfg: Dict[str, Any]): - model = MLP(cfg) - assert isinstance(model.hook_mid, HookPoint) - assert isinstance(model.ln, LayerNorm) - - input = torch.full((1, 1, 128), 0.85) - result = model(input) - assert result.shape == (1, 1, 128) diff --git a/tests/unit/components/mlps/test_moe.py b/tests/unit/components/mlps/test_moe.py deleted file mode 100644 index 4c56126ce1..0000000000 --- a/tests/unit/components/mlps/test_moe.py +++ /dev/null @@ -1,21 +0,0 @@ -import torch - -from transformer_lens.components import MoE - - -def test_forward(): - cfg = { - "d_model": 32, - "d_mlp": 14336, - "d_head": 4, - "num_experts": 32, - "n_layers": 16, - "n_ctx": 2048, - "experts_per_token": 4, - "gated_mlp": True, - "act_fn": "silu", - } - moe = MoE(cfg) - - x = torch.rand((1, 4, 32)) - moe(x) diff --git a/tests/unit/components/test_abstract_attention.py b/tests/unit/components/test_abstract_attention.py deleted file mode 100644 index 41db650b47..0000000000 --- a/tests/unit/components/test_abstract_attention.py +++ /dev/null @@ -1,234 +0,0 @@ -import math - -import torch - -from transformer_lens.components import AbstractAttention, Attention -from transformer_lens.config import HookedTransformerConfig - - -def _init_standalone(attn: Attention) -> Attention: - """Fill a standalone component's torch.empty weights; init_weights runs at model level.""" - with torch.random.fork_rng(devices=[]): - torch.manual_seed(0) - for parameter in attn.parameters(): - torch.nn.init.normal_(parameter, std=0.2) - return attn - - -def test_attention_clip_qkv_clamps_between_projection_and_scores(): - """cfg.clip_qkv must clamp Q/K/V after projection (OLMo v1 / OLMoE semantics): - output with clip_qkv equals a clip-free module fed pre-clamped Q/K/V.""" - cfg_kwargs = dict(n_layers=1, d_model=32, n_ctx=8, d_head=8, n_heads=4, act_fn="relu") - # fork_rng: deterministic setup without mutating the global RNG stream. - with torch.random.fork_rng(devices=[]): - torch.manual_seed(0) - attn_clip = Attention(HookedTransformerConfig(**cfg_kwargs, clip_qkv=0.5)) - attn_ref = Attention(HookedTransformerConfig(**cfg_kwargs)) - # Standalone components carry torch.empty weights (init_weights runs at - # model level), so fill them explicitly before sharing the state dict. - with torch.no_grad(): - for param in attn_clip.parameters(): - param.normal_(0, 0.02) - attn_ref.load_state_dict(attn_clip.state_dict()) - - batch, seq = 1, 4 - q0 = torch.randn(batch, seq, 4, 8) * 2 - k0 = torch.randn(batch, seq, 4, 8) * 2 - v0 = torch.randn(batch, seq, 4, 8) * 2 - x = torch.randn(batch, seq, 32) - # clamp is active on all three of Q/K/V - assert (q0.abs() > 0.5).any() and (k0.abs() > 0.5).any() and (v0.abs() > 0.5).any() - - attn_clip.calculate_qkv_matrices = lambda *args, **kwargs: (q0, k0, v0) - attn_ref.calculate_qkv_matrices = lambda *args, **kwargs: ( - q0.clamp(min=-0.5, max=0.5), - k0.clamp(min=-0.5, max=0.5), - v0.clamp(min=-0.5, max=0.5), - ) - - with torch.no_grad(): - out_clip = attn_clip(x, x, x) - out_ref = attn_ref(x, x, x) - - torch.testing.assert_close(out_clip, out_ref) - - -def test_create_alibi_slope(): - n_ctx = 100 - - # Expected result computed non-vectorized way - expected = torch.zeros((n_ctx, n_ctx)) - for row in range(n_ctx): - for col in range(n_ctx): - expected[row, col] = float(min(col - row, 0)) - - # Check against the method's vectorized version - result = AbstractAttention.create_alibi_slope(n_ctx) - assert torch.allclose(expected, result) - - -def test_create_alibi_bias(): - n_heads = 2 - n_ctx = 4 - - result = AbstractAttention.create_alibi_bias(n_heads, n_ctx, torch.device("cpu")) - - for matrix in result: - n_row, n_col = matrix.size() - slope = -matrix[1, 0] - # Check if upper triangle is all zeros - assert torch.equal(torch.triu(matrix), torch.zeros_like(matrix)) - - ref_lower_triangle = torch.zeros_like(matrix) - for i in range(1, n_row): - for j in range(i): - ref_lower_triangle[i, j] = -slope * (i - j) - - # Check if the lower triangle is decreasing by a constant slope (towards the bottom left corner). - assert torch.equal( - torch.tril(matrix, diagonal=-1), torch.tril(ref_lower_triangle, diagonal=-1) - ) - - -class TestLogNAttentionScaling: - """Qwen-1's log-n scaling: queries past the training length scale by - log_{train_len}(position), eval only. TL dropped it entirely.""" - - @staticmethod - def _attention(use_logn: bool) -> Attention: - # n_ctx larger than the training length: the only configuration in - # which contexts long enough to trigger log-n can run at all. - cfg = HookedTransformerConfig( - d_model=16, - d_head=4, - n_heads=4, - n_ctx=16, - n_layers=1, - d_vocab=32, - act_fn="relu", - positional_embedding_type="rotary", - rotary_dim=4, - use_logn_attn=use_logn, - train_seq_length=8, - ) - return _init_standalone(Attention(cfg).eval()) - - def test_matches_the_remote_formula_past_n_ctx(self) -> None: - attn = self._attention(True) - q = torch.ones(1, 12, 4, 4) - scaled = attn._apply_logn_scaling(q, kv_cache_pos_offset=0) - # Positions 1..8 (<= the training length) untouched; 9..12 scale by log_8(pos). - torch.testing.assert_close(scaled[:, :8], q[:, :8]) - for position in (9, 12): - expected = math.log(position, 8) - torch.testing.assert_close( - scaled[0, position - 1, 0, 0], torch.tensor(expected), atol=1e-6, rtol=0 - ) - - def test_cached_decode_uses_absolute_positions(self) -> None: - attn = self._attention(True) - q = torch.ones(1, 1, 4, 4) # single decode step at absolute position 11 - scaled = attn._apply_logn_scaling(q, kv_cache_pos_offset=10) - torch.testing.assert_close( - scaled[0, 0, 0, 0], torch.tensor(math.log(11, 8)), atol=1e-6, rtol=0 - ) - - def test_flag_changes_the_forward_and_default_does_not(self) -> None: - """End to end through the component forward, past the training length.""" - torch.manual_seed(1) - x = torch.randn(1, 12, 16) - on, off = self._attention(True), self._attention(False) - off.load_state_dict(on.state_dict()) - with torch.no_grad(): - out_on = on(x, x, x) - out_off = off(x, x, x) - # Positions within the training length agree exactly; beyond it they differ. - torch.testing.assert_close(out_on[:, :8], out_off[:, :8]) - assert not torch.allclose(out_on[:, 8:], out_off[:, 8:]) - - def test_training_mode_is_untouched(self) -> None: - attn = self._attention(True).train() - x = torch.randn(1, 12, 16) - reference = self._attention(False).train() - reference.load_state_dict(attn.state_dict()) - with torch.no_grad(): - torch.testing.assert_close(attn(x, x, x), reference(x, x, x)) - - -class TestDynamicNTKRotary: - """Qwen-1's dynamic NTK widens the rotary BASE past the training length, so - TL dropping it left long contexts on training-length frequencies.""" - - @staticmethod - def _attention(use_ntk: bool, n_ctx: int = 64) -> Attention: - cfg = HookedTransformerConfig( - d_model=16, - d_head=4, - n_heads=4, - n_ctx=n_ctx, - n_layers=1, - d_vocab=32, - act_fn="relu", - positional_embedding_type="rotary", - rotary_dim=4, - rotary_base=10000, - use_dynamic_ntk_rope=use_ntk, - train_seq_length=8, - ) - return _init_standalone(Attention(cfg).eval()) - - def test_alpha_matches_the_remote_step_function(self) -> None: - """The hub's get_ntk_alpha: 2**ceil(log2(len/train) + 1) - 1, floored at 1.""" - attn = self._attention(True) - assert attn._ntk_alpha(4, 8) == 1.0 # below the training length - assert attn._ntk_alpha(8, 8) == 1.0 # exactly at it - assert attn._ntk_alpha(9, 8) == 3 # just past -> first step - assert attn._ntk_alpha(16, 8) == 3 - assert attn._ntk_alpha(17, 8) == 7 # 2x past -> next step - assert attn._ntk_alpha(32, 8) == 7 - - def test_base_widens_by_the_remote_exponent(self) -> None: - attn = self._attention(True) - before = attn.rotary_cos.clone() - attn._rescale_rotary_for_ntk(32) # alpha = 7, rotary_dim = 4 - expected_base = 10000 * 7 ** (4 / (4 - 2)) - expected_sin, expected_cos = attn.calculate_sin_cos_rotary( - 4, attn.rotary_cos.shape[0], base=expected_base, dtype=attn.cfg.dtype - ) - torch.testing.assert_close(attn.rotary_cos, expected_cos) - torch.testing.assert_close(attn.rotary_sin, expected_sin) - assert not torch.allclose(attn.rotary_cos[: before.shape[0]], before) - - def test_short_context_leaves_the_table_alone(self) -> None: - attn = self._attention(True) - before = attn.rotary_cos.clone() - attn._rescale_rotary_for_ntk(8) # at the training length: alpha = 1 - torch.testing.assert_close(attn.rotary_cos, before) - - def test_forward_differs_only_past_the_training_length(self) -> None: - torch.manual_seed(1) - on, off = self._attention(True), self._attention(False) - off.load_state_dict(on.state_dict()) - with torch.no_grad(): - short = torch.randn(1, 8, 16) - torch.testing.assert_close(on(short, short, short), off(short, short, short)) - long = torch.randn(1, 32, 16) - assert not torch.allclose(on(long, long, long), off(long, long, long)) - - def test_training_mode_is_untouched(self) -> None: - on = self._attention(True).train() - off = self._attention(False).train() - off.load_state_dict(on.state_dict()) - x = torch.randn(1, 32, 16) - with torch.no_grad(): - torch.testing.assert_close(on(x, x, x), off(x, x, x)) - - def test_table_reverts_when_the_context_shrinks(self) -> None: - """alpha is recomputed per forward, so a later short sequence must not - keep the widened base from an earlier long one.""" - attn = self._attention(True) - baseline = attn.rotary_cos.clone() - attn._rescale_rotary_for_ntk(32) - assert not torch.allclose(attn.rotary_cos[: baseline.shape[0]], baseline) - attn._rescale_rotary_for_ntk(8) - torch.testing.assert_close(attn.rotary_cos[: baseline.shape[0]], baseline) diff --git a/tests/unit/components/test_transformer_block_olmo_post_norm.py b/tests/unit/components/test_transformer_block_olmo_post_norm.py deleted file mode 100644 index 6a5a36a042..0000000000 --- a/tests/unit/components/test_transformer_block_olmo_post_norm.py +++ /dev/null @@ -1,164 +0,0 @@ -"""OLMo 2/3 post-norm hook placement in HookedTransformer's TransformerBlock. - -The residual identities are weight-independent, so a random-weight model with -original_architecture="Olmo2ForCausalLM" pins the hook ordering (ln1/ln2 must -apply before hook_attn_out / hook_mlp_out). See issue #1648. -""" - -import pytest -import torch - -from transformer_lens import HookedTransformer, HookedTransformerConfig - -D_VOCAB = 100 - - -@pytest.fixture(scope="module", params=["Olmo2ForCausalLM", "Olmo3ForCausalLM"]) -def olmo_model(request) -> HookedTransformer: - torch.manual_seed(0) - cfg = HookedTransformerConfig( - n_layers=2, - d_model=64, - n_ctx=32, - d_head=16, - n_heads=4, - d_mlp=128, - d_vocab=D_VOCAB, - act_fn="silu", - gated_mlp=True, - normalization_type="RMS", - positional_embedding_type="rotary", - rotary_dim=16, - original_architecture=request.param, - ) - model = HookedTransformer(cfg) - model.init_weights() - return model - - -@pytest.fixture(scope="module") -def sample_tokens() -> torch.Tensor: - torch.manual_seed(0) - return torch.randint(0, D_VOCAB, (1, 10)) - - -def test_residual_branch_hooks_decompose_stream( - olmo_model: HookedTransformer, sample_tokens: torch.Tensor -) -> None: - with torch.no_grad(): - _, cache = olmo_model.run_with_cache(sample_tokens) - - for layer in range(olmo_model.cfg.n_layers): - torch.testing.assert_close( - cache[f"blocks.{layer}.hook_resid_mid"], - cache[f"blocks.{layer}.hook_resid_pre"] + cache[f"blocks.{layer}.hook_attn_out"], - ) - torch.testing.assert_close( - cache[f"blocks.{layer}.hook_resid_post"], - cache[f"blocks.{layer}.hook_resid_mid"] + cache[f"blocks.{layer}.hook_mlp_out"], - ) - - -def test_attn_out_ablation_collapses_residual_step( - olmo_model: HookedTransformer, sample_tokens: torch.Tensor -) -> None: - """Zeroing hook_attn_out must yield resid_mid == resid_pre — pins that writes - land on the additive contribution, with no norm applied afterwards.""" - captured = {} - - def grab(key: str): - def hook_fn(tensor: torch.Tensor, hook) -> torch.Tensor: - captured[key] = tensor.detach().clone() - return tensor - - return hook_fn - - with torch.no_grad(): - baseline = olmo_model(sample_tokens) - ablated = olmo_model.run_with_hooks( - sample_tokens, - fwd_hooks=[ - ("blocks.0.hook_attn_out", lambda tensor, hook: torch.zeros_like(tensor)), - ("blocks.0.hook_resid_pre", grab("resid_pre")), - ("blocks.0.hook_resid_mid", grab("resid_mid")), - ], - ) - - assert not torch.equal(ablated, baseline) - torch.testing.assert_close(captured["resid_mid"], captured["resid_pre"]) - - -def test_attn_out_write_lands_unmodified( - olmo_model: HookedTransformer, sample_tokens: torch.Tensor -) -> None: - """Writing v to hook_attn_out must make the contribution exactly v — a - post-hook norm would distort it (zero-ablation can't catch this: zero is a - fixed point of RMSNorm).""" - torch.manual_seed(1) - replacement = torch.randn(1, sample_tokens.shape[1], olmo_model.cfg.d_model) - captured = {} - - def grab(key: str): - def hook_fn(tensor: torch.Tensor, hook) -> torch.Tensor: - captured[key] = tensor.detach().clone() - return tensor - - return hook_fn - - with torch.no_grad(): - olmo_model.run_with_hooks( - sample_tokens, - fwd_hooks=[ - ("blocks.0.hook_attn_out", lambda tensor, hook: replacement.clone()), - ("blocks.0.hook_resid_pre", grab("resid_pre")), - ("blocks.0.hook_resid_mid", grab("resid_mid")), - ], - ) - - torch.testing.assert_close(captured["resid_mid"] - captured["resid_pre"], replacement) - - -def test_hook_mlp_in_exposes_mid_residual( - olmo_model: HookedTransformer, sample_tokens: torch.Tensor -) -> None: - """With use_hook_mlp_in, hook_mlp_in must equal resid_mid — post-norm OLMo - has no pre-MLP norm, so the MLP input is the mid-residual itself.""" - olmo_model.cfg.use_hook_mlp_in = True - try: - with torch.no_grad(): - _, cache = olmo_model.run_with_cache(sample_tokens) - for layer in range(olmo_model.cfg.n_layers): - torch.testing.assert_close( - cache[f"blocks.{layer}.hook_mlp_in"], - cache[f"blocks.{layer}.hook_resid_mid"], - ) - finally: - olmo_model.cfg.use_hook_mlp_in = False - - -def test_mlp_out_ablation_collapses_residual_step( - olmo_model: HookedTransformer, sample_tokens: torch.Tensor -) -> None: - """Zeroing hook_mlp_out must yield resid_post == resid_mid.""" - captured = {} - - def grab(key: str): - def hook_fn(tensor: torch.Tensor, hook) -> torch.Tensor: - captured[key] = tensor.detach().clone() - return tensor - - return hook_fn - - with torch.no_grad(): - baseline = olmo_model(sample_tokens) - ablated = olmo_model.run_with_hooks( - sample_tokens, - fwd_hooks=[ - ("blocks.0.hook_mlp_out", lambda tensor, hook: torch.zeros_like(tensor)), - ("blocks.0.hook_resid_mid", grab("resid_mid")), - ("blocks.0.hook_resid_post", grab("resid_post")), - ], - ) - - assert not torch.equal(ablated, baseline) - torch.testing.assert_close(captured["resid_post"], captured["resid_mid"]) diff --git a/tests/unit/factored_matrix/test_properties.py b/tests/unit/factored_matrix/test_properties.py index 68707ba6d6..ab8a9c5791 100644 --- a/tests/unit/factored_matrix/test_properties.py +++ b/tests/unit/factored_matrix/test_properties.py @@ -2,7 +2,8 @@ import torch from torch import randn -from transformer_lens import FactoredMatrix, utils +from transformer_lens import FactoredMatrix +from transformer_lens import utilities as utils @pytest.fixture(scope="module") diff --git a/tests/unit/model_bridge/compatibility/test_loading_from_pretrained_utilities.py b/tests/unit/model_bridge/compatibility/test_loading_from_pretrained_utilities.py deleted file mode 100644 index 09cfea8471..0000000000 --- a/tests/unit/model_bridge/compatibility/test_loading_from_pretrained_utilities.py +++ /dev/null @@ -1,54 +0,0 @@ -from unittest import mock - -from transformer_lens.loading_from_pretrained import fill_missing_keys -from transformer_lens.model_bridge import TransformerBridge - - -def get_transformer_bridge(): - """Get a TransformerBridge for testing.""" - return TransformerBridge.boot_transformers("gpt2", device="cpu") - - -# Successes - - -@mock.patch("logging.warning") -def test_fill_missing_keys(mock_warning: mock.MagicMock): - model = get_transformer_bridge() - default_state_dict = model.state_dict() - - incomplete_state_dict = {k: v for k, v in default_state_dict.items() if "W_" not in k} - - filled_state_dict = fill_missing_keys(model, incomplete_state_dict) - - assert set(filled_state_dict.keys()) == set(default_state_dict.keys()) - - # Check that warnings were issued for missing weight matrices - for key in default_state_dict: - if "W_" in key and key not in incomplete_state_dict: - mock_warning.assert_any_call( - f"Missing key for a weight matrix in pretrained, filled in with an empty tensor: {key}" - ) - - -def test_fill_missing_keys_with_hf_model_keys(): - model = get_transformer_bridge() - default_state_dict = model.state_dict() - - incomplete_state_dict = {k: v for k, v in default_state_dict.items() if "hf_model" not in k} - - filled_state_dict = fill_missing_keys(model, incomplete_state_dict) - - expected_keys = set(default_state_dict.keys()) - { - k for k in default_state_dict.keys() if "hf_model" in k - } - assert set(filled_state_dict.keys()) == expected_keys - - -def test_fill_missing_keys_no_missing_keys(): - model = get_transformer_bridge() - default_state_dict = model.state_dict() - - filled_state_dict = fill_missing_keys(model, default_state_dict) - - assert filled_state_dict == default_state_dict diff --git a/tests/unit/model_bridge/compatibility/test_next_sentence_prediction.py b/tests/unit/model_bridge/compatibility/test_next_sentence_prediction.py deleted file mode 100644 index b7999127fa..0000000000 --- a/tests/unit/model_bridge/compatibility/test_next_sentence_prediction.py +++ /dev/null @@ -1,213 +0,0 @@ -from unittest.mock import Mock - -import pytest -import torch - -from transformer_lens import BertNextSentencePrediction -from transformer_lens.model_bridge import TransformerBridge - - -@pytest.fixture -def mock_transformer_bridge(): - """Create a mock TransformerBridge that mimics HookedEncoder behavior for NSP.""" - mock_bridge = Mock(spec=TransformerBridge) - - mock_bridge.cfg = Mock() - mock_bridge.cfg.device = "cpu" - mock_bridge.cfg.n_ctx = 512 - - mock_bridge.tokenizer = Mock() - - mock_encodings = { - "input_ids": torch.tensor([[101, 2034, 102, 2035, 102]]), - "token_type_ids": torch.tensor([[0, 0, 0, 1, 1]]), - "attention_mask": torch.tensor([[1, 1, 1, 1, 1]]), - } - mock_bridge.tokenizer.return_value = mock_encodings - - # Mock bridge output - need to adapt this for TransformerBridge - # TransformerBridge doesn't have encoder_output method, so we'll mock the forward call - mock_bridge.return_value = torch.randn(1, 7, 768) # Mock forward output - - # Mock encoder_output method for BERT compatibility - mock_bridge.encoder_output = Mock() - mock_bridge.encoder_output.return_value = torch.randn(1, 7, 768) - - # Mock pooler and NSP head - these would need to be added to TransformerBridge for BERT - mock_bridge.pooler = Mock() - mock_bridge.pooler.return_value = torch.randn(1, 768) - mock_bridge.nsp_head = Mock() - mock_bridge.nsp_head.return_value = torch.tensor([[0.6, 0.4]]) - - # Mock run_with_cache - mock_bridge.run_with_cache = Mock() - - return mock_bridge - - -@pytest.fixture -def bert_nsp(mock_transformer_bridge): - """Create BertNextSentencePrediction with mocked TransformerBridge.""" - # Note: This test may need to be adapted when TransformerBridge supports BERT-style models - # For now, we'll test the interface compatibility - return BertNextSentencePrediction(mock_transformer_bridge) - - -def test_tokenizer_integration(bert_nsp, mock_transformer_bridge): - """Test that tokenizer integration works with TransformerBridge""" - input_sentences = ["First sentence.", "Second sentence."] - - mock_transformer_bridge.tokenizer = Mock() - - # Mock tokenizer output - mock_encodings = { - "input_ids": torch.tensor([[101, 2034, 102, 2035, 102]]), - "token_type_ids": torch.tensor([[0, 0, 0, 1, 1]]), - "attention_mask": torch.tensor([[1, 1, 1, 1, 1]]), - } - mock_transformer_bridge.tokenizer.return_value = mock_encodings - - tokens, type_ids, mask = bert_nsp.to_tokens(input_sentences) - - # Verify tokenizer was called correctly - mock_transformer_bridge.tokenizer.assert_called_once_with( - input_sentences[0], - input_sentences[1], - return_tensors="pt", - padding=True, - truncation=True, - max_length=mock_transformer_bridge.cfg.n_ctx, - ) - - # Verify outputs match tokenizer output - assert torch.equal(tokens, mock_encodings["input_ids"]) - assert torch.equal(type_ids, mock_encodings["token_type_ids"]) - assert torch.equal(mask, mock_encodings["attention_mask"]) - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires a CUDA device") -def test_device_handling_to_tokens(bert_nsp, mock_transformer_bridge): - """Test proper device handling in to_tokens with TransformerBridge""" - mock_transformer_bridge.cfg.device = "cuda" # Mock GPU device - - input_data = ["First sentence.", "Second sentence."] - - # Mock tokenizer output - mock_encodings = { - "input_ids": torch.tensor([[101, 2034, 102, 2035, 102]]), - "token_type_ids": torch.tensor([[0, 0, 0, 1, 1]]), - "attention_mask": torch.tensor([[1, 1, 1, 1, 1]]), - } - mock_transformer_bridge.tokenizer.return_value = mock_encodings - - tokens, type_ids, mask = bert_nsp.to_tokens(input_data, move_to_device=True) - - # Verify each tensor was moved to the correct device - for tensor in [tokens, type_ids, mask]: - assert tensor.device.type == mock_transformer_bridge.cfg.device - - tokens, type_ids, mask = bert_nsp.to_tokens(input_data, move_to_device=False) - - # Verify tensors remained on CPU - for tensor in [tokens, type_ids, mask]: - assert tensor.device.type == "cpu" - - -def test_output_for_prediction_return_type(bert_nsp, mock_transformer_bridge): - """Test that output for return_type='predictions' works with TransformerBridge""" - input_data = ["First sentence.", "Second sentence."] - - # Test case 1: Sequential prediction - mock_transformer_bridge.nsp_head.return_value = torch.tensor([[0.9, 0.1]]) - pred = bert_nsp.forward(input_data, return_type="predictions") - assert pred == "The sentences are sequential" - - # Test case 2: Non-sequential prediction - mock_transformer_bridge.nsp_head.return_value = torch.tensor([[0.2, 0.8]]) - pred = bert_nsp.forward(input_data, return_type="predictions") - assert pred == "The sentences are NOT sequential" - - -@pytest.mark.parametrize("return_type", [None, "logits", "predictions"]) -def test_forward_return_types(bert_nsp, mock_transformer_bridge, return_type): - """Test different return types from forward pass with TransformerBridge""" - # Setup mock logits that favor sequential prediction - mock_logits = torch.tensor([[0.7, 0.3]]) - mock_transformer_bridge.nsp_head.return_value = mock_logits - - input_data = ["She went to the grocery store.", "She bought an apple."] - output = bert_nsp.forward(input_data, return_type=return_type) - - if return_type is None: - assert output is None - elif return_type == "logits": - assert isinstance(output, torch.Tensor) - assert output.shape == (1, 2) - assert torch.equal(output, mock_logits) - elif return_type == "predictions": - assert isinstance(output, str) - assert output == "The sentences are sequential" # Based on mock logits - - -def test_to_tokens_validation(bert_nsp): - """Test input validation in to_tokens method with TransformerBridge""" - with pytest.raises( - ValueError, match="Next sentence prediction task requires exactly two sentences" - ): - bert_nsp.to_tokens(["Single sentence"]) - - with pytest.raises( - ValueError, match="Next sentence prediction task requires exactly two sentences" - ): - bert_nsp.to_tokens(["One", "Two", "Three"]) - - -def test_run_with_cache(bert_nsp, mock_transformer_bridge): - """Test run_with_cache compatibility with TransformerBridge""" - # Set up mock returns - mock_resid = torch.randn(1, 3, 768) - mock_cache = {"resid_pre": torch.randn(1, 3, 768), "attn_output": torch.randn(1, 3, 768)} - mock_transformer_bridge.run_with_cache.return_value = ( - torch.tensor([[0.6, 0.4]]), # Mock logits - mock_cache, - ) - - input_data = ["First sentence.", "Second sentence."] - - output, cache = bert_nsp.run_with_cache( - input_data, return_type="logits", return_cache_object=True - ) - - # Verify output shape and values - assert output.shape == (1, 2) - assert isinstance(cache, dict) or hasattr(cache, "cache_dict") - - # Verify the cache contains expected keys - if hasattr(cache, "cache_dict"): - cache_dict = cache.cache_dict - else: - cache_dict = cache - - assert "resid_pre" in cache_dict - assert "attn_output" in cache_dict - - -def test_return_type_consistency(bert_nsp, mock_transformer_bridge): - """Test consistency between logits and prediction outputs with TransformerBridge""" - # Setup mock logits that favor non-sequential prediction - mock_logits = torch.tensor([[0.2, 0.8]]) - mock_transformer_bridge.nsp_head.return_value = mock_logits - - input_data = ["She went to the grocery store.", "She bought an apple."] - - # Get predictions using different return types - logits = bert_nsp.forward(input_data, return_type="logits") - prediction_str = bert_nsp.forward(input_data, return_type="predictions") - - # Calculate predicted class from logits - predicted_class = logits.argmax(dim=-1).item() - expected_prediction = ["The sentences are sequential", "The sentences are NOT sequential"][ - predicted_class - ] - - assert prediction_str == expected_prediction # Based on mock logits diff --git a/tests/unit/model_bridge/supported_architectures/test_gemma3_config.py b/tests/unit/model_bridge/supported_architectures/test_gemma3_config.py deleted file mode 100644 index 9893595b9b..0000000000 --- a/tests/unit/model_bridge/supported_architectures/test_gemma3_config.py +++ /dev/null @@ -1,321 +0,0 @@ -"""Unit tests for Gemma 3 / MedGemma legacy `get_pretrained_model_config` lookup. - -Covers registration, config generation, hybrid attention, per-layer RoPE, -and the config's rotary_base_local field. -""" - -from unittest import mock - -import pytest - -from transformer_lens.loading_from_pretrained import get_pretrained_model_config -from transformer_lens.supported_models import OFFICIAL_MODEL_NAMES - -GEMMA3_MODELS = [ - "google/gemma-3-270m", - "google/gemma-3-270m-it", - "google/gemma-3-1b-pt", - "google/gemma-3-1b-it", - "google/gemma-3-4b-pt", - "google/gemma-3-4b-it", - "google/gemma-3-12b-pt", - "google/gemma-3-12b-it", - "google/gemma-3-27b-pt", - "google/gemma-3-27b-it", -] - -MEDGEMMA_MODELS = [ - "google/medgemma-4b-pt", - "google/medgemma-4b-it", - "google/medgemma-27b-it", - "google/medgemma-27b-text-it", -] - -GEMMA3_CONFIG_SPECS = { - "270m": { - "d_model": 640, - "n_heads": 4, - "n_layers": 18, - "d_head": 256, - "n_key_value_heads": 1, - "d_mlp": 2048, - "window_size": 512, - }, - "1b": { - "d_model": 1152, - "n_heads": 4, - "n_layers": 26, - "d_head": 256, - "n_key_value_heads": 1, - "d_mlp": 6912, - "window_size": 512, - }, - "4b": { - "d_model": 2560, - "n_heads": 8, - "n_layers": 34, - "d_head": 256, - "n_key_value_heads": 4, - "d_mlp": 10240, - "window_size": 1024, - }, - "12b": { - "d_model": 3840, - "n_heads": 16, - "n_layers": 48, - "d_head": 256, - "n_key_value_heads": 8, - "d_mlp": 15360, - "window_size": 1024, - }, - "27b": { - "d_model": 5376, - "n_heads": 32, - "n_layers": 62, - "d_head": 128, - "n_key_value_heads": 16, - "d_mlp": 21504, - "window_size": 1024, - }, -} - - -class TestGemma3ModelRegistration: - """All Gemma 3 / MedGemma models are listed in OFFICIAL_MODEL_NAMES.""" - - @pytest.mark.parametrize("model_name", GEMMA3_MODELS) - def test_gemma3_models_in_official_list(self, model_name: str): - assert model_name in OFFICIAL_MODEL_NAMES, f"{model_name} should be in OFFICIAL_MODEL_NAMES" - - @pytest.mark.parametrize("model_name", MEDGEMMA_MODELS) - def test_medgemma_models_in_official_list(self, model_name: str): - assert model_name in OFFICIAL_MODEL_NAMES, f"{model_name} should be in OFFICIAL_MODEL_NAMES" - - -class TestGemma3ConfigGeneration: - """get_pretrained_model_config generates correct configs for Gemma 3.""" - - @pytest.fixture(scope="class") - def mock_hf_config(self): - config = mock.Mock() - config.architectures = ["Gemma3ForCausalLM"] - return config - - @pytest.mark.parametrize( - "model_name,size_key", - [ - ("google/gemma-3-270m", "270m"), - ("google/gemma-3-270m-it", "270m"), - ("google/gemma-3-1b-pt", "1b"), - ("google/gemma-3-1b-it", "1b"), - ], - ) - def test_gemma3_small_model_config(self, model_name: str, size_key: str, mock_hf_config): - with mock.patch( - "transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained", - return_value=mock_hf_config, - ): - cfg = get_pretrained_model_config(model_name) - - expected = GEMMA3_CONFIG_SPECS[size_key] - assert cfg.d_model == expected["d_model"] - assert cfg.n_heads == expected["n_heads"] - assert cfg.n_layers == expected["n_layers"] - assert cfg.d_head == expected["d_head"] - assert cfg.n_key_value_heads == expected["n_key_value_heads"] - assert cfg.d_mlp == expected["d_mlp"] - - @pytest.mark.parametrize( - "model_name,size_key", - [ - ("google/gemma-3-4b-pt", "4b"), - ("google/gemma-3-4b-it", "4b"), - ("google/medgemma-4b-pt", "4b"), - ("google/medgemma-4b-it", "4b"), - ], - ) - def test_gemma3_4b_model_config(self, model_name: str, size_key: str, mock_hf_config): - mock_hf_config.architectures = ["Gemma3ForConditionalGeneration"] - with mock.patch( - "transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained", - return_value=mock_hf_config, - ): - cfg = get_pretrained_model_config(model_name) - - expected = GEMMA3_CONFIG_SPECS[size_key] - assert cfg.d_model == expected["d_model"] - assert cfg.n_heads == expected["n_heads"] - assert cfg.n_layers == expected["n_layers"] - assert cfg.n_key_value_heads == expected["n_key_value_heads"] - - -class TestGemma3HybridAttention: - """Hybrid local/global attention (5:1 pattern).""" - - @pytest.fixture(scope="class") - def mock_hf_config(self): - config = mock.Mock() - config.architectures = ["Gemma3ForCausalLM"] - return config - - def test_attn_types_pattern_270m(self, mock_hf_config): - with mock.patch( - "transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained", - return_value=mock_hf_config, - ): - cfg = get_pretrained_model_config("google/gemma-3-270m") - - assert cfg.use_local_attn is True - assert cfg.attn_types is not None - assert len(cfg.attn_types) == 18 - - for i, attn_type in enumerate(cfg.attn_types): - expected = "global" if (i + 1) % 6 == 0 else "local" - assert attn_type == expected, f"Layer {i}: expected {expected}, got {attn_type}" - - def test_attn_types_pattern_1b(self, mock_hf_config): - with mock.patch( - "transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained", - return_value=mock_hf_config, - ): - cfg = get_pretrained_model_config("google/gemma-3-1b-pt") - - assert cfg.use_local_attn is True - assert len(cfg.attn_types) == 26 - - global_count = cfg.attn_types.count("global") - local_count = cfg.attn_types.count("local") - assert global_count == 4 - assert local_count == 22 - - def test_window_size_small_models(self, mock_hf_config): - with mock.patch( - "transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained", - return_value=mock_hf_config, - ): - cfg = get_pretrained_model_config("google/gemma-3-270m") - assert cfg.window_size == 512 - - def test_window_size_large_models(self, mock_hf_config): - mock_hf_config.architectures = ["Gemma3ForConditionalGeneration"] - with mock.patch( - "transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained", - return_value=mock_hf_config, - ): - cfg = get_pretrained_model_config("google/gemma-3-4b-pt") - assert cfg.window_size == 1024 - - -class TestGemma3PerLayerRoPE: - """Per-layer RoPE base configuration.""" - - @pytest.fixture(scope="class") - def mock_hf_config(self): - config = mock.Mock() - config.architectures = ["Gemma3ForCausalLM"] - return config - - def test_rotary_base_global(self, mock_hf_config): - with mock.patch( - "transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained", - return_value=mock_hf_config, - ): - cfg = get_pretrained_model_config("google/gemma-3-270m") - assert cfg.rotary_base == 1_000_000 - - def test_rotary_base_local(self, mock_hf_config): - with mock.patch( - "transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained", - return_value=mock_hf_config, - ): - cfg = get_pretrained_model_config("google/gemma-3-270m") - assert cfg.rotary_base_local == 10_000 - - -class TestGemma3QKNorm: - """Q/K normalization configuration.""" - - @pytest.fixture(scope="class") - def mock_hf_config(self): - config = mock.Mock() - config.architectures = ["Gemma3ForCausalLM"] - return config - - def test_use_qk_norm_enabled(self, mock_hf_config): - with mock.patch( - "transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained", - return_value=mock_hf_config, - ): - cfg = get_pretrained_model_config("google/gemma-3-270m") - assert cfg.use_qk_norm is True - - -class TestGemma3Normalization: - """Gemma 2/3 style normalization (before and after blocks).""" - - @pytest.fixture(scope="class") - def mock_hf_config(self): - config = mock.Mock() - config.architectures = ["Gemma3ForCausalLM"] - return config - - def test_normalization_before_and_after(self, mock_hf_config): - with mock.patch( - "transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained", - return_value=mock_hf_config, - ): - cfg = get_pretrained_model_config("google/gemma-3-270m") - assert cfg.use_normalization_before_and_after is True - - -class TestGemma3VocabSize: - """Vocabulary size configuration.""" - - @pytest.fixture(scope="class") - def mock_hf_config(self): - config = mock.Mock() - config.architectures = ["Gemma3ForCausalLM"] - return config - - def test_vocab_size_small_models(self, mock_hf_config): - with mock.patch( - "transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained", - return_value=mock_hf_config, - ): - cfg = get_pretrained_model_config("google/gemma-3-270m") - assert cfg.d_vocab == 262144 - - def test_vocab_size_multimodal_models(self, mock_hf_config): - mock_hf_config.architectures = ["Gemma3ForConditionalGeneration"] - with mock.patch( - "transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained", - return_value=mock_hf_config, - ): - cfg = get_pretrained_model_config("google/gemma-3-4b-pt") - assert cfg.d_vocab == 262208 - - def test_vocab_size_medgemma_text_only(self, mock_hf_config): - with mock.patch( - "transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained", - return_value=mock_hf_config, - ): - cfg = get_pretrained_model_config("google/medgemma-27b-text-it") - assert cfg.d_vocab == 262144 - - -class TestGemma3ContextLength: - """Default context length configuration.""" - - @pytest.fixture(scope="class") - def mock_hf_config(self): - config = mock.Mock() - config.architectures = ["Gemma3ForCausalLM"] - return config - - def test_default_context_length(self, mock_hf_config): - with mock.patch( - "transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained", - return_value=mock_hf_config, - ): - cfg = get_pretrained_model_config("google/gemma-3-270m") - assert cfg.n_ctx == 8192 diff --git a/tests/unit/pretrained_weight_conversions/test_apertus.py b/tests/unit/pretrained_weight_conversions/test_apertus.py deleted file mode 100644 index 643c5ab7a1..0000000000 --- a/tests/unit/pretrained_weight_conversions/test_apertus.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Tests for Apertus weight conversion.""" - -from unittest import mock - -import torch - -from transformer_lens.config import TransformerBridgeConfig -from transformer_lens.pretrained.weight_conversions.apertus import ( - convert_apertus_weights, -) - - -def make_cfg(use_qk_norm=True, n_key_value_heads=4): - return TransformerBridgeConfig( - n_layers=1, - d_model=64, - d_head=16, - n_heads=4, - d_mlp=128, - n_ctx=32, - d_vocab=100, - act_fn="xielu", - normalization_type="RMS", - positional_embedding_type="rotary", - gated_mlp=False, - final_rms=True, - use_qk_norm=use_qk_norm, - n_key_value_heads=n_key_value_heads, - dtype=torch.float32, - device="cpu", - ) - - -def make_mock_model(cfg, has_act_fn=True): - """Build a minimal mock HF Apertus model.""" - d = cfg.d_model - d_mlp = cfg.d_mlp - n_heads = cfg.n_heads - n_kv = cfg.n_key_value_heads or cfg.n_heads - d_head = cfg.d_head - - model = mock.Mock() - model.model.embed_tokens.weight = torch.randn(cfg.d_vocab, d) - model.model.norm.weight = torch.randn(d) - model.lm_head.weight = torch.randn(cfg.d_vocab, d) - - layer = mock.Mock() - layer.attention_layernorm.weight = torch.randn(d) - layer.feedforward_layernorm.weight = torch.randn(d) - - # Attention - layer.self_attn.q_proj.weight = torch.randn(n_heads * d_head, d) - layer.self_attn.k_proj.weight = torch.randn(n_kv * d_head, d) - layer.self_attn.v_proj.weight = torch.randn(n_kv * d_head, d) - layer.self_attn.o_proj.weight = torch.randn(d, n_heads * d_head) - - # QK norm - layer.self_attn.q_norm.weight = torch.randn(d_head) - layer.self_attn.k_norm.weight = torch.randn(d_head) - - # Non-gated MLP - layer.mlp.up_proj.weight = torch.randn(d_mlp, d) - layer.mlp.down_proj.weight = torch.randn(d, d_mlp) - - # XIeLU activation parameters - if has_act_fn: - layer.mlp.act_fn.alpha_p = torch.tensor(0.3) - layer.mlp.act_fn.alpha_n = torch.tensor(0.4) - layer.mlp.act_fn.beta = torch.tensor(0.6) - else: - # Remove act_fn to test fallback - del layer.mlp.act_fn - - model.model.layers = [layer] - return model - - -class TestApertusWeightConversion: - def test_embedding_extraction(self): - cfg = make_cfg() - model = make_mock_model(cfg) - sd = convert_apertus_weights(model, cfg) - assert "embed.W_E" in sd - assert sd["embed.W_E"].shape == (cfg.d_vocab, cfg.d_model) - - def test_attention_weight_shapes(self): - cfg = make_cfg() - model = make_mock_model(cfg) - sd = convert_apertus_weights(model, cfg) - - assert sd["blocks.0.attn.W_Q"].shape == (cfg.n_heads, cfg.d_model, cfg.d_head) - assert sd["blocks.0.attn._W_K"].shape == (cfg.n_key_value_heads, cfg.d_model, cfg.d_head) - assert sd["blocks.0.attn._W_V"].shape == (cfg.n_key_value_heads, cfg.d_model, cfg.d_head) - assert sd["blocks.0.attn.W_O"].shape == (cfg.n_heads, cfg.d_head, cfg.d_model) - - def test_gqa_underscore_keys(self): - """GQA models should have _W_K/_W_V (with underscore prefix).""" - cfg = make_cfg(n_key_value_heads=4) - model = make_mock_model(cfg) - sd = convert_apertus_weights(model, cfg) - assert "blocks.0.attn._W_K" in sd - assert "blocks.0.attn._W_V" in sd - assert "blocks.0.attn._b_K" in sd - assert "blocks.0.attn._b_V" in sd - - def test_no_gqa_no_underscore(self): - """Non-GQA models should have W_K/W_V (no underscore).""" - cfg = make_cfg(n_key_value_heads=None) - model = make_mock_model(cfg) - sd = convert_apertus_weights(model, cfg) - assert "blocks.0.attn.W_K" in sd - assert "blocks.0.attn.W_V" in sd - - def test_qk_norm_extracted(self): - cfg = make_cfg(use_qk_norm=True) - model = make_mock_model(cfg) - sd = convert_apertus_weights(model, cfg) - assert "blocks.0.attn.q_norm.w" in sd - assert "blocks.0.attn.k_norm.w" in sd - assert sd["blocks.0.attn.q_norm.w"].shape == (cfg.d_head,) - - def test_no_qk_norm_when_disabled(self): - cfg = make_cfg(use_qk_norm=False) - model = make_mock_model(cfg) - sd = convert_apertus_weights(model, cfg) - assert "blocks.0.attn.q_norm.w" not in sd - assert "blocks.0.attn.k_norm.w" not in sd - - def test_non_gated_mlp_weights(self): - """Apertus uses non-gated MLP (up_proj + down_proj, no gate_proj).""" - cfg = make_cfg() - model = make_mock_model(cfg) - sd = convert_apertus_weights(model, cfg) - assert sd["blocks.0.mlp.W_in"].shape == (cfg.d_model, cfg.d_mlp) - assert sd["blocks.0.mlp.W_out"].shape == (cfg.d_mlp, cfg.d_model) - assert "blocks.0.mlp.W_gate" not in sd - - def test_xielu_params_extracted(self): - cfg = make_cfg() - model = make_mock_model(cfg, has_act_fn=True) - sd = convert_apertus_weights(model, cfg) - torch.testing.assert_close(sd["blocks.0.mlp.act_fn.alpha_p"], torch.tensor(0.3)) - torch.testing.assert_close(sd["blocks.0.mlp.act_fn.alpha_n"], torch.tensor(0.4)) - torch.testing.assert_close(sd["blocks.0.mlp.act_fn.beta"], torch.tensor(0.6)) - - def test_xielu_params_fallback_defaults(self): - """When activation params aren't found, defaults should be used.""" - cfg = make_cfg() - model = make_mock_model(cfg, has_act_fn=False) - # Also remove alternate attribute paths - layer = model.model.layers[0] - if hasattr(layer.mlp, "act"): - del layer.mlp.act - if hasattr(layer.mlp, "alpha_p"): - del layer.mlp.alpha_p - - sd = convert_apertus_weights(model, cfg) - torch.testing.assert_close( - sd["blocks.0.mlp.act_fn.alpha_p"], torch.tensor(0.8, dtype=cfg.dtype) - ) - torch.testing.assert_close( - sd["blocks.0.mlp.act_fn.alpha_n"], torch.tensor(0.8, dtype=cfg.dtype) - ) - torch.testing.assert_close( - sd["blocks.0.mlp.act_fn.beta"], torch.tensor(0.5, dtype=cfg.dtype) - ) - - def test_layer_norm_names(self): - """Apertus uses attention_layernorm/feedforward_layernorm.""" - cfg = make_cfg() - model = make_mock_model(cfg) - sd = convert_apertus_weights(model, cfg) - assert "blocks.0.ln1.w" in sd - assert "blocks.0.ln2.w" in sd - - def test_zero_biases_exist_with_correct_shapes(self): - """Apertus has no projection biases, so converter fills zeros with config-derived shapes.""" - cfg = make_cfg() - model = make_mock_model(cfg) - sd = convert_apertus_weights(model, cfg) - expected_shapes = { - "blocks.0.attn.b_Q": (cfg.n_heads, cfg.d_head), - "blocks.0.attn.b_O": (cfg.d_model,), - "blocks.0.mlp.b_in": (cfg.d_mlp,), - "blocks.0.mlp.b_out": (cfg.d_model,), - "unembed.b_U": (cfg.d_vocab,), - } - for key, shape in expected_shapes.items(): - assert sd[key].shape == shape, f"{key} wrong shape" - assert torch.count_nonzero(sd[key]) == 0, f"{key} should be all zeros" - - def test_unembed_shapes(self): - cfg = make_cfg() - model = make_mock_model(cfg) - sd = convert_apertus_weights(model, cfg) - assert sd["unembed.W_U"].shape == (cfg.d_model, cfg.d_vocab) - assert sd["ln_final.w"].shape == (cfg.d_model,) diff --git a/tests/unit/pretrained_weight_conversions/test_gemma.py b/tests/unit/pretrained_weight_conversions/test_gemma.py deleted file mode 100644 index 9a4c351328..0000000000 --- a/tests/unit/pretrained_weight_conversions/test_gemma.py +++ /dev/null @@ -1,358 +0,0 @@ -""" -Unit tests for Gemma weight conversion. - -Tests cover: -1. Multimodal vs text-only model detection -2. Weight extraction from multimodal models -3. Q/K normalization weight loading -4. Device compatibility (MPS/CUDA) -""" - -import torch -import torch.nn as nn - -from transformer_lens.config import TransformerBridgeConfig -from transformer_lens.pretrained.weight_conversions.gemma import convert_gemma_weights - - -def get_gemma3_config( - n_layers: int = 2, - use_qk_norm: bool = True, - use_normalization_before_and_after: bool = True, -): - """Create a Gemma 3 style config for testing.""" - return TransformerBridgeConfig( - d_model=128, - d_head=64, - n_heads=2, - n_key_value_heads=1, - d_mlp=512, - n_ctx=128, - n_layers=n_layers, - d_vocab=1024, - act_fn="gelu_pytorch_tanh", - use_qk_norm=use_qk_norm, - use_normalization_before_and_after=use_normalization_before_and_after, - normalization_type="RMS", - positional_embedding_type="rotary", - gated_mlp=True, - ) - - -class MockGemmaLayer(nn.Module): - """A mock Gemma layer with real nn.Module components.""" - - def __init__(self, cfg: TransformerBridgeConfig, use_qk_norm: bool = True): - super().__init__() - self.input_layernorm = nn.LayerNorm(cfg.d_model) - self.post_attention_layernorm = nn.LayerNorm(cfg.d_model) - - if cfg.use_normalization_before_and_after: - self.pre_feedforward_layernorm = nn.LayerNorm(cfg.d_model) - self.post_feedforward_layernorm = nn.LayerNorm(cfg.d_model) - - # Self attention - self.self_attn = nn.Module() - self.self_attn.q_proj = nn.Linear(cfg.d_model, cfg.n_heads * cfg.d_head, bias=False) - self.self_attn.k_proj = nn.Linear( - cfg.d_model, cfg.n_key_value_heads * cfg.d_head, bias=False - ) - self.self_attn.v_proj = nn.Linear( - cfg.d_model, cfg.n_key_value_heads * cfg.d_head, bias=False - ) - self.self_attn.o_proj = nn.Linear(cfg.n_heads * cfg.d_head, cfg.d_model, bias=False) - - # Q/K norms (Gemma 3) - if use_qk_norm: - self.self_attn.q_norm = nn.LayerNorm(cfg.d_head) - self.self_attn.k_norm = nn.LayerNorm(cfg.d_head) - - # MLP - self.mlp = nn.Module() - self.mlp.up_proj = nn.Linear(cfg.d_model, cfg.d_mlp, bias=False) - self.mlp.gate_proj = nn.Linear(cfg.d_model, cfg.d_mlp, bias=False) - self.mlp.down_proj = nn.Linear(cfg.d_mlp, cfg.d_model, bias=False) - - -class MockGemmaTextModel(nn.Module): - """A mock text-only Gemma3ForCausalLM model.""" - - def __init__(self, cfg: TransformerBridgeConfig): - super().__init__() - self.model = nn.Module() - self.model.embed_tokens = nn.Embedding(cfg.d_vocab, cfg.d_model) - self.model.norm = nn.LayerNorm(cfg.d_model) - self.model.layers = nn.ModuleList( - [MockGemmaLayer(cfg, use_qk_norm=cfg.use_qk_norm) for _ in range(cfg.n_layers)] - ) - self.lm_head = nn.Linear(cfg.d_model, cfg.d_vocab, bias=False) - - -class MockGemmaMultimodalModel(nn.Module): - """A mock multimodal Gemma3ForConditionalGeneration model.""" - - def __init__(self, cfg: TransformerBridgeConfig): - super().__init__() - # Multimodal structure: model.language_model.model - self.language_model = nn.Module() - self.language_model.model = nn.Module() - - base = self.language_model.model - base.embed_tokens = nn.Embedding(cfg.d_vocab, cfg.d_model) - base.norm = nn.LayerNorm(cfg.d_model) - base.layers = nn.ModuleList( - [MockGemmaLayer(cfg, use_qk_norm=cfg.use_qk_norm) for _ in range(cfg.n_layers)] - ) - # Note: No lm_head in multimodal, uses tied embeddings - - -# ============================================================================ -# Test: Text-only model weight conversion -# ============================================================================ - - -class TestGemmaTextOnlyConversion: - """Test weight conversion for text-only Gemma3ForCausalLM models.""" - - def test_convert_weights_basic(self): - """Test basic weight conversion produces expected keys.""" - cfg = get_gemma3_config() - model = MockGemmaTextModel(cfg) - - state_dict = convert_gemma_weights(model, cfg) - - # Check essential keys exist - assert "embed.W_E" in state_dict - assert "ln_final.w" in state_dict - assert "unembed.W_U" in state_dict - assert "unembed.b_U" in state_dict - - def test_convert_weights_layer_keys(self): - """Test that layer-specific keys are generated.""" - cfg = get_gemma3_config(n_layers=2) - model = MockGemmaTextModel(cfg) - - state_dict = convert_gemma_weights(model, cfg) - - for layer in range(cfg.n_layers): - # Attention weights - assert f"blocks.{layer}.attn.W_Q" in state_dict - assert f"blocks.{layer}.attn._W_K" in state_dict - assert f"blocks.{layer}.attn._W_V" in state_dict - assert f"blocks.{layer}.attn.W_O" in state_dict - - # Attention biases - assert f"blocks.{layer}.attn.b_Q" in state_dict - assert f"blocks.{layer}.attn._b_K" in state_dict - assert f"blocks.{layer}.attn._b_V" in state_dict - assert f"blocks.{layer}.attn.b_O" in state_dict - - # MLP weights - assert f"blocks.{layer}.mlp.W_in" in state_dict - assert f"blocks.{layer}.mlp.W_gate" in state_dict - assert f"blocks.{layer}.mlp.W_out" in state_dict - - # Layer norms - assert f"blocks.{layer}.ln1.w" in state_dict - assert f"blocks.{layer}.ln2.w" in state_dict - - def test_convert_weights_qk_norm(self): - """Test that Q/K norm weights are extracted when use_qk_norm=True.""" - cfg = get_gemma3_config(use_qk_norm=True) - model = MockGemmaTextModel(cfg) - - state_dict = convert_gemma_weights(model, cfg) - - for layer in range(cfg.n_layers): - assert f"blocks.{layer}.attn.q_norm.w" in state_dict - assert f"blocks.{layer}.attn.k_norm.w" in state_dict - - def test_convert_weights_no_qk_norm(self): - """Test that Q/K norm weights are not extracted when use_qk_norm=False.""" - cfg = get_gemma3_config(use_qk_norm=False) - model = MockGemmaTextModel(cfg) - - state_dict = convert_gemma_weights(model, cfg) - - for layer in range(cfg.n_layers): - assert f"blocks.{layer}.attn.q_norm.w" not in state_dict - assert f"blocks.{layer}.attn.k_norm.w" not in state_dict - - def test_convert_weights_normalization_before_and_after(self): - """Test Gemma 2/3 style normalization weights.""" - cfg = get_gemma3_config(use_normalization_before_and_after=True) - model = MockGemmaTextModel(cfg) - - state_dict = convert_gemma_weights(model, cfg) - - for layer in range(cfg.n_layers): - # Post-attention and post-feedforward norms (Gemma 2/3 style) - assert f"blocks.{layer}.ln1_post.w" in state_dict - assert f"blocks.{layer}.ln2_post.w" in state_dict - - -# ============================================================================ -# Test: Multimodal model weight conversion -# ============================================================================ - - -class TestGemmaMultimodalConversion: - """Test weight conversion for multimodal Gemma3ForConditionalGeneration models.""" - - def test_multimodal_detection(self): - """Test that multimodal models are correctly detected.""" - cfg = get_gemma3_config() - model = MockGemmaMultimodalModel(cfg) - - # Should have language_model attribute - assert hasattr(model, "language_model") - - # Should produce valid state dict - state_dict = convert_gemma_weights(model, cfg) - assert "embed.W_E" in state_dict - - def test_multimodal_embedding_extraction(self): - """Test that embeddings are extracted from language_model.model.""" - cfg = get_gemma3_config() - model = MockGemmaMultimodalModel(cfg) - - state_dict = convert_gemma_weights(model, cfg) - - # Check embedding exists and has correct shape - assert "embed.W_E" in state_dict - assert state_dict["embed.W_E"].shape[0] == cfg.d_vocab - - def test_multimodal_tied_embeddings_for_unembed(self): - """Test that multimodal models use tied embeddings for unembed.""" - cfg = get_gemma3_config() - model = MockGemmaMultimodalModel(cfg) - - state_dict = convert_gemma_weights(model, cfg) - - # Unembed should exist - assert "unembed.W_U" in state_dict - - -# ============================================================================ -# Test: Weight shapes -# ============================================================================ - - -class TestGemmaWeightShapes: - """Test that converted weights have correct shapes.""" - - def test_attention_weight_shapes(self): - """Test attention weight shapes are correct.""" - cfg = get_gemma3_config() - model = MockGemmaTextModel(cfg) - - state_dict = convert_gemma_weights(model, cfg) - - # W_Q: [n_heads, d_model, d_head] - assert state_dict["blocks.0.attn.W_Q"].shape == (cfg.n_heads, cfg.d_model, cfg.d_head) - - # W_K/V with GQA: [n_key_value_heads, d_model, d_head] - assert state_dict["blocks.0.attn._W_K"].shape == ( - cfg.n_key_value_heads, - cfg.d_model, - cfg.d_head, - ) - assert state_dict["blocks.0.attn._W_V"].shape == ( - cfg.n_key_value_heads, - cfg.d_model, - cfg.d_head, - ) - - # W_O: [n_heads, d_head, d_model] - assert state_dict["blocks.0.attn.W_O"].shape == (cfg.n_heads, cfg.d_head, cfg.d_model) - - def test_mlp_weight_shapes(self): - """Test MLP weight shapes are correct.""" - cfg = get_gemma3_config() - model = MockGemmaTextModel(cfg) - - state_dict = convert_gemma_weights(model, cfg) - - # W_in and W_gate: [d_model, d_mlp] - assert state_dict["blocks.0.mlp.W_in"].shape == (cfg.d_model, cfg.d_mlp) - assert state_dict["blocks.0.mlp.W_gate"].shape == (cfg.d_model, cfg.d_mlp) - - # W_out: [d_mlp, d_model] - assert state_dict["blocks.0.mlp.W_out"].shape == (cfg.d_mlp, cfg.d_model) - - def test_embedding_scaling(self): - """Test that embeddings are scaled by sqrt(d_model).""" - cfg = get_gemma3_config() - model = MockGemmaTextModel(cfg) - - original_embed = model.model.embed_tokens.weight.clone() - state_dict = convert_gemma_weights(model, cfg) - - # Embedding should be scaled by sqrt(d_model) - expected_scale = cfg.d_model**0.5 - # Check approximately equal (allowing for dtype conversion) - ratio = (state_dict["embed.W_E"] / original_embed).detach() - assert torch.allclose(ratio, torch.full_like(ratio, expected_scale), rtol=1e-4) - - -# ============================================================================ -# Test: Device consistency -# ============================================================================ - - -class TestGemmaDeviceConsistency: - """Test that bias tensors are created on the correct device.""" - - def test_bias_tensors_have_matching_device(self): - """Test that zero bias tensors match weight tensor devices.""" - cfg = get_gemma3_config() - model = MockGemmaTextModel(cfg) - - state_dict = convert_gemma_weights(model, cfg) - - # Check that bias tensors are on the same device as their weights - for layer in range(cfg.n_layers): - w_q = state_dict[f"blocks.{layer}.attn.W_Q"] - b_q = state_dict[f"blocks.{layer}.attn.b_Q"] - assert w_q.device == b_q.device - - w_out = state_dict[f"blocks.{layer}.mlp.W_out"] - b_out = state_dict[f"blocks.{layer}.mlp.b_out"] - assert w_out.device == b_out.device - - -class MockGemma5xMultimodalModel(nn.Module): - """transformers 5.x shape: language_model moved UNDER .model.""" - - def __init__(self, cfg: TransformerBridgeConfig): - super().__init__() - self.model = nn.Module() - self.model.language_model = nn.Module() - base = self.model.language_model - base.embed_tokens = nn.Embedding(cfg.d_vocab, cfg.d_model) - base.norm = nn.LayerNorm(cfg.d_model) - base.layers = nn.ModuleList( - [MockGemmaLayer(cfg, use_qk_norm=cfg.use_qk_norm) for _ in range(cfg.n_layers)] - ) - - -class TestGemma5xMultimodalDetection: - """5.x moved language_model under .model; the old top-level probe silently - reported text-only there and converted the multimodal model down the - wrong path (reading .model as if it were the text transformer).""" - - def test_5x_shape_converts_the_language_model(self): - cfg = get_gemma3_config() - model = MockGemma5xMultimodalModel(cfg) - state_dict = convert_gemma_weights(model, cfg) - expected = model.model.language_model.embed_tokens.weight - # Gemma scales embeddings by sqrt(d_model); undo it to pin the source. - import math - - torch.testing.assert_close(state_dict["embed.W_E"] / math.sqrt(cfg.d_model), expected) - - def test_4x_shape_still_converts(self): - cfg = get_gemma3_config() - model = MockGemmaMultimodalModel(cfg) - state_dict = convert_gemma_weights(model, cfg) - assert "embed.W_E" in state_dict diff --git a/tests/unit/pretrained_weight_conversions/test_hubert_weights.py b/tests/unit/pretrained_weight_conversions/test_hubert_weights.py deleted file mode 100644 index aaad47dbb4..0000000000 --- a/tests/unit/pretrained_weight_conversions/test_hubert_weights.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Tests for HuBERT weight conversion.""" - -from unittest import mock - -import pytest -import torch - -from transformer_lens.config import TransformerBridgeConfig -from transformer_lens.pretrained.weight_conversions.hubert import convert_hubert_weights - - -def make_cfg(): - return TransformerBridgeConfig( - n_layers=1, - d_model=64, - d_head=16, - n_heads=4, - d_mlp=128, - n_ctx=32, - d_vocab=100, - act_fn="gelu", - normalization_type="LN", - dtype=torch.float32, - ) - - -def make_mock_hubert(cfg, has_biases=True): - """Build a minimal mock HF HuBERT model.""" - d = cfg.d_model - d_mlp = cfg.d_mlp - n_heads = cfg.n_heads - d_head = cfg.d_head - - model = mock.Mock() - - layer = mock.Mock() - - # Attention projections - layer.attention.q_proj.weight = torch.randn(n_heads * d_head, d) - layer.attention.k_proj.weight = torch.randn(n_heads * d_head, d) - layer.attention.v_proj.weight = torch.randn(n_heads * d_head, d) - layer.attention.out_proj.weight = torch.randn(d, n_heads * d_head) - - if has_biases: - layer.attention.q_proj.bias = torch.randn(n_heads * d_head) - layer.attention.k_proj.bias = torch.randn(n_heads * d_head) - layer.attention.v_proj.bias = torch.randn(n_heads * d_head) - layer.attention.out_proj.bias = torch.randn(d) - else: - layer.attention.q_proj.bias = None - layer.attention.k_proj.bias = None - layer.attention.v_proj.bias = None - layer.attention.out_proj.bias = None - - # Layer norms - layer.layer_norm.weight = torch.randn(d) - layer.layer_norm.bias = torch.randn(d) - layer.final_layer_norm.weight = torch.randn(d) - layer.final_layer_norm.bias = torch.randn(d) - - # Feed-forward - layer.feed_forward.intermediate_dense.weight = torch.randn(d_mlp, d) - layer.feed_forward.intermediate_dense.bias = torch.randn(d_mlp) - layer.feed_forward.output_dense.weight = torch.randn(d, d_mlp) - layer.feed_forward.output_dense.bias = torch.randn(d) - - # Remove alternate attribute names so mock doesn't auto-create them - layer.self_attn = None - - model.encoder.layers = [layer] - model.encoder.layer_norm.weight = torch.randn(d) - model.encoder.layer_norm.bias = torch.randn(d) - - return model - - -class TestHubertWeightConversion: - def test_attention_weight_shapes(self): - cfg = make_cfg() - model = make_mock_hubert(cfg) - sd = convert_hubert_weights(model, cfg) - - assert sd["blocks.0.attn.W_Q"].shape == (cfg.n_heads, cfg.d_model, cfg.d_head) - assert sd["blocks.0.attn.W_K"].shape == (cfg.n_heads, cfg.d_model, cfg.d_head) - assert sd["blocks.0.attn.W_V"].shape == (cfg.n_heads, cfg.d_model, cfg.d_head) - assert sd["blocks.0.attn.W_O"].shape == (cfg.n_heads, cfg.d_head, cfg.d_model) - - def test_attention_bias_shapes(self): - cfg = make_cfg() - model = make_mock_hubert(cfg, has_biases=True) - sd = convert_hubert_weights(model, cfg) - - assert sd["blocks.0.attn.b_Q"].shape == (cfg.n_heads, cfg.d_head) - assert sd["blocks.0.attn.b_K"].shape == (cfg.n_heads, cfg.d_head) - assert sd["blocks.0.attn.b_V"].shape == (cfg.n_heads, cfg.d_head) - assert sd["blocks.0.attn.b_O"].shape == (cfg.d_model,) - - def test_no_bias_omits_bias_keys(self): - cfg = make_cfg() - model = make_mock_hubert(cfg, has_biases=False) - sd = convert_hubert_weights(model, cfg) - - assert "blocks.0.attn.b_Q" not in sd - assert "blocks.0.attn.b_K" not in sd - assert "blocks.0.attn.b_V" not in sd - assert "blocks.0.attn.b_O" not in sd - - def test_layer_norm_extraction(self): - cfg = make_cfg() - model = make_mock_hubert(cfg) - sd = convert_hubert_weights(model, cfg) - - assert "blocks.0.ln1.w" in sd - assert "blocks.0.ln1.b" in sd - assert "blocks.0.ln2.w" in sd - assert "blocks.0.ln2.b" in sd - assert sd["blocks.0.ln1.w"].shape == (cfg.d_model,) - - def test_ffn_weight_shapes(self): - """FFN weights should be transposed to TL convention.""" - cfg = make_cfg() - model = make_mock_hubert(cfg) - sd = convert_hubert_weights(model, cfg) - - # W_in: (d_model, d_mlp) — transposed from HF's (d_mlp, d_model) - assert sd["blocks.0.mlp.W_in"].shape == (cfg.d_model, cfg.d_mlp) - # W_out: (d_mlp, d_model) — transposed from HF's (d_model, d_mlp) - assert sd["blocks.0.mlp.W_out"].shape == (cfg.d_mlp, cfg.d_model) - assert sd["blocks.0.mlp.b_in"].shape == (cfg.d_mlp,) - assert sd["blocks.0.mlp.b_out"].shape == (cfg.d_model,) - - def test_final_layer_norm(self): - cfg = make_cfg() - model = make_mock_hubert(cfg) - sd = convert_hubert_weights(model, cfg) - - assert "ln_final.w" in sd - assert "ln_final.b" in sd - assert sd["ln_final.w"].shape == (cfg.d_model,) - - def test_no_embedding_keys(self): - """HuBERT is encoder-only — no embed or unembed keys.""" - cfg = make_cfg() - model = make_mock_hubert(cfg) - sd = convert_hubert_weights(model, cfg) - - assert "embed.W_E" not in sd - assert "unembed.W_U" not in sd - - def test_raises_on_missing_encoder(self): - model = mock.Mock(spec=[]) - model.encoder = None - cfg = make_cfg() - with pytest.raises((ValueError, AttributeError)): - convert_hubert_weights(model, cfg) diff --git a/tests/unit/pretrained_weight_conversions/test_mixtral_conversion.py b/tests/unit/pretrained_weight_conversions/test_mixtral_conversion.py deleted file mode 100644 index b46cce1d40..0000000000 --- a/tests/unit/pretrained_weight_conversions/test_mixtral_conversion.py +++ /dev/null @@ -1,229 +0,0 @@ -"""convert_mixtral_weights against the transformers 5.x Mixtral layout. - -5.x renamed the MoE block (``block_sparse_moe`` -> ``mlp``) and replaced the -per-expert ``w1``/``w2``/``w3`` Linears with batched Parameters on a single -``MixtralExperts`` module, so the previous converter raised AttributeError on -every Mixtral load. - -The model is built from a tiny config in memory — no download, no hub access. -""" - -from __future__ import annotations - -import pytest -import torch -import torch.nn.functional as F -from transformers import MixtralConfig, MixtralForCausalLM - -from transformer_lens.config import HookedTransformerConfig -from transformer_lens.pretrained.weight_conversions import convert_mixtral_weights - -D_MODEL, D_MLP, N_EXPERTS, N_LAYERS = 8, 16, 4, 1 -# top-k must be < N_EXPERTS or the softmax already sums to 1 and the -# top-k renormalization becomes a no-op no test could observe. -EXPERTS_PER_TOKEN = 2 - - -@pytest.fixture(scope="module") -def hf_model() -> MixtralForCausalLM: - """Tiny Mixtral with AMPLIFIED weights (std=0.3): SiLU is near-linear at - default init, so a swapped gate/up is ~1.6e-05 off (reads as noise) vs - ~8e-02 amplified. Do not lower without re-measuring the negative control. - """ - with torch.random.fork_rng(devices=[]): - torch.manual_seed(0) - model = MixtralForCausalLM( - MixtralConfig( - hidden_size=D_MODEL, - intermediate_size=D_MLP, - num_hidden_layers=N_LAYERS, - num_attention_heads=2, - num_key_value_heads=1, - vocab_size=32, - num_local_experts=N_EXPERTS, - num_experts_per_tok=EXPERTS_PER_TOKEN, - max_position_embeddings=32, - ) - ).eval() - for param in model.parameters(): - torch.nn.init.normal_(param, std=0.3) - return model - - -@pytest.fixture(scope="module") -def tl_cfg() -> HookedTransformerConfig: - return HookedTransformerConfig( - d_model=D_MODEL, - d_head=4, - n_heads=2, - n_key_value_heads=1, - n_layers=N_LAYERS, - n_ctx=32, - d_vocab=32, - d_mlp=D_MLP, - num_experts=N_EXPERTS, - experts_per_token=EXPERTS_PER_TOKEN, - act_fn="silu", - normalization_type="RMS", - positional_embedding_type="rotary", - # Mirrors what the MixtralForCausalLM config branch produces; that the - # branch really does set it is pinned separately by - # test_config_pins_topk_renormalization, so the two together chain the - # real load path to the behavior below. - norm_topk_prob=True, - ) - - -@pytest.fixture(scope="module") -def state_dict(hf_model, tl_cfg) -> dict: - return convert_mixtral_weights(hf_model, tl_cfg) - - -def test_converts_the_5x_batched_expert_layout(state_dict) -> None: - """The whole conversion runs — it previously raised AttributeError looking - for the removed `block_sparse_moe` attribute.""" - for expert in range(N_EXPERTS): - for name, shape in ( - ("W_gate", (D_MLP, D_MODEL)), - ("W_in", (D_MLP, D_MODEL)), - ("W_out", (D_MODEL, D_MLP)), - ): - key = f"blocks.0.mlp.experts.{expert}.{name}.weight" - assert key in state_dict, f"missing {key}" - assert state_dict[key].shape == shape - - -def test_expert_weights_reproduce_hf_expert_output(hf_model, state_dict) -> None: - """The decisive check on the fused split: shapes cannot catch a swapped - gate/up (both halves [d_mlp, d_model]), so compare numerically with a - negative control proving the swap would differ. - """ - with torch.random.fork_rng(devices=[]): - torch.manual_seed(1) - x = torch.randn(1, D_MODEL) - - experts = hf_model.model.layers[0].mlp.experts - for expert in range(N_EXPERTS): - # HF's own math (MixtralExperts.forward): the fused projection is split - # with .chunk(2, dim=-1) AFTER the linear, so the first half is the gate. - gate_hf, up_hf = F.linear(x, experts.gate_up_proj[expert]).chunk(2, dim=-1) - expected = F.linear(F.silu(gate_hf) * up_hf, experts.down_proj[expert]) - - w_gate = state_dict[f"blocks.0.mlp.experts.{expert}.W_gate.weight"] - w_in = state_dict[f"blocks.0.mlp.experts.{expert}.W_in.weight"] - w_out = state_dict[f"blocks.0.mlp.experts.{expert}.W_out.weight"] - actual = F.linear(F.silu(F.linear(x, w_gate)) * F.linear(x, w_in), w_out) - torch.testing.assert_close(actual, expected) - - swapped = F.linear(F.silu(F.linear(x, w_in)) * F.linear(x, w_gate), w_out) - assert not torch.allclose(swapped, expected, atol=1e-6), ( - f"expert {expert}: gate and up are interchangeable in this fixture, " - "so the assertion above cannot detect a swapped mapping" - ) - - -def test_router_weights_come_from_the_moe_gate(hf_model, state_dict) -> None: - torch.testing.assert_close( - state_dict["blocks.0.mlp.W_gate.weight"], - hf_model.model.layers[0].mlp.gate.weight, - ) - - -@pytest.mark.parametrize( - "dtype,reason", - [ - (torch.int8, "packed integer storage"), - (torch.uint8, "packed integer storage"), - # float8 reports is_floating_point=True, so a plain float check admits - # it — and it is the one family that slices without complaint. - (torch.float8_e4m3fn, "narrow float"), - ], -) -def test_quantized_expert_weights_are_refused(hf_model, tl_cfg, dtype, reason) -> None: - """Slicing packed or scale-separated expert weights would silently drop the - scales, so the converter must refuse rather than emit plausible garbage.""" - experts = hf_model.model.layers[0].mlp.experts - original = experts.gate_up_proj - try: - experts.gate_up_proj = torch.nn.Parameter(original.detach().to(dtype), requires_grad=False) - with pytest.raises(NotImplementedError, match=reason): - convert_mixtral_weights(hf_model, tl_cfg) - finally: - experts.gate_up_proj = original - - -def test_config_pins_topk_renormalization() -> None: - """HF's MixtralTopKRouter renormalizes top-k weights unconditionally, and - MixtralConfig has no field to read that from, so TL's config branch must pin - it — otherwise TL skips the renormalization and routing is silently wrong.""" - from transformer_lens.loading_from_pretrained import convert_hf_model_config - - cfg = convert_hf_model_config("mistralai/Mixtral-8x7B-v0.1") - assert cfg["norm_topk_prob"] is True - - -def test_converted_model_reproduces_hf_logits(hf_model, tl_cfg, state_dict) -> None: - """End-to-end logits parity — catches orientation and routing errors the - per-tensor assertions can't see (notably top-k renormalization). - """ - from transformer_lens import HookedTransformer - - tl_model = HookedTransformer(tl_cfg, tokenizer=None) - tl_model.load_state_dict(state_dict, strict=False) - tl_model.eval() - - ids = torch.tensor([[1, 5, 9, 2]]) - with torch.no_grad(): - tl_logits = tl_model(ids) - hf_logits = hf_model(ids).logits - - max_diff = (tl_logits - hf_logits).abs().max().item() - scale = max(1.0, hf_logits.abs().max().item()) - assert max_diff < 1e-4 * scale, ( - f"converted Mixtral drifts {max_diff:.3e} from HF (scale {scale:.3f}) — " - "a weight orientation or routing term is wrong" - ) - - -def test_routing_renormalization_matches_hf(hf_model, tl_cfg, state_dict) -> None: - """TL's MoE block must reproduce HF's unconditional top-k renormalization. - Compares blocks directly: hook_expert_weights fires pre-top-k, whose top-k - slice sums to 1 by construction, so asserting on it is vacuous. - """ - from transformer_lens import HookedTransformer - - tl_model = HookedTransformer(tl_cfg, tokenizer=None) - tl_model.load_state_dict(state_dict, strict=False) - tl_model.eval() - - with torch.random.fork_rng(devices=[]): - torch.manual_seed(3) - hidden = torch.randn(1, 4, D_MODEL) - - with torch.no_grad(): - tl_out = tl_model.blocks[0].mlp(hidden) - hf_out = hf_model.model.layers[0].mlp(hidden) - hf_out = hf_out[0] if isinstance(hf_out, tuple) else hf_out - - torch.testing.assert_close(tl_out, hf_out, atol=1e-5, rtol=1e-4) - - -@pytest.mark.parametrize( - "dtype,reason", - [ - (torch.int8, "packed integer storage"), - (torch.float8_e4m3fn, "narrow float"), - ], -) -def test_quantized_router_weight_is_refused(hf_model, tl_cfg, dtype, reason) -> None: - """The router sits next to guarded expert reads and had no guard; - load_state_dict accepts a same-shape int8/FP8 router and silently casts. - """ - moe = hf_model.model.layers[0].mlp - original = moe.gate.weight - try: - moe.gate.weight = torch.nn.Parameter(original.detach().to(dtype), requires_grad=False) - with pytest.raises(NotImplementedError, match=reason): - convert_mixtral_weights(hf_model, tl_cfg) - finally: - moe.gate.weight = original diff --git a/tests/unit/pretrained_weight_conversions/test_olmo3.py b/tests/unit/pretrained_weight_conversions/test_olmo3.py deleted file mode 100644 index 9d507c3d48..0000000000 --- a/tests/unit/pretrained_weight_conversions/test_olmo3.py +++ /dev/null @@ -1,411 +0,0 @@ -""" -Unit tests for OLMo 3/3.1 weight conversion. - -Tests cover: -1. Basic weight conversion produces expected keys -2. GQA (Grouped Query Attention) with underscore prefix for K/V weights -3. Q/K normalization weight loading -4. Device consistency across all tensors -5. Weight shapes match expected dimensions -""" - -import torch -import torch.nn as nn - -from transformer_lens import HookedTransformer -from transformer_lens.config import TransformerBridgeConfig -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.pretrained.weight_conversions.olmo3 import convert_olmo3_weights - - -def _olmo3_config_kwargs( - n_layers: int = 2, - use_qk_norm: bool = True, - n_key_value_heads: int | None = 1, -) -> dict: - return dict( - d_model=128, - d_head=64, - n_heads=2, - n_key_value_heads=n_key_value_heads, - d_mlp=512, - n_ctx=2048, - n_layers=n_layers, - d_vocab=50304, - act_fn="silu", - use_qk_norm=use_qk_norm, - use_normalization_before_and_after=True, - normalization_type="RMS", - positional_embedding_type="rotary", - gated_mlp=True, - ) - - -def get_olmo3_config( - n_layers: int = 2, - use_qk_norm: bool = True, - n_key_value_heads: int | None = 1, -): - """Create an OLMo 3 style config for testing.""" - return TransformerBridgeConfig(**_olmo3_config_kwargs(n_layers, use_qk_norm, n_key_value_heads)) - - -class MockOlmo3Layer(nn.Module): - """A mock OLMo 3 layer with real nn.Module components.""" - - def __init__(self, cfg: TransformerBridgeConfig, use_qk_norm: bool = True): - super().__init__() - - # OLMo 3 uses post-attention and post-feedforward layer norms (RMSNorm) - self.post_attention_layernorm = nn.RMSNorm(cfg.d_model, eps=cfg.eps) - self.post_feedforward_layernorm = nn.RMSNorm(cfg.d_model, eps=cfg.eps) - - # Self attention - n_kv_heads = cfg.n_key_value_heads if cfg.n_key_value_heads else cfg.n_heads - self.self_attn = nn.Module() - self.self_attn.q_proj = nn.Linear(cfg.d_model, cfg.n_heads * cfg.d_head, bias=False) - self.self_attn.k_proj = nn.Linear(cfg.d_model, n_kv_heads * cfg.d_head, bias=False) - self.self_attn.v_proj = nn.Linear(cfg.d_model, n_kv_heads * cfg.d_head, bias=False) - self.self_attn.o_proj = nn.Linear(cfg.n_heads * cfg.d_head, cfg.d_model, bias=False) - - # Q/K norms (OLMo 3) - if use_qk_norm: - self.self_attn.q_norm = nn.RMSNorm(cfg.d_head, eps=cfg.eps) - self.self_attn.k_norm = nn.RMSNorm(cfg.d_head, eps=cfg.eps) - - # MLP (Gated / SwiGLU-style) - self.mlp = nn.Module() - self.mlp.up_proj = nn.Linear(cfg.d_model, cfg.d_mlp, bias=False) - self.mlp.gate_proj = nn.Linear(cfg.d_model, cfg.d_mlp, bias=False) - self.mlp.down_proj = nn.Linear(cfg.d_mlp, cfg.d_model, bias=False) - - -class MockOlmo3Model(nn.Module): - """A mock Olmo3ForCausalLM model.""" - - def __init__(self, cfg: TransformerBridgeConfig): - super().__init__() - self.model = nn.Module() - self.model.embed_tokens = nn.Embedding(cfg.d_vocab, cfg.d_model) - self.model.norm = nn.RMSNorm(cfg.d_model, eps=cfg.eps) - self.model.layers = nn.ModuleList( - [MockOlmo3Layer(cfg, use_qk_norm=cfg.use_qk_norm) for _ in range(cfg.n_layers)] - ) - self.lm_head = nn.Linear(cfg.d_model, cfg.d_vocab, bias=False) - - -# ============================================================================ -# Test: Basic weight conversion -# ============================================================================ - - -class TestOlmo3BasicConversion: - """Test basic weight conversion for OLMo 3 models.""" - - def test_convert_weights_basic(self): - """Test basic weight conversion produces expected keys.""" - cfg = get_olmo3_config() - model = MockOlmo3Model(cfg) - - state_dict = convert_olmo3_weights(model, cfg) - - # Check essential keys exist - assert "embed.W_E" in state_dict - assert "ln_final.w" in state_dict - assert "unembed.W_U" in state_dict - assert "unembed.b_U" in state_dict - - def test_convert_weights_layer_keys(self): - """Test that layer-specific keys are generated.""" - cfg = get_olmo3_config(n_layers=2) - model = MockOlmo3Model(cfg) - - state_dict = convert_olmo3_weights(model, cfg) - - for layer in range(cfg.n_layers): - # Attention weights - assert f"blocks.{layer}.attn.W_Q" in state_dict - assert f"blocks.{layer}.attn._W_K" in state_dict # GQA underscore prefix - assert f"blocks.{layer}.attn._W_V" in state_dict # GQA underscore prefix - assert f"blocks.{layer}.attn.W_O" in state_dict - - # Attention biases - assert f"blocks.{layer}.attn.b_Q" in state_dict - assert f"blocks.{layer}.attn._b_K" in state_dict - assert f"blocks.{layer}.attn._b_V" in state_dict - assert f"blocks.{layer}.attn.b_O" in state_dict - - # MLP weights - assert f"blocks.{layer}.mlp.W_in" in state_dict - assert f"blocks.{layer}.mlp.W_gate" in state_dict - assert f"blocks.{layer}.mlp.W_out" in state_dict - - # Layer norms - assert f"blocks.{layer}.ln1.w" in state_dict - assert f"blocks.{layer}.ln2.w" in state_dict - - def test_convert_weights_no_biases_in_source(self): - """Test that OLMo 3 has no bias weights in the source model.""" - cfg = get_olmo3_config() - model = MockOlmo3Model(cfg) - - state_dict = convert_olmo3_weights(model, cfg) - - # All biases should be zero tensors created during conversion - assert torch.all(state_dict["blocks.0.attn.b_Q"] == 0) - assert torch.all(state_dict["blocks.0.attn._b_K"] == 0) - assert torch.all(state_dict["blocks.0.attn.b_O"] == 0) - assert torch.all(state_dict["unembed.b_U"] == 0) - - -# ============================================================================ -# Test: Q/K normalization -# ============================================================================ - - -class TestOlmo3QKNorm: - """Test Q/K normalization weight loading.""" - - def test_qk_norm_weights_extracted(self): - """Test that Q/K norm weights are extracted when use_qk_norm=True.""" - cfg = get_olmo3_config(use_qk_norm=True) - model = MockOlmo3Model(cfg) - - state_dict = convert_olmo3_weights(model, cfg) - - for layer in range(cfg.n_layers): - assert f"blocks.{layer}.attn.q_norm.w" in state_dict - assert f"blocks.{layer}.attn.k_norm.w" in state_dict - # Verify weights are not zeros (actual model weights) - assert not torch.all(state_dict[f"blocks.{layer}.attn.q_norm.w"] == 0) - assert not torch.all(state_dict[f"blocks.{layer}.attn.k_norm.w"] == 0) - - -# ============================================================================ -# Test: GQA (Grouped Query Attention) -# ============================================================================ - - -class TestOlmo3GQA: - """Test Grouped Query Attention (GQA) handling. - - The prefix must match TransformerBlock's attention choice: it builds - GroupedQueryAttention (underscore-prefixed _W_K/_W_V) whenever - n_key_value_heads is set — even when it equals n_heads (#1620). The - previous expectation of un-prefixed keys for that case zero-filled K/V - on checkpoints like allenai/Olmo-3-1025-7B (32 kv heads == 32 heads). - """ - - def test_kv_heads_equal_to_n_heads_keeps_underscore_prefix(self): - """n_key_value_heads == n_heads still instantiates GQA, so keys need the prefix.""" - cfg = get_olmo3_config(n_key_value_heads=2) # equals n_heads - model = MockOlmo3Model(cfg) - - state_dict = convert_olmo3_weights(model, cfg) - - for layer in range(cfg.n_layers): - assert f"blocks.{layer}.attn._W_K" in state_dict - assert f"blocks.{layer}.attn._W_V" in state_dict - assert f"blocks.{layer}.attn.W_K" not in state_dict - assert f"blocks.{layer}.attn.W_V" not in state_dict - - def test_no_kv_heads_means_plain_mha_keys(self): - """Without n_key_value_heads the block builds plain Attention — no prefix.""" - cfg = get_olmo3_config(n_key_value_heads=None) - model = MockOlmo3Model(cfg) - - state_dict = convert_olmo3_weights(model, cfg) - - for layer in range(cfg.n_layers): - assert f"blocks.{layer}.attn.W_K" in state_dict - assert f"blocks.{layer}.attn.W_V" in state_dict - assert f"blocks.{layer}.attn._W_K" not in state_dict - assert f"blocks.{layer}.attn._W_V" not in state_dict - - -# ============================================================================ -# Test: Device consistency -# ============================================================================ - - -class TestOlmo3DeviceConsistency: - """Test device consistency across converted weights.""" - - def test_all_tensors_on_same_device(self): - """Test that all converted tensors are on the same device as source.""" - cfg = get_olmo3_config() - - # Test on CPU - model = MockOlmo3Model(cfg) - state_dict = convert_olmo3_weights(model, cfg) - - for key, value in state_dict.items(): - assert value.device == model.model.embed_tokens.weight.device, ( - f"Tensor {key} is on {value.device} but should be on " - f"{model.model.embed_tokens.weight.device}" - ) - - -# ============================================================================ -# Test: Weight shapes -# ============================================================================ - - -class TestOlmo3WeightShapes: - """Test that converted weights have correct shapes.""" - - def test_attention_weight_shapes(self): - """Test attention weight shapes are correct.""" - cfg = get_olmo3_config(n_layers=1) - model = MockOlmo3Model(cfg) - - state_dict = convert_olmo3_weights(model, cfg) - - # W_Q: (n_heads, d_model, d_head) - assert state_dict["blocks.0.attn.W_Q"].shape == (cfg.n_heads, cfg.d_model, cfg.d_head) - # W_K: (n_kv_heads, d_model, d_head) - assert state_dict["blocks.0.attn._W_K"].shape == ( - cfg.n_key_value_heads, - cfg.d_model, - cfg.d_head, - ) - # W_V: (n_kv_heads, d_model, d_head) - assert state_dict["blocks.0.attn._W_V"].shape == ( - cfg.n_key_value_heads, - cfg.d_model, - cfg.d_head, - ) - # W_O: (n_heads, d_head, d_model) - assert state_dict["blocks.0.attn.W_O"].shape == (cfg.n_heads, cfg.d_head, cfg.d_model) - - def test_mlp_weight_shapes(self): - """Test MLP weight shapes are correct.""" - cfg = get_olmo3_config(n_layers=1) - model = MockOlmo3Model(cfg) - - state_dict = convert_olmo3_weights(model, cfg) - - # W_in: (d_model, d_mlp) - transposed from HF (d_mlp, d_model) - assert state_dict["blocks.0.mlp.W_in"].shape == (cfg.d_model, cfg.d_mlp) - # W_gate: (d_model, d_mlp) - transposed from HF (d_mlp, d_model) - assert state_dict["blocks.0.mlp.W_gate"].shape == (cfg.d_model, cfg.d_mlp) - # W_out: (d_mlp, d_model) - transposed from HF (d_model, d_mlp) - assert state_dict["blocks.0.mlp.W_out"].shape == (cfg.d_mlp, cfg.d_model) - - def test_embedding_shapes(self): - """Test embedding weight shapes are correct.""" - cfg = get_olmo3_config() - model = MockOlmo3Model(cfg) - - state_dict = convert_olmo3_weights(model, cfg) - - assert state_dict["embed.W_E"].shape == (cfg.d_vocab, cfg.d_model) - assert state_dict["unembed.W_U"].shape == (cfg.d_model, cfg.d_vocab) - - -# ============================================================================ -# Test: Converted dict loads into HookedTransformer (#1620 regression) -# ============================================================================ - - -class TestOlmo3StateDictLoads: - """The converted dict must feed the instantiated attention without - zero-filling K/V — the #1620 failure mode on kv_heads == n_heads.""" - - def test_converted_dict_loads_with_nonzero_kv(self): - cfg = get_olmo3_config(n_layers=1, n_key_value_heads=2) # equals n_heads - source_model = MockOlmo3Model(cfg) - state_dict = convert_olmo3_weights(source_model, cfg) - - # HookedTransformer needs its own config class, not the bridge config. - ht_cfg = HookedTransformerConfig(**_olmo3_config_kwargs(n_layers=1, n_key_value_heads=2)) - model = HookedTransformer(ht_cfg) - model.load_and_process_state_dict( - state_dict, - fold_ln=False, - center_writing_weights=False, - center_unembed=False, - fold_value_biases=False, - ) - - for block in model.blocks: - assert block.attn.W_K.abs().max() > 0, "K weights were zero-filled (#1620)" - assert block.attn.W_V.abs().max() > 0, "V weights were zero-filled (#1620)" - - -class TestOlmo3HookedTransformerParity: - """End-to-end HT-vs-HF logit parity on a tiny random Olmo-3. - - Olmo-3 declares per-layer-type rope in transformers 5.x — plain rope on - sliding_attention layers, YARN on full_attention layers — and the old - mapping read the defunct `rope_scaling` attribute, silently applying plain - rope everywhere and never wiring the sliding-window mask. This locks in - the per-type mapping via the production config path. - """ - - def test_tiny_model_logit_parity(self, tmp_path): - from transformers import AutoModelForCausalLM, Olmo3Config, Olmo3ForCausalLM - - from transformer_lens.loading_from_pretrained import convert_hf_model_config - - hf_config = Olmo3Config( - hidden_size=64, - num_hidden_layers=4, - num_attention_heads=4, - num_key_value_heads=4, # kv == heads, the #1620 trigger - intermediate_size=32, - vocab_size=256, - max_position_embeddings=256, - sliding_window=4, # < seq len so the sliding-window mask actually bites - layer_types=[ - "sliding_attention", - "sliding_attention", - "full_attention", - "sliding_attention", - ], - rope_parameters={ - "sliding_attention": {"rope_type": "default", "rope_theta": 500000.0}, - "full_attention": { - "rope_type": "yarn", - "rope_theta": 500000.0, - "factor": 8.0, - "attention_factor": 1.2079441541679836, - "beta_fast": 32, - "beta_slow": 1, - "original_max_position_embeddings": 32, - }, - }, - ) - torch.manual_seed(0) - hf_model = Olmo3ForCausalLM(hf_config).eval().float() - hf_model.save_pretrained(tmp_path) - - ids = torch.tensor([[5, 17, 29, 3, 11, 42, 7, 23]]) - reference = AutoModelForCausalLM.from_pretrained( - str(tmp_path), dtype=torch.float32, attn_implementation="eager" - ).eval() - with torch.inference_mode(): - ref_logits = reference(ids).logits - - cfg_dict = convert_hf_model_config(str(tmp_path)) - assert cfg_dict["use_yarn_rope"] is True - assert cfg_dict["yarn_global_attn_only"] is True - assert cfg_dict["use_local_attn"] is True - assert cfg_dict["window_size"] == 4 - cfg_dict["dtype"] = torch.float32 - cfg_dict["tokenizer_name"] = None # keep the test download-free - - cfg = HookedTransformerConfig.from_dict(cfg_dict) - model = HookedTransformer(cfg) - model.load_and_process_state_dict( - convert_olmo3_weights(hf_model, cfg), - fold_ln=False, - center_writing_weights=False, - center_unembed=False, - fold_value_biases=False, - ) - with torch.inference_mode(): - ht_logits = model(ids, return_type="logits") - - max_diff = (ref_logits - ht_logits).abs().max().item() - assert max_diff < 1e-4, f"HT vs HF logit drift {max_diff:.2e}" diff --git a/tests/unit/pretrained_weight_conversions/test_olmoe.py b/tests/unit/pretrained_weight_conversions/test_olmoe.py deleted file mode 100644 index 5426d5b190..0000000000 --- a/tests/unit/pretrained_weight_conversions/test_olmoe.py +++ /dev/null @@ -1,144 +0,0 @@ -"""convert_olmoe_weights: the batched-expert split and its quantization guard. - -OLMoE stores all experts in two batched Parameters and the converter slices -them, so a packed or scale-separated tensor would be reshaped into a -plausible-looking matrix rather than raising. These guards existed but nothing -exercised them — removing both left the whole suite green. - -The model is built from a tiny config in memory: no download, no hub access. -""" - -from __future__ import annotations - -import pytest -import torch -import torch.nn.functional as F -from transformers import OlmoeConfig, OlmoeForCausalLM - -from transformer_lens.config import HookedTransformerConfig -from transformer_lens.pretrained.weight_conversions import convert_olmoe_weights - -D_MODEL, D_MLP, N_EXPERTS, N_LAYERS = 8, 16, 4, 1 -EXPERTS_PER_TOKEN = 2 - - -@pytest.fixture(scope="module") -def hf_model() -> OlmoeForCausalLM: - """Tiny OLMoE with AMPLIFIED weights — same reason as the Mixtral fixture: - SiLU near-linearity hides a swapped gate/up at default init. Do not lower - without re-measuring the negative control. - """ - with torch.random.fork_rng(devices=[]): - torch.manual_seed(0) - model = OlmoeForCausalLM( - OlmoeConfig( - hidden_size=D_MODEL, - intermediate_size=D_MLP, - num_hidden_layers=N_LAYERS, - num_attention_heads=2, - num_key_value_heads=1, - vocab_size=32, - num_experts=N_EXPERTS, - num_experts_per_tok=EXPERTS_PER_TOKEN, - max_position_embeddings=32, - ) - ).eval() - for param in model.parameters(): - torch.nn.init.normal_(param, std=0.3) - return model - - -@pytest.fixture(scope="module") -def tl_cfg() -> HookedTransformerConfig: - return HookedTransformerConfig( - d_model=D_MODEL, - d_head=4, - n_heads=2, - n_key_value_heads=1, - n_layers=N_LAYERS, - n_ctx=32, - d_vocab=32, - d_mlp=D_MLP, - num_experts=N_EXPERTS, - experts_per_token=EXPERTS_PER_TOKEN, - act_fn="silu", - normalization_type="RMS", - positional_embedding_type="rotary", - use_qk_norm=True, - ) - - -def test_expert_weights_reproduce_hf_expert_output(hf_model, tl_cfg) -> None: - """Shapes cannot catch a swapped gate/up (both halves [d_mlp, d_model]); - compare numerically with a swap negative control. - """ - state_dict = convert_olmoe_weights(hf_model, tl_cfg) - - with torch.random.fork_rng(devices=[]): - torch.manual_seed(1) - x = torch.randn(1, D_MODEL) - - experts = hf_model.model.layers[0].mlp.experts - for expert in range(N_EXPERTS): - gate_hf, up_hf = F.linear(x, experts.gate_up_proj[expert]).chunk(2, dim=-1) - expected = F.linear(F.silu(gate_hf) * up_hf, experts.down_proj[expert]) - - w_gate = state_dict[f"blocks.0.mlp.experts.{expert}.W_gate.weight"] - w_in = state_dict[f"blocks.0.mlp.experts.{expert}.W_in.weight"] - w_out = state_dict[f"blocks.0.mlp.experts.{expert}.W_out.weight"] - actual = F.linear(F.silu(F.linear(x, w_gate)) * F.linear(x, w_in), w_out) - torch.testing.assert_close(actual, expected) - - swapped = F.linear(F.silu(F.linear(x, w_in)) * F.linear(x, w_gate), w_out) - assert not torch.allclose(swapped, expected, atol=1e-6), ( - f"expert {expert}: gate and up are interchangeable in this fixture, " - "so the assertion above cannot detect a swapped mapping" - ) - - -@pytest.mark.parametrize( - "dtype,reason", - [ - (torch.int8, "packed integer storage"), - (torch.uint8, "packed integer storage"), - # float8 reports is_floating_point=True, so a plain float check admits - # it — and it is the one family that slices without complaint. - (torch.float8_e4m3fn, "narrow float"), - (torch.float8_e5m2, "narrow float"), - ], -) -@pytest.mark.parametrize("tensor_name", ["gate_up_proj", "down_proj"]) -def test_quantized_expert_weights_are_refused(hf_model, tl_cfg, dtype, reason, tensor_name) -> None: - """Both batched expert tensors are sliced, so both must be guarded — - slicing either would silently drop the scales stored beside it.""" - experts = hf_model.model.layers[0].mlp.experts - original = getattr(experts, tensor_name) - try: - setattr( - experts, - tensor_name, - torch.nn.Parameter(original.detach().to(dtype), requires_grad=False), - ) - with pytest.raises(NotImplementedError, match=reason): - convert_olmoe_weights(hf_model, tl_cfg) - finally: - setattr(experts, tensor_name, original) - - -@pytest.mark.parametrize( - "dtype,reason", - [ - (torch.int8, "packed integer storage"), - (torch.float8_e4m3fn, "narrow float"), - ], -) -def test_quantized_router_weight_is_refused(hf_model, tl_cfg, dtype, reason) -> None: - """Same unguarded router read as Mixtral's, caught by the same guard.""" - moe = hf_model.model.layers[0].mlp - original = moe.gate.weight - try: - moe.gate.weight = torch.nn.Parameter(original.detach().to(dtype), requires_grad=False) - with pytest.raises(NotImplementedError, match=reason): - convert_olmoe_weights(hf_model, tl_cfg) - finally: - moe.gate.weight = original diff --git a/tests/unit/pretrained_weight_conversions/test_openai.py b/tests/unit/pretrained_weight_conversions/test_openai.py deleted file mode 100644 index dcb618e1b4..0000000000 --- a/tests/unit/pretrained_weight_conversions/test_openai.py +++ /dev/null @@ -1,332 +0,0 @@ -import pytest -import torch - -from transformer_lens.config import TransformerBridgeConfig -from transformer_lens.pretrained.weight_conversions.openai import ( - convert_gpt_oss_weights, -) - - -def make_cfg(): - return TransformerBridgeConfig( - n_layers=1, - d_model=32, - d_head=8, - n_heads=4, - d_mlp=16, - n_ctx=64, - d_vocab=128, - act_fn="silu", - normalization_type="RMS", - positional_embedding_type="rotary", - rotary_base=150000, - eps=1e-5, - n_key_value_heads=2, - gated_mlp=True, - use_local_attn=False, - rotary_dim=8, - num_experts=2, - experts_per_token=1, - dtype=torch.float32, - original_architecture="GptOssForCausalLM", - ) - - -def make_mock_model(cfg): - """Build a minimal mock HF GPT-OSS model with the right tensor shapes.""" - from unittest import mock - - d = cfg.d_model - d_mlp = cfg.d_mlp - n_experts = cfg.num_experts - n_heads = cfg.n_heads - n_kv = cfg.n_key_value_heads - d_head = cfg.d_head - - model = mock.Mock() - - # Embeddings - model.model.embed_tokens.weight = torch.randn(cfg.d_vocab, d) - - # Single layer - layer = mock.Mock() - - # LayerNorms - layer.input_layernorm.weight = torch.randn(d) - layer.post_attention_layernorm.weight = torch.randn(d) - - # Attention - layer.self_attn.sinks = torch.randn(n_heads) - layer.self_attn.q_proj.weight = torch.randn(n_heads * d_head, d) - layer.self_attn.k_proj.weight = torch.randn(n_kv * d_head, d) - layer.self_attn.v_proj.weight = torch.randn(n_kv * d_head, d) - layer.self_attn.o_proj.weight = torch.randn(d, n_heads * d_head) - layer.self_attn.q_proj.bias = torch.randn(n_heads * d_head) - layer.self_attn.k_proj.bias = torch.randn(n_kv * d_head) - layer.self_attn.v_proj.bias = torch.randn(n_kv * d_head) - layer.self_attn.o_proj.bias = torch.randn(d) - - # Router - layer.mlp.router.weight = torch.randn(n_experts, d) - layer.mlp.router.bias = torch.randn(n_experts) - - # Experts — interleaved gate_up_proj: (num_experts, d_model, 2*d_mlp) - layer.mlp.experts.gate_up_proj = torch.randn(n_experts, d, 2 * d_mlp) - layer.mlp.experts.gate_up_proj_bias = torch.randn(n_experts, 2 * d_mlp) - layer.mlp.experts.down_proj = torch.randn(n_experts, d_mlp, d) - layer.mlp.experts.down_proj_bias = torch.randn(n_experts, d) - - model.model.layers = [layer] - - # Final norm and lm_head - model.model.norm.weight = torch.randn(d) - model.lm_head.weight = torch.randn(cfg.d_vocab, d) - - return model - - -def test_interleaved_weight_split(): - """Even columns of gate_up_proj go to W_gate, odd columns go to W_in.""" - cfg = make_cfg() - model = make_mock_model(cfg) - - state_dict = convert_gpt_oss_weights(model, cfg) - - gate_up = model.model.layers[0].mlp.experts.gate_up_proj # (n_experts, d, 2*d_mlp) - gate_up_bias = model.model.layers[0].mlp.experts.gate_up_proj_bias # (n_experts, 2*d_mlp) - - for e in range(cfg.num_experts): - # W_gate gets even columns (::2), transposed for nn.Linear format - expected_gate_w = gate_up[e, :, ::2].T.contiguous() - actual_gate_w = state_dict[f"blocks.0.mlp.experts.{e}.W_gate.weight"] - assert torch.equal(actual_gate_w, expected_gate_w) - - expected_gate_b = gate_up_bias[e, ::2].contiguous() - actual_gate_b = state_dict[f"blocks.0.mlp.experts.{e}.W_gate.bias"] - assert torch.equal(actual_gate_b, expected_gate_b) - - # W_in gets odd columns (1::2), transposed for nn.Linear format - expected_in_w = gate_up[e, :, 1::2].T.contiguous() - actual_in_w = state_dict[f"blocks.0.mlp.experts.{e}.W_in.weight"] - assert torch.equal(actual_in_w, expected_in_w) - - expected_in_b = gate_up_bias[e, 1::2].contiguous() - actual_in_b = state_dict[f"blocks.0.mlp.experts.{e}.W_in.bias"] - assert torch.equal(actual_in_b, expected_in_b) - - -def test_down_proj_transposed(): - """down_proj is transposed for nn.Linear format.""" - cfg = make_cfg() - model = make_mock_model(cfg) - - state_dict = convert_gpt_oss_weights(model, cfg) - - down = model.model.layers[0].mlp.experts.down_proj # (n_experts, d_mlp, d) - - for e in range(cfg.num_experts): - expected = down[e].T.contiguous() - actual = state_dict[f"blocks.0.mlp.experts.{e}.W_out.weight"] - assert torch.equal(actual, expected) - - -def test_router_weight_and_bias(): - """Router weight and bias are correctly mapped.""" - cfg = make_cfg() - model = make_mock_model(cfg) - - state_dict = convert_gpt_oss_weights(model, cfg) - - assert torch.equal( - state_dict["blocks.0.mlp.W_gate.weight"], - model.model.layers[0].mlp.router.weight, - ) - assert torch.equal( - state_dict["blocks.0.mlp.W_gate.bias"], - model.model.layers[0].mlp.router.bias, - ) - - -def test_attention_weight_shapes(): - """Attention weights are correctly reshaped.""" - cfg = make_cfg() - model = make_mock_model(cfg) - - state_dict = convert_gpt_oss_weights(model, cfg) - - assert state_dict["blocks.0.attn.W_Q"].shape == (cfg.n_heads, cfg.d_model, cfg.d_head) - assert state_dict["blocks.0.attn._W_K"].shape == ( - cfg.n_key_value_heads, - cfg.d_model, - cfg.d_head, - ) - assert state_dict["blocks.0.attn._W_V"].shape == ( - cfg.n_key_value_heads, - cfg.d_model, - cfg.d_head, - ) - assert state_dict["blocks.0.attn.W_O"].shape == (cfg.n_heads, cfg.d_head, cfg.d_model) - - -def test_state_dict_completeness(): - """All expected keys are present in the converted state dict.""" - cfg = make_cfg() - model = make_mock_model(cfg) - - state_dict = convert_gpt_oss_weights(model, cfg) - - # Check essential keys exist - assert "embed.W_E" in state_dict - assert "ln_final.w" in state_dict - assert "unembed.W_U" in state_dict - assert "unembed.b_U" in state_dict - assert "blocks.0.ln1.w" in state_dict - assert "blocks.0.ln2.w" in state_dict - assert "blocks.0.mlp.W_gate.weight" in state_dict - assert "blocks.0.mlp.W_gate.bias" in state_dict - - for e in range(cfg.num_experts): - for name in [ - "W_gate.weight", - "W_gate.bias", - "W_in.weight", - "W_in.bias", - "W_out.weight", - "W_out.bias", - ]: - assert f"blocks.0.mlp.experts.{e}.{name}" in state_dict - - -class Tensor: - """Mimics triton_kernels.tensor.Tensor: named "Tensor", not a torch.Tensor, - not subscriptable — what packed-MXFP4 gpt-oss checkpoints expose (#1619).""" - - def __init__(self, shape): - self.shape = shape - self.dtype = "mxfp4" - self.data = object() - self.storage = object() - - -def test_packed_mxfp4_experts_raise_legible_error(): - """Packed MXFP4 expert weights must raise naming MXFP4, not a bare TypeError.""" - cfg = make_cfg() - model = make_mock_model(cfg) - model.model.layers[0].mlp.experts.gate_up_proj = Tensor( - (cfg.num_experts, cfg.d_model, 2 * cfg.d_mlp) - ) - - with pytest.raises(NotImplementedError, match="MXFP4"): - convert_gpt_oss_weights(model, cfg) - - -@pytest.mark.parametrize( - "dtype,reason", - [ - (torch.int8, "packed integer storage"), - (torch.uint8, "packed integer storage"), - # The one that a plain isinstance/is_floating_point check admits: FP8 - # reports is_floating_point=True and slices without complaint, so this - # converter used to emit scale-less garbage for it silently. - (torch.float8_e4m3fn, "narrow float"), - (torch.float8_e5m2, "narrow float"), - ], -) -@pytest.mark.parametrize("tensor_name", ["gate_up_proj", "down_proj"]) -def test_quantized_expert_weights_are_refused(dtype, reason, tensor_name): - """Both fused expert tensors are sliced, so both must be guarded — a packed - down_proj would silently drop its scales just as gate_up_proj would.""" - cfg = make_cfg() - model = make_mock_model(cfg) - experts = model.model.layers[0].mlp.experts - setattr(experts, tensor_name, getattr(experts, tensor_name).to(dtype)) - - with pytest.raises(NotImplementedError, match=reason) as excinfo: - convert_gpt_oss_weights(model, cfg) - # The gpt-oss remedy must survive the move onto the shared helper. - assert "Mxfp4Config(dequantize=True)" in str(excinfo.value) - - -def test_sinks_are_converted(): - """The learned attention-sink logits must reach the state dict (#1619 follow-up).""" - cfg = make_cfg() - model = make_mock_model(cfg) - - state_dict = convert_gpt_oss_weights(model, cfg) - - assert torch.equal(state_dict["blocks.0.attn.sinks"], model.model.layers[0].self_attn.sinks) - - -class TestGptOssHookedTransformerParity: - """End-to-end HT-vs-HF logit parity on a tiny random GPT-OSS. - - Loading used to crash on MXFP4 (#1619); once past that, HookedTransformer - silently disagreed with HF because its attention lacked sinks, sliding-window - layers, and yarn RoPE with truncate=False. This locks in all three via the - production config-mapping path (convert_hf_model_config on a local dir). - """ - - def test_tiny_model_logit_parity(self, tmp_path): - from transformers import AutoModelForCausalLM, GptOssConfig, GptOssForCausalLM - - from transformer_lens import HookedTransformer - from transformer_lens.config.hooked_transformer_config import ( - HookedTransformerConfig, - ) - from transformer_lens.loading_from_pretrained import convert_hf_model_config - - hf_config = GptOssConfig( - hidden_size=64, - num_hidden_layers=2, # one sliding_attention + one full_attention layer - num_attention_heads=4, - num_key_value_heads=2, - head_dim=16, - intermediate_size=32, - num_local_experts=4, - num_experts_per_tok=2, - vocab_size=256, - max_position_embeddings=128, - sliding_window=4, # < seq len so the sliding-window mask actually bites - rope_parameters={ - "rope_type": "yarn", - "rope_theta": 150000.0, - "factor": 4.0, - "original_max_position_embeddings": 32, - "beta_fast": 32.0, - "beta_slow": 1.0, - "truncate": False, - }, - ) - torch.manual_seed(0) - hf_model = GptOssForCausalLM(hf_config).eval().float() - hf_model.save_pretrained(tmp_path) - - ids = torch.tensor([[5, 17, 29, 3, 11, 42, 7, 23]]) - reference = AutoModelForCausalLM.from_pretrained( - str(tmp_path), dtype=torch.float32, attn_implementation="eager" - ).eval() - with torch.inference_mode(): - ref_logits = reference(ids).logits - - cfg_dict = convert_hf_model_config(str(tmp_path)) - assert cfg_dict["use_attention_sinks"] is True - assert cfg_dict["attn_types"] == ["local", "global"] - assert cfg_dict["use_yarn_rope"] is True - assert cfg_dict["yarn_truncate"] is False - cfg_dict["dtype"] = torch.float32 - cfg_dict["tokenizer_name"] = None # keep the test download-free - - cfg = HookedTransformerConfig.from_dict(cfg_dict) - model = HookedTransformer(cfg) - model.load_and_process_state_dict( - convert_gpt_oss_weights(hf_model, cfg), - fold_ln=False, - center_writing_weights=False, - center_unembed=False, - fold_value_biases=False, - ) - with torch.inference_mode(): - ht_logits = model(ids, return_type="logits") - - max_diff = (ref_logits - ht_logits).abs().max().item() - assert max_diff < 1e-4, f"HT vs HF logit drift {max_diff:.2e}" diff --git a/tests/unit/test_deprecation_warnings.py b/tests/unit/test_deprecation_warnings.py index b3219756ca..236d25497c 100644 --- a/tests/unit/test_deprecation_warnings.py +++ b/tests/unit/test_deprecation_warnings.py @@ -1,4 +1,9 @@ -"""Regression coverage for deprecated legacy entry points.""" +"""Import-time warning hygiene after the legacy Hooked* stack was removed. + +The per-class deprecation-warning tests are gone with the classes they covered +(4.0 deletion). What survives: importing the package must stay warning-clean, +and the KEPT HookedRootModule must NOT warn (it was never part of the removal). +""" from __future__ import annotations @@ -6,34 +11,9 @@ import sys from pathlib import Path -import pytest - PROJECT_ROOT = Path(__file__).parents[2] -def _small_config(): - from transformer_lens import HookedTransformerConfig - - return HookedTransformerConfig( - n_layers=1, - d_model=16, - d_head=4, - n_heads=4, - n_ctx=8, - d_vocab=20, - attn_only=True, - ) - - -def _assert_single_deprecation(constructor, class_name: str) -> None: - with pytest.warns(DeprecationWarning, match=class_name) as caught: - constructor() - - assert len(caught) == 1 - assert "TransformerBridge.boot_transformers" in str(caught[0].message) - assert "4.0" in str(caught[0].message) - - def test_importing_transformer_lens_emits_no_deprecation_warning(): code = "\n".join( [ @@ -57,58 +37,10 @@ def test_importing_transformer_lens_emits_no_deprecation_warning(): assert result.returncode == 0, result.stderr -def test_hooked_transformer_constructor_warns_once(): - from transformer_lens import HookedTransformer - - _assert_single_deprecation(lambda: HookedTransformer(_small_config()), "HookedTransformer") - - -def test_hooked_encoder_constructor_warns_once(): - from transformer_lens import HookedEncoder - - _assert_single_deprecation(lambda: HookedEncoder(_small_config()), "HookedEncoder") - - -def test_hooked_encoder_from_pretrained_warning_reaches_external_caller(): - code = "\n".join( - [ - "import importlib", - "hooked_encoder_module = importlib.import_module('transformer_lens.HookedEncoder')", - "HookedEncoder = hooked_encoder_module.HookedEncoder", - "def stop_loading(*args, **kwargs):", - " raise RuntimeError('stop after deprecation warning')", - "hooked_encoder_module.loading.get_official_model_name = stop_loading", - "try:", - " HookedEncoder.from_pretrained('bert-base-cased')", - "except RuntimeError as error:", - " assert str(error) == 'stop after deprecation warning'", - ] - ) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - cwd=PROJECT_ROOT, - text=True, - check=False, - ) - - assert result.returncode == 0, result.stderr - assert "<string>:" in result.stderr - assert "DeprecationWarning: HookedEncoder.from_pretrained is deprecated" in result.stderr - - -def test_bert_next_sentence_prediction_constructor_warns_once(): - from transformer_lens import BertNextSentencePrediction - - _assert_single_deprecation( - lambda: BertNextSentencePrediction(object()), "BertNextSentencePrediction" - ) - - def test_hooked_root_module_is_not_deprecated(): """HookedRootModule (with HookPoint) is KEPT infrastructure — the supported way to hook arbitrary nn.Modules — and must construct without any - DeprecationWarning, unlike the legacy model classes.""" + DeprecationWarning.""" import warnings as w from transformer_lens import HookedRootModule @@ -117,46 +49,3 @@ def test_hooked_root_module_is_not_deprecated(): w.simplefilter("always") HookedRootModule() assert not [x for x in caught if issubclass(x.category, DeprecationWarning)] - - -def test_hooked_encoder_decoder_constructor_warns_once(): - from transformer_lens import HookedEncoderDecoder, HookedTransformerConfig - - cfg = HookedTransformerConfig( - n_layers=1, - d_model=16, - d_head=4, - n_heads=4, - n_ctx=8, - d_vocab=20, - d_mlp=32, - act_fn="relu", - attention_dir="bidirectional", - tie_word_embeddings=False, - positional_embedding_type="relative_positional_bias", - relative_attention_num_buckets=4, - relative_attention_max_distance=8, - ) - _assert_single_deprecation(lambda: HookedEncoderDecoder(cfg), "HookedEncoderDecoder") - - -def test_hooked_audio_encoder_constructor_warns_once(): - from transformer_lens import HookedAudioEncoder - - cfg = _small_config() - _assert_single_deprecation(lambda: HookedAudioEncoder(cfg), "HookedAudioEncoder") - - -def test_from_pretrained_warns_exactly_once(): - """__init__ is suppressed under from_pretrained — one warning per entry point, - attributed to the caller, not two.""" - import warnings as w - - from transformer_lens import HookedTransformer - - with w.catch_warnings(record=True) as caught: - w.simplefilter("always") - HookedTransformer.from_pretrained("gpt2") - dep = [x for x in caught if issubclass(x.category, DeprecationWarning)] - assert len(dep) == 1, [str(d.message)[:60] for d in dep] - assert "from_pretrained" in str(dep[0].message) diff --git a/tests/unit/test_doctest_no_hooked_transformer.py b/tests/unit/test_doctest_no_hooked_transformer.py index 6cf200f816..ff89ea801c 100644 --- a/tests/unit/test_doctest_no_hooked_transformer.py +++ b/tests/unit/test_doctest_no_hooked_transformer.py @@ -1,13 +1,9 @@ -"""Guard: surviving modules' doctests must not construct HookedTransformer. +"""Guard: no module's doctests may construct HookedTransformer. -The v4 removal deletes the Hooked* implementation files; any OTHER module whose -docstring examples build a HookedTransformer would turn the docstring test tier -red the moment those files go. This guard makes the removal safe by -construction: it fails the instant such an example is (re)introduced. - -The Hooked* implementation files themselves (and the legacy train module) are -exempt — their docstrings legitimately describe the class they implement, and -they are deleted wholesale by the removal PR. +HookedTransformer was removed in 4.0. No surviving module's docstring examples +may build it — doing so would fail the docstring test tier — so this guard +fails the instant such an example is (re)introduced. With the class gone there +are no exempt files left; every ``.py`` under the package is checked. """ import re @@ -15,16 +11,9 @@ PACKAGE_ROOT = Path(__file__).parents[2] / "transformer_lens" -# Deleted wholesale by the Hooked* removal — their own docstrings are exempt. -EXEMPT = { - "HookedTransformer.py", - "HookedRootModule.py", - "HookedEncoder.py", - "HookedEncoderDecoder.py", - "HookedAudioEncoder.py", - "train.py", -} -EXEMPT_DIRS = {"components", "factories"} +# The Hooked* implementation files are deleted, so nothing is exempt anymore. +EXEMPT: set[str] = set() +EXEMPT_DIRS: set[str] = set() # A doctest example line (>>> or ... continuation) that references the class at # all — construction, import, or isinstance: any of them turns the docstring diff --git a/tests/unit/test_dropped_inference_ops.py b/tests/unit/test_dropped_inference_ops.py deleted file mode 100644 index 5899fe9419..0000000000 --- a/tests/unit/test_dropped_inference_ops.py +++ /dev/null @@ -1,347 +0,0 @@ -"""Inference-time ops HookedTransformer must not silently drop. - -HT re-implements each architecture's forward from converted weights, so an op HF -applies but TL's config never represents diverges silently — no error, just wrong -numbers. Each test here pins one such op found by sweeping the HF modeling -sources against TL's flags. -""" - -from types import SimpleNamespace -from unittest import mock - -import torch - -from transformer_lens.loading_from_pretrained import convert_hf_model_config - - -def _llama_shaped_config(**extra): - return SimpleNamespace( - architectures=["LlamaForCausalLM"], - hidden_size=128, - num_attention_heads=4, - num_key_value_heads=4, - intermediate_size=256, - num_hidden_layers=2, - max_position_embeddings=4096, - rms_norm_eps=1e-5, - vocab_size=32000, - hidden_act="silu", - **extra, - ) - - -class TestGenericLlamaRotaryBase: - """Llama-arch checkpoints with no name-matched branch (Yi ships rope_theta - 5e6) fell through to the config default 10000 — every rotary angle wrong.""" - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_rope_theta_is_read(self, mock_auto_config: mock.MagicMock) -> None: - mock_auto_config.from_pretrained.return_value = _llama_shaped_config(rope_theta=5000000.0) - assert convert_hf_model_config("01-ai/Yi-6B")["rotary_base"] == 5000000.0 - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_rope_theta_from_transformers_5_dict(self, mock_auto_config: mock.MagicMock) -> None: - mock_auto_config.from_pretrained.return_value = _llama_shaped_config( - rope_parameters={"rope_type": "default", "rope_theta": 5000000.0} - ) - assert convert_hf_model_config("01-ai/Yi-6B")["rotary_base"] == 5000000.0 - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_llama3_scaling_is_threaded(self, mock_auto_config: mock.MagicMock) -> None: - mock_auto_config.from_pretrained.return_value = _llama_shaped_config( - rope_theta=500000.0, - rope_scaling={ - "rope_type": "llama3", - "factor": 8.0, - "low_freq_factor": 1.0, - "high_freq_factor": 4.0, - "original_max_position_embeddings": 8192, - }, - ) - cfg = convert_hf_model_config("01-ai/Yi-34B") - assert cfg["use_NTK_by_parts_rope"] is True - assert cfg["NTK_by_parts_factor"] == 8.0 - assert cfg["NTK_original_ctx_len"] == 8192 - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_default_theta_when_config_is_silent(self, mock_auto_config: mock.MagicMock) -> None: - mock_auto_config.from_pretrained.return_value = _llama_shaped_config() - assert convert_hf_model_config("01-ai/Yi-6B-Chat")["rotary_base"] == 10000.0 - - -class TestGemma2SlidingWindowPhase: - """HF makes layer 0 sliding; TL alternated the other way, windowing exactly - the layers HF leaves global past 4096 tokens.""" - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_even_layers_are_local(self, mock_auto_config: mock.MagicMock) -> None: - mock_auto_config.from_pretrained.return_value = SimpleNamespace( - architectures=["Gemma2ForCausalLM"] - ) - for name, n_layers in [ - ("google/gemma-2-2b", 26), - ("google/gemma-2-9b", 42), - ("google/gemma-2-27b", 46), - ]: - cfg = convert_hf_model_config(name) - types = cfg["attn_types"] - assert len(types) == n_layers == cfg["n_layers"], name - assert types[0] == "local", name - assert types[1] == "global", name - # Windowing only binds because n_ctx outruns the window. - assert cfg["n_ctx"] > cfg["window_size"], name - - def test_hf_derivation_still_puts_sliding_on_layer_zero(self) -> None: - """Pin the upstream rule this fix mirrors, so a phase flip in - transformers is caught here rather than by drifting logits.""" - from transformers.models.gemma2.configuration_gemma2 import Gemma2Config - - derived = Gemma2Config(num_hidden_layers=4).layer_types - assert derived[:2] == ["sliding_attention", "full_attention"], derived - - -class TestApertusQKNorm: - """HF applies q_norm/k_norm unconditionally; TL gated them on a config field - that does not exist, so they were never built or converted.""" - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_qk_norm_is_on(self, mock_auto_config: mock.MagicMock) -> None: - mock_auto_config.from_pretrained.return_value = SimpleNamespace( - architectures=["ApertusForCausalLM"], - hidden_size=128, - num_attention_heads=4, - num_key_value_heads=4, - intermediate_size=256, - num_hidden_layers=2, - max_position_embeddings=4096, - rms_norm_eps=1e-5, - vocab_size=32000, - hidden_act="xielu", - rope_theta=12000000.0, - ) - assert convert_hf_model_config("swiss-ai/Apertus-8B-2509")["use_qk_norm"] is True - - def test_hf_config_still_has_no_gate(self) -> None: - """The fix hardcodes True because HF has nothing to read; if a gate ever - appears, this fails and the branch should read it instead.""" - from transformers.models.apertus.configuration_apertus import ApertusConfig - - assert not hasattr(ApertusConfig(), "qk_norm") - - -class TestPhi3SlidingWindow: - """Phi-3-mini-4k ships sliding_window=2047 inside a 4096 n_ctx; TL never read - the field, leaving attention full-causal past 2047 tokens.""" - - @staticmethod - def _phi3_config(**extra): - return SimpleNamespace( - architectures=["Phi3ForCausalLM"], - hidden_size=128, - num_attention_heads=4, - num_key_value_heads=4, - intermediate_size=256, - num_hidden_layers=3, - max_position_embeddings=4096, - rms_norm_eps=1e-5, - vocab_size=32064, - hidden_act="silu", - initializer_range=0.02, - rope_theta=10000.0, - **extra, - ) - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_window_is_populated(self, mock_auto_config: mock.MagicMock) -> None: - mock_auto_config.from_pretrained.return_value = self._phi3_config(sliding_window=2047) - cfg = convert_hf_model_config("microsoft/Phi-3-mini-4k-instruct") - assert cfg["use_local_attn"] is True - assert cfg["window_size"] == 2047 - assert cfg["attn_types"] == ["local"] * 3 - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_no_window_stays_global(self, mock_auto_config: mock.MagicMock) -> None: - mock_auto_config.from_pretrained.return_value = self._phi3_config(sliding_window=None) - cfg = convert_hf_model_config("microsoft/Phi-3-mini-4k-instruct") - assert cfg["use_local_attn"] is False - assert cfg["attn_types"] is None - - -class TestMixtralAndGPT2ConfigReads: - """Two fields the branches hardcoded instead of reading.""" - - @staticmethod - def _mixtral_config(**extra): - return SimpleNamespace( - architectures=["MixtralForCausalLM"], - hidden_size=128, - num_attention_heads=4, - num_key_value_heads=2, - intermediate_size=256, - num_hidden_layers=3, - max_position_embeddings=4096, - rms_norm_eps=1e-5, - vocab_size=32000, - hidden_act="silu", - rope_theta=1000000.0, - num_local_experts=8, - num_experts_per_tok=2, - **extra, - ) - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_mixtral_window_follows_the_config(self, mock_auto_config: mock.MagicMock) -> None: - mock_auto_config.from_pretrained.return_value = self._mixtral_config(sliding_window=4096) - cfg = convert_hf_model_config("mistralai/Mixtral-8x7B-Instruct-v0.1") - assert cfg["use_local_attn"] is True - assert cfg["attn_types"] == ["local"] * 3 - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_mixtral_attn_types_length_tracks_n_layers( - self, mock_auto_config: mock.MagicMock - ) -> None: - """The old ["global"] * 32 was inert only because use_local_attn was - hardcoded False; reading the config makes its length load-bearing.""" - mock_auto_config.from_pretrained.return_value = self._mixtral_config(sliding_window=None) - cfg = convert_hf_model_config("mistralai/Mixtral-8x7B-v0.1") - assert cfg["use_local_attn"] is False - assert len(cfg["attn_types"]) == cfg["n_layers"] == 3 - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_gpt2_reads_scale_attn_weights(self, mock_auto_config: mock.MagicMock) -> None: - mock_auto_config.from_pretrained.return_value = SimpleNamespace( - architectures=["GPT2LMHeadModel"], - n_embd=128, - n_head=4, - n_layer=2, - n_ctx=1024, - layer_norm_epsilon=1e-5, - vocab_size=50257, - activation_function="gelu_new", - scale_attn_by_inverse_layer_idx=False, - scale_attn_weights=False, - ) - assert convert_hf_model_config("gpt2")["use_attn_scale"] is False - - -class TestT5DecoderRelativeBias: - """HF buckets decoder self-attention with bidirectional=False; TL hardcoded - True, so every decoder forward used the encoder's scheme.""" - - @staticmethod - def _cfg(): - from transformer_lens import HookedTransformerConfig - - return HookedTransformerConfig.from_dict( - { - "d_model": 32, - "d_head": 8, - "n_heads": 4, - "d_mlp": 64, - "n_layers": 2, - "n_ctx": 16, - "d_vocab": 100, - "act_fn": "relu", - "normalization_type": "RMS", - "positional_embedding_type": "relative_positional_bias", - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_attn_scale": False, - # What the real T5 branch sets — one cfg shared by both stacks. - "attention_dir": "bidirectional", - } - ) - - def _tl_buckets(self, is_decoder: bool, seq: int = 8) -> torch.Tensor: - from transformer_lens.components.t5_attention import T5Attention - - attn = T5Attention(self._cfg(), has_relative_attention_bias=True, is_decoder=is_decoder) - context = torch.arange(seq)[:, None] - memory = torch.arange(seq)[None, :] - return T5Attention._relative_position_bucket( - memory - context, - bidirectional=not attn.is_decoder, - num_buckets=32, - max_distance=128, - ) - - def _hf_buckets(self, is_decoder: bool, seq: int = 8) -> torch.Tensor: - from transformers.models.t5.modeling_t5 import T5Attention as HFT5Attention - - context = torch.arange(seq)[:, None] - memory = torch.arange(seq)[None, :] - return HFT5Attention._relative_position_bucket( - memory - context, - bidirectional=not is_decoder, - num_buckets=32, - max_distance=128, - ) - - def test_decoder_buckets_match_hf(self) -> None: - torch.testing.assert_close(self._tl_buckets(True), self._hf_buckets(True)) - - def test_encoder_buckets_match_hf(self) -> None: - torch.testing.assert_close(self._tl_buckets(False), self._hf_buckets(False)) - - def test_the_two_schemes_actually_differ(self) -> None: - """Engagement check: if decoder and encoder bucketing coincided, the two - tests above would pass no matter which flag the component used.""" - assert not torch.equal(self._tl_buckets(True), self._tl_buckets(False)) - - def test_decoder_blocks_build_decoder_attention(self) -> None: - from transformer_lens.components.t5_block import T5Block - - assert T5Block(self._cfg(), 0, is_decoder=True).attn.is_decoder is True - assert T5Block(self._cfg(), 0, is_decoder=False).attn.is_decoder is False - - def _pattern(self, is_decoder: bool, seq: int = 6) -> torch.Tensor: - from transformer_lens.components.t5_block import T5Block - - attn = T5Block(self._cfg(), 0, is_decoder=is_decoder).attn - with torch.random.fork_rng(): - torch.manual_seed(0) - for p in attn.parameters(): - torch.nn.init.normal_(p, std=0.2) - resid = torch.randn(1, seq, 32) - seen: dict = {} - attn.hook_pattern.add_hook(lambda t, hook: seen.__setitem__("p", t)) - bias = attn.compute_relative_attention_bias(seq, seq) - with torch.no_grad(): - attn(resid, resid, resid, position_bias=bias) - return seen["p"] - - def test_decoder_self_attention_masks_the_future(self) -> None: - """T5's cfg says attention_dir="bidirectional" for the encoder, and - HookedEncoderDecoder never built a causal mask — so decoder self-attention - read future tokens on every multi-token forward.""" - pattern = self._pattern(is_decoder=True) - seq = pattern.shape[-1] - future = torch.triu(torch.ones(seq, seq, dtype=torch.bool), diagonal=1) - assert pattern[..., future].abs().max() == 0.0 - - def test_encoder_self_attention_stays_bidirectional(self) -> None: - pattern = self._pattern(is_decoder=False) - seq = pattern.shape[-1] - future = torch.triu(torch.ones(seq, seq, dtype=torch.bool), diagonal=1) - assert pattern[..., future].abs().max() > 0.0 - - def test_cross_attention_is_not_masked(self) -> None: - """Only decoder SELF-attention is causal; cross-attention sees the whole - encoder sequence.""" - from transformer_lens.components.t5_block import T5Block - - assert T5Block(self._cfg(), 0, is_decoder=True).cross_attn._attention_dir_override is None - - def test_bias_values_differ_between_stacks(self) -> None: - """End to end through the component that actually feeds attn_scores.""" - from transformer_lens.components.t5_block import T5Block - - with torch.random.fork_rng(): - torch.manual_seed(0) - enc = T5Block(self._cfg(), 0, is_decoder=False).attn - dec = T5Block(self._cfg(), 0, is_decoder=True).attn - dec.rel_pos_bias.weight.data.copy_(enc.rel_pos_bias.weight.data) - assert not torch.equal( - enc.compute_relative_attention_bias(8, 8), - dec.compute_relative_attention_bias(8, 8), - ) diff --git a/tests/unit/test_goldens_infra.py b/tests/unit/test_goldens_infra.py index 585b140ad5..3e1e31e209 100644 --- a/tests/unit/test_goldens_infra.py +++ b/tests/unit/test_goldens_infra.py @@ -7,16 +7,12 @@ from __future__ import annotations import json -import sys -from pathlib import Path import pytest import torch -sys.path.insert(0, str(Path(__file__).parents[2] / "scripts")) -import capture_ht_goldens as capture # noqa: E402 - -from tests import goldens # noqa: E402 +from tests import goldens +from tests import goldens_capture_spec as capture # noqa: E402 @pytest.fixture() diff --git a/tests/unit/test_hubert.py b/tests/unit/test_hubert.py deleted file mode 100644 index f7a1869477..0000000000 --- a/tests/unit/test_hubert.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Tests for HookedAudioEncoder (HuBERT) basic functionality.""" - -import math - -import numpy as np -import pytest -import torch - -from transformer_lens import HookedAudioEncoder - -SAMPLE_RATE = 16000 -DURATION_S = 1.0 -DEVICE = "cuda" if torch.cuda.is_available() else "cpu" -HF_CHECKPOINT = "facebook/hubert-base-ls960" - - -def make_sine(frequency=440.0, sr=SAMPLE_RATE, duration=DURATION_S, amplitude=0.1): - t = np.linspace(0, duration, int(sr * duration), endpoint=False, dtype=np.float32) - return amplitude * np.sin(2 * math.pi * frequency * t) - - -@pytest.fixture(scope="module") -def audio_model(): - return HookedAudioEncoder.from_pretrained(HF_CHECKPOINT, device=DEVICE) - - -@pytest.fixture(scope="module") -def waveform(): - return make_sine(frequency=440.0, sr=SAMPLE_RATE, duration=DURATION_S) - - -def _get_output_tensor(out): - """Extract tensor from model output (handles dict or tensor).""" - if isinstance(out, torch.Tensor): - return out - try: - return out["predictions"] - except (KeyError, TypeError): - return out - - -class TestHubertForwardPass: - def test_output_is_finite(self, audio_model, waveform): - audio_model.eval() - x = torch.from_numpy(waveform).unsqueeze(0).to(DEVICE) - with torch.no_grad(): - out = audio_model(x) - out_tensor = _get_output_tensor(out) - assert torch.isfinite(out_tensor).all(), "Found NaNs or Infs in forward output" - - def test_output_shape(self, audio_model, waveform): - audio_model.eval() - x = torch.from_numpy(waveform).unsqueeze(0).to(DEVICE) - with torch.no_grad(): - out = audio_model(x) - out_tensor = _get_output_tensor(out) - assert out_tensor.ndim == 3, f"Expected 3D output, got {out_tensor.ndim}D" - assert out_tensor.shape[0] == 1, f"Expected batch=1, got {out_tensor.shape[0]}" - - def test_deterministic_eval(self, audio_model, waveform): - audio_model.eval() - x = torch.from_numpy(waveform).unsqueeze(0).to(DEVICE) - with torch.no_grad(): - out1 = _get_output_tensor(audio_model(x)) - out2 = _get_output_tensor(audio_model(x)) - assert torch.allclose( - out1, out2, atol=1e-6 - ), f"Outputs differ between eval runs, max diff: {(out1 - out2).abs().max().item()}" - - def test_gradient_flow(self, audio_model, waveform): - audio_model.train() - for p in audio_model.parameters(): - if p.grad is not None: - p.grad.detach_() - p.grad.zero_() - x = torch.from_numpy(waveform).unsqueeze(0).to(DEVICE) - out = _get_output_tensor(audio_model(x)) - loss = out.mean() - loss.backward() - grads_found = any( - p.grad is not None and torch.isfinite(p.grad).all() - for p in audio_model.parameters() - if p.requires_grad - ) - assert grads_found, "No finite gradients found after backward()" - - -class TestHubertHFComparison: - def test_cosine_similarity_to_hf(self, audio_model, waveform): - try: - from transformers import HubertModel, Wav2Vec2FeatureExtractor - except ImportError: - pytest.skip("transformers HubertModel not available") - - hf_feat = Wav2Vec2FeatureExtractor(sampling_rate=SAMPLE_RATE, do_normalize=True) - hf_model = HubertModel.from_pretrained(HF_CHECKPOINT).to(DEVICE).eval() - - input_values = hf_feat(waveform, sampling_rate=SAMPLE_RATE, return_tensors="pt").get( - "input_values" - ) - input_values = input_values.to(DEVICE) - - with torch.no_grad(): - hf_out = hf_model(input_values).last_hidden_state.mean(dim=1) - - audio_model.eval() - our_out = _get_output_tensor( - audio_model(torch.from_numpy(waveform).unsqueeze(0).to(DEVICE)) - ) - if our_out.ndim == 3: - our_out = our_out.mean(dim=1) - - if hf_out.shape[1] != our_out.shape[1]: - pytest.skip(f"Dimension mismatch (HF {hf_out.shape[1]} vs ours {our_out.shape[1]})") - - cos = torch.nn.functional.cosine_similarity(hf_out, our_out, dim=1) - assert cos.item() > 0.99, f"Cosine similarity too low: {cos.item()}" - - -class TestEncoderOutputPaddingMask: - def test_masked_frame_entry_matches_hf_encoder(self, audio_model): - """encoder_output under a padding mask must match HF on the real frames. - - HF zeroes pad frames before pos_conv_embed; without that, the kernel-128 - conv smears pad content into real frames (~23% relative error on this - input). Frames come from an unmasked feature pass — HF mutates - hidden_states in place, so masked-run frames are already pre-zeroed and - would make this comparison self-fulfilling. - """ - from transformers import HubertModel - - hf = HubertModel.from_pretrained(HF_CHECKPOINT).to(DEVICE).eval() - - wave = torch.from_numpy(make_sine()).unsqueeze(0) - padded = torch.cat([wave, torch.zeros(1, 4000)], dim=1).to(DEVICE) - sample_mask = ( - torch.cat([torch.ones(1, wave.shape[1]), torch.zeros(1, 4000)], dim=1).long().to(DEVICE) - ) - - with torch.no_grad(): - ref = hf(padded, attention_mask=sample_mask).last_hidden_state - feats = hf.feature_extractor(padded).transpose(1, 2) - frames = hf.feature_projection(feats) - frame_mask = hf._get_feature_vector_attention_mask(frames.shape[1], sample_mask) - out = audio_model.encoder_output(frames, one_zero_attention_mask=frame_mask.long()) - - real = frame_mask[0].bool() - assert not real.all(), "padding produced no masked frames; test setup is broken" - torch.testing.assert_close(out[:, real], ref[:, real], atol=1e-4, rtol=1e-4) - - def test_caller_frames_survive(self, audio_model): - """masked_fill, not HF's in-place write: the caller's tensor is untouched.""" - frames = torch.randn(1, 8, audio_model.cfg.d_model, device=DEVICE) - keep = frames.clone() - mask = torch.tensor([[1, 1, 1, 1, 0, 0, 0, 0]], device=DEVICE) - with torch.no_grad(): - audio_model.encoder_output(frames, one_zero_attention_mask=mask) - assert torch.equal(frames, keep) diff --git a/tests/unit/test_hubert_hooks.py b/tests/unit/test_hubert_hooks.py deleted file mode 100644 index 6cdf6303c0..0000000000 --- a/tests/unit/test_hubert_hooks.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Tests for HookedAudioEncoder (HuBERT) hook and ablation functionality.""" - -import math - -import numpy as np -import pytest -import torch - -from transformer_lens import HookedAudioEncoder -from transformer_lens import utilities as utils - -SAMPLE_RATE = 16000 -DURATION_S = 1.0 -DEVICE = "cuda" if torch.cuda.is_available() else "cpu" - - -def _get_output_tensor(out): - """Extract tensor from model output (handles dict or tensor).""" - if isinstance(out, torch.Tensor): - return out - try: - return out["predictions"] - except (KeyError, TypeError): - return out - - -def make_sine(frequency=440.0, sr=SAMPLE_RATE, duration=DURATION_S, amplitude=0.1): - t = np.linspace(0, duration, int(sr * duration), endpoint=False, dtype=np.float32) - return amplitude * np.sin(2 * math.pi * frequency * t) - - -@pytest.fixture(scope="module") -def audio_model(): - return HookedAudioEncoder.from_pretrained("facebook/hubert-base-ls960", device=DEVICE) - - -@pytest.fixture(scope="module") -def frames_and_mask(audio_model): - wav = make_sine() - frames, frame_mask = audio_model.to_frames( - [wav], sampling_rate=SAMPLE_RATE, move_to_device=True - ) - return frames, frame_mask - - -class TestHubertRunWithCache: - def test_cache_contains_attention_pattern(self, audio_model, frames_and_mask): - frames, frame_mask = frames_and_mask - _, cache = audio_model.run_with_cache( - frames, one_zero_attention_mask=frame_mask, remove_batch_dim=True - ) - layer = 0 - pattern_name = utils.get_act_name("pattern", layer) - if pattern_name in cache: - attn = cache[pattern_name] - elif ("pattern", layer, "attn") in cache: - attn = cache["pattern", layer, "attn"] - else: - pytest.fail(f"Attention pattern not found in cache. Keys: {list(cache.keys())[:10]}") - - def test_cache_attention_pattern_shape(self, audio_model, frames_and_mask): - frames, frame_mask = frames_and_mask - _, cache = audio_model.run_with_cache( - frames, one_zero_attention_mask=frame_mask, remove_batch_dim=True - ) - pattern_name = utils.get_act_name("pattern", 0) - if pattern_name in cache: - attn = cache[pattern_name] - elif ("pattern", 0, "attn") in cache: - attn = cache["pattern", 0, "attn"] - else: - pytest.fail(f"Attention pattern not found in cache. Keys: {list(cache.keys())[:10]}") - # Should be (n_heads, seq, seq) or (seq, n_heads, seq) - assert attn.ndim == 3, f"Expected 3D attention pattern, got {attn.ndim}D" - - -class TestHubertHeadAblation: - def test_ablation_changes_output(self, audio_model, frames_and_mask): - frames, frame_mask = frames_and_mask - head_to_ablate = 0 - layer_to_ablate = 0 - v_act_name = utils.get_act_name("v", layer_to_ablate) - - def head_ablation_hook(value, hook): - v = value.clone() - if v.ndim == 4: - v[:, :, head_to_ablate, :] = 0.0 - elif v.ndim == 3: - v[:, head_to_ablate, :] = 0.0 - return v - - # Baseline - baseline_out = audio_model.run_with_hooks( - frames, fwd_hooks=[], one_zero_attention_mask=frame_mask - ) - baseline_tensor = _get_output_tensor(baseline_out) - - # Ablated - ablated_out = audio_model.run_with_hooks( - frames, - fwd_hooks=[(v_act_name, head_ablation_hook)], - one_zero_attention_mask=frame_mask, - ) - ablated_tensor = _get_output_tensor(ablated_out) - - # Outputs should differ after ablation - assert not torch.allclose( - baseline_tensor, ablated_tensor, atol=1e-6 - ), "Ablating a head had no effect on the output" diff --git a/tests/unit/test_loading_from_pretrained_utilities.py b/tests/unit/test_loading_from_pretrained_utilities.py deleted file mode 100644 index 66a5364ca6..0000000000 --- a/tests/unit/test_loading_from_pretrained_utilities.py +++ /dev/null @@ -1,525 +0,0 @@ -from types import SimpleNamespace -from unittest import mock - -import pytest -import torch - -from transformer_lens import HookedTransformer -from transformer_lens.config import HookedTransformerConfig -from transformer_lens.loading_from_pretrained import ( - _mxfp4_dequantize_config, - fill_missing_keys, - get_pretrained_model_config, -) - - -def get_default_config(): - return HookedTransformerConfig( - d_model=128, d_head=8, n_heads=16, n_ctx=128, n_layers=1, d_vocab=50257, attn_only=True - ) - - -def _config_with_architecture( - architecture: str, quantization_method: str | None = None -) -> HookedTransformerConfig: - return HookedTransformerConfig( - d_model=128, - d_head=8, - n_heads=16, - n_ctx=128, - n_layers=1, - d_vocab=50257, - attn_only=True, - original_architecture=architecture, - quantization_method=quantization_method, - ) - - -class TestMxfp4DequantizeConfig: - """Packed-MXFP4 checkpoints must load dequantized so the weight converter - sees plain torch.Tensors instead of triton-kernels wrappers (#1619).""" - - def test_mxfp4_gets_dequantize_config(self): - result = _mxfp4_dequantize_config(_config_with_architecture("GptOssForCausalLM", "mxfp4")) - assert result is not None - assert result.dequantize is True - - def test_applies_beyond_gpt_oss(self): - """The architecture gate this used to carry existed only to dodge a - second AutoConfig fetch. MXFP4 Qwen3-MoE checkpoints are in the registry - and pack their experts the same way, so they must dequantize too.""" - result = _mxfp4_dequantize_config(_config_with_architecture("Qwen3MoeForCausalLM", "mxfp4")) - assert result is not None - assert result.dequantize is True - - def test_unquantized_finetune_untouched(self): - assert _mxfp4_dequantize_config(_config_with_architecture("GptOssForCausalLM")) is None - - def test_other_quantizations_are_not_dequantized_here(self): - """Only MXFP4 has a dequantize-on-load path; the rest are refused later - by the converter guards, which give a better-targeted message.""" - for method in ("bitsandbytes", "gptq", "awq", "fp8"): - assert ( - _mxfp4_dequantize_config(_config_with_architecture("LlamaForCausalLM", method)) - is None - ) - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_costs_no_hub_round_trip(self, mock_auto_config: mock.MagicMock): - """The whole point of carrying the method on the cfg: this runs on every - load, and a fetch here would be a Hub HEAD request per model load.""" - _mxfp4_dequantize_config(_config_with_architecture("GptOssForCausalLM", "mxfp4")) - mock_auto_config.from_pretrained.assert_not_called() - - -class TestUnsupportedQuantizationRefusal: - """The ordinary from_pretrained("<name>") path must refuse quantized weights: - the old refusal lived under `if hf_model is not None`, and same-shape - int8/FP8 survives converter AND load_state_dict (cast to fp32) silently. - """ - - @staticmethod - def _model(dtype=None, quant_method="gptq"): - class _Tiny(torch.nn.Module): - def __init__(self) -> None: - super().__init__() - self.q_proj = torch.nn.Linear(4, 4, bias=False) - if dtype is not None: - self.q_proj.weight = torch.nn.Parameter( - torch.zeros(4, 4, dtype=dtype), requires_grad=False - ) - - model = _Tiny() - quantization_config = {"quant_method": quant_method} if quant_method is not None else None - model.config = SimpleNamespace(quantization_config=quantization_config) - return model - - @pytest.mark.parametrize( - "dtype", - [torch.int8, torch.uint8, torch.float8_e4m3fn], - ) - def test_quantized_storage_is_refused(self, dtype): - from transformer_lens.loading_from_pretrained import ( - _refuse_unsupported_quantization, - ) - - cfg = _config_with_architecture("LlamaForCausalLM") - with pytest.raises(NotImplementedError, match="gptq"): - _refuse_unsupported_quantization(cfg, self._model(dtype)) - - def test_unquantized_model_passes(self): - """The positive control — every ordinary load goes through this.""" - from transformer_lens.loading_from_pretrained import ( - _refuse_unsupported_quantization, - ) - - cfg = _config_with_architecture("LlamaForCausalLM") - _refuse_unsupported_quantization(cfg, self._model(quant_method=None)) - - def test_dequantized_checkpoint_still_loads(self): - """A model loaded with dequantize=True keeps advertising its original - quant_method while holding real bf16 tensors. Refusing on the - declaration alone would break the MXFP4 auto-dequantize path.""" - from transformer_lens.loading_from_pretrained import ( - _refuse_unsupported_quantization, - ) - - cfg = _config_with_architecture("GptOssForCausalLM", "mxfp4") - _refuse_unsupported_quantization(cfg, self._model(quant_method="mxfp4")) - - def test_bitsandbytes_4bit_llama_flow_is_preserved(self): - """The one supported quantized HT flow: weight conversion and - abstract_attention's matmul_4bit both depend on it.""" - from transformer_lens.loading_from_pretrained import ( - _refuse_unsupported_quantization, - ) - - cfg = _config_with_architecture("LlamaForCausalLM") - cfg.load_in_4bit = True - _refuse_unsupported_quantization(cfg, self._model(torch.uint8, quant_method="bitsandbytes")) - - @mock.patch("transformer_lens.loading_from_pretrained.AutoModelForCausalLM") - def test_refusal_is_wired_into_the_internal_load_path(self, mock_auto_model: mock.MagicMock): - """The wiring, not just the helper: the refusal must fire where TL loads the - model itself and the caller never sees an hf_model. - """ - from transformer_lens.loading_from_pretrained import get_pretrained_state_dict - - mock_auto_model.from_pretrained.return_value = self._model(torch.int8) - - cfg = _config_with_architecture("LlamaForCausalLM") - with pytest.raises(NotImplementedError, match="gptq"): - get_pretrained_state_dict("meta-llama/Llama-2-7b-hf", cfg) - - -class TestQuantizationMethodCapture: - """`convert_hf_model_config` must record the checkpoint's quant_method while - the HF config is in hand — that capture is what lets the loader act on the - quantization without refetching.""" - - @pytest.mark.parametrize( - "quantization_config", - [ - {"quant_method": "mxfp4"}, - SimpleNamespace(quant_method="mxfp4"), - ], - ids=["dict-style", "object-style"], - ) - def test_capture_handles_both_config_shapes(self, quantization_config): - from transformer_lens.utilities.quantization import quantization_method - - assert ( - quantization_method(SimpleNamespace(quantization_config=quantization_config)) == "mxfp4" - ) - - def test_unquantized_config_captures_none(self): - from transformer_lens.utilities.quantization import quantization_method - - assert quantization_method(SimpleNamespace()) is None - assert quantization_method(None) is None - - @staticmethod - def _gpt2_shaped_config(**extra): - """The attributes convert_hf_model_config's GPT2 branch reads, so the - real function can run without touching the Hub.""" - return SimpleNamespace( - architectures=["GPT2LMHeadModel"], - n_embd=128, - n_head=4, - n_layer=2, - n_ctx=1024, - layer_norm_epsilon=1e-5, - vocab_size=50257, - activation_function="gelu_new", - scale_attn_by_inverse_layer_idx=False, - **extra, - ) - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_convert_hf_model_config_records_the_method(self, mock_auto_config: mock.MagicMock): - """The wiring, not just the extractor: without this the loader would - silently stop dequantizing MXFP4 and the converter would raise instead.""" - from transformer_lens.loading_from_pretrained import convert_hf_model_config - - mock_auto_config.from_pretrained.return_value = self._gpt2_shaped_config( - quantization_config={"quant_method": "mxfp4"} - ) - assert convert_hf_model_config("gpt2")["quantization_method"] == "mxfp4" - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_convert_hf_model_config_records_none_when_unquantized( - self, mock_auto_config: mock.MagicMock - ): - from transformer_lens.loading_from_pretrained import convert_hf_model_config - - mock_auto_config.from_pretrained.return_value = self._gpt2_shaped_config() - assert convert_hf_model_config("gpt2")["quantization_method"] is None - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_user_supplied_hf_model_config_wins(self, mock_auto_config: mock.MagicMock): - """A user passing hf_model= says how the weights in hand are stored, - which can differ from the Hub repo's declaration (they may have loaded - it dequantized, or quantized one that ships in bf16).""" - mock_auto_config.from_pretrained.return_value = self._gpt2_shaped_config() - - cfg = get_pretrained_model_config( - "gpt2", hf_cfg={"quantization_config": {"quant_method": "bitsandbytes"}} - ) - assert cfg.quantization_method == "bitsandbytes" - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_unquantized_hf_model_does_not_erase_the_repo_method( - self, mock_auto_config: mock.MagicMock - ): - """hf_cfg is often present but carries no quantization_config at all; - that absence must not wipe what the repo config declared.""" - mock_auto_config.from_pretrained.return_value = self._gpt2_shaped_config( - quantization_config={"quant_method": "mxfp4"} - ) - - cfg = get_pretrained_model_config("gpt2", hf_cfg={"vocab_size": 50257}) - assert cfg.quantization_method == "mxfp4" - - @mock.patch("transformer_lens.loading_from_pretrained.convert_neel_model_config") - def test_config_builders_that_never_see_an_hf_config(self, mock_neel: mock.MagicMock): - """convert_neel_model_config builds cfg_dict from the name alone — reading - quantization_method with [] broke every NeelNanda load. - """ - mock_neel.return_value = { - "d_model": 128, - "d_head": 8, - "n_heads": 16, - "n_ctx": 128, - "n_layers": 1, - "d_vocab": 50257, - "attn_only": True, - "original_architecture": "neel", - } - - cfg = get_pretrained_model_config("NeelNanda/SoLU_2L512W_C4_Code", hf_cfg={}) - assert cfg.quantization_method is None - - def test_name_based_architecture_branches_capture_none(self): - """Llama/gemma names never fetch a config, so nothing is there to read. - Pinned because the field must be present-and-None rather than missing — - HookedTransformerConfig.from_dict passes cfg_dict through unfiltered.""" - from transformer_lens.loading_from_pretrained import convert_hf_model_config - - cfg_dict = convert_hf_model_config("llama-7b-hf") - assert cfg_dict["quantization_method"] is None - - -# Successes - - -def test_fill_missing_keys_raises_on_missing_attention_weights(): - """A missing attention W means converter/component naming mismatch (#1620) — - zero-filling it silently breaks the model, so it must raise instead. - (The warn-and-fill behavior is covered by the bridge-based tests in - tests/unit/model_bridge/compatibility/test_loading_from_pretrained_utilities.py.)""" - cfg = HookedTransformerConfig( - d_model=128, d_head=8, n_heads=16, n_ctx=128, n_layers=1, d_vocab=50257, attn_only=True - ) - model = HookedTransformer(cfg) - default_state_dict = model.state_dict() - - incomplete_state_dict = { - k: v for k, v in default_state_dict.items() if not k.endswith("attn.W_K") - } - - with pytest.raises(ValueError, match="missing weight matrices"): - fill_missing_keys(model, incomplete_state_dict) - - -def test_n_ctx_override_reduces_context(): - """ - n_ctx override should work when reducing below the model default. - Uses the config loader directly — no model loading needed. - Fixes #1006. - """ - from transformer_lens.loading_from_pretrained import get_pretrained_model_config - - cfg = get_pretrained_model_config("gpt2", n_ctx=256) - assert cfg.n_ctx == 256, f"Expected n_ctx=256, got {cfg.n_ctx}" - - -@mock.patch("logging.warning") -def test_n_ctx_override_larger_than_default_warns(mock_warning: mock.MagicMock): - """ - A warning should be issued when n_ctx exceeds the model's default. - GPT-2 default n_ctx is 1024 — requesting 2048 should trigger the warning. - Fixes #1006. - """ - from transformer_lens.loading_from_pretrained import get_pretrained_model_config - - cfg = get_pretrained_model_config("gpt2", n_ctx=2048) - assert cfg.n_ctx == 2048, f"Expected n_ctx=2048, got {cfg.n_ctx}" - mock_warning.assert_any_call( - "You are setting n_ctx=2048 which is larger than this model's " - "default context length of 1024. The model was not " - "trained on sequences this long and may produce unreliable results. " - "Ensure you have sufficient memory for this context length." - ) - - -def _minimal_qwen_config(architecture: str, **overrides: object) -> SimpleNamespace: - values = { - "architectures": [architecture], - "hidden_size": 128, - "num_attention_heads": 4, - "intermediate_size": 512, - "num_hidden_layers": 2, - "layer_norm_epsilon": 1e-6, - "rms_norm_eps": 1e-6, - "vocab_size": 1000, - "scale_attn_weights": True, - "initializer_range": 0.02, - "kv_channels": 32, - "num_key_value_heads": 4, - "hidden_act": "silu", - "rope_theta": 1000000.0, - } - values.update(overrides) - return SimpleNamespace(**values) - - -@mock.patch("transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained") -def test_qwen_uses_hf_seq_length_for_n_ctx(mock_from_pretrained: mock.MagicMock): - mock_from_pretrained.return_value = _minimal_qwen_config( - "QWenLMHeadModel", - seq_length=8192, - max_position_embeddings=32768, - ) - - cfg = get_pretrained_model_config("Qwen/Qwen-7B") - - assert cfg.n_ctx == 8192 - - -@mock.patch("transformer_lens.loading_from_pretrained.AutoConfig.from_pretrained") -@pytest.mark.parametrize( - ("model_name", "max_position_embeddings"), - [ - ("Qwen/Qwen1.5-0.5B", 32768), - ("Qwen/Qwen2-0.5B", 131072), - ], -) -def test_qwen2_uses_hf_max_position_embeddings_for_n_ctx( - mock_from_pretrained: mock.MagicMock, - model_name: str, - max_position_embeddings: int, -): - mock_from_pretrained.return_value = _minimal_qwen_config( - "Qwen2ForCausalLM", - max_position_embeddings=max_position_embeddings, - ) - - cfg = get_pretrained_model_config(model_name) - - assert cfg.n_ctx == max_position_embeddings - - -# --- Architecture config tests --- - - -class TestArchitectureConfigs: - """Verify that convert_hf_model_config produces correct configs for new architectures.""" - - def test_apertus_config(self): - try: - cfg = get_pretrained_model_config("apertus-8b") - except ValueError as e: - if "does not recognize this architecture" in str(e): - pytest.skip(f"transformers version too old: {e}") - raise - assert cfg.original_architecture == "ApertusForCausalLM" - assert cfg.normalization_type == "RMS" - assert cfg.positional_embedding_type == "rotary" - assert cfg.gated_mlp is False - assert cfg.final_rms is True - assert cfg.act_fn == "xielu" - assert cfg.use_qk_norm is True - assert cfg.n_key_value_heads is not None - assert cfg.d_model > 0 - assert cfg.n_heads > 0 - - def test_gpt_oss_config(self): - try: - cfg = get_pretrained_model_config("gpt-oss-20b") - except ValueError as e: - if "does not recognize this architecture" in str(e): - pytest.skip(f"transformers version too old: {e}") - raise - assert cfg.original_architecture == "GptOssForCausalLM" - assert cfg.normalization_type == "RMS" - assert cfg.positional_embedding_type == "rotary" - assert cfg.gated_mlp is True - assert cfg.final_rms is True - assert cfg.num_experts is not None - assert cfg.num_experts > 0 - assert cfg.experts_per_token is not None - assert cfg.n_key_value_heads is not None - - def test_apertus_instruct_config(self): - try: - cfg = get_pretrained_model_config("apertus-8b-instruct") - except ValueError as e: - if "does not recognize this architecture" in str(e): - pytest.skip(f"transformers version too old: {e}") - raise - assert cfg.original_architecture == "ApertusForCausalLM" - assert cfg.act_fn == "xielu" - - @pytest.mark.parametrize("use_parallel_residual", [True, False]) - def test_gpt_neox_parallel_residual_follows_hf_config( - self, tmp_path, use_parallel_residual: bool - ): - """GPTNeoX ships sequential variants; parallel_attn_mlp must not be hardcoded.""" - from transformers import GPTNeoXConfig - - from transformer_lens.loading_from_pretrained import convert_hf_model_config - - hf_config = GPTNeoXConfig( - vocab_size=128, - hidden_size=64, - intermediate_size=128, - num_hidden_layers=2, - num_attention_heads=4, - max_position_embeddings=128, - use_parallel_residual=use_parallel_residual, - ) - hf_config.architectures = ["GPTNeoXForCausalLM"] - hf_config.save_pretrained(tmp_path) - - cfg_dict = convert_hf_model_config(str(tmp_path)) - - assert cfg_dict["parallel_attn_mlp"] is use_parallel_residual - - -class TestHetConfigThroughConvertHfModelConfig: - """convert_hf_model_config had zero het protection: transformers>=5.15 - per-layer fields raise (not AttributeError) on global reads, so the branch - chain's own hasattr probes were crash sites.""" - - @mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") - def test_per_layer_field_resolves_to_majority(self, mock_auto_config: mock.MagicMock): - from tests.unit.model_bridge.test_config_mapping import _HeterogeneousConfig - from transformer_lens.loading_from_pretrained import convert_hf_model_config - - mock_auto_config.from_pretrained.return_value = _HeterogeneousConfig( - {"scale_attn_by_inverse_layer_idx": [False, False]}, - architectures=["GPT2LMHeadModel"], - n_embd=128, - n_head=4, - n_layer=2, - num_hidden_layers=2, - n_ctx=1024, - layer_norm_epsilon=1e-5, - vocab_size=50257, - activation_function="gelu_new", - ) - cfg_dict = convert_hf_model_config("gpt2") - assert cfg_dict["scale_attn_by_inverse_layer_idx"] is False - - -class TestFillMissingKeysFailLoud: - """Zero-filling an MLP weight matrix is the same silent-sublayer-death as - the attention case; norm fills are frequently right but must be named.""" - - @staticmethod - def _model_and_state(missing_leaf): - cfg = get_default_config() - cfg.attn_only = False - model = HookedTransformer( - HookedTransformerConfig( - d_model=128, - d_head=8, - n_heads=16, - n_ctx=128, - n_layers=1, - d_vocab=50257, - d_mlp=256, - act_fn="relu", - ) - ) - state_dict = model.state_dict() - removed = [key for key in state_dict if key.rsplit(".", 1)[-1] == missing_leaf] - assert removed, f"fixture produced no {missing_leaf} keys" - for key in removed: - del state_dict[key] - return model, state_dict - - @pytest.mark.parametrize("leaf", ["W_in", "W_out"]) - def test_missing_mlp_matrices_raise(self, leaf): - model, state_dict = self._model_and_state(leaf) - with pytest.raises(ValueError, match=leaf): - fill_missing_keys(model, state_dict) - - @mock.patch("logging.warning") - def test_missing_norm_keys_warn_by_name(self, mock_warning: mock.MagicMock): - model, state_dict = self._model_and_state("w") - fill_missing_keys(model, state_dict) - norm_warnings = [ - call for call in mock_warning.call_args_list if "normalization" in str(call.args[0]) - ] - assert norm_warnings, "norm fills must be named, not silent" diff --git a/tests/unit/test_make_docs.py b/tests/unit/test_make_docs.py index 354c1782b1..50e9cd4b72 100644 --- a/tests/unit/test_make_docs.py +++ b/tests/unit/test_make_docs.py @@ -1,56 +1,5 @@ """Make Docs Tests.""" -import pytest - -from docs.make_docs import get_config, get_model_info, get_property - - -def test_get_config(): - """Test get config with attn-only-1l model.""" - config = get_config("attn-only-1l") - assert config.attn_only is True - - -def test_get_property(): - """Test get property with attn-only-1l model.""" - act_fn = get_property("act_fn", "attn-only-1l") - assert act_fn == "attn_only" - - n_params = get_property("n_params", "attn-only-1l") - assert n_params == "1.0M" - - n_layers = get_property("n_layers", "attn-only-1l") - assert n_layers == 1 - - d_model = get_property("d_model", "attn-only-1l") - assert d_model == 512 - - n_heads = get_property("n_heads", "attn-only-1l") - assert n_heads == 8 - - n_ctx = get_property("n_ctx", "attn-only-1l") - assert n_ctx == 1024 - - d_vocab = get_property("d_vocab", "attn-only-1l") - assert d_vocab == 48262 - - d_head = get_property("d_head", "attn-only-1l") - assert d_head == 64 - - d_mlp = get_property("d_mlp", "attn-only-1l") - assert d_mlp == 2048 - - n_key_value_heads = get_property("n_key_value_heads", "attn-only-1l") - assert n_key_value_heads is None - - # Test an unknown property - with pytest.raises(KeyError): - get_property("unknown_property", "attn-only-1l") - - -def test_get_model_info(): - get_model_info("attn-only-1l") - def test_render_bridge_models_page_interpolates_registry_constants(): """Download-free: the bridge-models page template resolves every placeholder diff --git a/tests/unit/test_moe_expert_fold.py b/tests/unit/test_moe_expert_fold.py new file mode 100644 index 0000000000..b369368829 --- /dev/null +++ b/tests/unit/test_moe_expert_fold.py @@ -0,0 +1,100 @@ +"""ProcessWeights folds ln2 into an MoE layer's router and every expert. + +The MoE weight-processing fold once refused to fold the experts, silently +dropping the gains — OLMoE sat 20.5 off HF in log-softmax with 0% argmax +agreement. The fold now engages: ln2's gain multiplies the router gate and each +expert's W_in / W_gate, and ln2 is set to identity only once real expert +weights were folded. + +Re-anchored on ``ProcessWeights._fold_mlp_layer_norm`` directly (the surviving +weight-processing path) after the legacy ``get_pretrained_model_config`` + +HookedTransformer end-to-end test was removed at 4.0. +""" + +import torch + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.weight_processing import ProcessWeights + +N_EXPERTS = 4 +D_MODEL = 8 +D_MLP = 16 + + +def _moe_cfg() -> TransformerBridgeConfig: + cfg = TransformerBridgeConfig( + n_layers=1, + d_model=D_MODEL, + d_head=4, + n_heads=2, + d_mlp=D_MLP, + d_vocab=10, + n_ctx=16, + act_fn="silu", + normalization_type="RMS", + gated_mlp=True, + ) + cfg.num_experts = N_EXPERTS + return cfg + + +def _moe_state_dict(ln2_w: torch.Tensor) -> dict: + torch.manual_seed(0) + sd = { + "blocks.0.ln2.w": ln2_w.clone(), + "blocks.0.mlp.W_gate.weight": torch.randn(N_EXPERTS, D_MODEL), + } + for e in range(N_EXPERTS): + sd[f"blocks.0.mlp.experts.{e}.W_in.weight"] = torch.randn(D_MLP, D_MODEL) + sd[f"blocks.0.mlp.experts.{e}.W_gate.weight"] = torch.randn(D_MLP, D_MODEL) + return sd + + +def test_ln2_gain_folds_into_router_and_every_expert(): + """Each expert's W_in/W_gate is scaled by ln2's gain, and ln2 becomes ones.""" + ln2_w = torch.rand(D_MODEL) + 0.5 # non-trivial gain; folding a ones vector is vacuous + cfg = _moe_cfg() + original = _moe_state_dict(ln2_w) + processed = ProcessWeights._fold_mlp_layer_norm( + {k: v.clone() for k, v in original.items()}, + cfg, + layer=0, + fold_biases=False, + center_weights=False, + adapter=None, + ) + + # Router gate folded. + torch.testing.assert_close( + processed["blocks.0.mlp.W_gate.weight"], + original["blocks.0.mlp.W_gate.weight"] * ln2_w[None, :], + ) + # Every expert folded — the bug was these being skipped. + for e in range(N_EXPERTS): + for suffix in ("W_in", "W_gate"): + key = f"blocks.0.mlp.experts.{e}.{suffix}.weight" + torch.testing.assert_close(processed[key], original[key] * ln2_w[None, :]) + # ln2 set to identity, since real expert weights were folded. + torch.testing.assert_close(processed["blocks.0.ln2.w"], torch.ones_like(ln2_w)) + + +def test_ln2_not_zeroed_when_no_expert_weights_present(): + """Guard against a false fold: with no experts.* keys, ln2 must be left as-is + (and the router-gate fold undone), not silently identity-ied.""" + ln2_w = torch.rand(D_MODEL) + 0.5 + cfg = _moe_cfg() + sd = { + "blocks.0.ln2.w": ln2_w.clone(), + "blocks.0.mlp.W_gate.weight": torch.randn(N_EXPERTS, D_MODEL), + } + router_before = sd["blocks.0.mlp.W_gate.weight"].clone() + processed = ProcessWeights._fold_mlp_layer_norm( + {k: v.clone() for k, v in sd.items()}, + cfg, + layer=0, + fold_biases=False, + center_weights=False, + adapter=None, + ) + torch.testing.assert_close(processed["blocks.0.ln2.w"], ln2_w) + torch.testing.assert_close(processed["blocks.0.mlp.W_gate.weight"], router_before) diff --git a/tests/unit/test_moe_fold_guard.py b/tests/unit/test_moe_fold_guard.py deleted file mode 100644 index f6afddf5c5..0000000000 --- a/tests/unit/test_moe_fold_guard.py +++ /dev/null @@ -1,104 +0,0 @@ -"""MoE models fold their norms like dense models do. - -History: HT once switched MoE models to the gain-less *Pre norm while its -process step refused to fold the experts, silently dropping the gains entirely -(OLMoE sat 20.5 off HF in log-softmax, 0% argmax). A guard then refused folding -outright, which diverged from the bridge (which folds) at unembed.hook_in. The -shared ProcessWeights fold handles the router and every expert's W_in/W_gate, -and HT-with-MoE-fold measures bit-exact against HF (0.0000 log-softmax, -100% argmax on OLMoE-1B-7B), so folding is simply enabled. -""" - -from types import SimpleNamespace -from unittest import mock - -import pytest - -from transformer_lens.loading_from_pretrained import get_pretrained_model_config - - -def _moe_config(architecture: str, num_experts: int) -> SimpleNamespace: - return SimpleNamespace( - architectures=[architecture], - hidden_size=64, - num_attention_heads=4, - num_key_value_heads=4, - intermediate_size=128, - num_hidden_layers=2, - max_position_embeddings=512, - rms_norm_eps=1e-6, - vocab_size=100, - hidden_act="silu", - rope_theta=500000.0, - sliding_window=None, - num_experts=num_experts, - num_local_experts=num_experts, - num_experts_per_tok=2, - norm_topk_prob=False, - tie_word_embeddings=False, - initializer_range=0.02, - ) - - -@pytest.mark.parametrize( - "model_name,architecture,n_experts", - [ - ("allenai/OLMoE-1B-7B-0924", "OlmoeForCausalLM", 64), - ("mistralai/Mixtral-8x7B-v0.1", "MixtralForCausalLM", 8), - ], -) -@mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") -def test_moe_folds_like_dense(mock_auto_config, caplog, model_name, architecture, n_experts): - """fold_ln=True switches MoE to the folded *Pre norm, same as dense, without - a refusal warning — the gains are folded into router and experts, not dropped.""" - mock_auto_config.from_pretrained.return_value = _moe_config(architecture, n_experts) - with caplog.at_level("WARNING"): - cfg = get_pretrained_model_config(model_name, fold_ln=True) - assert cfg.num_experts == n_experts - assert cfg.normalization_type == "RMSPre", cfg.normalization_type - assert not any("MoE" in r.getMessage() for r in caplog.records), [ - r.getMessage() for r in caplog.records - ] - - -def test_process_weights_folds_moe_and_preserves_outputs() -> None: - """The fold must engage (norms swap to *Pre) and be equivalence-preserving on - a MoE model — a skipped fold leaves RMS modules; a broken fold moves logits.""" - import torch - - from transformer_lens import HookedTransformer, HookedTransformerConfig - from transformer_lens.components import RMSNormPre - - torch.manual_seed(0) - cfg = HookedTransformerConfig( - n_layers=2, - d_model=32, - d_head=8, - n_heads=4, - d_mlp=64, - d_vocab=50, - n_ctx=16, - act_fn="silu", - normalization_type="RMS", - gated_mlp=True, - num_experts=4, - experts_per_token=2, - ) - model = HookedTransformer(cfg) - with torch.no_grad(): - for name, param in model.named_parameters(): - torch.nn.init.normal_(param, std=0.2) - # Non-trivial gains, or folding is a vacuous multiply-by-one. - for block in model.blocks: - block.ln1.w.copy_(torch.rand_like(block.ln1.w) + 0.5) - block.ln2.w.copy_(torch.rand_like(block.ln2.w) + 0.5) - model.ln_final.w.copy_(torch.rand_like(model.ln_final.w) + 0.5) - model.eval() - tokens = torch.randint(0, 50, (1, 8)) - with torch.no_grad(): - before = model(tokens) - model.process_weights_(fold_ln=True, center_writing_weights=False, center_unembed=False) - assert isinstance(model.blocks[0].ln2, RMSNormPre), type(model.blocks[0].ln2).__name__ - with torch.no_grad(): - after = model(tokens) - torch.testing.assert_close(after, before, atol=1e-4, rtol=1e-4) diff --git a/tests/unit/test_next_sentence_prediction.py b/tests/unit/test_next_sentence_prediction.py deleted file mode 100644 index 90683ced42..0000000000 --- a/tests/unit/test_next_sentence_prediction.py +++ /dev/null @@ -1,296 +0,0 @@ -from unittest.mock import Mock - -import pytest -import torch -from transformers import AutoTokenizer, BertForNextSentencePrediction - -from transformer_lens import ActivationCache, BertNextSentencePrediction, HookedEncoder - - -@pytest.fixture -def mock_hooked_encoder(): - mock_encoder = Mock(spec=HookedEncoder) - - mock_encoder.cfg = Mock() - mock_encoder.cfg.device = "cpu" - mock_encoder.cfg.n_ctx = 512 - - mock_encoder.tokenizer = Mock() - - mock_encodings = { - "input_ids": torch.tensor([[101, 2034, 102, 2035, 102]]), - "token_type_ids": torch.tensor([[0, 0, 0, 1, 1]]), - "attention_mask": torch.tensor([[1, 1, 1, 1, 1]]), - } - mock_encoder.tokenizer.return_value = mock_encodings - - # Mock encoder output - mock_encoder.encoder_output.return_value = (torch.randn(1, 7, 768), {}) # resid # cache - - # Mock pooler and NSP head - mock_encoder.pooler = Mock() - mock_encoder.pooler.return_value = torch.randn(1, 768) - mock_encoder.nsp_head = Mock() - mock_encoder.nsp_head.return_value = torch.tensor([[0.6, 0.4]]) - - # Mock run_with_cache - mock_encoder.run_with_cache = Mock() - - return mock_encoder - - -@pytest.fixture -def bert_nsp(mock_hooked_encoder): - return BertNextSentencePrediction(mock_hooked_encoder) - - -@pytest.fixture -def huggingface_bert(): - return BertForNextSentencePrediction.from_pretrained("bert-base-cased") - - -@pytest.fixture -def encodings(): - tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") - sentence_a = "She went to the grocery store." - sentence_b = "She bought an apple." - return tokenizer(sentence_a, sentence_b, return_tensors="pt") - - -def test_init(mock_hooked_encoder): - """Test initialization of BertNextSentencePrediction""" - bert_nsp = BertNextSentencePrediction(mock_hooked_encoder) - assert bert_nsp.model == mock_hooked_encoder - - -def test_call_chain(bert_nsp, mock_hooked_encoder): - """Test that encoder_output, pooler and nsp_head are called with the correct parameters""" - input_tensor = torch.tensor([[1, 2, 3]]) - token_type_ids = torch.tensor([[0, 0, 1]]) - attention_mask = torch.tensor([[1, 1, 1]]) - - # Set up specific mock returns - mock_resid = torch.randn(1, 3, 768) - mock_hooked_encoder.encoder_output.return_value = mock_resid - - mock_pooled = torch.randn(1, 768) - mock_hooked_encoder.pooler.return_value = mock_pooled - - mock_nsp_output = torch.tensor([[0.7, 0.3]]) - mock_hooked_encoder.nsp_head.return_value = mock_nsp_output - - # Call forward - output = bert_nsp.forward( - input_tensor, token_type_ids=token_type_ids, one_zero_attention_mask=attention_mask - ) - - # Verify the entire chain of calls - mock_hooked_encoder.encoder_output.assert_called_once_with( - input_tensor, token_type_ids, attention_mask - ) - mock_hooked_encoder.pooler.assert_called_once_with(mock_resid) - mock_hooked_encoder.nsp_head.assert_called_once_with(mock_pooled) - - # Verify output matches the mock NSP head output - assert torch.equal(output, mock_nsp_output) - - -def test_tokenizer_integration(bert_nsp, mock_hooked_encoder): - """Test that tokenizer is properly integrated and called""" - input_sentences = ["First sentence.", "Second sentence."] - - mock_hooked_encoder.tokenizer = Mock() - - # Mock tokenizer output - mock_encodings = { - "input_ids": torch.tensor([[101, 2034, 102, 2035, 102]]), - "token_type_ids": torch.tensor([[0, 0, 0, 1, 1]]), - "attention_mask": torch.tensor([[1, 1, 1, 1, 1]]), - } - mock_hooked_encoder.tokenizer.return_value = mock_encodings - - # Call to_tokens - tokens, type_ids, mask = bert_nsp.to_tokens(input_sentences) - - # Verify tokenizer was called correctly - mock_hooked_encoder.tokenizer.assert_called_once_with( - input_sentences[0], - input_sentences[1], - return_tensors="pt", - padding=True, - truncation=True, - max_length=mock_hooked_encoder.cfg.n_ctx, - ) - - # Verify outputs match tokenizer output - assert torch.equal(tokens, mock_encodings["input_ids"]) - assert torch.equal(type_ids, mock_encodings["token_type_ids"]) - assert torch.equal(mask, mock_encodings["attention_mask"]) - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires a CUDA device") -def test_device_handling_to_tokens(bert_nsp, mock_hooked_encoder): - """Test proper device handling in to_tokens""" - mock_hooked_encoder.cfg.device = "cuda" # Mock GPU device - - input_data = ["First sentence.", "Second sentence."] - - # Mock tokenizer output - mock_encodings = { - "input_ids": torch.tensor([[101, 2034, 102, 2035, 102]]), - "token_type_ids": torch.tensor([[0, 0, 0, 1, 1]]), - "attention_mask": torch.tensor([[1, 1, 1, 1, 1]]), - } - mock_hooked_encoder.tokenizer.return_value = mock_encodings - - # Call to_tokens with move_to_device=True - tokens, type_ids, mask = bert_nsp.to_tokens(input_data, move_to_device=True) - - # Verify each tensor was moved to the correct device - for tensor in [tokens, type_ids, mask]: - assert tensor.device.type == mock_hooked_encoder.cfg.device - - # Call with move_to_device=False - tokens, type_ids, mask = bert_nsp.to_tokens(input_data, move_to_device=False) - - # Verify tensors remained on CPU - for tensor in [tokens, type_ids, mask]: - assert tensor.device.type == "cpu" - - -def test_output_for_prediction_return_type(bert_nsp, mock_hooked_encoder): - """Test that output for return_type='predictions' is correct based on NSP head output""" - input_data = ["First sentence.", "Second sentence."] - - # Test case 1: Sequential prediction - mock_hooked_encoder.nsp_head.return_value = torch.tensor([[0.9, 0.1]]) - pred = bert_nsp.forward(input_data, return_type="predictions") - assert pred == "The sentences are sequential" - - # Test case 2: Non-sequential prediction - mock_hooked_encoder.nsp_head.return_value = torch.tensor([[0.2, 0.8]]) - pred = bert_nsp.forward(input_data, return_type="predictions") - assert pred == "The sentences are NOT sequential" - - -@pytest.mark.parametrize("input_type", ["str_list", "tensor"]) -def test_forward_input_types(bert_nsp, encodings, input_type): - """Test forward pass with different input types""" - sentence_a = "She went to the grocery store." - sentence_b = "She bought an apple." - if input_type == "str_list": - output = bert_nsp([sentence_a, sentence_b]) - else: - output = bert_nsp.forward( - encodings.input_ids, - token_type_ids=encodings.token_type_ids, - one_zero_attention_mask=encodings.attention_mask, - ) - - assert isinstance(output, torch.Tensor) - assert output.shape == (1, 2) # Batch size 1, binary classification - - -def test_forward_tokens_as_input_without_token_type_ids_error(bert_nsp): - """Test that forward raises error when tokens are input directly but no token_type_ids are provided""" - with pytest.raises( - ValueError, match="You are using the NSP task without specifying token_type_ids" - ): - bert_nsp.forward(torch.tensor([[1, 2, 768]])) - - -@pytest.mark.parametrize("return_type", [None, "logits", "predictions"]) -def test_forward_return_types(bert_nsp, mock_hooked_encoder, return_type): - """Test different return types from forward pass""" - # Setup mock logits that favor sequential prediction - mock_logits = torch.tensor([[0.7, 0.3]]) - mock_hooked_encoder.nsp_head.return_value = mock_logits - - input_data = ["She went to the grocery store.", "She bought an apple."] - output = bert_nsp.forward(input_data, return_type=return_type) - - if return_type is None: - assert output is None - elif return_type == "logits": - assert isinstance(output, torch.Tensor) - assert output.shape == (1, 2) - assert torch.equal(output, mock_logits) - elif return_type == "predictions": - assert isinstance(output, str) - assert output == "The sentences are sequential" # Based on mock logits - - -def test_to_tokens_validation(bert_nsp): - """Test input validation in to_tokens method""" - with pytest.raises( - ValueError, match="Next sentence prediction task requires exactly two sentences" - ): - bert_nsp.to_tokens(["Single sentence"]) - - with pytest.raises( - ValueError, match="Next sentence prediction task requires exactly two sentences" - ): - bert_nsp.to_tokens(["One", "Two", "Three"]) - - -def test_run_with_cache(bert_nsp, mock_hooked_encoder): - """run_with_cache must wrap the dict in an ActivationCache and restore model.forward.""" - # Sentinel forward that must be restored after the cache run. - original_forward = Mock(name="original_forward") - mock_hooked_encoder.forward = original_forward - - mock_cache = { - "resid_pre": torch.randn(1, 3, 768), - "attn_output": torch.randn(1, 3, 768), - } - captured = {} - - def fake_run_with_cache(*args, **kwargs): - # While inside the wrapper, model.forward must point at the NSP forward, not the original. - captured["forward_during"] = mock_hooked_encoder.forward - return (torch.tensor([[0.6, 0.4]]), mock_cache) - - mock_hooked_encoder.run_with_cache.side_effect = fake_run_with_cache - - input_data = ["First sentence.", "Second sentence."] - - output, cache = bert_nsp.run_with_cache( - input_data, return_type="logits", return_cache_object=True - ) - - # Effect 1: return_cache_object=True wraps the raw dict in an ActivationCache object, - # it is NOT the plain dict the encoder returned. - assert isinstance(cache, ActivationCache) - assert not isinstance(cache, dict) - assert cache.model is bert_nsp - - assert cache["resid_pre"].shape == (1, 3, 768) - assert cache["attn_output"].shape == (1, 3, 768) - - # Effect 3: ForwardWrapper swaps model.forward to the NSP forward during the run... - assert captured["forward_during"] is not original_forward - # ...and restores the original forward afterwards. - assert mock_hooked_encoder.forward is original_forward - - assert output.shape == (1, 2) - - -def test_return_type_consistency(bert_nsp, mock_hooked_encoder): - """Test consistency between logits and prediction outputs""" - # Setup mock logits that favor non-sequential prediction - mock_logits = torch.tensor([[0.2, 0.8]]) - mock_hooked_encoder.nsp_head.return_value = mock_logits - - input_data = ["She went to the grocery store.", "She bought an apple."] - - # Get predictions using different return types - logits = bert_nsp.forward(input_data, return_type="logits") - prediction_str = bert_nsp.forward(input_data, return_type="predictions") - - # Calculate predicted class from logits - predicted_class = logits.argmax(dim=-1).item() - expected_prediction = ["The sentences are sequential", "The sentences are NOT sequential"][ - predicted_class - ] - - assert prediction_str == expected_prediction # Based on mock logits diff --git a/tests/unit/test_post_norm_processing_guards.py b/tests/unit/test_post_norm_processing_guards.py index deb7684a04..a9a84e9b90 100644 --- a/tests/unit/test_post_norm_processing_guards.py +++ b/tests/unit/test_post_norm_processing_guards.py @@ -1,27 +1,31 @@ -"""LN folding and writing-weight centering must stay off for post-norm decoders. +"""Post-norm decoders must not have their embedding writing-weights centered. -Both transforms assume the norm gain sits on a sublayer's INPUT. OLMo 2/3 apply -ln1/ln2 to the sublayer OUTPUT, so folding is the wrong algebra: on the real -allenai/Olmo-3-1025-7B it moved log-softmax by 19.73 and dropped argmax agreement -with HF to 0%. The guard existed but named only OLMo 2, so OLMo 3 folded silently. -""" +Centering assumes the first attention's input is normalized; OLMo 2/3 apply +ln1/ln2 to the sublayer OUTPUT, leaving the residual stream un-normed there, so +centering shifts a stream nothing re-normalizes. On the real allenai/Olmo-3-1025-7B +this moved log-softmax by 19.73 and dropped argmax agreement with HF to 0%. The +guard once named only OLMo 2, so OLMo 3 was silently mis-centered. -from types import SimpleNamespace -from unittest import mock +Re-anchored on ``ProcessWeights`` directly (the surviving weight-processing path) +after the legacy ``get_pretrained_model_config`` wrapper was removed at 4.0; the +``POST_NORM_ARCHITECTURES`` membership it keys on is unchanged. +""" import pytest -import torch -from transformer_lens.loading_from_pretrained import get_pretrained_model_config +from transformer_lens.config import TransformerBridgeConfig from transformer_lens.utilities.architectures import POST_NORM_ARCHITECTURES from transformer_lens.weight_processing import ProcessWeights +POST_NORM_MODELS = [ + "Olmo3ForCausalLM", + "Olmo2ForCausalLM", +] -def _tl_config(architecture: str): - from transformer_lens import HookedTransformerConfig - return HookedTransformerConfig( - n_layers=0, +def _cfg(architecture: str) -> TransformerBridgeConfig: + cfg = TransformerBridgeConfig( + n_layers=2, d_model=8, n_ctx=16, d_head=4, @@ -29,91 +33,43 @@ def _tl_config(architecture: str): d_vocab=10, act_fn="silu", normalization_type="RMS", - original_architecture=architecture, positional_embedding_type="rotary", ) + cfg.original_architecture = architecture + return cfg -POST_NORM_MODELS = [ - ("allenai/Olmo-3-1025-7B", "Olmo3ForCausalLM"), - ("allenai/OLMo-2-0425-1B", "Olmo2ForCausalLM"), -] +@pytest.mark.parametrize("architecture", POST_NORM_MODELS) +def test_post_norm_architectures_are_registered(architecture): + """Both OLMo generations are in the guard set — OLMo 3 being absent is the + exact regression this file guards against.""" + assert architecture in POST_NORM_ARCHITECTURES -def _olmo_hf_config(architecture: str) -> SimpleNamespace: - return SimpleNamespace( - architectures=[architecture], - hidden_size=64, - num_attention_heads=4, - num_key_value_heads=4, - intermediate_size=128, - num_hidden_layers=4, - max_position_embeddings=512, - rms_norm_eps=1e-6, - vocab_size=100, - hidden_act="silu", - rope_theta=500000.0, - layer_types=["sliding_attention"] * 3 + ["full_attention"], - sliding_window=4096, - initializer_range=0.02, - tie_word_embeddings=False, - rope_parameters={ - "sliding_attention": {"rope_type": "default", "rope_theta": 500000.0}, - "full_attention": {"rope_type": "default", "rope_theta": 500000.0}, - }, - ) +@pytest.mark.parametrize("architecture", POST_NORM_MODELS) +def test_center_writing_weights_skips_post_norm(architecture, capsys): + """center_writing_weights leaves a post-norm model's embedding untouched.""" + import torch + cfg = _cfg(architecture) + W_E = torch.randn(cfg.d_vocab, cfg.d_model) + state_dict = {"embed.W_E": W_E.clone()} -@pytest.mark.parametrize("model_name,architecture", POST_NORM_MODELS) -@mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") -def test_fold_ln_is_refused(mock_auto_config, caplog, model_name, architecture) -> None: - mock_auto_config.from_pretrained.return_value = _olmo_hf_config(architecture) - with caplog.at_level("WARNING"): - get_pretrained_model_config(model_name, fold_ln=True) - assert any( - "fold_ln=True is incompatible" in record.getMessage() for record in caplog.records - ), [r.getMessage() for r in caplog.records] - - -@mock.patch("transformer_lens.loading_from_pretrained.AutoConfig") -def test_pre_norm_architecture_still_folds(mock_auto_config, caplog) -> None: - """Negative control: the guard must not disable folding for everyone.""" - mock_auto_config.from_pretrained.return_value = SimpleNamespace( - architectures=["LlamaForCausalLM"], - hidden_size=64, - num_attention_heads=4, - num_key_value_heads=4, - intermediate_size=128, - num_hidden_layers=2, - max_position_embeddings=512, - rms_norm_eps=1e-6, - vocab_size=100, - hidden_act="silu", - rope_theta=10000.0, - ) - with caplog.at_level("WARNING"): - get_pretrained_model_config("01-ai/Yi-6B", fold_ln=True) - assert not any("fold_ln=True is incompatible" in r.getMessage() for r in caplog.records) - - -@pytest.mark.parametrize("architecture", sorted(POST_NORM_ARCHITECTURES)) -def test_embeddings_are_not_centered(architecture) -> None: - """The first attention's input is un-normed, so centering W_E shifts a residual - stream nothing re-normalizes.""" - torch.manual_seed(0) - embedding = torch.randn(10, 8) - state = {"embed.W_E": embedding.clone()} - cfg = _tl_config(architecture) - out = ProcessWeights.center_writing_weights(state, cfg) - torch.testing.assert_close(out["embed.W_E"], embedding) - - -def test_embeddings_are_centered_for_pre_norm() -> None: - """Engagement check: centering is a no-op above only because of the guard.""" - torch.manual_seed(0) - embedding = torch.randn(10, 8) - state = {"embed.W_E": embedding.clone()} - cfg = _tl_config("LlamaForCausalLM") - out = ProcessWeights.center_writing_weights(state, cfg) - assert not torch.allclose(out["embed.W_E"], embedding) - torch.testing.assert_close(out["embed.W_E"].mean(-1), torch.zeros(10), atol=1e-6, rtol=0) + result = ProcessWeights.center_writing_weights(state_dict, cfg, adapter=None) + + assert torch.equal(result["embed.W_E"], W_E), "post-norm embedding was centered" + assert f"Not centering embedding weights for {architecture}" in capsys.readouterr().out + + +def test_pre_norm_architecture_is_still_centered(capsys): + """Negative control: the guard must not disable centering for everyone.""" + import torch + + cfg = _cfg("LlamaForCausalLM") + assert "LlamaForCausalLM" not in POST_NORM_ARCHITECTURES + W_E = torch.randn(cfg.d_vocab, cfg.d_model) + state_dict = {"embed.W_E": W_E.clone()} + + result = ProcessWeights.center_writing_weights(state_dict, cfg, adapter=None) + + assert not torch.equal(result["embed.W_E"], W_E), "pre-norm embedding was not centered" diff --git a/tests/unit/test_supported_models.py b/tests/unit/test_supported_models.py index 56a59d2054..552172145b 100644 --- a/tests/unit/test_supported_models.py +++ b/tests/unit/test_supported_models.py @@ -1,5 +1,8 @@ -from transformer_lens.loading_from_pretrained import get_official_model_name -from transformer_lens.supported_models import MODEL_ALIASES, OFFICIAL_MODEL_NAMES +from transformer_lens.supported_models import ( + MODEL_ALIASES, + OFFICIAL_MODEL_NAMES, + get_official_model_name, +) OLMO3_BASE_MODELS = { "allenai/Olmo-3-1025-7B": "olmo-3-1025-7b", @@ -21,6 +24,20 @@ def test_model_aliases_is_alphabetical(): assert actual_keys == expected_keys, "MODEL_ALIASES keys are not in alphabetical order. " +def test_get_official_model_name_is_case_insensitive(): + """The deleted loading_from_pretrained resolver lowercased both sides; the + rehomed one must too, or ~900 previously-accepted case variants regress.""" + assert get_official_model_name("GPT2") == "gpt2" + assert get_official_model_name("gpt2-small") == get_official_model_name("GPT2-Small") + + +def test_get_official_model_name_raises_on_unknown(): + import pytest + + with pytest.raises(ValueError, match="not an official model name"): + get_official_model_name("definitely-not-a-real-model-xyz") + + def test_olmo3_base_models_have_supported_aliases(): for model_name, alias in OLMO3_BASE_MODELS.items(): assert model_name in OFFICIAL_MODEL_NAMES diff --git a/tests/unit/test_train_config_isolation.py b/tests/unit/test_train_config_isolation.py deleted file mode 100644 index 0107d35694..0000000000 --- a/tests/unit/test_train_config_isolation.py +++ /dev/null @@ -1,37 +0,0 @@ -"""train() must not mutate the caller's config (device/wandb defaults).""" - -from __future__ import annotations - -import torch -from torch.utils.data import Dataset - -from transformer_lens import HookedTransformer, HookedTransformerConfig -from transformer_lens.train import HookedTransformerTrainConfig, train - - -def test_train_leaves_caller_config_untouched() -> None: - model = HookedTransformer( - HookedTransformerConfig( - n_layers=1, d_model=16, d_head=8, n_heads=2, n_ctx=8, d_vocab=16, act_fn="gelu" - ) - ) - - class _TokensDataset(Dataset): - def __len__(self) -> int: - return 1 - - def __getitem__(self, idx: int) -> dict: - return {"tokens": torch.tensor([1, 2, 3, 4])} - - dataset = _TokensDataset() - config = HookedTransformerTrainConfig( - num_epochs=1, - batch_size=1, - lr=1e-3, - seed=0, - device=None, - ) - - train(model, config, dataset) - - assert config.device is None, "train() wrote its resolved device onto the caller's config" diff --git a/tests/unit/tools/test_model_registry.py b/tests/unit/tools/test_model_registry.py index 1a456962d8..26e9877477 100644 --- a/tests/unit/tools/test_model_registry.py +++ b/tests/unit/tools/test_model_registry.py @@ -1088,15 +1088,6 @@ def test_no_alias_maps_to_two_officials(self): ), f"alias {alias!r} under both {seen[alias]!r} and {official!r}" seen[alias] = official - def test_legacy_loader_agrees_with_registry(self): - """The legacy loader and the registry must resolve aliases identically until 4.0.""" - from transformer_lens.loading_from_pretrained import get_official_model_name - from transformer_lens.tools.model_registry.registry_io import ( - resolve_model_alias, - ) - - assert get_official_model_name("gpt2-small") == resolve_model_alias("gpt2-small") - class TestCheckpointLabels: """Registry-canonical checkpoint schedules (checkpoints.py).""" @@ -1144,25 +1135,3 @@ def test_non_checkpointed_model_raises(self): with pytest.raises(ValueError, match="not checkpointed"): get_checkpoint_labels("gpt2") - - def test_matches_legacy_loader(self): - """Bridge checkpoint resolution must not drift from the legacy HT path until 4.0.""" - from transformer_lens.loading_from_pretrained import ( - get_checkpoint_labels as legacy_labels, - ) - from transformer_lens.tools.model_registry.checkpoints import ( - get_checkpoint_labels, - ) - - for name in ("EleutherAI/pythia-70m", "stanford-crfm/alias-gpt2-small-x21"): - assert get_checkpoint_labels(name) == legacy_labels(name) - - -class TestRemoteCodePrefixes: - def test_covers_legacy_remote_code_models(self): - """Registry tooling loads legacy-listed models too; prefixes must stay a superset until 4.0.""" - from transformer_lens.loading_from_pretrained import NEED_REMOTE_CODE_MODELS - from transformer_lens.tools.model_registry import REMOTE_CODE_MODEL_PREFIXES - - missing = set(NEED_REMOTE_CODE_MODELS) - set(REMOTE_CODE_MODEL_PREFIXES) - assert not missing, f"legacy remote-code prefixes not in registry list: {missing}" diff --git a/transformer_lens/ActivationCache.py b/transformer_lens/ActivationCache.py index 53da85cb4e..b49cbd9d85 100644 --- a/transformer_lens/ActivationCache.py +++ b/transformer_lens/ActivationCache.py @@ -203,7 +203,7 @@ def __getitem__(self, key) -> torch.Tensor: shorthand naming conventions. It also supports tuples for advanced indexing, with the dimension order as (name, layer_index, layer_type). - See :func:`transformer_lens.utils.get_act_name` for how shorthand is converted to a full name. + See :func:`transformer_lens.utilities.get_act_name` for how shorthand is converted to a full name. Args: diff --git a/transformer_lens/BertNextSentencePrediction.py b/transformer_lens/BertNextSentencePrediction.py deleted file mode 100644 index a55f0d862d..0000000000 --- a/transformer_lens/BertNextSentencePrediction.py +++ /dev/null @@ -1,280 +0,0 @@ -"""Next Sentence Prediction. - -Contains a BERT style model specifically for Next Sentence Prediction. This is separate from -:class:`transformer_lens.HookedTransformer` because it has a significantly different architecture -to e.g. GPT style transformers. -""" - -import warnings -from typing import Any, Dict, List, Optional, Tuple, Union, overload - -import torch -from jaxtyping import Float, Int -from typing_extensions import Literal - -from transformer_lens.ActivationCache import ActivationCache - - -class BertNextSentencePrediction: - """A BERT-style model for Next Sentence Prediction (NSP) that extends HookedEncoder. - - This class implements a BERT model specifically designed for the Next Sentence Prediction task, - where the model predicts whether two input sentences naturally follow each other in the original text. - It inherits from HookedEncoder and adds NSP-specific components like the NSP head and pooler layer. - - The model processes pairs of sentences and outputs either logits or human-readable predictions - indicating whether the sentences are sequential. String inputs are automatically tokenized with - appropriate token type IDs to distinguish between the two sentences. - - Note: - This model expects inputs to be provided as pairs of sentences. Single sentence inputs - or inputs without proper sentence separation will raise errors. - """ - - def __init__(self, model: Any): - warnings.warn( - "BertNextSentencePrediction is deprecated and will be removed in 4.0. Use " - "TransformerBridge.boot_transformers(...) for BERT-style models; see " - "demos/BERT.ipynb.", - DeprecationWarning, - stacklevel=2, - ) - self.model = model - - def __call__( - self, - input: Union[ - List[str], - Int[torch.Tensor, "batch pos"], - ], - return_type: Optional[Union[Literal["logits"], Literal["predictions"]]] = "logits", - token_type_ids: Optional[Int[torch.Tensor, "batch pos"]] = None, - one_zero_attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, - ) -> Optional[Union[Float[torch.Tensor, "batch 2"], str]]: - """Makes the NextSentencePrediction instance callable. - - This method delegates to the forward method, allowing the model to be called directly. - The arguments and return types match the forward method exactly. - """ - return self.forward( - input, - return_type=return_type, - token_type_ids=token_type_ids, - one_zero_attention_mask=one_zero_attention_mask, - ) - - def to_tokens( - self, - input: List[str], - move_to_device: bool = True, - truncate: bool = True, - ) -> Tuple[ - Int[torch.Tensor, "batch pos"], - Int[torch.Tensor, "batch pos"], - Int[torch.Tensor, "batch pos"], - ]: - """Converts a string to a tensor of tokens. - Taken mostly from the HookedTransformer implementation, but does not support default padding - sides or prepend_bos. - Args: - input: List[str]]: The input to tokenize. - move_to_device (bool): Whether to move the output tensor of tokens to the device the model lives on. Defaults to True - truncate (bool): If the output tokens are too long, whether to truncate the output - tokens to the model's max context window. Does nothing for shorter inputs. Defaults to - True. - """ - - if len(input) != 2: - raise ValueError( - "Next sentence prediction task requires exactly two sentences, please provide a list of strings with each sentence as an element." - ) - - # We need to input the two sentences separately for NSP - encodings = self.model.tokenizer( - input[0], - input[1], - return_tensors="pt", - padding=True, - truncation=truncate, - max_length=self.model.cfg.n_ctx if truncate else None, - ) - - tokens = encodings["input_ids"] - token_type_ids = encodings["token_type_ids"] - attention_mask = encodings["attention_mask"] - - if move_to_device: - tokens = tokens.to(self.model.cfg.device) - token_type_ids = token_type_ids.to(self.model.cfg.device) - attention_mask = attention_mask.to(self.model.cfg.device) - - return tokens, token_type_ids, attention_mask - - @overload - def forward( - self, - input: Union[ - List[str], - Int[torch.Tensor, "batch pos"], - ], - return_type: Union[Literal["logits"], Literal["predictions"]], - token_type_ids: Optional[Int[torch.Tensor, "batch pos"]] = None, - one_zero_attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, - ) -> Union[Float[torch.Tensor, "batch 2"], str]: - ... - - @overload - def forward( - self, - input: Union[ - List[str], - Int[torch.Tensor, "batch pos"], - ], - return_type: Literal[None], - token_type_ids: Optional[Int[torch.Tensor, "batch pos"]] = None, - one_zero_attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, - ) -> Optional[Union[Float[torch.Tensor, "batch 2"], str]]: - ... - - def forward( - self, - input: Union[ - List[str], - Int[torch.Tensor, "batch pos"], - ], - return_type: Optional[Union[Literal["logits"], Literal["predictions"]]] = "logits", - token_type_ids: Optional[Int[torch.Tensor, "batch pos"]] = None, - one_zero_attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, - ) -> Optional[Union[Float[torch.Tensor, "batch 2"], str]]: - """Forward pass through the NextSentencePrediction module. Performs Next Sentence Prediction on a pair of sentences. - - Args: - input: The input to process. Can be one of: - - List[str]: A list of two strings representing the two sentences NSP should be performed on - - torch.Tensor: Input tokens as integers with shape (batch, position) - return_type: Optional[str]: The type of output to return. Can be one of: - - None: Return nothing, don't calculate logits - - 'logits': Return logits tensor - - 'predictions': Return human-readable predictions - token_type_ids: Optional[torch.Tensor]: Binary ids indicating whether a token belongs - to sequence A or B. For example, for two sentences: - "[CLS] Sentence A [SEP] Sentence B [SEP]", token_type_ids would be - [0, 0, ..., 0, 1, ..., 1, 1]. `0` represents tokens from Sentence A, - `1` from Sentence B. If not provided, BERT assumes a single sequence input. - This parameter gets inferred from the tokenizer if input is a string or list of strings. - Shape is (batch_size, sequence_length). - one_zero_attention_mask: Optional[torch.Tensor]: A binary mask which indicates - which tokens should be attended to (1) and which should be ignored (0). - Primarily used for padding variable-length sentences in a batch. - For instance, in a batch with sentences of differing lengths, shorter - sentences are padded with 0s on the right. If not provided, the model - assumes all tokens should be attended to. - This parameter gets inferred from the tokenizer if input is a string or list of strings. - Shape is (batch_size, sequence_length). - - Returns: - Optional[torch.Tensor]: Depending on return_type: - - None: Returns None if return_type is None - - torch.Tensor: Returns logits if return_type is 'logits' (or if return_type is not explicitly provided) - - Shape is (batch_size, 2) - - str or List[str]: Returns string indicating if sentences are sequential if return_type is 'predictions' - - Raises: - ValueError: If using NSP task without proper input format or token_type_ids - AssertionError: If using string input without a tokenizer - """ - - if isinstance(input, list): - assert self.model.tokenizer is not None, "Must provide a tokenizer if input is a string" - tokens, token_type_ids_from_tokenizer, attention_mask = self.to_tokens(input) - - # If token_type_ids or attention mask are not provided, use the ones from the tokenizer - token_type_ids = ( - token_type_ids_from_tokenizer if token_type_ids is None else token_type_ids - ) - one_zero_attention_mask = ( - attention_mask if one_zero_attention_mask is None else one_zero_attention_mask - ) - elif token_type_ids == None and isinstance(input, torch.Tensor): - raise ValueError( - "You are using the NSP task without specifying token_type_ids." - "This means that the model will treat the input as a single sequence which will lead to incorrect results." - "Please provide token_type_ids or use a string input." - ) - else: - tokens = input - - resid = self.model.encoder_output(tokens, token_type_ids, one_zero_attention_mask) - - # NSP requires pooling (for more information see BertPooler) - resid = self.model.pooler(resid) - logits = self.model.nsp_head(resid) - - if return_type == "predictions": - logprobs = logits.log_softmax(dim=-1) - predictions = [ - "The sentences are sequential", - "The sentences are NOT sequential", - ] - return predictions[logprobs.argmax(dim=-1).item()] - - elif return_type == None: - return None - - return logits - - @overload - def run_with_cache( - self, *model_args, return_cache_object: Literal[True] = True, **kwargs - ) -> Tuple[Float[torch.Tensor, "batch 2"], ActivationCache,]: - ... - - @overload - def run_with_cache( - self, *model_args, return_cache_object: Literal[False], **kwargs - ) -> Tuple[Float[torch.Tensor, "batch 2"], Dict[str, torch.Tensor],]: - ... - - def run_with_cache( - self, - *model_args, - return_cache_object: bool = True, - remove_batch_dim: bool = False, - **kwargs, - ) -> Tuple[Float[torch.Tensor, "batch 2"], Union[ActivationCache, Dict[str, torch.Tensor]],]: - """ - Wrapper around run_with_cache in HookedRootModule. If return_cache_object is True, - this will return an ActivationCache object, with a bunch of useful HookedTransformer specific methods, - otherwise it will return a dictionary of activations as in HookedRootModule. - This function was copied directly from HookedTransformer. - """ - - # Create wrapper for forward function, such that run_with_cache uses - # the forward function of this class and not HookedEncoder - - class ForwardWrapper: - def __init__(self, nsp): - self.nsp = nsp - self.original_forward = nsp.model.forward - - def __enter__(self): - # Store reference to wrapper function - def wrapped_forward(*fargs, **fkwargs): - return self.nsp.forward(*fargs, **fkwargs) - - self.nsp.model.forward = wrapped_forward - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - # Restore original forward - self.nsp.model.forward = self.original_forward - - with ForwardWrapper(self): - out, cache_dict = self.model.run_with_cache( - *model_args, remove_batch_dim=remove_batch_dim, **kwargs - ) - if return_cache_object: - cache = ActivationCache(cache_dict, self, has_batch_dim=not remove_batch_dim) - return out, cache - else: - return out, cache_dict diff --git a/transformer_lens/HookedAudioEncoder.py b/transformer_lens/HookedAudioEncoder.py deleted file mode 100644 index e81fb45134..0000000000 --- a/transformer_lens/HookedAudioEncoder.py +++ /dev/null @@ -1,537 +0,0 @@ -"""Hooked Audio Encoder. - -Contains a HuBERT style model. This is separate from :class:`transformer_lens.HookedTransformer` -because it has a significantly different architecture to e.g. GPT style transformers. -""" - -from __future__ import annotations - -import logging -import warnings -from typing import Any, Dict, List, Optional, Tuple, TypeVar, Union, cast, overload - -import numpy as np -import torch -from einops import repeat -from jaxtyping import Float, Int -from transformers import AutoFeatureExtractor, HubertModel, Wav2Vec2Model -from typing_extensions import Literal - -from transformer_lens import loading_from_pretrained as loading -from transformer_lens.ActivationCache import ActivationCache -from transformer_lens.components import MLP, BertBlock -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.FactoredMatrix import FactoredMatrix -from transformer_lens.HookedRootModule import HookedRootModule -from transformer_lens.utilities import TypedModuleList, devices - -T = TypeVar("T", bound="HookedAudioEncoder") - - -class HookedAudioEncoder(HookedRootModule): - """ - This class implements a BERT-style encoder using the components in ./components.py, with HookPoints on every interesting activation. It inherits from HookedRootModule. - - Limitations: - - The model does not include dropouts, which may lead to inconsistent results from training or fine-tuning. - - Like HookedTransformer, it can have a pretrained Transformer's weights loaded via `.from_pretrained`. There are a few features you might know from HookedTransformer which are not yet supported: - - There is no preprocessing (e.g. LayerNorm folding) when loading a pretrained model - """ - - # Set by from_pretrained while constructing, so each public entry point - # emits exactly one DeprecationWarning. - _suppress_init_deprecation: bool = False - - processor: Any # AutoFeatureExtractor — HF auto class, not typed as callable in stubs - hubert_model: Union[HubertModel, Wav2Vec2Model] - blocks: TypedModuleList[BertBlock] - - def __init__( - self, - cfg: Union[HookedTransformerConfig, Dict], - move_to_device: bool = True, - model_name: str = "facebook/hubert-base-ls960", - **kwargs: Any, - ): - super().__init__() - # from_pretrained warns at its own level (so the warning attributes to - # the caller and survives the default DeprecationWarning filter) and - # suppresses this one — each entry point fires exactly once. - if not getattr(type(self), "_suppress_init_deprecation", False): - warnings.warn( - "HookedAudioEncoder is deprecated and will be removed in 4.0. Use " - "TransformerBridge.boot_transformers(...) instead — " - "HuBERT/Wav2Vec2 are supported via the bridge's audio adapter. " - "See docs/source/content/migrating_to_v3.md.", - DeprecationWarning, - stacklevel=2, - ) - if isinstance(cfg, Dict): - cfg = HookedTransformerConfig(**cfg) - elif isinstance(cfg, str): - raise ValueError( - "Please pass in a config dictionary or HookedTransformerConfig object. If you want to load a pretrained model, use HookedAudioEncoder.from_pretrained() instead." - ) - self.cfg = cfg - - assert self.cfg.n_devices == 1, "Multiple devices not supported for HookedEncoder" - - self.blocks = TypedModuleList([BertBlock(self.cfg) for _ in range(self.cfg.n_layers)]) - - if move_to_device: - if self.cfg.device is None: - raise ValueError("Cannot move to device when device is None") - self.to(self.cfg.device) - - self.setup() - - def _ensure_numpy(self, wave): - """ - Convert torch.Tensor / np.ndarray / list -> 1D np.float32 array on CPU. - """ - if isinstance(wave, torch.Tensor): - arr = wave.detach().cpu().numpy() - elif isinstance(wave, np.ndarray): - arr = wave - elif isinstance(wave, list): - arr = np.asarray(wave) - else: - raise TypeError("wave must be torch.Tensor, np.ndarray or list of floats") - - # force 1-D (if stereo or shape (N,1) etc) - if arr.ndim > 1: - # if shape (n_samples, n_channels) average channels -> mono - if arr.shape[1] <= arr.shape[0]: - arr = arr.mean(axis=1) - else: - arr = arr.reshape(-1) - - return arr.astype(np.float32, copy=False) - - def to_frames( - self, - raw_inputs: Union[torch.Tensor, List[Union[torch.Tensor, np.ndarray]]], - sampling_rate: int = 16000, - move_to_device: bool = True, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: - """ - Convert raw audio batch -> (projected frames, frame_attention_mask) - - Args: - raw_inputs: one of: - - a 1D torch.Tensor or numpy array (single waveform) - - a list of 1D torch.Tensors / numpy arrays (batch) - self.processor: HF AutoProcessor (creates input_values + sample-level attention_mask) - self.model: pretrained HubertModel (provides feature_extractor and feature_projection) - sampling_rate: sample rate of the audio (default 16k) - move_to_device: move outputs to model.device - - Returns: - frames: torch.Tensor of shape (batch, frames, hidden_size) <- after feature_projection - frame_attention_mask: torch.LongTensor of shape (batch, frames) with 1 for real frames, 0 for padding - """ - # AutoFeatureExtractor works better onnumpy array where it pads automatically. If passing in tensors, it does not pad properly, giving inhomogeneous arts error - if isinstance(raw_inputs, (torch.Tensor, np.ndarray)): - waves = [self._ensure_numpy(raw_inputs)] - elif isinstance(raw_inputs, list): - waves = [self._ensure_numpy(w) for w in raw_inputs] - else: - raise TypeError("Unsupported raw_inputs type") - - # Use HF processor to create input_values (padded) + sample-level attention_mask - # Processor will do padding so we can pass a variable-length batch - proc_out = self.processor( - waves, - sampling_rate=sampling_rate, - return_tensors="pt", - padding=True, - return_attention_mask=True, - ) - input_values = proc_out["input_values"] # (batch, samples), float - sample_attention_mask = proc_out.get( - "attention_mask" - ) # (batch, samples), 1 for valid, 0 for padding; may be None - - # move to device - device = self.cfg.device - if move_to_device: - input_values = input_values.to(device) - if sample_attention_mask is not None: - sample_attention_mask = sample_attention_mask.to(device) - - # 1) convolutional frontend -> (batch, conv_dim, conv_time) - if input_values.ndim > 2: - input_values = input_values.squeeze() - if input_values.ndim == 1: - input_values = input_values.unsqueeze(0) # (1, T) - with torch.no_grad(): - conv_feats = self.hubert_model.feature_extractor(input_values) # (B, C, T_conv) - - # 2) transpose to (batch, T_conv, C) - extract_features = conv_feats.transpose(1, 2) - - # 3) compute reduced frame-level attention mask (if sample mask provided) - frame_attention_mask = None - if sample_attention_mask is not None: - # model should provide helper _get_feature_vector_attention_mask - try: - frame_attention_mask = self.hubert_model._get_feature_vector_attention_mask( - extract_features.shape[1], sample_attention_mask - ) - except AttributeError: - # fallback: compute output lengths and create mask similarly to HF implementation - # compute output lengths (downsampled lengths) from sample attention mask (sums per example) - input_lengths = sample_attention_mask.sum(dim=-1) # (batch,) - # compute output lengths through conv layers using model._get_feat_extract_output_lengths if exists - if hasattr(self.hubert_model, "_get_feat_extract_output_lengths"): - output_lengths = self.hubert_model._get_feat_extract_output_lengths( - input_lengths - ).to(torch.long) - else: - # fallback to naive downsample ratio: output_frames = extract_features.shape[1] - output_lengths = torch.full( - (sample_attention_mask.shape[0],), - extract_features.shape[1], - device=device, - dtype=torch.long, - ) - - batch_size = sample_attention_mask.shape[0] - feat_len = extract_features.shape[1] - frame_attention_mask = torch.zeros( - (batch_size, feat_len), dtype=sample_attention_mask.dtype, device=device - ) - # mark the last valid index for each example and then cumsum trick to fill ones before it - idx = (torch.arange(batch_size, device=device), (output_lengths - 1).clamp(min=0)) - frame_attention_mask[idx] = 1 - frame_attention_mask = ( - frame_attention_mask.flip([-1]).cumsum(-1).flip([-1]).bool().long() - ) - - # 4) feature projection -> (batch, frames, hidden_size) - with torch.no_grad(): - hidden_states = self.hubert_model.feature_projection( - extract_features - ) # typically returns (B, T, hidden) - # In HF's hubert, feature_projection is a module that returns a tensor (not tuple). If it returns tuple, adjust. - - # convert bool mask to long (1/0) if needed - if frame_attention_mask is not None: - frame_attention_mask = frame_attention_mask.to(dtype=torch.long) - - return hidden_states, frame_attention_mask - - def encoder_output( - self, - frames: torch.Tensor, # (batch, frames, d_model) <-- precomputed conv features - one_zero_attention_mask: Optional[torch.Tensor] = None, # (batch, frames) - ): - # Ensure device - if frames.device.type != self.cfg.device: - frames = frames.to(self.cfg.device) - if one_zero_attention_mask is not None: - one_zero_attention_mask = one_zero_attention_mask.to(self.cfg.device) - - if one_zero_attention_mask is not None: - # HF zeroes pad frames before the positional conv (kernel 128 smears - # pad content into real frames otherwise); masked_fill, not HF's - # in-place write, so the caller's tensor survives. - frames = frames.masked_fill(~one_zero_attention_mask.bool().unsqueeze(-1), 0.0) - - position_embeddings = self.hubert_model.encoder.pos_conv_embed(frames) - resid = frames + position_embeddings - resid = self.hubert_model.encoder.layer_norm(resid) - - large_negative_number = -torch.inf - mask = ( - repeat(1 - one_zero_attention_mask, "batch pos -> batch 1 1 pos") - if one_zero_attention_mask is not None - else None - ) - additive_attention_mask = ( - torch.where(mask == 1, large_negative_number, 0) if mask is not None else None - ) - for block in self.blocks: - resid = block(resid, additive_attention_mask) - - return resid - - def forward( - self, - inputs: Union[ - torch.Tensor, # waveform (1D) OR precomputed frames (3D) - List[Union[torch.Tensor, np.ndarray]], # list of waveforms - Tuple[torch.Tensor, torch.Tensor], # (frames, frame_mask) - ], - one_zero_attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, - sampling_rate: int = 16000, - move_to_device: bool = True, - ) -> Optional[torch.Tensor]: - """ - HuBERT-like forward (Transformer-Lens style). - - Args: - input: one of: - - 1D torch.Tensor or numpy array (single waveform) OR list of 1D waveforms -> will call self.to_frames(...) - - 3D torch.Tensor shaped (batch, frames, d_model) -> treated as precomputed frames (skip to_frames) - - tuple (frames, frame_mask) -> use directly - sampling_rate: sampling rate for to_frames when converting raw audio. - use_proj: Whether to use the final head of HubertCTC - move_to_device: move tensors to self.cfg.device (to match your other code). - - Returns: - Depending on return_type: - - "hidden": (batch, frames, d_model) final encoder hidden states - """ - # ---------- 1) Normalize input: get (frames, frame_mask) ---------- - frames = None - frame_mask = None # one_zero_attention_mask: 1 = valid, 0 = padding - # If user passed (frames, mask) tuple - if isinstance(inputs, tuple) and len(inputs) == 2 and isinstance(inputs[0], torch.Tensor): - frames, frame_mask = inputs - - # If user passed a 3D tensor -> assume (B, T, D) frames (pre-projected) - elif isinstance(inputs, torch.Tensor) and inputs.ndim == 3: - frames = inputs - # frame_mask stays whatever was passed as separate argument (None here) - - # Else treat as raw waveform(s) -> call to_frames - else: - # allow single 1D tensor or numpy array or list of tensors/arrays - frames, frame_mask = self.to_frames(inputs) - # to_frames should already place tensors on device if move_to_device=True - if isinstance(frames, tuple): - frames = frames[0] - frame_mask = frame_mask if one_zero_attention_mask is None else one_zero_attention_mask - # ---------- 2) Ensure device & dtype consistency ---------- - device = self.cfg.device - if frames.device.type != device: - frames = frames.to(device) - if frame_mask is not None: - frame_mask = frame_mask.to(device) - - # ---------- 3) Run encoder (respects pos_conv_embed / layer_norm / dropout inside encoder_output) ---------- - resid = self.encoder_output(frames, frame_mask) # (B, T, d_model) - - return resid - - @overload - def run_with_cache( - self, *model_args: Any, return_cache_object: Literal[True] = True, **kwargs: Any - ) -> Tuple[Float[torch.Tensor, "batch pos d_vocab"], ActivationCache]: - ... - - @overload - def run_with_cache( - self, *model_args: Any, return_cache_object: Literal[False], **kwargs: Any - ) -> Tuple[Float[torch.Tensor, "batch pos d_vocab"], Dict[str, torch.Tensor]]: - ... - - def run_with_cache( - self, - *model_args: Any, - return_cache_object: bool = True, - remove_batch_dim: bool = False, - **kwargs: Any, - ) -> Tuple[ - Float[torch.Tensor, "batch pos d_vocab"], - Union[ActivationCache, Dict[str, torch.Tensor]], - ]: - """ - Wrapper around run_with_cache in HookedRootModule. If return_cache_object is True, this will return an ActivationCache object, with a bunch of useful HookedTransformer specific methods, otherwise it will return a dictionary of activations as in HookedRootModule. This function was copied directly from HookedTransformer. - """ - out, cache_dict = super().run_with_cache( - *model_args, remove_batch_dim=remove_batch_dim, **kwargs - ) - if return_cache_object: - cache = ActivationCache(cache_dict, self, has_batch_dim=not remove_batch_dim) - return out, cache - else: - return out, cache_dict - - def to( # type: ignore - self, - device_or_dtype: Union[torch.device, str, torch.dtype], - print_details: bool = True, - ): - return devices.move_to_and_update_config(self, device_or_dtype, print_details) - - def cuda(self: T, device: Optional[Union[int, torch.device]] = None) -> T: - if isinstance(device, int): - return self.to(f"cuda:{device}") - elif device is None: - return self.to("cuda") - else: - return self.to(device) - - def cpu(self: T) -> T: - return self.to("cpu") - - def mps(self: T) -> T: - return self.to(torch.device("mps")) - - @classmethod - def from_pretrained( - cls, - model_name: str, - checkpoint_index: Optional[int] = None, - checkpoint_value: Optional[int] = None, - hf_model: Optional[Any] = None, - device: Optional[str] = None, - move_to_device: bool = True, - dtype: torch.dtype = torch.float32, - **from_pretrained_kwargs: Any, - ) -> "HookedAudioEncoder": - """Loads in the pretrained weights from huggingface. Currently supports loading weight from HuggingFace BertForMaskedLM. Unlike HookedTransformer, this does not yet do any preprocessing on the model.""" - import warnings - - warnings.warn( - "HookedAudioEncoder.from_pretrained is deprecated and will be removed in " - "4.0. Use TransformerBridge.boot_transformers(...) instead — " - "HuBERT/Wav2Vec2 are supported via the bridge's audio adapter. See " - "docs/source/content/migrating_to_v3.md.", - DeprecationWarning, - stacklevel=2, - ) - - logging.warning( - "Support for HuBERT in TransformerLens is currently experimental, until such a time when it has feature " - "parity with HookedTransformer and has been tested on real research tasks. Until then, backward " - "compatibility is not guaranteed. Please see the docs for information on the limitations of the current " - "implementation." - "\n" - "If using HuBERT for interpretability research, keep in mind that HuBERT has some significant architectural " - "differences to GPT. For example, LayerNorms are applied *after* the attention and MLP components, meaning " - "that the last LayerNorm in a block cannot be folded." - ) - - assert not ( - from_pretrained_kwargs.get("load_in_8bit", False) - or from_pretrained_kwargs.get("load_in_4bit", False) - ), "Quantization not supported" - - if "torch_dtype" in from_pretrained_kwargs: - dtype = from_pretrained_kwargs["torch_dtype"] - - official_model_name = loading.get_official_model_name(model_name) - - cfg = loading.get_pretrained_model_config( - official_model_name, - checkpoint_index=checkpoint_index, - checkpoint_value=checkpoint_value, - fold_ln=False, - device=device, - n_devices=1, - dtype=dtype, - **from_pretrained_kwargs, - ) - - state_dict = loading.get_pretrained_state_dict( - official_model_name, cfg, hf_model, dtype=dtype, **from_pretrained_kwargs - ) - - _prev_suppress = cls._suppress_init_deprecation - cls._suppress_init_deprecation = True - try: - model = cls(cfg, move_to_device=False, model_name=official_model_name) - finally: - cls._suppress_init_deprecation = _prev_suppress - model.load_state_dict(state_dict, strict=False) - - model.processor = AutoFeatureExtractor.from_pretrained(official_model_name) - - hubert_model: Union[Wav2Vec2Model, HubertModel] - if "wav2vec2" in model_name: - hubert_model = Wav2Vec2Model.from_pretrained(official_model_name) - else: - hubert_model = HubertModel.from_pretrained(official_model_name) - - hubert_model.eval() - model.hubert_model = hubert_model - - if move_to_device: - device = cfg.device - if device is None: - raise ValueError("Cannot move to device when device is None") - hubert_model.to(torch.device(device)) # type: ignore[arg-type] - model.to(device) - - print(f"Loaded pretrained model {model_name} into HookedEncoder") - - return model - - @property - def W_K(self) -> Float[torch.Tensor, "n_layers n_heads d_model d_head"]: - """Stacks the key weights across all layers""" - return torch.stack([block.attn.W_K for block in self.blocks], dim=0) - - @property - def W_Q(self) -> Float[torch.Tensor, "n_layers n_heads d_model d_head"]: - """Stacks the query weights across all layers""" - return torch.stack([block.attn.W_Q for block in self.blocks], dim=0) - - @property - def W_V(self) -> Float[torch.Tensor, "n_layers n_heads d_model d_head"]: - """Stacks the value weights across all layers""" - return torch.stack([block.attn.W_V for block in self.blocks], dim=0) - - @property - def W_O(self) -> Float[torch.Tensor, "n_layers n_heads d_head d_model"]: - """Stacks the attn output weights across all layers""" - return torch.stack([block.attn.W_O for block in self.blocks], dim=0) - - @property - def W_in(self) -> Float[torch.Tensor, "n_layers d_model d_mlp"]: - """Stacks the MLP input weights across all layers""" - return torch.stack([block.mlp.W_in for block in self.blocks], dim=0) - - @property - def W_out(self) -> Float[torch.Tensor, "n_layers d_mlp d_model"]: - """Stacks the MLP output weights across all layers""" - return torch.stack([block.mlp.W_out for block in self.blocks], dim=0) - - @property - def b_K(self) -> Float[torch.Tensor, "n_layers n_heads d_head"]: - """Stacks the key biases across all layers""" - return torch.stack([block.attn.b_K for block in self.blocks], dim=0) - - @property - def b_Q(self) -> Float[torch.Tensor, "n_layers n_heads d_head"]: - """Stacks the query biases across all layers""" - return torch.stack([block.attn.b_Q for block in self.blocks], dim=0) - - @property - def b_V(self) -> Float[torch.Tensor, "n_layers n_heads d_head"]: - """Stacks the value biases across all layers""" - return torch.stack([block.attn.b_V for block in self.blocks], dim=0) - - @property - def b_O(self) -> Float[torch.Tensor, "n_layers d_model"]: - """Stacks the attn output biases across all layers""" - return torch.stack([block.attn.b_O for block in self.blocks], dim=0) - - @property - def b_in(self) -> Float[torch.Tensor, "n_layers d_mlp"]: - """Stacks the MLP input biases across all layers""" - return torch.stack([cast(MLP, block.mlp).b_in for block in self.blocks], dim=0) - - @property - def b_out(self) -> Float[torch.Tensor, "n_layers d_model"]: - """Stacks the MLP output biases across all layers""" - return torch.stack([cast(MLP, block.mlp).b_out for block in self.blocks], dim=0) - - @property - def QK(self) -> FactoredMatrix: # [n_layers, n_heads, d_model, d_model] - """Returns a FactoredMatrix object with the product of the Q and K matrices for each layer and head. - Useful for visualizing attention patterns.""" - return FactoredMatrix(self.W_Q, self.W_K.transpose(-2, -1)) - - @property - def OV(self) -> FactoredMatrix: # [n_layers, n_heads, d_model, d_model] - """Returns a FactoredMatrix object with the product of the O and V matrices for each layer and head.""" - return FactoredMatrix(self.W_V, self.W_O) - - def all_head_labels(self) -> List[str]: - """Returns a list of strings with the format "L{l}H{h}", where l is the layer index and h is the head index.""" - return [f"L{l}H{h}" for l in range(self.cfg.n_layers) for h in range(self.cfg.n_heads)] diff --git a/transformer_lens/HookedEncoder.py b/transformer_lens/HookedEncoder.py deleted file mode 100644 index ae155387c0..0000000000 --- a/transformer_lens/HookedEncoder.py +++ /dev/null @@ -1,572 +0,0 @@ -"""Hooked Encoder. - -Contains a BERT style model. This is separate from :class:`transformer_lens.HookedTransformer` -because it has a significantly different architecture to e.g. GPT style transformers. -""" - -from __future__ import annotations - -import logging -import os -import warnings -from typing import Any, Dict, List, Optional, Tuple, TypeVar, Union, cast, overload - -import torch -from einops import repeat -from jaxtyping import Float, Int -from transformers.models.auto.tokenization_auto import AutoTokenizer -from typing_extensions import Literal - -import transformer_lens.loading_from_pretrained as loading -from transformer_lens.ActivationCache import ActivationCache -from transformer_lens.components import ( - MLP, - BertBlock, - BertEmbed, - BertMLMHead, - BertNSPHead, - BertPooler, - Unembed, -) -from transformer_lens.components.mlps.gated_mlp import GatedMLP -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.FactoredMatrix import FactoredMatrix -from transformer_lens.hook_points import HookPoint -from transformer_lens.HookedRootModule import HookedRootModule -from transformer_lens.utilities import TypedModuleList, devices - -T = TypeVar("T", bound="HookedEncoder") - - -class HookedEncoder(HookedRootModule): - """ - This class implements a BERT-style encoder using the components in ./components.py, with HookPoints on every interesting activation. It inherits from HookedRootModule. - - Limitations: - - The model does not include dropouts, which may lead to inconsistent results from training or fine-tuning. - - Like HookedTransformer, it can have a pretrained Transformer's weights loaded via `.from_pretrained`. There are a few features you might know from HookedTransformer which are not yet supported: - - There is no preprocessing (e.g. LayerNorm folding) when loading a pretrained model - """ - - # Set by from_pretrained while constructing, so each public entry point - # emits exactly one DeprecationWarning. - _suppress_init_deprecation: bool = False - - blocks: TypedModuleList[BertBlock] - - def __init__( - self, - cfg: Union[HookedTransformerConfig, Dict], - tokenizer: Optional[Any] = None, - move_to_device: bool = True, - **kwargs: Any, - ): - super().__init__() - # from_pretrained warns at its own level (so the warning attributes to - # the caller and survives the default DeprecationWarning filter) and - # suppresses this one — each entry point fires exactly once. - if not getattr(type(self), "_suppress_init_deprecation", False): - warnings.warn( - "HookedEncoder is deprecated and will be removed in 4.0. Use " - "TransformerBridge.boot_transformers(...) instead.", - DeprecationWarning, - stacklevel=2, - ) - if isinstance(cfg, Dict): - cfg = HookedTransformerConfig(**cfg) - elif isinstance(cfg, str): - raise ValueError( - "Please pass in a config dictionary or HookedTransformerConfig object. If you want to load a pretrained model, use HookedEncoder.from_pretrained() instead." - ) - self.cfg = cfg - - assert self.cfg.n_devices == 1, "Multiple devices not supported for HookedEncoder" - if tokenizer is not None: - self.tokenizer = tokenizer - elif self.cfg.tokenizer_name is not None: - huggingface_token = os.environ.get("HF_TOKEN", "") - self.tokenizer = AutoTokenizer.from_pretrained( - self.cfg.tokenizer_name, - token=huggingface_token if len(huggingface_token) > 0 else None, - ) - else: - self.tokenizer = None - - if self.cfg.d_vocab == -1: - # If we have a tokenizer, vocab size can be inferred from it. - assert self.tokenizer is not None, "Must provide a tokenizer if d_vocab is not provided" - self.cfg.d_vocab = max(self.tokenizer.vocab.values()) + 1 - if self.cfg.d_vocab_out == -1: - self.cfg.d_vocab_out = self.cfg.d_vocab - - self.embed = BertEmbed(self.cfg) - self.blocks = TypedModuleList([BertBlock(self.cfg) for _ in range(self.cfg.n_layers)]) - self.mlm_head = BertMLMHead(self.cfg) - self.unembed = Unembed(self.cfg) - self.nsp_head = BertNSPHead(self.cfg) - self.pooler = BertPooler(self.cfg) - - self.hook_full_embed = HookPoint() - - if move_to_device: - if self.cfg.device is None: - raise ValueError("Cannot move to device when device is None") - self.to(self.cfg.device) - - self.setup() - - def to_tokens( - self, - input: Union[str, List[str]], - move_to_device: bool = True, - truncate: bool = True, - ) -> Tuple[ - Int[torch.Tensor, "batch pos"], - Int[torch.Tensor, "batch pos"], - Int[torch.Tensor, "batch pos"], - ]: - """Converts a string to a tensor of tokens. - Taken mostly from the HookedTransformer implementation, but does not support default padding - sides or prepend_bos. - Args: - input (Union[str, List[str]]): The input to tokenize. - move_to_device (bool): Whether to move the output tensor of tokens to the device the model lives on. Defaults to True - truncate (bool): If the output tokens are too long, whether to truncate the output - tokens to the model's max context window. Does nothing for shorter inputs. Defaults to - True. - """ - - assert self.tokenizer is not None, "Cannot use to_tokens without a tokenizer" - - encodings = self.tokenizer( - input, - return_tensors="pt", - padding=True, - truncation=truncate, - max_length=self.cfg.n_ctx if truncate else None, - ) - - tokens = encodings.input_ids - token_type_ids = encodings.token_type_ids - attention_mask = encodings.attention_mask - - if move_to_device: - tokens = tokens.to(self.cfg.device) - token_type_ids = token_type_ids.to(self.cfg.device) - attention_mask = attention_mask.to(self.cfg.device) - - return tokens, token_type_ids, attention_mask - - def encoder_output( - self, - tokens: Int[torch.Tensor, "batch pos"], - token_type_ids: Optional[Int[torch.Tensor, "batch pos"]] = None, - one_zero_attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, - ) -> Float[torch.Tensor, "batch pos d_vocab"]: - """Processes input through the encoder layers and returns the resulting residual stream. - - Args: - input: Input tokens as integers with shape (batch, position) - token_type_ids: Optional binary ids indicating segment membership. - Shape (batch_size, sequence_length). For example, with input - "[CLS] Sentence A [SEP] Sentence B [SEP]", token_type_ids would be - [0, 0, ..., 0, 1, ..., 1, 1] where 0 marks tokens from sentence A - and 1 marks tokens from sentence B. - one_zero_attention_mask: Optional binary mask of shape (batch_size, sequence_length) - where 1 indicates tokens to attend to and 0 indicates tokens to ignore. - Used primarily for handling padding in batched inputs. - - Returns: - resid: Final residual stream tensor of shape (batch, position, d_model) - - Raises: - AssertionError: If using string input without a tokenizer - """ - - if tokens.device.type != self.cfg.device: - tokens = tokens.to(self.cfg.device) - if one_zero_attention_mask is not None: - one_zero_attention_mask = one_zero_attention_mask.to(self.cfg.device) - - resid = self.hook_full_embed(self.embed(tokens, token_type_ids)) - - large_negative_number = -torch.inf - mask = ( - repeat(1 - one_zero_attention_mask, "batch pos -> batch 1 1 pos") - if one_zero_attention_mask is not None - else None - ) - additive_attention_mask = ( - torch.where(mask == 1, large_negative_number, 0) if mask is not None else None - ) - - for block in self.blocks: - resid = block(resid, additive_attention_mask) - - return resid - - @overload - def forward( - self, - input: Union[ - str, - List[str], - Int[torch.Tensor, "batch pos"], - ], - return_type: Union[Literal["logits"], Literal["predictions"]], - token_type_ids: Optional[Int[torch.Tensor, "batch pos"]] = None, - one_zero_attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, - ) -> Union[Float[torch.Tensor, "batch pos d_vocab"], str, List[str]]: - ... - - @overload - def forward( - self, - input: Union[ - str, - List[str], - Int[torch.Tensor, "batch pos"], - ], - return_type: Literal[None], - token_type_ids: Optional[Int[torch.Tensor, "batch pos"]] = None, - one_zero_attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, - ) -> Optional[Union[Float[torch.Tensor, "batch pos d_vocab"], str, List[str]]]: - ... - - def forward( - self, - input: Union[ - str, - List[str], - Int[torch.Tensor, "batch pos"], - ], - return_type: Optional[Union[Literal["logits"], Literal["predictions"]]] = "logits", - token_type_ids: Optional[Int[torch.Tensor, "batch pos"]] = None, - one_zero_attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, - ) -> Optional[Union[Float[torch.Tensor, "batch pos d_vocab"], str, List[str]]]: - """Forward pass through the HookedEncoder. Performs Masked Language Modelling on the given input. - - Args: - input: The input to process. Can be one of: - - str: A single text string - - List[str]: A list of text strings - - torch.Tensor: Input tokens as integers with shape (batch, position) - return_type: Optional[str]: The type of output to return. Can be one of: - - None: Return nothing, don't calculate logits - - 'logits': Return logits tensor - - 'predictions': Return human-readable predictions - token_type_ids: Optional[torch.Tensor]: Binary ids indicating whether a token belongs - to sequence A or B. For example, for two sentences: - "[CLS] Sentence A [SEP] Sentence B [SEP]", token_type_ids would be - [0, 0, ..., 0, 1, ..., 1, 1]. `0` represents tokens from Sentence A, - `1` from Sentence B. If not provided, BERT assumes a single sequence input. - This parameter gets inferred from the tokenizer if input is a string or list of strings. - Shape is (batch_size, sequence_length). - one_zero_attention_mask: Optional[torch.Tensor]: A binary mask which indicates - which tokens should be attended to (1) and which should be ignored (0). - Primarily used for padding variable-length sentences in a batch. - For instance, in a batch with sentences of differing lengths, shorter - sentences are padded with 0s on the right. If not provided, the model - assumes all tokens should be attended to. - This parameter gets inferred from the tokenizer if input is a string or list of strings. - Shape is (batch_size, sequence_length). - - Returns: - Optional[torch.Tensor]: Depending on return_type: - - None: Returns None if return_type is None - - torch.Tensor: Returns logits if return_type is 'logits' (or if return_type is not explicitly provided) - - Shape is (batch_size, sequence_length, d_vocab) - - str or List[str]: Returns predicted words for masked tokens if return_type is 'predictions'. - Returns a list of strings if input is a list of strings, otherwise a single string. - - Raises: - AssertionError: If using string input without a tokenizer - """ - - if isinstance(input, str) or isinstance(input, list): - assert self.tokenizer is not None, "Must provide a tokenizer if input is a string" - tokens, token_type_ids_from_tokenizer, attention_mask = self.to_tokens(input) - - # If token_type_ids or attention mask are not provided, use the ones from the tokenizer - token_type_ids = ( - token_type_ids_from_tokenizer if token_type_ids is None else token_type_ids - ) - one_zero_attention_mask = ( - attention_mask if one_zero_attention_mask is None else one_zero_attention_mask - ) - - else: - tokens = input - - resid = self.encoder_output(tokens, token_type_ids, one_zero_attention_mask) - - # MLM requires an unembedding step - resid = self.mlm_head(resid) - logits = self.unembed(resid) - - if return_type == "predictions": - assert ( - self.tokenizer is not None - ), "Must have a tokenizer to use return_type='predictions'" - # Get predictions for masked tokens - logprobs = logits[tokens == self.tokenizer.mask_token_id].log_softmax(dim=-1) - predictions = self.tokenizer.decode(logprobs.argmax(dim=-1)) - - # If input was a list of strings, split predictions into a list - if " " in predictions: - predictions = predictions.split(" ") - predictions = [f"Prediction {i}: {p}" for i, p in enumerate(predictions)] - return predictions - - elif return_type == None: - return None - - return logits - - @overload - def run_with_cache( - self, *model_args: Any, return_cache_object: Literal[True] = True, **kwargs: Any - ) -> Tuple[Float[torch.Tensor, "batch pos d_vocab"], ActivationCache]: - ... - - @overload - def run_with_cache( - self, *model_args: Any, return_cache_object: Literal[False], **kwargs: Any - ) -> Tuple[Float[torch.Tensor, "batch pos d_vocab"], Dict[str, torch.Tensor]]: - ... - - def run_with_cache( - self, - *model_args: Any, - return_cache_object: bool = True, - remove_batch_dim: bool = False, - **kwargs: Any, - ) -> Tuple[ - Float[torch.Tensor, "batch pos d_vocab"], - Union[ActivationCache, Dict[str, torch.Tensor]], - ]: - """ - Wrapper around run_with_cache in HookedRootModule. If return_cache_object is True, this will return an ActivationCache object, with a bunch of useful HookedTransformer specific methods, otherwise it will return a dictionary of activations as in HookedRootModule. This function was copied directly from HookedTransformer. - """ - out, cache_dict = super().run_with_cache( - *model_args, remove_batch_dim=remove_batch_dim, **kwargs - ) - if return_cache_object: - cache = ActivationCache(cache_dict, self, has_batch_dim=not remove_batch_dim) - return out, cache - else: - return out, cache_dict - - def to( # type: ignore - self, - device_or_dtype: Union[torch.device, str, torch.dtype], - print_details: bool = True, - ): - return devices.move_to_and_update_config(self, device_or_dtype, print_details) - - def cuda(self: T, device: Optional[Union[int, torch.device]] = None) -> T: - if isinstance(device, int): - return self.to(f"cuda:{device}") - elif device is None: - return self.to("cuda") - else: - return self.to(device) - - def cpu(self: T) -> T: - return self.to("cpu") - - def mps(self: T) -> T: - """Warning: MPS may produce silently incorrect results. See #1178.""" - return self.to(torch.device("mps")) - - @classmethod - def from_pretrained( - cls, - model_name: str, - checkpoint_index: Optional[int] = None, - checkpoint_value: Optional[int] = None, - hf_model: Optional[Any] = None, - device: Optional[str] = None, - tokenizer: Optional[Any] = None, - move_to_device: bool = True, - dtype: torch.dtype = torch.float32, - **from_pretrained_kwargs: Any, - ) -> HookedEncoder: - """Loads in the pretrained weights from huggingface. Currently supports loading weight from HuggingFace BertForMaskedLM. Unlike HookedTransformer, this does not yet do any preprocessing on the model.""" - warnings.warn( - "HookedEncoder.from_pretrained is deprecated and will be removed in 4.0. Use " - "TransformerBridge.boot_transformers(...) instead.", - DeprecationWarning, - stacklevel=2, - ) - logging.warning( - "Support for BERT in TransformerLens is currently experimental, until such a time when it has feature " - "parity with HookedTransformer and has been tested on real research tasks. Until then, backward " - "compatibility is not guaranteed. Please see the docs for information on the limitations of the current " - "implementation." - "\n" - "If using BERT for interpretability research, keep in mind that BERT has some significant architectural " - "differences to GPT. For example, LayerNorms are applied *after* the attention and MLP components, meaning " - "that the last LayerNorm in a block cannot be folded." - ) - - assert not ( - from_pretrained_kwargs.get("load_in_8bit", False) - or from_pretrained_kwargs.get("load_in_4bit", False) - ), "Quantization not supported" - - if "torch_dtype" in from_pretrained_kwargs: - dtype = from_pretrained_kwargs["torch_dtype"] - - official_model_name = loading.get_official_model_name(model_name) - - cfg = loading.get_pretrained_model_config( - official_model_name, - checkpoint_index=checkpoint_index, - checkpoint_value=checkpoint_value, - fold_ln=False, - device=device, - n_devices=1, - dtype=dtype, - **from_pretrained_kwargs, - ) - - state_dict = loading.get_pretrained_state_dict( - official_model_name, cfg, hf_model, dtype=dtype, **from_pretrained_kwargs - ) - - _prev_suppress = cls._suppress_init_deprecation - cls._suppress_init_deprecation = True - try: - model = cls(cfg, tokenizer, move_to_device=False) - finally: - cls._suppress_init_deprecation = _prev_suppress - - model.load_state_dict(state_dict, strict=False) - - if move_to_device: - if cfg.device is not None: - model.to(cfg.device) - - print(f"Loaded pretrained model {model_name} into HookedEncoder") - - return model - - @property - def W_U(self) -> Float[torch.Tensor, "d_model d_vocab"]: - """ - Convenience to get the unembedding matrix (ie the linear map from the final residual stream to the output logits) - """ - return self.unembed.W_U - - @property - def b_U(self) -> Float[torch.Tensor, "d_vocab"]: - """ - Convenience to get the unembedding bias - """ - return self.unembed.b_U - - @property - def W_E(self) -> Float[torch.Tensor, "d_vocab d_model"]: - """ - Convenience to get the embedding matrix - """ - return self.embed.embed.W_E - - @property - def W_pos(self) -> Float[torch.Tensor, "n_ctx d_model"]: - """ - Convenience function to get the positional embedding. Only works on models with absolute positional embeddings! - """ - return self.embed.pos_embed.W_pos - - @property - def W_E_pos(self) -> Float[torch.Tensor, "d_vocab+n_ctx d_model"]: - """ - Concatenated W_E and W_pos. Used as a full (overcomplete) basis of the input space, useful for full QK and full OV circuits. - """ - return torch.cat([self.W_E, self.W_pos], dim=0) - - @property - def W_K(self) -> Float[torch.Tensor, "n_layers n_heads d_model d_head"]: - """Stacks the key weights across all layers""" - return torch.stack([block.attn.W_K for block in self.blocks], dim=0) - - @property - def W_Q(self) -> Float[torch.Tensor, "n_layers n_heads d_model d_head"]: - """Stacks the query weights across all layers""" - return torch.stack([block.attn.W_Q for block in self.blocks], dim=0) - - @property - def W_V(self) -> Float[torch.Tensor, "n_layers n_heads d_model d_head"]: - """Stacks the value weights across all layers""" - return torch.stack([block.attn.W_V for block in self.blocks], dim=0) - - @property - def W_O(self) -> Float[torch.Tensor, "n_layers n_heads d_head d_model"]: - """Stacks the attn output weights across all layers""" - return torch.stack([block.attn.W_O for block in self.blocks], dim=0) - - @property - def W_in(self) -> Float[torch.Tensor, "n_layers d_model d_mlp"]: - """Stacks the MLP input weights across all layers""" - return torch.stack( - [cast(Union[MLP, GatedMLP], block.mlp).W_in for block in self.blocks], dim=0 - ) - - @property - def W_out(self) -> Float[torch.Tensor, "n_layers d_mlp d_model"]: - """Stacks the MLP output weights across all layers""" - return torch.stack( - [cast(Union[MLP, GatedMLP], block.mlp).W_out for block in self.blocks], dim=0 - ) - - @property - def b_K(self) -> Float[torch.Tensor, "n_layers n_heads d_head"]: - """Stacks the key biases across all layers""" - return torch.stack([block.attn.b_K for block in self.blocks], dim=0) - - @property - def b_Q(self) -> Float[torch.Tensor, "n_layers n_heads d_head"]: - """Stacks the query biases across all layers""" - return torch.stack([block.attn.b_Q for block in self.blocks], dim=0) - - @property - def b_V(self) -> Float[torch.Tensor, "n_layers n_heads d_head"]: - """Stacks the value biases across all layers""" - return torch.stack([block.attn.b_V for block in self.blocks], dim=0) - - @property - def b_O(self) -> Float[torch.Tensor, "n_layers d_model"]: - """Stacks the attn output biases across all layers""" - return torch.stack([block.attn.b_O for block in self.blocks], dim=0) - - @property - def b_in(self) -> Float[torch.Tensor, "n_layers d_mlp"]: - """Stacks the MLP input biases across all layers""" - return torch.stack( - [cast(Union[MLP, GatedMLP], block.mlp).b_in for block in self.blocks], dim=0 - ) - - @property - def b_out(self) -> Float[torch.Tensor, "n_layers d_model"]: - """Stacks the MLP output biases across all layers""" - return torch.stack( - [cast(Union[MLP, GatedMLP], block.mlp).b_out for block in self.blocks], dim=0 - ) - - @property - def QK(self) -> FactoredMatrix: # [n_layers, n_heads, d_model, d_model] - """Returns a FactoredMatrix object with the product of the Q and K matrices for each layer and head. - Useful for visualizing attention patterns.""" - return FactoredMatrix(self.W_Q, self.W_K.transpose(-2, -1)) - - @property - def OV(self) -> FactoredMatrix: # [n_layers, n_heads, d_model, d_model] - """Returns a FactoredMatrix object with the product of the O and V matrices for each layer and head.""" - return FactoredMatrix(self.W_V, self.W_O) - - def all_head_labels(self) -> List[str]: - """Returns a list of strings with the format "L{l}H{h}", where l is the layer index and h is the head index.""" - return [f"L{l}H{h}" for l in range(self.cfg.n_layers) for h in range(self.cfg.n_heads)] diff --git a/transformer_lens/HookedEncoderDecoder.py b/transformer_lens/HookedEncoderDecoder.py deleted file mode 100644 index 8ba8b53f21..0000000000 --- a/transformer_lens/HookedEncoderDecoder.py +++ /dev/null @@ -1,801 +0,0 @@ -"""Hooked EncoderDecoder - -Contains a T5 style model. This is separate from :class:`transformer_lens.HookedTransformer` -because it has a significantly different architecture to e.g. GPT style transformers. -""" - -from __future__ import annotations - -import logging -import os -import warnings -from itertools import chain -from pathlib import Path -from typing import ( - Any, - Dict, - List, - Optional, - Tuple, - Type, - TypeVar, - Union, - cast, - overload, -) - -import torch -import tqdm -from einops import repeat -from jaxtyping import Float, Int -from transformers import AutoTokenizer, PreTrainedTokenizerBase -from typing_extensions import Literal - -import transformer_lens.loading_from_pretrained as loading -from transformer_lens.ActivationCache import ActivationCache -from transformer_lens.components import MLP, Embed, GatedMLP, RMSNorm, T5Block, Unembed -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.FactoredMatrix import FactoredMatrix -from transformer_lens.hook_points import HookPoint -from transformer_lens.HookedRootModule import HookedRootModule -from transformer_lens.utilities import TypedModuleList, sample_logits, warn_if_mps -from transformer_lens.utilities.multi_gpu import get_device_for_block_index - -T = TypeVar("T", bound="HookedEncoderDecoder") - - -class HookedEncoderDecoder(HookedRootModule): - """ - This class implements a T5 encoder-decoder using the components in ./components.py, with HookPoints on every interesting activation. It inherits from HookedRootModule. - - Limitations: - - Also note that model does not include dropouts, which may lead to inconsistent results from training or fine-tuning. - - Like HookedTransformer, it can have a pretrained Transformer's weights loaded via `.from_pretrained`. There are a few features you might know from HookedTransformer which are not yet supported: - - There is no preprocessing (e.g. LayerNorm folding) when loading a pretrained model - - The model only accepts tokens as inputs, and not strings, or lists of strings - """ - - # Set by from_pretrained while constructing, so each public entry point - # emits exactly one DeprecationWarning. - _suppress_init_deprecation: bool = False - - tokenizer: Optional[PreTrainedTokenizerBase] - encoder: TypedModuleList[T5Block] - decoder: TypedModuleList[T5Block] - - def __init__( - self, - cfg: Union[HookedTransformerConfig, Dict], - tokenizer: Optional[PreTrainedTokenizerBase] = None, - move_to_device: bool = True, - **kwargs: Any, - ): - super().__init__() - # from_pretrained warns at its own level (so the warning attributes to - # the caller and survives the default DeprecationWarning filter) and - # suppresses this one — each entry point fires exactly once. - if not getattr(type(self), "_suppress_init_deprecation", False): - warnings.warn( - "HookedEncoderDecoder is deprecated and will be removed in 4.0. Use " - "TransformerBridge.boot_transformers(...) instead — " - "the bridge supports T5-style encoder-decoder models. " - "See docs/source/content/migrating_to_v3.md.", - DeprecationWarning, - stacklevel=2, - ) - if isinstance(cfg, Dict): - cfg = HookedTransformerConfig(**cfg) - elif isinstance(cfg, str): - raise ValueError( - "Please pass in a config dictionary or HookedTransformerConfig object. If you want to load a pretrained model, use HookedEncoderDecoder.from_pretrained() instead." - ) - self.cfg: HookedTransformerConfig = cfg - - if self.cfg.n_devices != 1: - raise ValueError("Multiple devices not supported for HookedEncoderDecoder") - if tokenizer is not None: - self.tokenizer = tokenizer - elif self.cfg.tokenizer_name is not None: - huggingface_token = os.environ.get("HF_TOKEN", "") - self.tokenizer = AutoTokenizer.from_pretrained( - self.cfg.tokenizer_name, - token=huggingface_token if len(huggingface_token) > 0 else None, - ) - else: - self.tokenizer = None - - if self.cfg.d_vocab == -1: - # If we have a tokenizer, vocab size can be inferred from it. - if self.tokenizer is None: - raise ValueError("Must provide a tokenizer if d_vocab is not provided") - - self.cfg.d_vocab = len(self.tokenizer) - if self.cfg.d_vocab_out == -1: - self.cfg.d_vocab_out = self.cfg.d_vocab - - self.embed = Embed(self.cfg) - self.encoder = TypedModuleList( - [ - T5Block(self.cfg, num_layer, is_decoder=False) - for num_layer in range(self.cfg.n_layers) - ] - ) - self.encoder_final_ln = RMSNorm(self.cfg) - self.decoder = TypedModuleList( - [ - T5Block(self.cfg, num_layer, is_decoder=True) - for num_layer in range(self.cfg.n_layers) - ] - ) - self.decoder_final_ln = RMSNorm(self.cfg) - self.unembed = Unembed(self.cfg) - - self.hook_embed = HookPoint() - - if move_to_device and self.cfg.device is not None: - self.to(self.cfg.device) - - self.setup() - - def to_tokens( - self, - input: Union[str, List[str]], - move_to_device: bool = True, - truncate: bool = True, - ) -> Tuple[Int[torch.Tensor, "batch pos"], Int[torch.Tensor, "batch pos"]]: - """Converts a string to a tensor of tokens. - Taken mostly from the HookedTransformer implementation, but does not support default padding - sides or prepend_bos. - - Args: - input (Union[str, List[str]]): The input to tokenize. - move_to_device (bool): Whether to move the output tensor of tokens to the device the - model lives on. Defaults to True - truncate (bool): If the output tokens are too long, whether to truncate the output - tokens to the model's max context window. Does nothing for shorter inputs. - Defaults to True. - """ - - assert self.tokenizer is not None, "Cannot use to_tokens without a tokenizer" - - encodings = self.tokenizer( - input, - return_tensors="pt", - padding=True, - truncation=truncate, - max_length=self.cfg.n_ctx if truncate else None, - ) - - tokens = encodings.input_ids - attention_mask = encodings.attention_mask - - if move_to_device: - tokens = tokens.to(self.cfg.device) - attention_mask = attention_mask.to(self.cfg.device) - return tokens, attention_mask - - @overload - def forward( - self, - input: Union[str, List[str], Int[torch.Tensor, "batch pos"]], - decoder_input: Optional[Int[torch.Tensor, "batch decoder_pos"]] = None, - return_type: Literal["logits"] = "logits", - one_zero_attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, - ) -> Float[torch.Tensor, "batch pos d_vocab"]: - ... - - @overload - def forward( - self, - input: Union[str, List[str], Int[torch.Tensor, "batch pos"]], - decoder_input: Optional[Int[torch.Tensor, "batch decoder_pos"]] = None, - return_type: Optional[Literal[None]] = None, - one_zero_attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, - ) -> Optional[Float[torch.Tensor, "batch pos d_vocab"]]: - ... - - def forward( - self, - input: Union[str, List[str], Int[torch.Tensor, "batch pos"]], - decoder_input: Optional[Int[torch.Tensor, "batch decoder_pos"]] = None, - return_type: Optional[str] = "logits", - one_zero_attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, - ) -> Optional[Float[torch.Tensor, "batch decoder_pos d_vocab"]]: - """Forward pass of the T5 model. - - Args: - input: Input to be processed. Can be one of: - - str: A single string input - - List[str]: A batch of string inputs - - Int[torch.Tensor, "batch pos"]: A batch of token IDs - decoder_input: Tensor of shape (batch, decoder_pos) containing the decoder input sequence. - If None and input is of type str or List[str], starts with batch of beginning-of-sequence (BOS) tokens. - return_type: Specifies the model output type: - - "logits": Return logits tensor - - None: Returns nothing - one_zero_attention_mask: A binary mask which indicates - which tokens should be attended to (1) and which should be ignored (0). - Primarily used for padding variable-length sentences in a batch. - For instance, in a batch with sentences of differing lengths, shorter - sentences are padded with 0s on the right. If not provided, the model - assumes all tokens should be attended to. - This parameter gets inferred from the tokenizer if input is a string or list of strings. - Shape is (batch_size, sequence_length). - - Returns: - Optional[Float[torch.Tensor, "batch decoder_pos d_vocab"]]: - If return_type="logits": Returns logits tensor of shape (batch, decoder_pos, vocab_size) - If return_type=None: Returns None - """ - - if isinstance(input, (str, list)): - tokens, attention_mask = self.to_tokens(input) - - # If attention mask is not provided, use the ones from the tokenizer - one_zero_attention_mask = ( - attention_mask if one_zero_attention_mask is None else one_zero_attention_mask - ) - - # If decoder_input is not provided, start with tensor of PAD tokens of shape (batch, 1) - if decoder_input is None: - assert self.tokenizer is not None - decoder_input = torch.full( - (tokens.shape[0], 1), - self.tokenizer.pad_token_id, - device=self.cfg.device, - ) - else: - tokens = input - - if one_zero_attention_mask is None: - logging.warning( - "No attention mask provided. Assuming all tokens should be attended to." - ) - - if decoder_input is None: - raise ValueError( - "Must provide decoder_input if input is not a string or list of strings" - ) - - if tokens.device.type != self.cfg.device: - tokens = tokens.to(self.cfg.device) - - if one_zero_attention_mask is not None: - one_zero_attention_mask = one_zero_attention_mask.to(self.cfg.device) - - resid = self.hook_embed(self.embed(tokens)) - - if one_zero_attention_mask is not None: - additive_attention_mask = ( - repeat(1 - one_zero_attention_mask, "batch pos -> batch 1 1 pos") - ) * torch.finfo(self.cfg.dtype).min - else: - additive_attention_mask = None - - query_len = key_len = tokens.shape[1] - - encoder_positional_bias = self.encoder[0].attn.compute_relative_attention_bias( - query_len, key_len, device=self.cfg.device - ) - - for encoder_block in self.encoder: - resid = encoder_block( - resid_pre=resid, - additive_attention_mask=additive_attention_mask, - position_bias=encoder_positional_bias, - ) - - encoder_resid = self.encoder_final_ln(resid) - - if decoder_input is None: - raise ValueError("decoder_input cannot be None when input is not a string") - decoder_resid = self.embed(decoder_input) - decoder_query_len = decoder_key_len = decoder_input.shape[1] - decoder_positional_bias = self.decoder[0].attn.compute_relative_attention_bias( - decoder_query_len, decoder_key_len, device=self.cfg.device - ) - - for decoder_block in self.decoder: - decoder_resid = decoder_block( - resid_pre=decoder_resid, - position_bias=decoder_positional_bias, - encoder_hidden_states=encoder_resid, - encoder_additive_attention_mask=additive_attention_mask, - ) - - decoder_resid = self.decoder_final_ln(decoder_resid) - - if self.cfg.tie_word_embeddings: - # Rescale output before projecting on vocab - # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/transformer/transformer.py#L586 - decoder_resid *= self.cfg.d_model**-0.5 - - logits = self.unembed(decoder_resid) - if return_type is None: - return None - return logits - - @torch.inference_mode() - def generate( - self, - input: Union[str, Int[torch.Tensor, "batch pos"]] = "", - one_zero_attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, - max_new_tokens: int = 10, - stop_at_eos: bool = True, - eos_token_id: Optional[Union[int, List[int]]] = None, - do_sample: bool = True, - top_k: Optional[int] = None, - top_p: Optional[float] = None, - temperature: float = 1.0, - freq_penalty: float = 0.0, - return_type: Optional[str] = "input", - verbose: bool = True, - ) -> Union[Int[torch.Tensor, "batch new_tokens"], str]: - """Sample tokens from the T5 encoder-decoder model. - - Sample tokens from the model until the model outputs eos_token or max_new_tokens is reached. - This function is primarily taken from HookedTransformer but adjusted for the HookedEncoderDecoder - architecture. - This function does not support key value caching and no default padding sides or prepend_bos. - - To avoid fiddling with ragged tensors, if we input a batch of text and some sequences finish - (by producing an EOT token), we keep running the model on the entire batch, but throw away - the output for a finished sequence and just keep adding EOTs to pad. - - This supports entering a single string, but not a list of strings - if the strings don't - tokenize to exactly the same length, this gets messy. If that functionality is needed, - convert them to a batch of tokens and input that instead. - - Args: - input (Union[str, Int[torch.Tensor, "batch pos"])]): Either a batch of tokens ([batch, - pos]) or a text string (this will be converted to a batch of tokens with batch size - 1). - max_new_tokens (int): Maximum number of tokens to generate. - stop_at_eos (bool): If True, stop generating tokens when the model outputs eos_token. - eos_token_id (Optional[Union[int, Sequence]]): The token ID to use for end - of sentence. If None, use the tokenizer's eos_token_id - required if using - stop_at_eos. It's also possible to provide a list of token IDs (not just the - eos_token_id), in which case the generation will stop when any of them are output - (useful e.g. for stable_lm). - do_sample (bool): If True, sample from the model's output distribution. Otherwise, use - greedy search (take the max logit each time). - top_k (int): Number of tokens to sample from. If None, sample from all tokens. - top_p (float): Probability mass to sample from. If 1.0, sample from all tokens. If <1.0, - we take the top tokens with cumulative probability >= top_p. - temperature (float): Temperature for sampling. Higher values will make the model more - random (limit of temp -> 0 is just taking the top token, limit of temp -> inf is - sampling from a uniform distribution). - freq_penalty (float): Frequency penalty for sampling - how much to penalise previous - tokens. Higher values will make the model more random. - return_type (Optional[str]): The type of the output to return - either a string (str), - a tensor of tokens (tensor) or whatever the format of the input was (input). - verbose (bool): If True, show tqdm progress bars for generation. - - Returns: - outputs (torch.Tensor): [batch, new_tokens], generated sequence of new tokens - (by default returns same type as input). - """ - - if isinstance(input, str): - # If text, convert to tokens (batch_size=1) - assert ( - self.tokenizer is not None - ), "Must provide a tokenizer if passing a string to the model" - encoder_input, attention_mask = self.to_tokens(input) - - # If attention mask is not provided, use the one from the tokenizer - one_zero_attention_mask = ( - attention_mask if one_zero_attention_mask is None else one_zero_attention_mask - ) - else: - assert isinstance(input, torch.Tensor) # keep mypy happy - encoder_input = input - - # If tokens are provided, user should be aware that attention mask will not be inferred - if one_zero_attention_mask is None: - logging.warning( - "No attention mask provided. Assuming all tokens should be attended to." - ) - - if return_type == "input": - if isinstance(input, str): - return_type = "str" - else: - return_type = "tensor" - - assert isinstance(encoder_input, torch.Tensor) - batch_size = encoder_input.shape[0] - device = get_device_for_block_index(0, self.cfg) - - # For the decoder input, we start with a tensor of PAD tokens of shape (batch, 1) - assert self.tokenizer is not None - decoder_input = torch.full((batch_size, 1), self.tokenizer.pad_token_id).to(device) - - stop_tokens: List[int] = [] - eos_token_for_padding = 0 - if stop_at_eos: - tokenizer_has_eos_token = self.tokenizer.eos_token_id is not None - - local_eos_token_id: Optional[Union[int, List[int]]] = eos_token_id - if local_eos_token_id is None: - assert ( - tokenizer_has_eos_token - ), "Must pass a eos_token_id if stop_at_eos is True and tokenizer is None or has no eos_token_id" - - local_eos_token_id = self.tokenizer.eos_token_id - - if isinstance(local_eos_token_id, int): - stop_tokens = [local_eos_token_id] - eos_token_for_padding = local_eos_token_id - else: - # eos_token_id is a Sequence (e.g. list or tuple) - if local_eos_token_id is None: - raise ValueError("eos_token_id cannot be None here") - stop_tokens = local_eos_token_id - eos_token_for_padding = ( - self.tokenizer.eos_token_id - if tokenizer_has_eos_token - else local_eos_token_id[0] - ) - - # An array to track which sequences in the batch have finished. - finished_sequences = torch.zeros(batch_size, dtype=torch.bool, device=self.cfg.device) - - # Currently nothing in HookedTransformer changes with eval, but this is here in case - # that changes in the future. - self.eval() - for _ in tqdm.tqdm(range(max_new_tokens), disable=not verbose): - # While generating, we keep generating logits, throw away all but the final logits, - # and then use those logits to sample from the distribution We keep adding the - # sampled tokens to the end of tokens. - # We input the entire sequence, as a [batch, pos] tensor, since we aren't using - # the cache. - - # Encoder input will be the same for all iterations - # Decoder input will be appended with the new token each iteration - logits = self.forward( - encoder_input, - decoder_input=decoder_input, - one_zero_attention_mask=one_zero_attention_mask, - ) - assert logits is not None - final_logits = logits[:, -1, :] - - if do_sample: - sampled_tokens = sample_logits( - final_logits, - top_k=top_k, - top_p=top_p, - temperature=temperature, - freq_penalty=freq_penalty, - tokens=decoder_input, - ).to(get_device_for_block_index(0, self.cfg)) - else: - sampled_tokens = final_logits.argmax(-1).to(get_device_for_block_index(0, self.cfg)) - - if stop_at_eos: - # For all unfinished sequences, add on the next token. If a sequence was - # finished, throw away the generated token and add eos_token_for_padding - # instead. - sampled_tokens[finished_sequences] = eos_token_for_padding - finished_sequences.logical_or_( - torch.isin( - sampled_tokens.to(self.cfg.device), - torch.tensor(stop_tokens).to(self.cfg.device), - ) - ) - - decoder_input = torch.cat([decoder_input, sampled_tokens.unsqueeze(-1)], dim=-1) - - if stop_at_eos and finished_sequences.all(): - break - - if return_type == "str": - assert self.tokenizer is not None - return cast(str, self.tokenizer.decode(decoder_input[0], skip_special_tokens=True)) - - else: - return decoder_input - - @overload # type: ignore[overload-overlap] - def run_with_cache( - self, *model_args: Any, return_cache_object: Literal[True] = True, **kwargs: Any - ) -> Tuple[Float[torch.Tensor, "batch pos d_vocab"], ActivationCache]: - ... - - @overload # type: ignore[overload-overlap] - def run_with_cache( - self, *model_args: Any, return_cache_object: Literal[False] = False, **kwargs: Any - ) -> Tuple[Float[torch.Tensor, "batch pos d_vocab"], Dict[str, torch.Tensor]]: - ... - - def run_with_cache( - self, - *model_args: Any, - return_cache_object: bool = True, - remove_batch_dim: bool = False, - **kwargs: Any, - ) -> Tuple[ - Float[torch.Tensor, "batch pos d_vocab"], - Union[ActivationCache, Dict[str, torch.Tensor]], - ]: - """ - Wrapper around run_with_cache in HookedRootModule. If return_cache_object is True, this will return an ActivationCache object, with a bunch of useful HookedTransformer specific methods, otherwise it will return a dictionary of activations as in HookedRootModule. This function was copied directly from HookedTransformer. - """ - out, cache_dict = super().run_with_cache( - *model_args, remove_batch_dim=remove_batch_dim, **kwargs - ) - if return_cache_object: - cache = ActivationCache(cache_dict, self, has_batch_dim=not remove_batch_dim) - return out, cache - else: - return out, cache_dict - - def to(self: T, *args: Any, **kwargs: Any) -> T: - return super().to(*args, **kwargs) - - def cuda(self: T, device: Optional[Union[int, torch.device]] = None) -> T: - if isinstance(device, int): - return self.to(f"cuda:{device}") - elif device is None: - return self.to("cuda") - else: - return self.to(device) - - def cpu(self: T) -> T: - return self.to("cpu") - - def mps(self: T) -> T: - """Warning: MPS may produce silently incorrect results. See #1178.""" - warn_if_mps("mps") - return self.to(torch.device("mps")) - - @classmethod - def from_pretrained( - cls: Type[T], - model_name: str, - checkpoint_index: Optional[int] = None, - checkpoint_value: Optional[int] = None, - hf_model: Optional[Any] = None, - device: Optional[str] = None, - tokenizer: Optional[Any] = None, - move_to_device: bool = True, - dtype: Optional[torch.dtype] = torch.float32, - **from_pretrained_kwargs: Any, - ) -> T: - """Loads in the pretrained weights from huggingface. Currently supports loading weight from HuggingFace BertForMaskedLM. Unlike HookedTransformer, this does not yet do any preprocessing on the model.""" - import warnings - - warnings.warn( - "HookedEncoderDecoder.from_pretrained is deprecated and will be removed in " - "4.0. Use TransformerBridge.boot_transformers(...) instead — " - "the bridge supports T5-style encoder-decoder models. See " - "docs/source/content/migrating_to_v3.md.", - DeprecationWarning, - stacklevel=2, - ) - - logging.warning( - "Support for T5 in TransformerLens is currently experimental, until such a time when it has feature " - "parity with HookedTransformer and has been tested on real research tasks. Until then, backward " - "compatibility is not guaranteed. Please see the docs for information on the limitations of the current " - "implementation." - "\n" - "If using T5 for interpretability research, keep in mind that T5 has some significant architectural " - "differences to GPT. The major one is that T5 is an Encoder-Decoder model" - "Also, it uses relative positional embeddings, different types of Attention (without bias) and LayerNorm" - ) - - if from_pretrained_kwargs.get("load_in_8bit", False) or from_pretrained_kwargs.get( - "load_in_4bit", False - ): - raise ValueError("Quantization not supported") - - if "torch_dtype" in from_pretrained_kwargs: - dtype = from_pretrained_kwargs["torch_dtype"] - - if dtype is None: - dtype = torch.float32 - - name_or_path = ( - model_name if Path(model_name).exists() else loading.get_official_model_name(model_name) - ) - - cfg = loading.get_pretrained_model_config( - name_or_path, - checkpoint_index=checkpoint_index, - checkpoint_value=checkpoint_value, - fold_ln=False, - device=device, - n_devices=1, - dtype=dtype, - **from_pretrained_kwargs, - ) - - state_dict = loading.get_pretrained_state_dict( - name_or_path, cfg, hf_model, dtype=dtype, **from_pretrained_kwargs - ) - - _prev_suppress = cls._suppress_init_deprecation - cls._suppress_init_deprecation = True - try: - model = cls(cfg, tokenizer, move_to_device=False) - finally: - cls._suppress_init_deprecation = _prev_suppress - - model.load_state_dict(state_dict, strict=False) - - if move_to_device and cfg.device is not None: - model.to(cfg.device) - - print(f"Loaded pretrained model {model_name} into HookedTransformer") - - return model - - @property - def W_U(self) -> Float[torch.Tensor, "d_model d_vocab"]: - """ - Convenience to get the unembedding matrix (ie the linear map from the final residual stream to the output logits) - """ - return self.unembed.W_U - - @property - def b_U(self) -> Float[torch.Tensor, "d_vocab"]: - """ - Convenience to get the unembedding bias - """ - return self.unembed.b_U - - @property - def W_E(self) -> Float[torch.Tensor, "d_vocab d_model"]: - """ - Convenience to get the embedding matrix - """ - return self.embed.W_E - - @property - def W_pos(self) -> None: - """ - Convenience function to get the positional embedding. Only works on models with absolute positional embeddings! - """ - raise NotImplementedError( - "T5 does not have absolute positional embeddings. Uses relative positional embeddings instead." - ) - - @property - def W_K(self) -> Float[torch.Tensor, "n_layers n_heads d_model d_head"]: - """Stacks the key weights across all layers""" - return torch.stack( - [block.attn.W_K for block in chain(self.encoder, self.decoder)], - dim=0, - ) - - @property - def W_Q(self) -> Float[torch.Tensor, "n_layers n_heads d_model d_head"]: - """Stacks the query weights across all layers""" - return torch.stack( - [block.attn.W_Q for block in chain(self.encoder, self.decoder)], - dim=0, - ) - - @property - def W_V(self) -> Float[torch.Tensor, "n_layers n_heads d_model d_head"]: - """Stacks the value weights across all layers""" - return torch.stack( - [block.attn.W_V for block in chain(self.encoder, self.decoder)], - dim=0, - ) - - @property - def W_O(self) -> Float[torch.Tensor, "n_layers n_heads d_head d_model"]: - """Stacks the attn output weights across all layers""" - return torch.stack( - [block.attn.W_O for block in chain(self.encoder, self.decoder)], - dim=0, - ) - - @property - def W_in(self) -> Float[torch.Tensor, "n_layers d_model d_mlp"]: - """Stacks the MLP input weights across all layers""" - weights: List[torch.Tensor] = [] - for block in chain(self.encoder, self.decoder): - mlp = block.mlp - if isinstance(mlp, (MLP, GatedMLP)): - weights.append(mlp.W_in) - else: - raise NotImplementedError( - f"W_in property is not supported for MLP of type {type(mlp).__name__}" - ) - return torch.stack(weights, dim=0) - - @property - def W_out(self) -> Float[torch.Tensor, "n_layers d_mlp d_model"]: - """Stacks the MLP output weights across all layers""" - weights: List[torch.Tensor] = [] - for block in chain(self.encoder, self.decoder): - mlp = block.mlp - if isinstance(mlp, (MLP, GatedMLP)): - weights.append(mlp.W_out) - else: - raise NotImplementedError( - f"W_out property is not supported for MLP of type {type(mlp).__name__}" - ) - return torch.stack(weights, dim=0) - - @property - def b_K(self) -> Float[torch.Tensor, "n_layers n_heads d_head"]: - """Stacks the key biases across all layers""" - return torch.stack( - [block.attn.b_K for block in chain(self.encoder, self.decoder)], - dim=0, - ) - - @property - def b_Q(self) -> Float[torch.Tensor, "n_layers n_heads d_head"]: - """Stacks the query biases across all layers""" - return torch.stack( - [block.attn.b_Q for block in chain(self.encoder, self.decoder)], - dim=0, - ) - - @property - def b_V(self) -> Float[torch.Tensor, "n_layers n_heads d_head"]: - """Stacks the value biases across all layers""" - return torch.stack( - [block.attn.b_V for block in chain(self.encoder, self.decoder)], - dim=0, - ) - - @property - def b_O(self) -> Float[torch.Tensor, "n_layers d_model"]: - """Stacks the attn output biases across all layers""" - return torch.stack( - [block.attn.b_O for block in chain(self.encoder, self.decoder)], - dim=0, - ) - - @property - def b_in(self) -> Float[torch.Tensor, "n_layers d_mlp"]: - """Stacks the MLP input biases across all layers""" - biases: List[torch.Tensor] = [] - for block in chain(self.encoder, self.decoder): - mlp = block.mlp - if isinstance(mlp, (MLP, GatedMLP)): - biases.append(mlp.b_in) - else: - raise NotImplementedError( - f"b_in property is not supported for MLP of type {type(mlp).__name__}" - ) - return torch.stack(biases, dim=0) - - @property - def b_out(self) -> Float[torch.Tensor, "n_layers d_model"]: - """Stacks the MLP output biases across all layers""" - biases: List[torch.Tensor] = [] - for block in chain(self.encoder, self.decoder): - mlp = block.mlp - if isinstance(mlp, (MLP, GatedMLP)): - biases.append(mlp.b_out) - else: - raise NotImplementedError( - f"b_out property is not supported for MLP of type {type(mlp).__name__}" - ) - return torch.stack(biases, dim=0) - - @property - def QK(self) -> FactoredMatrix: # [n_layers, n_heads, d_model, d_model] - """Returns a FactoredMatrix object with the product of the Q and K matrices for each layer and head. - Useful for visualizing attention patterns.""" - return FactoredMatrix(self.W_Q, self.W_K.transpose(-2, -1)) - - @property - def OV(self) -> FactoredMatrix: # [n_layers, n_heads, d_model, d_model] - """Returns a FactoredMatrix object with the product of the O and V matrices for each layer and head.""" - return FactoredMatrix(self.W_V, self.W_O) - - def all_head_labels(self) -> List[str]: - """Returns a list of strings with the format "L{l}H{h}", where l is the layer index and h is the head index.""" - return [f"EL{l}H{h}" for l in range(self.cfg.n_layers) for h in range(self.cfg.n_heads)] + [ - f"DL{l}H{h}" for l in range(self.cfg.n_layers) for h in range(self.cfg.n_heads) - ] diff --git a/transformer_lens/HookedTransformer.py b/transformer_lens/HookedTransformer.py deleted file mode 100644 index 5d700c6841..0000000000 --- a/transformer_lens/HookedTransformer.py +++ /dev/null @@ -1,2835 +0,0 @@ -"""Hooked Transformer. - -The Hooked Transformer is the core part of TransformerLens. - -In common PyTorch model implementations (e.g. ones from HuggingFace) it's fairly easy to extract -model weights, but much harder to extract activations. TransformerLens aims to simplify this task by -attaching hooks to every notable activation within the model. This enables the inspection and/or -alteration of activations in individual components like attention heads and MLP layers, facilitating -a deeper understanding of the internal workings of transformers like GPT-2. -""" - -from __future__ import annotations - -import logging -import os -import warnings -from collections.abc import Generator -from typing import ( - Any, - Dict, - List, - NamedTuple, - Optional, - Tuple, - Type, - TypeVar, - Union, - cast, - overload, -) - -import einops -import numpy as np -import torch -import torch.nn as nn -import tqdm.auto as tqdm -from jaxtyping import Float, Int -from transformers import AutoTokenizer, PreTrainedModel, PreTrainedTokenizerBase -from transformers.models.auto.tokenization_auto import AutoTokenizer -from transformers.tokenization_utils_base import PreTrainedTokenizerBase -from typing_extensions import Literal - -import transformer_lens.loading_from_pretrained as loading -import transformer_lens.utilities as utils -from transformer_lens.ActivationCache import ActivationCache - -# Activation cache for run_with_cache; KV cache for generation -from transformer_lens.cache.key_value_cache import TransformerLensKeyValueCache -from transformer_lens.components import ( - Embed, - LayerNorm, - LayerNormPre, - PosEmbed, - RMSNorm, - RMSNormPre, - TransformerBlock, - Unembed, -) -from transformer_lens.components.mlps.gated_mlp import GatedMLP -from transformer_lens.components.mlps.mlp import MLP -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.FactoredMatrix import FactoredMatrix -from transformer_lens.hook_points import HookPoint -from transformer_lens.HookedRootModule import HookedRootModule -from transformer_lens.loading_from_pretrained import NON_HF_HOSTED_MODEL_NAMES -from transformer_lens.utilities import ( - USE_DEFAULT_VALUE, - TypedModuleList, - apply_softcap, - get_best_available_device, - get_device_for_block_index, - init_kaiming_normal_, - init_kaiming_uniform_, - init_xavier_normal_, - init_xavier_uniform_, - softcap_enabled, -) -from transformer_lens.utilities.architectures import POST_NORM_ARCHITECTURES -from transformer_lens.utilities.devices import move_to_and_update_config -from transformer_lens.weight_processing import ProcessWeights - -SingleLoss = Float[torch.Tensor, ""] # Type alias for a single element tensor -LossPerToken = Float[torch.Tensor, "batch pos-1"] -Loss = Union[SingleLoss, LossPerToken] - -DTYPE_FROM_STRING = { - "float32": torch.float32, - "fp32": torch.float32, - "float16": torch.float16, - "fp16": torch.float16, - "bfloat16": torch.bfloat16, - "bf16": torch.bfloat16, -} - -T = TypeVar("T", bound="HookedTransformer") - - -class Output(NamedTuple): - """Output Named Tuple. - - Named tuple object for if we want to output both logits and loss. - """ - - logits: Float[torch.Tensor, "batch pos d_vocab"] - loss: Loss - - -class HookedTransformer(HookedRootModule): - """Hooked Transformer. - - Implements a full Transformer using the components :doc:`here <transformer_lens.components>`, - with a :class:`transformer_lens.hook_points.HookPoint` on every interesting activation. - - TransformerLens comes loaded with >50 GPT-style models. Typically you initialise it with one of - these via :meth:`from_pretrained`, although it can also be instantiated with randomly - initialized weights via :meth:`__init__`. - - Once you've initialized the model, a common next step is to test it can do the task you're - investigating. This can be done with :func:`transformer_lens.utils.test_prompt`. - - Tokenization notes - ------------------ - - :meth:`to_tokens`, :meth:`to_str_tokens`, :meth:`get_token_position`, - :meth:`forward` (string input), and :meth:`generate` accept ``prepend_bos`` - to control BOS prepending. Resolution: explicit arg → - ``cfg.default_prepend_bos`` (defaults ``True``, even for non-BOS-trained - models — attention heads tend to use position 0 as a resting state). - **Pass ``prepend_bos=False`` when tokenizing a fragment of a larger - prompt** — off-by-one position errors usually trace back here. - - Reconciliation with ``cfg.tokenizer_prepends_bos`` (set by - :meth:`set_tokenizer` for tokenizers that add BOS automatically) is - handled internally — pass the value you want; the framework adds or - strips manually as needed. - - BPE/SentencePiece tokenizers treat ``"hello"``, ``" hello"``, and - ``"Hello"`` as distinct tokens. Concatenated prompts may not tokenize - as the sum of parts — inspect with :meth:`to_str_tokens` when in doubt. - """ - - # Set by from_pretrained while constructing, so each public entry point - # emits exactly one DeprecationWarning. - _suppress_init_deprecation: bool = False - # Same mechanism for from_pretrained_no_processing's delegation to - # from_pretrained. - _suppress_fp_deprecation: bool = False - - ln_final: nn.Module - tokenizer: Optional[PreTrainedTokenizerBase] - blocks: TypedModuleList[TransformerBlock] - - def __init__( - self, - cfg: Union[HookedTransformerConfig, Dict], - tokenizer: Optional[PreTrainedTokenizerBase] = None, - move_to_device: bool = True, - default_padding_side: Optional[Literal["left", "right"]] = None, - ): - """Model initialization. - - Note that if you want to load the model from pretrained weights, you should use - :meth:`from_pretrained` instead. - - Args: - cfg: The config to use for the model. - tokenizer: The tokenizer to use for the model. If not provided, it is inferred from - `cfg.tokenizer_name` or initialized to `None`. If `None`, then the model cannot be - passed strings, and d_vocab must be explicitly set. - move_to_device: Whether to move the model to the device specified in cfg. - device. Must be true if `n_devices` in the config is greater than 1, since the - model's layers will be split across multiple devices. - default_padding_side: Which side to pad on. - """ - super().__init__() - # from_pretrained warns at its own level (so the warning attributes to - # the caller and survives the default DeprecationWarning filter) and - # suppresses this one — each entry point fires exactly once. - if not getattr(type(self), "_suppress_init_deprecation", False): - warnings.warn( - "HookedTransformer is deprecated and will be removed in 4.0. Use " - "TransformerBridge.boot_transformers(...) instead, then call " - "enable_compatibility_mode() for HookedTransformer-equivalent numerics.", - DeprecationWarning, - stacklevel=2, - ) - if isinstance(cfg, str): - raise ValueError( - "Please pass in a config dictionary or HookedTransformerConfig object. If you want to load a " - "pretrained model, use HookedTransformer.from_pretrained() instead." - ) - - self.cfg = HookedTransformerConfig.unwrap(cfg) - if tokenizer is not None: - self.set_tokenizer(tokenizer, default_padding_side=default_padding_side) - elif self.cfg.tokenizer_name is not None: - # If we have a tokenizer name, we can load it from HuggingFace - if self.cfg.tokenizer_name in NON_HF_HOSTED_MODEL_NAMES: - logging.warning( - "%s tokenizer not loaded. Please load manually.", - self.cfg.tokenizer_name, - ) - else: - # Hugging Face defaults to use_fast to True - use_fast = True - # Phi model's fast tokenizer does not support adding a BOS token, use_fast - # should be False - if "phi" in self.cfg.tokenizer_name.lower(): - use_fast = False - huggingface_token = os.environ.get("HF_TOKEN", "") - add_bos_token = self.cfg.original_architecture not in [ - "OlmoForCausalLM", - "OlmoeForCausalLM", - "Olmo2ForCausalLM", - "Qwen3ForCausalLM", - "PhiForCausalLM", - ] - self.set_tokenizer( - AutoTokenizer.from_pretrained( - self.cfg.tokenizer_name, - add_bos_token=add_bos_token, - trust_remote_code=self.cfg.trust_remote_code, - use_fast=use_fast, - token=huggingface_token if len(huggingface_token) > 0 else None, - ), - default_padding_side=default_padding_side, - ) - else: - # If no tokenizer name is provided, we assume we're training on an algorithmic task and - # will pass in tokens directly. In this case, we don't need a tokenizer. - assert self.cfg.d_vocab != -1, "Must provide a tokenizer if d_vocab is not provided" - self.tokenizer = None - if default_padding_side != None: - logging.warning( - "default_padding_side is explicitly given but ignored because tokenizer is not set." - ) - - self.embed = Embed(self.cfg) - self.hook_embed = HookPoint() # [batch, pos, d_model] - - if self.cfg.positional_embedding_type != "rotary": - self.pos_embed = PosEmbed(self.cfg) - self.hook_pos_embed = HookPoint() # [batch, pos, d__dictmodel] - - if self.cfg.use_hook_tokens: - self.hook_tokens = HookPoint() # [batch, pos] - - self.blocks = TypedModuleList( - [TransformerBlock(self.cfg, block_index) for block_index in range(self.cfg.n_layers)] - ) - - if self.cfg.normalization_type == "RMS": - self.ln_final = RMSNorm(self.cfg) - elif self.cfg.normalization_type == "RMSPre": - self.ln_final = RMSNormPre(self.cfg) - elif self.cfg.normalization_type == "LN": - if self.cfg.final_rms: - self.ln_final = RMSNorm(self.cfg) - else: - self.ln_final = LayerNorm(self.cfg) - elif self.cfg.normalization_type == "LNPre": - # We've folded in LayerNorm weights, so just need the center + scale parts - if self.cfg.final_rms: - self.ln_final = RMSNormPre(self.cfg) - else: - self.ln_final = LayerNormPre(self.cfg) - elif self.cfg.normalization_type is None: - # If it's None, don't create either layer - pass - else: - logging.warning("Invalid normalization_type passed in %s", self.cfg.normalization_type) - self.unembed = Unembed(self.cfg) - - if self.cfg.init_weights: - self.init_weights() - - if move_to_device: - # We load the devices in a pipeline manner - the first device gets the embed and - # pos_embed layers and the first n_layers // n_devices blocks, the second gets the next - # n_layers // n_devices blocks ... the last gets the last n_layers // n_devices blocks, - # the final normalization layer (if it exists) and the unembed layer - self.move_model_modules_to_device() - - # Helper variable to store a small (10K-20K) dataset of training data. Empty by default, can - # be loaded with load_sample_training_dataset - self.dataset = None - - # Gives each module a parameter with its name (relative to this root module) - # Needed for HookPoints to work - self.setup() - - def check_hooks_to_add( - self, - hook_point, - hook_point_name, - hook, - dir="fwd", - is_permanent=False, - prepend=False, - ) -> None: - if hook_point_name.endswith("attn.hook_result"): - assert ( - self.cfg.use_attn_result - ), f"Cannot add hook {hook_point_name} if use_attn_result_hook is False" - if hook_point_name.endswith(("hook_q_input", "hook_k_input", "hook_v_input")): - assert ( - self.cfg.use_split_qkv_input - ), f"Cannot add hook {hook_point_name} if use_split_qkv_input is False" - if hook_point_name.endswith("mlp_in"): - assert ( - self.cfg.use_hook_mlp_in - ), f"Cannot add hook {hook_point_name} if use_hook_mlp_in is False" - if hook_point_name.endswith("attn_in"): - assert ( - self.cfg.use_attn_in - ), f"Cannot add hook {hook_point_name} if use_attn_in is False" - - def get_pos_offset(self, past_kv_cache, batch_size): - # If we're doing caching, then we reuse keys and values from previous runs, as that's the - # only way that past activations will affect the final logits. The cache contains those so - # we don't need to recompute them. This is useful for generating text. As we have absolute - # positional encodings, to implement this we have a `pos_offset` variable, defaulting to - # zero, which says to offset which positional encodings are used (cached keys and values - # were calculated with their own positional encodings). - if past_kv_cache is None: - pos_offset = 0 - else: - ( - cached_batch_size, - cache_ctx_length, - num_heads_in_cache, - d_head_in_cache, - ) = past_kv_cache[0].past_keys.shape - assert cached_batch_size == batch_size - if self.cfg.n_key_value_heads is None: - assert num_heads_in_cache == self.cfg.n_heads - else: - assert num_heads_in_cache == self.cfg.n_key_value_heads - assert d_head_in_cache == self.cfg.d_head - pos_offset = cache_ctx_length - return pos_offset - - def get_residual( - self, - embed, - pos_offset, - prepend_bos=USE_DEFAULT_VALUE, - attention_mask=None, - tokens=None, - return_shortformer_pos_embed=True, - device=None, - ): - if device is None: - device = get_device_for_block_index(0, self.cfg) - - if tokens is None: - # Because tokens only need for defining batch size and sequence length, we can simply synthesize them - tokens = torch.ones((embed.size(0), embed.size(1))).int().to(device) - - if self.cfg.positional_embedding_type == "standard": - pos_embed = self.hook_pos_embed( - self.pos_embed(tokens, pos_offset, attention_mask) - ) # [batch, pos, d_model] - residual = embed + pos_embed # [batch, pos, d_model] - shortformer_pos_embed = None - elif self.cfg.positional_embedding_type == "shortformer": - # If we're using shortformer style attention, we don't add the positional embedding to - # the residual stream. See HookedTransformerConfig for details - pos_embed = self.hook_pos_embed( - self.pos_embed(tokens, pos_offset, attention_mask) - ) # [batch, pos, d_model] - residual = embed - shortformer_pos_embed = pos_embed - elif self.cfg.positional_embedding_type == "rotary": - # Rotary doesn't use positional embeddings, instead they're applied when dot producting - # keys and queries. See HookedTransformerConfig for details - residual = embed - shortformer_pos_embed = None - elif self.cfg.positional_embedding_type == "alibi": - # ALiBi does not add positional embeddings to word embeddings,instead it biases QK attention scores. - residual = embed - shortformer_pos_embed = None - else: - raise ValueError( - f"Invalid positional_embedding_type passed in {self.cfg.positional_embedding_type}" - ) - - if return_shortformer_pos_embed: - return residual, shortformer_pos_embed - else: - return residual - - def input_to_embed( - self, - input: Union[str, List[str], Int[torch.Tensor, "batch pos"]], - prepend_bos: Optional[Union[bool, None]] = USE_DEFAULT_VALUE, - padding_side: Optional[Union[Literal["left", "right"], None]] = USE_DEFAULT_VALUE, - attention_mask: Optional[torch.Tensor] = None, - past_kv_cache: Optional[TransformerLensKeyValueCache] = None, - ) -> Tuple[ - Float[torch.Tensor, "batch pos d_model"], # residual - Optional[Int[torch.Tensor, "batch pos"]], # tokens - Optional[Float[torch.Tensor, "batch pos d_model"]], # shortformer_pos_embed - Optional[torch.Tensor], # attention_mask [batch pos] - ]: - """Convert input to first residual stream. - - Args: - input (Union[str, List[str], Int[torch.Tensor, "batch pos"]]): The input to the model. - prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend - the BOS token to the input (only applies when input is a string). Defaults to None, - implying usage of self.cfg.default_prepend_bos which is set to True unless specified - otherwise. Pass True or False to locally override the default. - padding_side ([Literal["left", "right"], optional): Overrides - self.tokenizer.padding_side. Specifies which side to pad when tokenizing - multiple strings of different lengths. - past_kv_cache (TransformerLensKeyValueCache, optional): If passed, we're doing caching - and attention_mask will be stored in the cache. - """ - if isinstance(input, str) or isinstance(input, list): - # If text, convert to tokens (batch_size=1) - assert ( - self.tokenizer is not None - ), "Must provide a tokenizer if passing a string to the model" - # This is only intended to support passing in a single string - tokens = self.to_tokens(input, prepend_bos=prepend_bos, padding_side=padding_side) - else: - tokens = input - if len(tokens.shape) == 1: - # If tokens are a rank 1 tensor, add a dummy batch dimension to avoid things breaking. - tokens = tokens[None] - if tokens.device.type != self.cfg.device: - tokens = tokens.to(get_device_for_block_index(0, self.cfg)) - - if ( - (self.tokenizer and self.tokenizer.padding_side == "left") - or attention_mask is not None - or past_kv_cache is not None - ): - # This means we need to have an explicit attention mask. - if attention_mask is None: - # If the padding side is left or we are using caching, we need to compute the attention - # mask for the adjustment of absolute positional embeddings and attention masking so - # that pad tokens are not attended. - if prepend_bos is USE_DEFAULT_VALUE: - prepend_bos = self.cfg.default_prepend_bos - if self.tokenizer is None: - raise ValueError("Cannot compute attention mask without a tokenizer.") - attention_mask = utils.get_attention_mask(self.tokenizer, tokens, prepend_bos) - - assert attention_mask.shape == tokens.shape, ( - f"Attention mask shape {attention_mask.shape} does not match tokens shape " - f"{tokens.shape}" - ) - attention_mask = attention_mask.to(get_device_for_block_index(0, self.cfg)) - if past_kv_cache is not None: - # past_kv_cache is not None, so we're doing caching. - # We need to extend the previous attention_mask. - # Update the past_kv_cache with the new attention_mask (unless it's frozen) - attention_mask = past_kv_cache.append_attention_mask(attention_mask) - else: - # We separate this case from for computational efficiency. - attention_mask = None - - batch_size = tokens.shape[0] - pos_offset = self.get_pos_offset(past_kv_cache, batch_size) - - if self.cfg.use_hook_tokens: - tokens = self.hook_tokens(tokens) - - embed = self.hook_embed(self.embed(tokens)) # [batch, pos, d_model] - residual, shortformer_pos_embed = self.get_residual( - embed, - pos_offset, - prepend_bos, - attention_mask, - tokens, - return_shortformer_pos_embed=True, - ) - return residual, tokens, shortformer_pos_embed, attention_mask - - @overload - def forward( - self, - input, - return_type: Literal["logits"], - loss_per_token: bool = False, - prepend_bos: Optional[Union[bool, None]] = USE_DEFAULT_VALUE, - padding_side: Optional[Union[Literal["left", "right"], None]] = USE_DEFAULT_VALUE, - start_at_layer: Optional[int] = None, - tokens: Optional[Int[torch.Tensor, "batch pos"]] = None, - shortformer_pos_embed: Optional[Float[torch.Tensor, "batch pos d_model"]] = None, - attention_mask: Optional[torch.Tensor] = None, # [batch pos] - stop_at_layer: Optional[int] = None, - past_kv_cache: Optional[TransformerLensKeyValueCache] = None, - ) -> Loss: - ... - - @overload - def forward( - self, - input, - return_type: Literal["loss"], - loss_per_token: bool = False, - prepend_bos: Optional[Union[bool, None]] = USE_DEFAULT_VALUE, - padding_side: Optional[Union[Literal["left", "right"], None]] = USE_DEFAULT_VALUE, - start_at_layer: Optional[int] = None, - tokens: Optional[Int[torch.Tensor, "batch pos"]] = None, - shortformer_pos_embed: Optional[Float[torch.Tensor, "batch pos d_model"]] = None, - attention_mask: Optional[torch.Tensor] = None, # [batch pos] - stop_at_layer: Optional[int] = None, - past_kv_cache: Optional[TransformerLensKeyValueCache] = None, - ) -> Loss: - ... - - @overload - def forward( - self, - input, - return_type: Literal["both"], - loss_per_token: bool = False, - prepend_bos: Optional[Union[bool, None]] = USE_DEFAULT_VALUE, - padding_side: Optional[Union[Literal["left", "right"], None]] = USE_DEFAULT_VALUE, - start_at_layer: Optional[int] = None, - tokens: Optional[Int[torch.Tensor, "batch pos"]] = None, - shortformer_pos_embed: Optional[Float[torch.Tensor, "batch pos d_model"]] = None, - attention_mask: Optional[torch.Tensor] = None, # [batch pos] - stop_at_layer: Optional[int] = None, - past_kv_cache: Optional[TransformerLensKeyValueCache] = None, - ) -> Tuple[Float[torch.Tensor, "batch pos d_vocab"], Loss]: - ... - - @overload - def forward( - self, - input, - return_type: Literal[None], - loss_per_token: bool = False, - prepend_bos: Optional[Union[bool, None]] = USE_DEFAULT_VALUE, - padding_side: Optional[Union[Literal["left", "right"], None]] = USE_DEFAULT_VALUE, - start_at_layer: Optional[int] = None, - tokens: Optional[Int[torch.Tensor, "batch pos"]] = None, - shortformer_pos_embed: Optional[Float[torch.Tensor, "batch pos d_model"]] = None, - attention_mask: Optional[torch.Tensor] = None, # [batch pos] - stop_at_layer: Optional[int] = None, - past_kv_cache: Optional[TransformerLensKeyValueCache] = None, - ) -> None: - ... - - def forward( - self, - input: Union[ - str, - List[str], - Int[torch.Tensor, "batch pos"], - Float[torch.Tensor, "batch pos d_model"], - ], - return_type: Optional[str] = "logits", - loss_per_token: bool = False, - prepend_bos: Optional[Union[bool, None]] = USE_DEFAULT_VALUE, - padding_side: Optional[Literal["left", "right"]] = USE_DEFAULT_VALUE, - start_at_layer: Optional[int] = None, - tokens: Optional[Int[torch.Tensor, "batch pos"]] = None, - shortformer_pos_embed: Optional[Float[torch.Tensor, "batch pos d_model"]] = None, - attention_mask: Optional[torch.Tensor] = None, # [batch pos] - stop_at_layer: Optional[int] = None, - past_kv_cache: Optional[TransformerLensKeyValueCache] = None, - ) -> Union[ - None, - Float[torch.Tensor, "batch pos d_vocab"], - Loss, - Tuple[Float[torch.Tensor, "batch pos d_vocab"], Loss], - ]: - """Forward Pass. - - Input is either a batch of tokens ([batch, pos]) or a text string, a string is automatically - tokenized to a batch of a single element. The prepend_bos flag only applies when inputting a - text string. - - Note that loss is the standard "predict the next token" cross-entropy loss for GPT-2 style - language models - if you want a custom loss function, the recommended behaviour is returning - the logits and then applying your custom loss function. - - Args: - return_type Optional[str]: The type of output to return. Can be one of: None (return - nothing, don't calculate logits), 'logits' (return logits), 'loss' (return - cross-entropy loss), 'both' (return logits and loss). - loss_per_token bool: Whether to return the (next token prediction) loss per token (True) - or average (False). Average loss is a scalar (averaged over position *and* batch), - per-token loss is a tensor ([batch, position-1]) - position-1 because we're - predicting the next token, and there's no specified next token for the final token. - Defaults to False. - prepend_bos Optional[bool]: Overrides self.cfg.default_prepend_bos. Whether to prepend - the BOS token to the input (only applies when input is a string). Defaults to None, - implying usage of self.cfg.default_prepend_bos which is set to True unless specified - otherwise. (Even for models not explicitly trained with a prepended BOS token, heads - often use the first position as a resting position and accordingly lose information - from the first token, so this empirically seems to give better results.) Pass True - or False to locally override the default. - padding_side Optional[Literal["left", "right"]]: Overrides self.tokenizer.padding_side. - Specifies which side to pad on when tokenizing multiple strings of different - lengths. - start_at_layer Optional[int]: If not None, start the forward pass at the specified - layer. Requires input to be the residual stream before the specified layer with - shape [batch, pos, d_model]. Inclusive - ie, start_at_layer = 0 skips the embedding - then runs the rest of the model. Supports negative indexing. start_at_layer = -1 - only runs the final block and the unembedding. Defaults to None (run the full - model). - tokens: Optional[Int[torch.Tensor, "batch pos"]]: Tokenized input. Only use if - start_at_layer is not None and return type is "loss" or "both". - shortformer_pos_embed: Optional[Float[torch.Tensor, "batch pos d_model"]]: Positional - embedding for shortformer models. Only use if start_at_layer is not None and - self.cfg.positional_embedding_type == "shortformer". - attention_mask: Optional[torch.Tensor]: Override the attention mask used to ignore - padded tokens. If start_at_layer is not None and (self.tokenizer.padding_side == - "left" or past_kv_cache is not None), this should be passed as the attention mask - is not computed automatically. Defaults to None. - stop_at_layer Optional[int]: If not None, stop the forward pass at the specified layer. - Exclusive - ie, stop_at_layer = 0 will only run the embedding layer, stop_at_layer = - 1 will run the embedding layer and the first transformer block, etc. Supports - negative indexing. Useful for analysis of intermediate layers, eg finding neuron - activations in layer 3 of a 24 layer model. Defaults to None (run the full model). - If not None, we return the last residual stream computed. - past_kv_cache Optional[TransformerLensKeyValueCache]: If not None, keys and values - will be stored for every attention head (unless the cache is frozen). If there are - keys and values already in the cache, these will be prepended to the keys and values - for the new input, so that the new tokens can pay attention to previous tokens. This - is useful for generating text, because we don't need to repeat computation for - tokens that have already been through the model. Also caches attention_mask so - previous tokens are masked correctly (unless frozen). Padding should be ignored in - all cases, so it's okay to eg. pass in left padded tokens twice in a row. - Warning: Don't accidentally prepend_bos to the second half of a prompt. - Defaults to None (don't use caching). - """ - - with utils.LocallyOverridenDefaults( - self, prepend_bos=prepend_bos, padding_side=padding_side - ): - if start_at_layer is None: - ( - residual, - tokens, - shortformer_pos_embed, - attention_mask, - ) = self.input_to_embed( - input, - prepend_bos=prepend_bos, - padding_side=padding_side, - attention_mask=attention_mask, - past_kv_cache=past_kv_cache, - ) - else: - assert type(input) == torch.Tensor - residual = input - - if start_at_layer is None: - start_at_layer = 0 - # If we explicitly want to start or stop at a layer, we only iterate through the blocks - # between those indices. Note that start_at_layer is inclusive and stop_at_layer is - # exclusive. - # Eg: start_at_layer==None + stop_at_layer==0 means to only run the embed. - # Eg: start_at_layer==3 + stop_at_layer==-1 means to run from layer 3 until the end of the PENULTIMATE layer - blocks_and_idxs = list(zip(range(self.cfg.n_layers), self.blocks)) - for i, block in blocks_and_idxs[start_at_layer:stop_at_layer]: - # Note that each block includes skip connections, so we don't need - # residual + block(residual) - # If we're using multiple GPUs, we need to send the residual and shortformer_pos_embed to the correct GPU - residual = residual.to(get_device_for_block_index(i, self.cfg)) - if shortformer_pos_embed is not None: - shortformer_pos_embed = shortformer_pos_embed.to( - get_device_for_block_index(i, self.cfg) - ) - - residual = block( - residual, - # Cache contains a list of TransformerLensKeyValueCache objects, one for each - # block - past_kv_cache_entry=past_kv_cache[i] if past_kv_cache is not None else None, - shortformer_pos_embed=shortformer_pos_embed, - attention_mask=attention_mask, - ) # [batch, pos, d_model] - - if stop_at_layer is not None: - # When we stop at an early layer, we end here rather than doing further computation - return residual - - if self.cfg.normalization_type is not None: - residual = self.ln_final(residual) # [batch, pos, d_model] - if return_type is None: - return None - else: - logits = self.unembed(residual) # [batch, pos, d_vocab] - logits = apply_softcap(logits, self.cfg.output_logits_soft_cap) - if return_type == "logits": - return logits - else: - assert ( - tokens is not None - ), "tokens must be passed in if return_type is 'loss' or 'both'" - loss = self.loss_fn(logits, tokens, attention_mask, per_token=loss_per_token) - if return_type == "loss": - return loss - elif return_type == "both": - return Output(logits, loss) - else: - logging.warning(f"Invalid return_type passed in: {return_type}") - return None - - def loss_fn( - self, - logits: Float[torch.Tensor, "batch pos d_vocab"], - tokens: Int[torch.Tensor, "batch pos"], - attention_mask: Optional[Int[torch.Tensor, "batch pos"]] = None, - per_token: bool = False, - ): - """Wrapper around `utils.lm_cross_entropy_loss`. - - Used in forward() with return_type=="loss" or "both". - """ - if tokens.device != logits.device: - tokens = tokens.to(logits.device) - return utils.lm_cross_entropy_loss(logits, tokens, attention_mask, per_token) - - @overload - def run_with_cache( - self, *model_args, return_cache_object: Literal[True] = True, **kwargs - ) -> Tuple[Output, ActivationCache]: - ... - - @overload - def run_with_cache( - self, *model_args, return_cache_object: Literal[False], **kwargs - ) -> Tuple[Output, Dict[str, torch.Tensor]]: - ... - - def run_with_cache( - self, *model_args, return_cache_object=True, remove_batch_dim=False, **kwargs - ) -> Tuple[ - Union[ - None, - Float[torch.Tensor, "batch pos d_vocab"], - Loss, - Tuple[Float[torch.Tensor, "batch pos d_vocab"], Loss], - ], - Union[ActivationCache, Dict[str, torch.Tensor]], - ]: - """Wrapper around `run_with_cache` in HookedRootModule. - - If return_cache_object is True, this will return an ActivationCache object, with a bunch of - useful HookedTransformer specific methods, otherwise it will return a dictionary of - activations as in HookedRootModule. - """ - out, cache_dict = super().run_with_cache( - *model_args, remove_batch_dim=remove_batch_dim, **kwargs - ) - if return_cache_object: - cache = ActivationCache(cache_dict, self, has_batch_dim=not remove_batch_dim) - return out, cache - else: - return out, cache_dict - - def set_tokenizer( - self, - tokenizer, - default_padding_side=None, - ): - """Set the tokenizer to use for this model. - - Args: - tokenizer (PreTrainedTokenizer): a pretrained HuggingFace tokenizer. - default_padding_side (str): "right" or "left", which side to pad on. - - """ - assert isinstance( - tokenizer, PreTrainedTokenizerBase - ), f"{type(tokenizer)} is not a supported tokenizer, please use PreTrainedTokenizer or PreTrainedTokenizerFast" - - assert default_padding_side in [ - "right", - "left", - None, - ], f"padding_side must be 'right', 'left' or 'None', got {default_padding_side}" - - # Use a tokenizer that is initialized with add_bos_token=True as the default tokenizer. - # Such a tokenizer should be set as the default tokenizer because the tokenization of some - # tokenizers like LlamaTokenizer are different when bos token is automatically/manually - # prepended, and add_bos_token cannot be dynamically controlled after initialization - # (https://github.com/huggingface/transformers/issues/25886). - tokenizer_with_bos = tokenizer - if self.cfg.original_architecture not in [ - "OlmoForCausalLM", - "OlmoeForCausalLM", - "Olmo2ForCausalLM", - ]: - tokenizer_with_bos = utils.get_tokenizer_with_bos(tokenizer) - - self.tokenizer = tokenizer_with_bos - assert self.tokenizer is not None # keep mypy happy - - # Use explicit value, else tokenizer default, else "right" - if default_padding_side is not None: - self.tokenizer.padding_side = default_padding_side - if self.tokenizer.padding_side is None: - self.tokenizer.padding_side = "right" - - # Detect whether tokenizer actually prepends BOS to control prepend_bos dynamically - self.cfg.tokenizer_prepends_bos = len(self.tokenizer.encode("")) > 0 - - if self.tokenizer.eos_token is None: - self.tokenizer.eos_token = "<|endoftext|>" - if self.tokenizer.pad_token is None: - self.tokenizer.pad_token = self.tokenizer.eos_token - if self.tokenizer.bos_token is None: - self.tokenizer.bos_token = self.tokenizer.eos_token - - # Infer vocab size from tokenizer - if self.cfg.d_vocab == -1: - self.cfg.d_vocab = max(self.tokenizer.vocab.values()) + 1 - if self.cfg.d_vocab_out == -1: - self.cfg.d_vocab_out = self.cfg.d_vocab - - def to_tokens( - self, - input: Union[str, List[str]], - prepend_bos: Optional[Union[bool, None]] = USE_DEFAULT_VALUE, - padding_side: Optional[Union[Literal["left", "right"], None]] = USE_DEFAULT_VALUE, - move_to_device: bool = True, - truncate: bool = True, - ) -> Int[torch.Tensor, "batch pos"]: - """Converts a string to a tensor of tokens. - - See the class-level "Tokenization notes" for full ``prepend_bos`` - semantics, the ``default_prepend_bos`` / - ``tokenizer_prepends_bos`` interaction, and the whitespace- - sensitivity gotcha. **Pass ``prepend_bos=False`` whenever you're - tokenizing only part of a prompt.** - - Args: - input (Union[str, List[str]]): The input to tokenize. - prepend_bos (bool, optional): Overrides ``self.cfg.default_prepend_bos``. - Defaults to ``USE_DEFAULT_VALUE`` (use the cfg setting). Pass ``True`` - or ``False`` to override locally. - padding_side (Union[Literal["left", "right"], None], optional): Overrides - self.tokenizer.padding_side. Specifies which side to pad when tokenizing - multiple strings of different lengths. - move_to_device (bool): Whether to move the output tensor of tokens to the device the - model lives on. Defaults to True - truncate (bool): If the output tokens are too long, - whether to truncate the output tokens to the model's max context window. Does nothing - for shorter inputs. Defaults to True. - """ - with utils.LocallyOverridenDefaults( - self, prepend_bos=prepend_bos, padding_side=padding_side - ): - assert self.tokenizer is not None, "Cannot use to_tokens without a tokenizer" - assert ( - self.cfg.tokenizer_prepends_bos is not None - ), "Set the tokenizer for the model by calling set_tokenizer" - - if self.cfg.default_prepend_bos and not self.cfg.tokenizer_prepends_bos: - # We want to prepend bos but the tokenizer doesn't automatically do it, so we add it manually - input = utils.get_input_with_manually_prepended_bos(self.tokenizer.bos_token, input) - - tokens = self.tokenizer( - input, - return_tensors="pt", - padding=True, - truncation=truncate, - max_length=self.cfg.n_ctx if truncate else None, - )["input_ids"] - - if not self.cfg.default_prepend_bos and self.cfg.tokenizer_prepends_bos: - # We don't want to prepend bos but the tokenizer does it automatically, so we remove it manually - tokens = utils.get_tokens_with_bos_removed(self.tokenizer, tokens) - - if move_to_device: - tokens = tokens.to(self.cfg.device) - return tokens - - def to_string( - self, - tokens: Union[ - List[int], - Int[torch.Tensor, ""], - Int[torch.Tensor, "batch pos"], - Int[torch.Tensor, "pos"], - np.ndarray, - List[Int[torch.Tensor, "pos"]], - ], - ) -> Union[str, List[str]]: - """Tokens to String(s). - - Converts a tensor of tokens to a string (if rank 1) or a list of strings (if rank 2). - - Accepts lists of tokens and numpy arrays as inputs too (and converts to tensors internally) - """ - assert self.tokenizer is not None, "Cannot use to_string without a tokenizer" - - if not isinstance(tokens, torch.Tensor): - # We allow lists to be input - tokens = torch.tensor(tokens) - - # I'm not sure what exactly clean_up_tokenization_spaces does, but if - # it's set, then tokenization is no longer invertible, and some tokens - # with a bunch of whitespace get collapsed together - if len(tokens.shape) == 2: - return self.tokenizer.batch_decode(tokens, clean_up_tokenization_spaces=False) - elif len(tokens.shape) <= 1: - return self.tokenizer.decode(tokens, clean_up_tokenization_spaces=False) - else: - raise ValueError(f"Invalid shape passed in: {tokens.shape}") - - def to_str_tokens( - self, - input: Union[ - str, - Int[torch.Tensor, "pos"], - Int[torch.Tensor, "1 pos"], - Int[np.ndarray, "pos"], - Int[np.ndarray, "1 pos"], - list, - ], - prepend_bos: Optional[Union[bool, None]] = USE_DEFAULT_VALUE, - padding_side: Optional[Union[Literal["left", "right"], None]] = USE_DEFAULT_VALUE, - ) -> Union[List[str], List[List[str]]]: - """Map text, a list of text or tokens to a list of tokens as strings. - - See the class-level "Tokenization notes" for full ``prepend_bos`` - semantics. **Pass ``prepend_bos=False`` whenever you're tokenizing - only part of a prompt.** - - String inputs that exceed ``model.cfg.n_ctx`` are truncated. - - Args: - input (Union[str, list, torch.Tensor]): The input - either a string or a tensor of - tokens. If tokens, should be a tensor of shape [pos] or [1, pos]. - prepend_bos (bool, optional): Overrides ``self.cfg.default_prepend_bos``. Only - applies when ``input`` is a string. Defaults to ``USE_DEFAULT_VALUE`` - (use the cfg setting). Pass ``True`` or ``False`` to override locally. - padding_side (Union[Literal["left", "right"], None], optional): Overrides - self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple - strings of different lengths. - - Returns: - str_tokens: List of individual tokens as strings - """ - with utils.LocallyOverridenDefaults( - self, prepend_bos=prepend_bos, padding_side=padding_side - ): - assert self.tokenizer is not None # keep mypy happy - tokens: Union[np.ndarray, torch.Tensor] - if isinstance(input, list): - return list( - map( - lambda tokens: self.to_str_tokens(tokens, prepend_bos, padding_side), - input, - ) - ) # type: ignore - elif isinstance(input, str): - tokens = self.to_tokens(input, prepend_bos=prepend_bos, padding_side=padding_side)[ - 0 - ] - # Gemma tokenizer expects a batch dimension - if "gemma" in self.tokenizer.name_or_path and tokens.ndim == 1: - tokens = tokens.unsqueeze(1) - elif isinstance(input, torch.Tensor): - tokens = input - tokens = tokens.squeeze() # Get rid of a trivial batch dimension - if tokens.dim() == 0: - # Don't pass dimensionless tensor - tokens = tokens.unsqueeze(0) - assert ( - tokens.dim() == 1 - ), f"Invalid tokens input to to_str_tokens, has shape: {tokens.shape}" - elif isinstance(input, np.ndarray): - tokens = input - tokens = tokens.squeeze() # Get rid of a trivial batch dimension - if tokens.ndim == 0: - # Don't pass dimensionless tensor - tokens = np.expand_dims(tokens, axis=0) - assert ( - tokens.ndim == 1 - ), f"Invalid tokens input to to_str_tokens, has shape: {tokens.shape}" - else: - raise ValueError(f"Invalid input type to to_str_tokens: {type(input)}") - # v5 compat: wrap each token so batch_decode decodes them individually - if isinstance(tokens, np.ndarray): - tokens_list = [[int(t)] for t in tokens] - else: - tokens_list = [[int(t)] for t in tokens.tolist()] - str_tokens = self.tokenizer.batch_decode( - tokens_list, clean_up_tokenization_spaces=False - ) - return str_tokens - - def to_single_token(self, string): - """Map a string that makes up a single token to the id for that token. - - Raises an error for strings that are not a single token! If uncertain use to_tokens. - """ - - # We use the to_tokens method, do not append a BOS token - token = self.to_tokens(string, prepend_bos=False).squeeze() - # If token shape is non-empty, raise error - assert not token.shape, f"Input string: {string} is not a single token!" - return token.item() - - def to_single_str_token(self, int_token: int) -> str: - # Gives the single token corresponding to an int in string form - assert isinstance(int_token, int) - token = self.to_str_tokens(torch.tensor([int_token])) - assert len(token) == 1 - return cast(str, token[0]) - - def get_token_position( - self, - single_token: Union[str, int], - input: Union[str, Union[Float[torch.Tensor, "pos"], Float[torch.Tensor, "1 pos"]]], - mode="first", - prepend_bos: Optional[Union[bool, None]] = USE_DEFAULT_VALUE, - padding_side: Optional[Union[Literal["left", "right"], None]] = USE_DEFAULT_VALUE, - ): - """Get the position of a single_token in a string or sequence of tokens. - - Raises an error if the token is not present. - - When ``input`` is a string it's tokenized internally — see the - class-level "Tokenization notes" for ``prepend_bos`` semantics. - Off-by-one position errors usually mean ``prepend_bos`` is on - when it shouldn't be (or vice versa); pass ``prepend_bos=False`` - when ``input`` is a fragment of a larger prompt. - - Args: - single_token (Union[str, int]): The token to search for. Can - be a token index, or a string (but the string must correspond to a single token). - input (Union[str, torch.Tensor]): The sequence to - search in. Can be a string or a rank 1 tensor of tokens or a rank 2 tensor of tokens - with a dummy batch dimension. - mode (str, optional): If there are multiple matches, which match to return. Supports - "first" or "last". Defaults to "first". - prepend_bos (bool, optional): Overrides ``self.cfg.default_prepend_bos``. Only - applies when ``input`` is a string. Defaults to ``USE_DEFAULT_VALUE`` - (use the cfg setting). Pass ``True`` or ``False`` to override locally. - padding_side (Union[Literal["left", "right"], None], optional): Overrides - self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple - strings of different lengths. - """ - if isinstance(input, str): - # If the input is a string, convert to tensor - tokens = self.to_tokens(input, prepend_bos=prepend_bos, padding_side=padding_side) - else: - tokens = input - - if len(tokens.shape) == 2: - # If the tokens have shape [1, seq_len], flatten to [seq_len] - assert ( - tokens.shape[0] == 1 - ), f"If tokens are rank two, they must have shape [1, seq_len], not {tokens.shape}" - tokens = tokens[0] - - if isinstance(single_token, str): - # If the single token is a string, convert to an integer - single_token = self.to_single_token(single_token) - elif isinstance(single_token, torch.Tensor): - single_token = single_token.item() - - indices = torch.arange(len(tokens), device=tokens.device)[tokens == single_token] - assert len(indices) > 0, "The token does not occur in the prompt" - if mode == "first": - return indices[0].item() - elif mode == "last": - return indices[-1].item() - else: - raise ValueError(f"mode must be 'first' or 'last', not {mode}") - - def tokens_to_residual_directions( - self, - tokens: Union[ - str, - int, - Int[torch.Tensor, ""], - Int[torch.Tensor, "pos"], - Int[torch.Tensor, "batch pos"], - ], - ) -> Union[ - Float[torch.Tensor, "d_model"], - Float[torch.Tensor, "pos d_model"], - Float[torch.Tensor, "batch pos d_model"], - ]: - """Map tokens to a tensor with the unembedding vector for those tokens. - - I.e. the vector in the residual stream that we dot with to the get the logit for that token. - - WARNING: If you use this without folding in LayerNorm, the results will be misleading and - may be incorrect, as the LN weights change the unembed map. This is done automatically with - the fold_ln flag on from_pretrained - - WARNING 2: LayerNorm scaling will scale up or down the effective direction in the residual - stream for each output token on any given input token position. - ActivationCache.apply_ln_to_stack will apply the appropriate scaling to these directions. - - Args: - tokens (Union[str, int, torch.Tensor]): The token(s). If a single token, can be a single - element tensor, an integer, or string. If string, will be mapped to a single token - using to_single_token, and an error raised if it's multiple tokens. The method also - works for a batch of input tokens. - - Returns: - residual_direction torch.Tensor: The unembedding vector for the token(s), a stack of - [d_model] tensor. - """ - if isinstance(tokens, torch.Tensor) and tokens.numel() > 1: - # If the tokens are a tensor, and have more than one element, assume they are a batch of - # tokens. - residual_directions = self.W_U[:, tokens] - residual_directions = einops.rearrange( - residual_directions, "d_model ... -> ... d_model" - ) - return residual_directions - else: - # Otherwise there is a single token - if isinstance(tokens, str): - token = self.to_single_token(tokens) - elif isinstance(tokens, int): - token = tokens - elif isinstance(tokens, torch.Tensor) and tokens.numel() == 1: - token = tokens.item() - else: - raise ValueError(f"Invalid token type: {type(tokens)}") - residual_direction = self.W_U[:, token] - return residual_direction - - def to( # type: ignore - self, - device_or_dtype: Union[torch.device, str, torch.dtype], - print_details: bool = True, - ): - return move_to_and_update_config(self, device_or_dtype, print_details) - - def cuda(self: T, device: Optional[Union[int, torch.device]] = None) -> T: - # TODO: Add support for kwargs - if isinstance(device, int): - return self.to(f"cuda:{device}") - elif device is None: - return self.to("cuda") - else: - return self.to(device) - - def cpu(self: T) -> T: - return self.to(torch.device("cpu")) - - def mps(self: T) -> T: - """Warning: MPS may produce silently incorrect results. See #1178.""" - return self.to(torch.device("mps")) - - def move_model_modules_to_device(self): - self.embed.to(get_best_available_device(self.cfg)) - self.hook_embed.to(get_best_available_device(self.cfg)) - if self.cfg.positional_embedding_type != "rotary": - self.pos_embed.to(get_best_available_device(self.cfg)) - self.hook_pos_embed.to(get_best_available_device(self.cfg)) - - if hasattr(self, "ln_final"): - self.ln_final.to(get_best_available_device(self.cfg)) - self.unembed.to(get_best_available_device(self.cfg)) - for i, block in enumerate(self.blocks): - block.to(get_best_available_device(self.cfg)) - - @classmethod - def from_pretrained( - cls: Type[T], - model_name: str, - fold_ln: bool = True, - center_writing_weights: bool = True, - center_unembed: bool = True, - refactor_factored_attn_matrices: bool = False, - checkpoint_index: Optional[int] = None, - checkpoint_value: Optional[int] = None, - checkpoint_label: Optional[int] = None, - hf_model: Optional[PreTrainedModel] = None, - device: Optional[Union[str, torch.device]] = None, - n_devices: int = 1, - tokenizer: Optional[PreTrainedTokenizerBase] = None, - move_to_device: bool = True, - fold_value_biases: bool = True, - default_prepend_bos: Optional[bool] = None, - default_padding_side: Optional[Literal["left", "right"]] = None, - dtype="float32", - first_n_layers: Optional[int] = None, - n_ctx: Optional[int] = None, - **from_pretrained_kwargs, - ) -> T: - """Load in a Pretrained Model. - - Load in pretrained model weights to the HookedTransformer format and optionally to do some - processing to make the model easier to interpret. Currently supports loading from most - autoregressive HuggingFace models (``gpt2``, ``neo``, ``gptj``, ``opt``...) and from a range - of toy models and SoLU models trained by Neel Nanda. The full list is available in the docs - under :doc:`model properties</generated/model_properties_table>`. Also supports loading from - a checkpoint for checkpointed models (currently, models trained by NeelNanda and the - stanford-crfm models (using parameters ``checkpoint_index`` and ``checkpoint_value``). - - See :meth:`load_and_process_state_dict` for details on the processing (folding layer norm, - centering the unembedding and centering the writing weights). - - Example: - - >>> from transformer_lens import HookedTransformer - >>> model = HookedTransformer.from_pretrained("tiny-stories-1M") - Loaded pretrained model tiny-stories-1M into HookedTransformer - - Args: - model_name: The model name - must be an element of - :const:`transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES` or an alias - of one. The full list of available models can be found in the docs under :doc:`model - properties</generated/model_properties_table>`. - fold_ln: Whether to fold in the LayerNorm weights to the - subsequent linear layer. This does not change the computation. - - `LayerNorm - <https://wandb.ai/wandb_fc/LayerNorm/reports/Layer-Normalization-in-Pytorch-With-Examples---VmlldzoxMjk5MTk1>`_ - is a common regularization technique used in transformers. Unlike BatchNorm, it - cannot be turned off at inference time, as it significantly alters the mathematical - function implemented by the transformer. - - When `fold_ln` is set to True, LayerNorm (with weights :math:`w_{ln}` and - :math:`b_{ln}`) followed by a linear layer (:math:`W + b`) is optimized to - LayerNormPre (just centering & normalizing) followed by a new linear layer with - :math:`W_{eff} = w[:, \text{None}] * W` (element-wise multiplication) and - :math:`b_{eff} = b + b_{ln} @ W`. This transformation is computationally equivalent - and simplifies the model's interpretability. It essentially merges LayerNorm weights - into the subsequent linear layer's weights, which is handled by HookedTransformer - when loading pre-trained weights. Set `fold_ln` to False when loading a state dict - if you wish to turn this off. - - Mathematically, LayerNorm is defined as follows: - - .. math:: - x_1 &= x_0 - \\text{mean}(x_0) - - x_2 &= \\frac{x_1}{\\sqrt{\\text{mean}(x_1^2)}} - - x_3 &= x_2 \\cdot w - - x_4 &= x_3 + b - - For further details, refer to `this document - <https://transformer-circuits.pub/2021/framework/index.html#:~:text=Handling%20Layer%20Normalization>`_. - center_writing_weights: Whether to center weights - writing to the residual stream (ie set mean to be zero). Due to LayerNorm this - doesn't change the computation. - - A related idea to folding layernorm (``fold_ln``) - *every* component reading an - input from the residual stream is preceded by a LayerNorm, which means that the mean - of a residual stream vector (ie the component in the direction of all ones) never - matters. This means we can remove the all ones component of weights and biases whose - output *writes* to the residual stream. Mathematically, ``W_writing -= - W_writing.mean(dim=1, keepdim=True)``. - center_unembed: Whether to center W_U (ie set mean - to be zero). Softmax is translation invariant so this doesn't affect log probs or - loss, but does change logits. - - The logits are fed into a softmax. Softmax is translation invariant (eg, adding 1 to - every logit doesn't change the output), so we can simplify things by setting the - mean of the logits to be zero. This is equivalent to setting the mean of every - output vector of ``W_U`` to zero. In code, ``W_U -= W_U.mean(dim=-1, - keepdim=True)``. - refactor_factored_attn_matrices: Whether to convert the factored - matrices (W_Q & W_K, and W_O & W_V) to be "even". Defaults to False - checkpoint_index: If loading from a checkpoint, the index of - the checkpoint to load. - checkpoint_value: If loading from a checkpoint, the value of - the checkpoint to load, ie the step or token number (each model has checkpoints - labelled with exactly one of these). E.g. ``1000`` for a checkpoint taken at step - 1000 or after 1000 tokens. If `checkpoint_index` is also specified, this will be - ignored. - checkpoint_label: Alias for ``checkpoint_value`` kept for backwards compatibility with - older docs and downstream code. Cannot be combined with ``checkpoint_value``. - hf_model: If you have already loaded in the - HuggingFace model, you can pass it in here rather than needing to recreate the - object. Defaults to None. - device: The device to load the model onto. By - default will load to CUDA if available, else CPU. - n_devices: The number of devices to split the model - across. Defaults to 1. If greater than 1, `device` must be cuda. - tokenizer: The tokenizer to use for the model. If not - provided, it is inferred from cfg.tokenizer_name or initialized to None. If None, - then the model cannot be passed strings, and d_vocab must be explicitly set. - move_to_device: Whether to move the model to the device specified in - cfg. device. Must be true if `n_devices` in the config is greater than 1, since the - model's layers will be split across multiple devices. - fold_value_biases: Each attention head has a value bias. Values are averaged to create - mixed values (``z``), weighted by the attention pattern, but as the bias is - constant, its contribution to ``z`` is exactly the same. The output of a head is ``z - @ W_O``, and so the value bias just linearly adds to the output of the head. This - means that the value bias of a head has nothing to do with the head, and is just a - constant added to the attention layer outputs. We can take the sum across these and - b_O to get an "effective bias" for the layer. In code, we set ``b_V=0``. and ``b_O = - (b_V @ W_O).sum(dim=0) + b_O``. - - The technical derivation of this is as follows. ``v = residual @ W_V[h] + - broadcast_b_V[h]`` for each head ``h`` (where ``b_V`` is broadcast up from shape - ``d_head`` to shape ``[position, d_head]``). And ``z = pattern[h] @ v = pattern[h] @ - residual @ W_V[h] + pattern[h] @ broadcast_b_V[h]``. Because ``pattern[h]`` is - ``[destination_position, source_position]`` and ``broadcast_b_V`` is constant along - the ``(source_)position`` dimension, we're basically just multiplying it by the sum - of the pattern across the ``source_position`` dimension, which is just ``1``. So it - remains exactly the same, and so is just broadcast across the destination positions. - default_prepend_bos: Default behavior of whether to prepend the BOS - token when the methods of HookedTransformer process input text to tokenize (only - when input is a string). - Resolution order for default_prepend_bos: - 1. If user passes value explicitly, use that value - 2. Model-specific default from cfg_dict if it exists (e.g. for bloom models it's False) - 3. Global default (True) - - Even for models not explicitly trained with the BOS token, heads often use the first position as a resting position - and accordingly lose information from the first token, so this empirically seems to give better - results. Note that you can also locally override the default behavior by passing in - prepend_bos=True/False when you call a method that processes the input string. - from_pretrained_kwargs: Any other optional argument passed to - HuggingFace's from_pretrained (e.g. "cache_dir" or "torch_dtype"). Also passed to - other HuggingFace functions when compatible. For some models or arguments it doesn't - work, especially for models that are not internally loaded with HuggingFace's - from_pretrained (e.g. SoLU models). - dtype: What data type to load the model in (also sets the dtype of - the HuggingFace model). Set to bfloat16 or float16 if you get out of memory errors when loading - the model. - default_padding_side: Which side to pad on when tokenizing. - Resolution order for default_padding_side: - 1. If user passes value explicitly, use that value - 2. If tokenizer has a default padding side, use that value - 3. Global default ("right") - first_n_layers: If specified, only load the first n layers of the model. - """ - import warnings - - # Delegating wrappers (from_pretrained_no_processing) warn at their own - # stack level — a warning raised here would attribute to the wrapper's - # internal call and be dropped by the default DeprecationWarning filter. - if not getattr(cls, "_suppress_fp_deprecation", False): - warnings.warn( - "HookedTransformer.from_pretrained is deprecated and will be removed in " - "4.0. Use TransformerBridge.boot_transformers(...) instead, " - "then call enable_compatibility_mode() for HookedTransformer-equivalent " - "numerics. See docs/source/content/migrating_to_v3.md.", - DeprecationWarning, - stacklevel=2, - ) - - if checkpoint_value is not None and checkpoint_label is not None: - raise ValueError( - "Specify checkpoint_value or checkpoint_label, not both — they are aliases." - ) - elif checkpoint_label is not None: - checkpoint_value = checkpoint_label - - if model_name.lower().startswith("t5"): - raise RuntimeError( - "Execution stopped: Please use HookedEncoderDecoder to load T5 models instead of HookedTransformer." - ) - - if model_name.lower().startswith("bert"): - raise RuntimeError( - "Execution stopped: Please use HookedEncoder to load BERT-style models instead of HookedTransformer." - ) - - assert not ( - from_pretrained_kwargs.get("load_in_8bit", False) - or from_pretrained_kwargs.get("load_in_4bit", False) - ), "Quantization not supported" - - if hf_model is not None: - assert hasattr(hf_model, "config"), "PreTrainedModel must have a config attribute" - hf_cfg = hf_model.config.to_dict() - qc = hf_cfg.get("quantization_config", {}) - load_in_4bit = qc.get("load_in_4bit", False) - load_in_8bit = qc.get("load_in_8bit", False) - quant_method = qc.get("quant_method", "") - assert not load_in_8bit, "8-bit quantization is not supported" - assert not ( - load_in_4bit and ("llama" not in model_name.lower()) - ), "Quantization is only supported for Llama models" - if load_in_4bit: - assert ( - qc.get("quant_method", "") == "bitsandbytes" - ), "Only bitsandbytes quantization is supported" - elif quant_method: - # Anything other than the supported bitsandbytes 4-bit Llama - # flow reaches the converters, which slice `.weight` directly: - # packed or scale-separated storage yields wrong numbers rather - # than an error. Refuse instead. - raise NotImplementedError( - f"HookedTransformer cannot convert a {quant_method!r}-quantized " - "checkpoint: its weight converters read weights directly, and " - "packed or scale-separated storage would silently produce wrong " - "values. Load the model dequantized, or use TransformerBridge " - "for a quantized forward pass." - ) - else: - hf_cfg = {} - - if isinstance(dtype, str): - # Convert from string to a torch dtype - dtype = DTYPE_FROM_STRING[dtype] - if "torch_dtype" in from_pretrained_kwargs: - # Backwards compat: torch_dtype overrides dtype - dtype = from_pretrained_kwargs["torch_dtype"] - - if ( - (from_pretrained_kwargs.get("torch_dtype", None) == torch.float16) - or dtype == torch.float16 - ) and device in ["cpu", None]: - logging.warning("float16 models may not work on CPU. Consider using a GPU or bfloat16.") - - # Get the model name used in HuggingFace, rather than the alias. - official_model_name = loading.get_official_model_name(model_name) - - # Load config (includes checkpoint info if applicable) - cfg = loading.get_pretrained_model_config( - official_model_name, - hf_cfg=hf_cfg, - checkpoint_index=checkpoint_index, - checkpoint_value=checkpoint_value, - fold_ln=fold_ln, - device=device, - n_devices=n_devices, - default_prepend_bos=default_prepend_bos, - dtype=dtype, - first_n_layers=first_n_layers, - n_ctx=n_ctx, - **from_pretrained_kwargs, - ) - - if cfg.positional_embedding_type == "shortformer": - if fold_ln: - logging.warning( - "You tried to specify fold_ln=True for a shortformer model, but this can't be done! Setting fold_" - "ln=False instead." - ) - fold_ln = False - if center_unembed: - logging.warning( - "You tried to specify center_unembed=True for a shortformer model, but this can't be done! " - "Setting center_unembed=False instead." - ) - center_unembed = False - if center_writing_weights: - logging.warning( - "You tried to specify center_writing_weights=True for a shortformer model, but this can't be done! " - "Setting center_writing_weights=False instead." - ) - center_writing_weights = False - # Post-norm architectures are incompatible with fold_ln/center_writing_weights, - # both of which assume the norm gain sits on a sublayer's input. - if cfg.original_architecture in POST_NORM_ARCHITECTURES: - if fold_ln: - logging.warning( - f"fold_ln=True is incompatible with {cfg.original_architecture}'s " - "post-norm architecture. Setting fold_ln=False." - ) - fold_ln = False - if center_writing_weights: - logging.warning( - f"center_writing_weights=True is incompatible with " - f"{cfg.original_architecture}'s post-norm architecture. " - "Setting center_writing_weights=False." - ) - center_writing_weights = False - if center_unembed and softcap_enabled(cfg.output_logits_soft_cap): - logging.warning( - "You tried to specify center_unembed=True for a model using logit softcap, but this can't be done! Softcapping is not invariant upon adding a constant " - "Setting center_unembed=False instead." - ) - center_unembed = False - - # Get the state dict of the model (ie a mapping of parameter names to tensors), processed to - # match the HookedTransformer parameter names. - state_dict = loading.get_pretrained_state_dict( - official_model_name, cfg, hf_model, dtype=dtype, **from_pretrained_kwargs - ) - - # Create the HookedTransformer object (suppressing the __init__ warning: - # this entry point already warned above, at the caller's stack level). - _prev_suppress = cls._suppress_init_deprecation - cls._suppress_init_deprecation = True - try: - model = cls( - cfg, - tokenizer, - move_to_device=False, - default_padding_side=default_padding_side, - ) - finally: - cls._suppress_init_deprecation = _prev_suppress - - model.load_and_process_state_dict( - state_dict, - fold_ln=fold_ln, - center_writing_weights=center_writing_weights, - center_unembed=center_unembed, - fold_value_biases=fold_value_biases, - refactor_factored_attn_matrices=refactor_factored_attn_matrices, - ) - - if move_to_device: - model.move_model_modules_to_device() - - print(f"Loaded pretrained model {model_name} into HookedTransformer") - return model - - @classmethod - def from_pretrained_no_processing( - cls, - model_name: str, - fold_ln=False, - center_writing_weights=False, - center_unembed=False, - refactor_factored_attn_matrices=False, - fold_value_biases=False, - dtype=torch.float32, - default_prepend_bos=None, - default_padding_side=None, - **from_pretrained_kwargs, - ): - """Wrapper for from_pretrained. - - Wrapper for from_pretrained with all boolean flags related to simplifying the model set to - False. Refer to from_pretrained for details. - """ - warnings.warn( - "HookedTransformer.from_pretrained_no_processing is deprecated and will be " - "removed in 4.0. Use TransformerBridge.boot_transformers(...) instead. " - "See docs/source/content/migrating_to_v3.md.", - DeprecationWarning, - stacklevel=2, - ) - prev = getattr(cls, "_suppress_fp_deprecation", False) - cls._suppress_fp_deprecation = True - try: - return cls.from_pretrained( - model_name, - fold_ln=fold_ln, - center_writing_weights=center_writing_weights, - center_unembed=center_unembed, - fold_value_biases=fold_value_biases, - refactor_factored_attn_matrices=refactor_factored_attn_matrices, - dtype=dtype, - default_prepend_bos=default_prepend_bos, - default_padding_side=default_padding_side, - **from_pretrained_kwargs, - ) - finally: - cls._suppress_fp_deprecation = prev - - def init_weights(self): - """Initialize weights. - - LayerNorm weights are already initialized to 1.0, and all biases are initialized to 0.0 - (including LayerNorm), so this just initializes weight matrices. - - Weight matrices are set to empty by default (to save space + compute, since they're the bulk - of the parameters), so it is important to call this if you are not loading in pretrained - weights! Note that this function assumes that weight names being with `W_`. - - Set seed here to ensure determinism. - """ - - if self.cfg.seed is not None: - torch.manual_seed(self.cfg.seed) - - if self.cfg.init_mode == "gpt2": - self._init_weights_gpt2() - elif self.cfg.init_mode == "xavier_uniform": - self._init_weights_xavier(dist_type="uniform") - elif self.cfg.init_mode == "xavier_normal": - self._init_weights_xavier(dist_type="normal") - elif self.cfg.init_mode == "kaiming_uniform": - self._init_weights_kaiming(dist_type="uniform") - elif self.cfg.init_mode == "kaiming_normal": - self._init_weights_kaiming(dist_type="normal") - elif self.cfg.init_mode == "muP": - self._init_weights_muP(dist_type="normal") # muP uses normal initialization - - def _init_weights_gpt2(self): - """Initialize weights with GPT-2 initialization. Biases are initialized to 0.0 and weights - are initialized to N(0, 0.64/d_model) if initializer_range is not set, otherwise std is initializer_range. - """ - for name, param in self.named_parameters(): - if "W_" in name: - nn.init.normal_(param, std=self.cfg.initializer_range) - - def _init_weights_xavier(self, dist_type="normal"): - """ - Initialize weights with Xavier initialization -- that is, scale the weights by sqrt(6 / - (fan_in + fan_out)) for a [-1, 1] uniform distribution, or sqrt(2 / (fan_in + fan_out)) for a - standard normal. - - Note that since TransformerLens implements the matrices in the opposite orientation to what - torch does (e.g. it's d_in x d_out, not d_out x d_in as in torch), we need to calculate it - ourselves. - """ - gain = self.cfg.initializer_range - for name, param in self.named_parameters(): - if "W_" in name: - if dist_type == "uniform": - init_xavier_uniform_(param, gain=gain) - elif dist_type == "normal": - init_xavier_normal_(param, gain=gain) - - def _init_weights_kaiming(self, dist_type="uniform"): - """ - Initialize weights with Kaiming initialization -- that is, scale the weights by - c / sqrt(fan_in), where c = sqrt(2) if the params were immediately preceded by a relu and 1 for - everything else. - - Note that the numbers are actually incorrect here when you're using a nonlinearity other - than relu, e.g. the correct c for SiLu is ~1.74, for tanh it's 5/3 ~= 1.67, and for GeLU it's ~1.57. - But this is unlikely to matter in practice. - - I'm just using fan_mode = "fan_in" for now, but it should be trivial to add fan_out. - - Again, we have to implement it ourselves because of the orientation of the matrices. - """ - gain = self.cfg.initializer_range - for name, param in self.named_parameters(): - if "W_" in name: - if dist_type == "uniform": - init_kaiming_uniform_(param, gain=gain, nonlinearity="relu", mode="fan_in") - elif dist_type == "normal": - init_kaiming_normal_(param, gain=gain, nonlinearity="relu", mode="fan_in") - - def _init_weights_muP(self, dist_type="uniform"): - """ - Initialize weights with muParameterization. This involves scaling output weights by a factor - of 1/fan_in, input weights and biases by 1, everything else by a factor of 1/sqrt(fan_in). - - Also, you need to use muAdamW, which rescales the learning rate for output weights and - hidden weights by a factor of 1/fan_in. - - All biases are still assumed to be initialized to 0.0, so we only need to change the - weights. - """ - for name, param in self.named_parameters(): - if "W_" in name: - fan_in, _ = utils.calc_fan_in_and_fan_out(param) - if "embed" in name: - scale = float(1) - elif "unembed" in name: - scale = 1 / fan_in - else: - scale = 1 / fan_in**0.5 - - if dist_type == "uniform": - scale *= 3**0.5 - nn.init.uniform_(param, -scale, scale) - elif dist_type == "normal": - nn.init.normal_(param, std=scale) - - def load_and_process_state_dict( - self, - state_dict: Dict[str, torch.Tensor], - fold_ln: bool = True, - center_writing_weights: bool = True, - center_unembed: bool = True, - fold_value_biases: bool = True, - refactor_factored_attn_matrices: bool = False, - ): - """Load & Process State Dict. - - Load a state dict into the model, and to apply processing to simplify it. The state dict is - assumed to be in the HookedTransformer format. - - See the relevant method (same name as the flag) for more details on the folding, centering - and processing flags. - - Args: - state_dict (dict): The state dict of the model, in HookedTransformer format. fold_ln - fold_ln (bool, optional): Whether to fold in the LayerNorm weights to the - subsequent linear layer. This does not change the computation. Defaults to True. - center_writing_weights (bool, optional): Whether to center weights writing to the - residual stream (ie set mean to be zero). Due to LayerNorm this doesn't change the - computation. Defaults to True. - center_unembed (bool, optional): Whether to center W_U (ie set mean to be zero). - Softmax is translation invariant so this doesn't affect log probs or loss, but does - change logits. Defaults to True. - fold_value_biases (bool, optional): Whether to fold the value biases into the output - bias. Because attention patterns add up to 1, the value biases always have a - constant effect on a layer's output, and it doesn't matter which head a bias is - associated with. We can factor this all into a single output bias to the layer, and - make it easier to interpret the head's output. - refactor_factored_attn_matrices (bool, optional): Whether to convert the factored - matrices (W_Q & W_K, and W_O & W_V) to be "even". Defaults to False. - model_name (str, optional): checks the model name for special cases of state dict - loading. Only used for Redwood 2L model currently. - """ - if self.cfg.dtype not in [torch.float32, torch.float64] and fold_ln: - logging.warning( - "With reduced precision, it is advised to use `from_pretrained_no_processing` instead of `from_pretrained`." - ) - - if ( - self.cfg.dtype not in [torch.float32, torch.float64] - and self.cfg.num_experts - and self.cfg.num_experts > 1 - ): - logging.warning( - "When running MoE models, it is advised to use a higher precision data type. See docs for more info." - ) - - state_dict = self.fill_missing_keys(state_dict) - if fold_ln: - if self.cfg.normalization_type not in ["LN", "LNPre", "RMS", "RMSPre"]: - logging.warning( - "You are not using LayerNorm or RMSNorm, so the layer norm weights can't be folded! Skipping" - ) - fold_ln = False - else: - ln_keys_present = any( - k.endswith((".ln1.w", ".ln2.w", "ln_final.w")) for k in state_dict - ) - if not ln_keys_present: - logging.warning( - "fold_ln=True but no LayerNorm weights found in state_dict. " - "The model may have been saved with already-folded LayerNorms. " - "Skipping fold." - ) - fold_ln = False - else: - if self.cfg.normalization_type == "LN": - self.cfg.normalization_type = "LNPre" - self.ln_final = LayerNormPre(self.cfg) - for layer in self.blocks: - layer.ln1 = LayerNormPre(self.cfg) - layer.ln2 = LayerNormPre(self.cfg) - if self.cfg.is_layer_norm_activation(): - layer.mlp.ln = LayerNormPre(self.cfg) - elif self.cfg.normalization_type == "RMS": - self.cfg.normalization_type = "RMSPre" - self.ln_final = RMSNormPre(self.cfg) - for layer in self.blocks: - layer.ln1 = RMSNormPre(self.cfg) - layer.ln2 = RMSNormPre(self.cfg) - if self.cfg.is_layer_norm_activation(): - layer.mlp.ln = RMSNormPre(self.cfg) - - # Use the centralized ProcessWeights class for all weight processing - # (fold_ln is passed through — if we skipped above, it's now False) - state_dict = ProcessWeights.process_weights( - state_dict, - self.cfg, - fold_ln=fold_ln, - center_writing_weights=center_writing_weights, - center_unembed=center_unembed, - fold_value_biases=fold_value_biases, - refactor_factored_attn_matrices=refactor_factored_attn_matrices, - ) - - if self.cfg.load_in_4bit: - # with quantization, parameters should be assigned - # so that quantization settings are not lost - self.load_state_dict(state_dict, assign=True, strict=False) - else: - state_dict_keys = list(state_dict.keys()) - for key in state_dict_keys: - self.load_state_dict({key: state_dict[key]}, strict=False) - del state_dict[key] - - if fold_ln: - self.setup() - - def fill_missing_keys(self, state_dict): - return loading.fill_missing_keys(self, state_dict) - - def fold_layer_norm( - self, state_dict: Dict[str, torch.Tensor], fold_biases=True, center_weights=True - ): - """Fold Layer Norm. Can also be used to fold RMS Norm, when fold_biases and center_weights are set to False. - - Takes in a state dict from a pretrained model, formatted to be consistent with - HookedTransformer but with LayerNorm weights and biases. Folds these into the neighbouring - weights. See further_comments.md for more details. - - Args: - state_dict (Dict[str, torch.Tensor]): State dict of pretrained model. - fold_biases (bool): Enables folding of LN biases. Should be disabled when RMS Norm is used. - center_weights (bool): Enables the centering of weights after folding in LN. Should be disabled when RMS Norm is used. - """ - return ProcessWeights.fold_layer_norm(state_dict, self.cfg, fold_biases, center_weights) - - def center_writing_weights(self, state_dict: Dict[str, torch.Tensor]): - """Center Writing Weights. - - Centers the weights of the model that write to the residual stream - W_out, W_E, W_pos and - W_out. This is done by subtracting the mean of the weights from the weights themselves. This - is done in-place. See fold_layer_norm for more details. - """ - return ProcessWeights.center_writing_weights(state_dict, self.cfg) - - def center_unembed(self, state_dict: Dict[str, torch.Tensor]): - """Center the unembedding weights W_U. - - This is done by subtracting the mean of the weights from the weights themselves. This is - done in-place. As softmax is translation invariant, this changes the logits but not the log - probs, and makes the model logits (slightly) more interpretable - when trying to understand - how components contribute to the logits, we'll be less misled by components that just add - something to every logit. - """ - return ProcessWeights.center_unembed(state_dict) - - def fold_value_biases(self, state_dict: Dict[str, torch.Tensor]): - """Fold the value biases into the output bias. - - Because attention patterns add up to 1, the value biases always have a constant effect on a - head's output. Further, as the outputs of each head in a layer add together, each head's - value bias has a constant effect on the *layer's* output, which can make it harder to - interpret the effect of any given head, and it doesn't matter which head a bias is - associated with. We can factor this all into a single output bias to the layer, and make it - easier to interpret the head's output. Formally, we take b_O_new = b_O_original + - sum_head(b_V_head @ W_O_head). - """ - return ProcessWeights.fold_value_biases(state_dict, self.cfg) - - def refactor_factored_attn_matrices(self, state_dict: Dict[str, torch.Tensor]): - """Experimental method for managing queries, keys and values. - - As argued in [A Mathematical Framework for Transformer - Circuits](https://transformer-circuits.pub/2021/framework/index.html), queries, keys and - values are somewhat arbitrary intermediate terms when computing with the low rank factored - matrices W_QK = W_Q @ W_K.T and W_OV = W_V @ W_O, and these matrices are the only thing - determining head behaviour. But there are many ways to find a low rank factorization to a - given matrix, and hopefully some of these are more interpretable than others! This method is - one attempt, which makes all of the matrices have orthogonal rows or columns, W_O into a - rotation and W_Q and W_K having the nth column in each having the same norm. The formula is - $W_V = U @ S,W_O=Vh.T,W_Q=U@S.sqrt(),W_K=Vh@S.sqrt()$. - - More details: - - If W_OV = U @ S @ Vh.T in its singular value decomposition, (where S is in R^d_head not - R^d_model, as W_OV is low rank), W_OV = (U @ S) @ (Vh.T) is an equivalent low rank - factorisation, where rows/columns of each matrix are orthogonal! So setting $W_V=US$ and - $W_O=Vh.T$ works just as well. I *think* this is a more interpretable setup, because now - $W_O$ is just a rotation, and doesn't change the norm, so $z$ has the same norm as the - result of the head. - - For $W_QK = W_Q @ W_K.T$ we use the refactor $W_Q = U @ S.sqrt()$ and $W_K = Vh @ S.sqrt()$, - which is also equivalent ($S==S.sqrt() @ S.sqrt()$ as $S$ is diagonal). Here we keep the - matrices as having the same norm, since there's not an obvious asymmetry between the keys - and queries. - - Biases are more fiddly to deal with. For OV it's pretty easy - we just need (x @ W_V + b_V) - @ W_O + b_O to be preserved, so we can set b_V' = 0. and b_O' = b_V @ W_O + b_O (note that - b_V in R^{head_index x d_head} while b_O in R^{d_model}, so we need to sum b_V @ W_O along - the head_index dimension too). - - For QK it's messy - we need to preserve the bilinear form of (x @ W_Q + b_Q) * (y @ W_K + - b_K), which is fairly messy. To deal with the biases, we concatenate them to W_Q and W_K to - simulate a d_model+1 dimensional input (whose final coordinate is always 1), do the SVD - factorization on this effective matrix, then separate out into final weights and biases. - """ - return ProcessWeights.refactor_factored_attn_matrices(state_dict, self.cfg) - - def set_use_attn_result(self, use_attn_result: bool): - """Toggle whether to explicitly calculate and expose the result for each attention head. - - Useful for interpretability but can easily burn through GPU memory. - """ - self.cfg.use_attn_result = use_attn_result - - def set_use_split_qkv_input(self, use_split_qkv_input: bool): - """ - Toggles whether to allow editing of the separate Q, K, and V inputs to each attention head. - """ - self.cfg.use_split_qkv_input = use_split_qkv_input - - def set_use_hook_mlp_in(self, use_hook_mlp_in: bool): - """Toggles whether to allow storing and editing inputs to each MLP layer.""" - - assert not self.cfg.attn_only, "Can't use hook_mlp_in with attn_only model" - self.cfg.use_hook_mlp_in = use_hook_mlp_in - - def set_use_attn_in(self, use_attn_in: bool): - """ - Toggles whether to allow editing of inputs to each attention head. - """ - assert ( - self.cfg.n_key_value_heads is None - ), "Can't use attn_in with GroupedQueryAttention, please use split_qkv_input instead" - self.cfg.use_attn_in = use_attn_in - - def set_ungroup_grouped_query_attention(self, ungroup_grouped_query_attention: bool): - """ - Toggles whether to ungroup the grouped key and value heads in models with grouped query attention (GQA). - """ - self.cfg.ungroup_grouped_query_attention = ungroup_grouped_query_attention - - def process_weights_( - self, - fold_ln: bool = True, - center_writing_weights: bool = True, - center_unembed: bool = True, - refactor_factored_attn_matrices: bool = False, - ): - """Wrapper around `load_and_process_state_dict`. - - Wrapper around load_and_process_state_dict to allow for in-place processing of the weights. - This is useful if using HookedTransformer for training, if we then want to analyse a cleaner - version of the same model. - """ - state_dict = self.state_dict() - self.load_and_process_state_dict( - state_dict, - fold_ln=fold_ln, - center_writing_weights=center_writing_weights, - center_unembed=center_unembed, - refactor_factored_attn_matrices=refactor_factored_attn_matrices, - ) - - @torch.inference_mode() - def generate( - self, - input: Union[ - str, - List[str], - Int[torch.Tensor, "batch pos"], - Float[torch.Tensor, "batch pos hidden_size"], - ] = "", - max_new_tokens: int = 10, - stop_at_eos: bool = True, - eos_token_id: Optional[int] = None, - do_sample: bool = True, - top_k: Optional[int] = None, - top_p: Optional[float] = None, - temperature: float = 1.0, - freq_penalty: float = 0.0, - use_past_kv_cache: bool = True, - prepend_bos: Optional[bool] = USE_DEFAULT_VALUE, - padding_side: Optional[Literal["left", "right"]] = USE_DEFAULT_VALUE, - return_type: Optional[str] = "input", - verbose: bool = True, - **generation_kwargs, - ) -> Union[ - str, - List[str], - Int[torch.Tensor, "batch pos_plus_new_tokens"], - Float[torch.Tensor, "batch pos_plus_new_tokens hidden_size"], - Any, # transformers.utils.ModelOutput to accommodate output_logits=True. - # Using Any due to beartype's forward reference resolution limitations. - # See: https://github.com/beartype/beartype/issues/546 - ]: - """Sample Tokens from the Model. - - Sample tokens from the model until the model outputs eos_token or max_new_tokens is reached. - - To avoid fiddling with ragged tensors, if we input a batch of text and some sequences finish - (by producing an EOT token), we keep running the model on the entire batch, but throw away - the output for a finished sequence and just keep adding EOTs to pad. - - Args: - input (Union[str, List[str], Int[torch.Tensor, "batch pos"], Float[torch.Tensor, "batch pos hidden_size"]]): - A text string (this will be converted to a batch of tokens with batch - size 1), a list of strings, batch of tokens or a tensor of precomputed embeddings of shape - [batch, pos, hidden_size]. - max_new_tokens (int): Maximum number of tokens to generate. - stop_at_eos (bool): If True, stop generating tokens when the model outputs eos_token. - eos_token_id (Optional[Union[int, Sequence]]): The token ID to use for end - of sentence. If None, use the tokenizer's eos_token_id - required if using - stop_at_eos. It's also possible to provide a list of token IDs (not just the - eos_token_id), in which case the generation will stop when any of them are output - (useful e.g. for stable_lm). - do_sample (bool): If True, sample from the model's output distribution. Otherwise, use - greedy search (take the max logit each time). - top_k (int): Number of tokens to sample from. If None, sample from all tokens. - top_p (float): Probability mass to sample from. If 1.0, sample from all tokens. If <1.0, - we take the top tokens with cumulative probability >= top_p. - temperature (float): Temperature for sampling. Higher values will make the model more - random (limit of temp -> 0 is just taking the top token, limit of temp -> inf is - sampling from a uniform distribution). - freq_penalty (float): Frequency penalty for sampling - how much to penalise previous - tokens. Higher values will make the model more random. Works only with str and tokens input. - use_past_kv_cache (bool): If True, create and use cache to speed up generation. - prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend - the BOS token to the input (applicable when input is a string). Defaults to None, - implying usage of self.cfg.default_prepend_bos (default is True unless specified - otherwise). Pass True or False to override the default. - padding_side (Union[Literal["left", "right"], None], optional): Overrides - self.tokenizer.padding_side. Specifies which side to pad when tokenizing - multiple strings of different lengths. For batched list inputs, left-padding - is forced internally for correct generation behavior. - return_type (Optional[str]): The type of the output to return - a string or a list of strings ('str'), - a tensor of tokens ('tokens'), a tensor of output embeddings ('embeds') or whatever the format of the - input was ('input'). - verbose (bool): If True, show tqdm progress bars for generation. - - Returns: - outputs (str, List[str], Int[torch.Tensor, "batch pos_plus_new_tokens"], Float[torch.Tensor, - "batch pos_plus_new_tokens hidden_size"]): generated sequence. Str, tokens or embeddings. - If input is embeddings and return type is tokens or string, returns only new generated sequence. - In other cases returns sequence including input sequence. - """ - - with utils.LocallyOverridenDefaults( - self, prepend_bos=prepend_bos, padding_side=padding_side - ): - assert isinstance(input, (str, torch.Tensor, list)) and ( - isinstance(input, list) - and all(isinstance(i, str) for i in input) - or not isinstance(input, list) - ), "Input must be either string, torch.Tensor, or List[str]" - - assert return_type in [ - "input", - "str", - "tokens", - "embeds", - ], "return_type must be one of ['input', 'str', 'tokens', 'embeds']" - - if return_type == "input": - if isinstance(input, (str, list)): - return_type = "str" - elif input.ndim == 2: - return_type = "tokens" - else: - return_type = "embeds" - - # initial_attention_mask is always computed so that single-prompt and - # batched generation go through the same masked code path, producing - # consistent results for the same prompt regardless of batching. - initial_attention_mask: Optional[torch.Tensor] = None - _is_batched_list = isinstance(input, list) and len(input) > 1 - - if isinstance(input, (str, list)): - input_type = "str" - assert ( - self.tokenizer is not None - ), "Must provide a tokenizer if passing a string to the model" - if _is_batched_list: - # Force left-padding for batched generation so real tokens - # are flush-right and logits[:, -1, :] is always correct. - input = self.to_tokens(input, prepend_bos=prepend_bos, padding_side="left") - else: - input = self.to_tokens( - input, prepend_bos=prepend_bos, padding_side=padding_side - ) - elif input.ndim == 2: - input_type = "tokens" - else: - input_type = "embeds" - - input_tokens = input if input_type in ["str", "tokens"] else None - batch_size, ctx_length = input.shape[0], input.shape[1] - - # Compute initial attention mask. For batched inputs with padding, - # this correctly masks pad tokens. For single/unpadded inputs, this - # is all-ones which matches the no-mask code path but ensures both - # go through the same PosEmbed/attention logic for consistency. - if input_tokens is not None and self.tokenizer is not None: - _prepend_bos = ( - self.cfg.default_prepend_bos - if prepend_bos is USE_DEFAULT_VALUE - else (False if prepend_bos is None else prepend_bos) - ) - # Temporarily set padding_side="left" so get_attention_mask - # scans for leading pads (matching the left-padded tokens). - _orig_padding_side = self.tokenizer.padding_side - if _is_batched_list: - self.tokenizer.padding_side = "left" - initial_attention_mask = utils.get_attention_mask( - self.tokenizer, input_tokens, _prepend_bos - ) - if _is_batched_list: - self.tokenizer.padding_side = _orig_padding_side - device = get_device_for_block_index(0, self.cfg) - input = input.to(device) - if input_tokens is not None: - # Re-alias to the moved tensor: input_tokens must live on the model's - # device so later concatenations with sampled tokens (freq_penalty - # sampling and the final output) don't mix devices. - input_tokens = input - if use_past_kv_cache: - past_kv_cache = TransformerLensKeyValueCache.init_cache( - self.cfg, self.cfg.device, batch_size - ) - else: - past_kv_cache = None - - # Only `output_logits` is supported from HF generation kwargs - output_logits_flag = False - if generation_kwargs: - if "output_logits" in generation_kwargs: - output_logits_flag = bool(generation_kwargs.pop("output_logits")) - # Warn about unsupported keys - accepted_keys = {"output_logits", "return_dict_in_generate"} - unsupported_keys = [k for k in generation_kwargs.keys() if k not in accepted_keys] - # Ignore `return_dict_in_generate` - if "return_dict_in_generate" in generation_kwargs: - generation_kwargs.pop("return_dict_in_generate") - # Warn and drop unsupported keys - if unsupported_keys: - import warnings - - warnings.warn( - f"HookedTransformer.generate received unsupported generation kwargs; ignoring: {unsupported_keys}", - UserWarning, - ) - # Remove unsupported keys - for k in unsupported_keys: - generation_kwargs.pop(k, None) - - # Collect per-step logits if requested - logits_seq_list: Optional[List[torch.Tensor]] = [] if output_logits_flag else None - - shortformer_pos_embed = None - embeds = input if input_type == "embeds" else self.embed(input) - - assert isinstance(embeds, torch.Tensor) and embeds.ndim == 3 - - stop_tokens: List[int] = [] - eos_token_for_padding = 0 - if stop_at_eos: - tokenizer_has_eos_token = ( - self.tokenizer is not None and self.tokenizer.eos_token_id is not None - ) - if eos_token_id is None: - assert ( - tokenizer_has_eos_token - ), "Must pass a eos_token_id if stop_at_eos is True and tokenizer is None or has no eos_token_id" - assert self.tokenizer is not None - eos_token_id = self.tokenizer.eos_token_id - - if isinstance(eos_token_id, int): - stop_tokens = [eos_token_id] - eos_token_for_padding = eos_token_id - else: - # eos_token_id is a Sequence (e.g. list or tuple) - stop_tokens = eos_token_id - if tokenizer_has_eos_token: - assert self.tokenizer is not None - eos_token_for_padding = self.tokenizer.eos_token_id - else: - eos_token_for_padding = eos_token_id[0] - - # An array to track which sequences in the batch have finished. - finished_sequences = torch.zeros(batch_size, dtype=torch.bool, device=self.cfg.device) - - # Currently nothing in HookedTransformer changes with eval, but this is here in case - # that changes in the future. - self.eval() - sampled_tokens_list: List[torch.Tensor] = [] - for index in tqdm.tqdm(range(max_new_tokens), disable=not verbose): - pos_offset = self.get_pos_offset(past_kv_cache, batch_size) - - # Extend the initial attention mask with 1s for generated tokens. - attention_mask: Optional[torch.Tensor] = None - if initial_attention_mask is not None: - n_new = len(sampled_tokens_list) - if n_new > 0: - ones = torch.ones( - batch_size, - n_new, - dtype=initial_attention_mask.dtype, - device=device, - ) - attention_mask = torch.cat([initial_attention_mask.to(device), ones], dim=1) - else: - attention_mask = initial_attention_mask.to(device) - residual, shortformer_pos_embed = self.get_residual( - embeds, - pos_offset, - return_shortformer_pos_embed=True, - device=device, - attention_mask=attention_mask, - ) - - # While generating, we keep generating logits, throw away all but the final logits, - # and then use those logits to sample from the distribution We keep adding the - # sampled tokens to the end of tokens. - start_at_layer = 0 # Make forward returns embeddings - if use_past_kv_cache: - # We just take the final tokens, as a [batch, 1] tensor - if index > 0: - logits = self.forward( - residual[:, -1:], - return_type="logits", - prepend_bos=prepend_bos, - padding_side=padding_side, - past_kv_cache=past_kv_cache, - start_at_layer=start_at_layer, - shortformer_pos_embed=shortformer_pos_embed, - attention_mask=attention_mask, - ) - else: - logits = self.forward( - residual, - return_type="logits", - prepend_bos=prepend_bos, - padding_side=padding_side, - past_kv_cache=past_kv_cache, - start_at_layer=start_at_layer, - shortformer_pos_embed=shortformer_pos_embed, - attention_mask=attention_mask, - ) - else: - # We input the entire sequence, as a [batch, pos] tensor, since we aren't using - # the cache. - logits = self.forward( - residual, - return_type="logits", - prepend_bos=prepend_bos, - padding_side=padding_side, - start_at_layer=start_at_layer, - shortformer_pos_embed=shortformer_pos_embed, - attention_mask=attention_mask, - ) - final_logits = logits[:, -1, :] - - if output_logits_flag: - assert logits_seq_list is not None - logits_seq_list.append(final_logits.clone()) - - if do_sample: - if input_type in [ - "str", - "tokens", - ]: # Those types of inputs support frequency penalty - assert input_tokens is not None - sampled_tokens = utils.sample_logits( - final_logits, - top_k=top_k, - top_p=top_p, - temperature=temperature, - freq_penalty=freq_penalty, - tokens=torch.cat( - (input_tokens, torch.cat(sampled_tokens_list, dim=1)), dim=1 - ) - if "sampled_tokens" in locals() - else input_tokens, - ).to(get_device_for_block_index(0, self.cfg)) - else: - sampled_tokens = utils.sample_logits( - final_logits, top_k=top_k, top_p=top_p, temperature=temperature - ).to(get_device_for_block_index(0, self.cfg)) - else: - sampled_tokens = final_logits.argmax(-1).to( - get_device_for_block_index(0, self.cfg) - ) - sampled_tokens_list.append(sampled_tokens.unsqueeze(1)) - if stop_at_eos: - # For all unfinished sequences, add on the next token. If a sequence was - # finished, throw away the generated token and add eos_token_for_padding - # instead. - sampled_tokens[finished_sequences] = eos_token_for_padding - finished_sequences.logical_or_( - torch.isin( - sampled_tokens.to(self.cfg.device), - torch.tensor(stop_tokens).to(self.cfg.device), - ) - ) - - embeds = torch.hstack([embeds, self.embed(sampled_tokens.unsqueeze(-1))]) - - if stop_at_eos and finished_sequences.all(): - break - - sampled_tokens = torch.cat(sampled_tokens_list, dim=1) - if input_type in ["str", "tokens"]: - assert input_tokens is not None - output_tokens = torch.cat((input_tokens, sampled_tokens), dim=1) - else: - output_tokens = sampled_tokens - - if return_type == "str": - assert self.tokenizer is not None - decoded_texts: List[str] = [ - cast(str, self.tokenizer.decode(tokens, skip_special_tokens=True)) - for tokens in output_tokens - ] - result: Any = decoded_texts[0] if len(decoded_texts) == 1 else decoded_texts - elif return_type == "tokens": - result = cast(Any, output_tokens) - else: - result = cast(Any, embeds) - - if output_logits_flag: - # Return HF ModelOutput format - from transformers.utils import ModelOutput # type: ignore - - def _logits_to_tuple(logits_list: list[torch.Tensor]) -> tuple[torch.Tensor, ...]: - assert logits_list is not None - return tuple(logits_list) - - try: - from transformers.generation.utils import GenerateDecoderOnlyOutput - - return GenerateDecoderOnlyOutput( - sequences=cast(torch.LongTensor, output_tokens), - # HF's type hint tuple[FloatTensor] is really tuple[FloatTensor, ...] - logits=_logits_to_tuple(logits_seq_list), # type: ignore[arg-type] - ) - except (ImportError, AttributeError): - # Fallback for older transformers versions - # `sequences` expects a tensor of token ids - return ModelOutput(sequences=output_tokens, logits=_logits_to_tuple(logits_seq_list)) # type: ignore[arg-type] - else: - return result - - @torch.inference_mode() - def generate_stream( - self, - input: Union[str, Float[torch.Tensor, "batch pos"]] = "", - max_new_tokens: int = 10, - max_tokens_per_yield: int = 25, - stop_at_eos: bool = True, - eos_token_id: Optional[int] = None, - do_sample: bool = True, - top_k: Optional[int] = None, - top_p: Optional[float] = None, - temperature: float = 1.0, - freq_penalty: float = 0.0, - use_past_kv_cache: bool = True, - prepend_bos: Optional[bool] = USE_DEFAULT_VALUE, - padding_side: Optional[Literal["left", "right"]] = USE_DEFAULT_VALUE, - return_type: Optional[str] = "input", - verbose: bool = True, - ) -> Generator[Union[Int[torch.Tensor, "batch"], str], None, None]: - """Stream tokens from the Model as they are generated. - - Sample tokens from the model until the model outputs eos_token or max_new_tokens is reached, - yielding batches of tokens progressively during generation rather than waiting for the entire - sequence to be generated. - - To avoid fiddling with ragged tensors, if we input a batch of text and some sequences finish - (by producing an EOT token), we keep running the model on the entire batch, but throw away - the output for a finished sequence and just keep adding EOTs to pad. - - This supports entering a single string, but not a list of strings - if the strings don't - tokenize to exactly the same length, this gets messy. If that functionality is needed, - convert them to a batch of tokens and input that instead. - - Args: - input (Union[str, Int[torch.Tensor, "batch pos"])]): Either a batch of tokens ([batch, - pos]) or a text string (this will be converted to a batch of tokens with batch size - 1). - max_new_tokens (int): Maximum number of tokens to generate. - max_tokens_per_yield (int): Maximum number of tokens to accumulate before yielding. - Controls how frequently the function yields tokens during generation. - stop_at_eos (bool): If True, stop generating tokens when the model outputs eos_token. - eos_token_id (Optional[Union[int, Sequence]]): The token ID to use for end - of sentence. If None, use the tokenizer's eos_token_id - required if using - stop_at_eos. It's also possible to provide a list of token IDs (not just the - eos_token_id), in which case the generation will stop when any of them are output - (useful e.g. for stable_lm). - do_sample (bool): If True, sample from the model's output distribution. Otherwise, use - greedy search (take the max logit each time). - top_k (int): Number of tokens to sample from. If None, sample from all tokens. - top_p (float): Probability mass to sample from. If 1.0, sample from all tokens. If <1.0, - we take the top tokens with cumulative probability >= top_p. - temperature (float): Temperature for sampling. Higher values will make the model more - random (limit of temp -> 0 is just taking the top token, limit of temp -> inf is - sampling from a uniform distribution). - freq_penalty (float): Frequency penalty for sampling - how much to penalise previous - tokens. Higher values will make the model more random. - use_past_kv_cache (bool): If True, create and use cache to speed up generation. - prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend - the BOS token to the input (applicable when input is a string). Defaults to None, - implying usage of self.cfg.default_prepend_bos (default is True unless specified - otherwise). Pass True or False to override the default. - padding_side (Union[Literal["left", "right"], None], optional): Overrides - self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple - strings of different lengths. - return_type (Optional[str]): The type of the output to return - either a string (str), - a tensor of tokens (tensor) or whatever the format of the input was (input). - verbose (bool): If True, show tqdm progress bars for generation. - - Yields: - outputs (Union[Int[torch.Tensor, "batch"], str]): Batches of generated tokens, yielded - progressively during generation. Each yield contains accumulated tokens since the last - yield, up to max_tokens_per_yield. - """ - - with utils.LocallyOverridenDefaults( - self, prepend_bos=prepend_bos, padding_side=padding_side - ): - if type(input) == str: - # If text, convert to tokens (batch_size=1) - assert ( - self.tokenizer is not None - ), "Must provide a tokenizer if passing a string to the model" - tokens = self.to_tokens(input, prepend_bos=prepend_bos, padding_side=padding_side) - else: - assert isinstance(input, torch.Tensor), "Input must be a tensor when not a string" - tokens = input - - if return_type == "input": - if type(input) == str: - return_type = "str" - else: - return_type = "tensor" - - assert isinstance(tokens, torch.Tensor) - batch_size, ctx_length = tokens.shape - device = get_device_for_block_index(0, self.cfg) - tokens = tokens.to(device) - if use_past_kv_cache: - past_kv_cache = TransformerLensKeyValueCache.init_cache( - self.cfg, self.cfg.device, batch_size - ) - else: - past_kv_cache = None - - stop_tokens: List[int] = [] - eos_token_for_padding = 0 - if stop_at_eos: - tokenizer_has_eos_token = ( - self.tokenizer is not None and self.tokenizer.eos_token_id is not None - ) - if eos_token_id is None: - assert ( - tokenizer_has_eos_token - ), "Must pass a eos_token_id if stop_at_eos is True and tokenizer is None or has no eos_token_id" - assert self.tokenizer is not None - eos_token_id = self.tokenizer.eos_token_id - - if isinstance(eos_token_id, int): - stop_tokens = [eos_token_id] - eos_token_for_padding = eos_token_id - else: - # eos_token_id is a Sequence (e.g. list or tuple) - stop_tokens = eos_token_id - if tokenizer_has_eos_token: - assert self.tokenizer is not None - eos_token_for_padding = self.tokenizer.eos_token_id - else: - eos_token_for_padding = eos_token_id[0] - - # An array to track which sequences in the batch have finished. - finished_sequences = torch.zeros(batch_size, dtype=torch.bool, device=self.cfg.device) - - accumulated_tokens: Optional[torch.Tensor] = None - tokens_since_last_yield = 0 - - # Currently nothing in HookedTransformer changes with eval, but this is here in case - # that changes in the future. - self.eval() - for index in tqdm.tqdm(range(max_new_tokens), disable=not verbose): - # While generating, we keep generating logits, throw away all but the final logits, - # and then use those logits to sample from the distribution We keep adding the - # sampled tokens to the end of tokens. - if use_past_kv_cache: - # We just take the final tokens, as a [batch, 1] tensor - if index > 0: - logits = self.forward( - tokens[:, -1:], - return_type="logits", - prepend_bos=prepend_bos, - padding_side=padding_side, - past_kv_cache=past_kv_cache, - ) - else: - logits = self.forward( - tokens, - return_type="logits", - prepend_bos=prepend_bos, - padding_side=padding_side, - past_kv_cache=past_kv_cache, - ) - else: - # We input the entire sequence, as a [batch, pos] tensor, since we aren't using - # the cache. - logits = self.forward( - tokens, - return_type="logits", - prepend_bos=prepend_bos, - padding_side=padding_side, - ) - final_logits = logits[:, -1, :] - - if do_sample: - sampled_tokens = utils.sample_logits( - final_logits, - top_k=top_k, - top_p=top_p, - temperature=temperature, - freq_penalty=freq_penalty, - tokens=tokens, - ).to(get_device_for_block_index(0, self.cfg)) - else: - sampled_tokens = final_logits.argmax(-1).to( - get_device_for_block_index(0, self.cfg) - ) - - if stop_at_eos: - # For all unfinished sequences, add on the next token. If a sequence was - # finished, throw away the generated token and add eos_token_for_padding - # instead. - sampled_tokens[finished_sequences] = eos_token_for_padding - finished_sequences.logical_or_( - torch.isin( - sampled_tokens.to(self.cfg.device), - torch.tensor(stop_tokens).to(self.cfg.device), - ) - ) - - new_tokens = sampled_tokens.unsqueeze(-1) - - # Accumulate tokens until we hit max_tokens_per_yield - if index == 0: - accumulated_tokens = torch.cat([tokens, new_tokens], dim=-1) - tokens_since_last_yield = accumulated_tokens.shape[1] - else: - if accumulated_tokens is None: - accumulated_tokens = new_tokens - else: - accumulated_tokens = torch.cat([accumulated_tokens, new_tokens], dim=-1) - tokens_since_last_yield += 1 - - if tokens_since_last_yield >= max_tokens_per_yield: - yield accumulated_tokens - tokens_since_last_yield = 0 - accumulated_tokens = None - - tokens = torch.cat([tokens, new_tokens], dim=-1) - - if stop_at_eos and finished_sequences.all(): - # Yield any remaining accumulated tokens before breaking - if accumulated_tokens is not None: - yield accumulated_tokens - break - - # Only yield remaining tokens if we didn't already yield them in the break case - if accumulated_tokens is not None and not (stop_at_eos and finished_sequences.all()): - yield accumulated_tokens - - @property - def n_params_total(self) -> int: - """Total number of parameters in the model, including embeddings, biases, - and layer norm weights. - - This complements ``self.cfg.n_params``, which counts only the "hidden - weight" parameters (attention projections + MLP weights, excluding - embeddings/biases/layer norms) following the - `scaling laws paper <https://arxiv.org/pdf/2001.08361.pdf>`_ convention. - - Use this when you want the actual parameter count for memory budgeting, - comparison with HuggingFace's ``model.num_parameters()``, or alignment - with reported model sizes in papers (e.g. the Pythia suite). - - Returns: - int: ``sum(p.numel() for p in self.parameters())`` - """ - return sum(p.numel() for p in self.parameters()) - - # Give access to all weights as properties. - @property - def W_U(self) -> Float[torch.Tensor, "d_model d_vocab"]: - """Convenience to get the unembedding matrix. - - I.e. the linear map from the final residual stream to the output logits). - """ - return self.unembed.W_U - - @property - def b_U(self) -> Float[torch.Tensor, "d_vocab"]: - return self.unembed.b_U - - @property - def W_E(self) -> Float[torch.Tensor, "d_vocab d_model"]: - """Convenience to get the embedding matrix.""" - return self.embed.W_E - - @property - def W_pos(self) -> Float[torch.Tensor, "n_ctx d_model"]: - """Convenience function to get the positional embedding. - - Only works on models with absolute positional embeddings! - """ - return self.pos_embed.W_pos - - @property - def W_E_pos(self) -> Float[torch.Tensor, "d_vocab+n_ctx d_model"]: - """Concatenated W_E and W_pos. - - Used as a full (overcomplete) basis of the input space, useful for full QK and full OV - circuits. - """ - return torch.cat([self.W_E, self.W_pos], dim=0) - - # Layer-specific weights are stacked into one massive tensor and given as properties for - # convenience and a cache is used to avoid repeated computation. Often a useful convenience when - # we want to do analysis on weights across all layers. If GPU memory is a bottleneck, don't use - # these properties! - - @property - def W_K(self) -> Float[torch.Tensor, "n_layers n_heads d_model d_head"]: - """Stack the key weights across all layers.""" - return torch.stack([block.attn.W_K for block in self.blocks], dim=0) - - @property - def W_Q(self) -> Float[torch.Tensor, "n_layers n_heads d_model d_head"]: - """Stack the query weights across all layers.""" - return torch.stack([block.attn.W_Q for block in self.blocks], dim=0) - - @property - def W_V(self) -> Float[torch.Tensor, "n_layers n_heads d_model d_head"]: - """Stack the value weights across all layers.""" - return torch.stack([block.attn.W_V for block in self.blocks], dim=0) - - @property - def W_O(self) -> Float[torch.Tensor, "n_layers n_heads d_head d_model"]: - """Stack the attn output weights across all layers.""" - return torch.stack([block.attn.W_O for block in self.blocks], dim=0) - - @property - def W_in(self) -> Float[torch.Tensor, "n_layers d_model d_mlp"]: - """Stack the MLP input weights across all layers.""" - return torch.stack( - [cast(Union[MLP, GatedMLP], block.mlp).W_in for block in self.blocks], dim=0 - ) - - @property - def W_gate(self) -> Union[Float[torch.Tensor, "n_layers d_model d_mlp"], None]: - """Stack the MLP gate weights across all layers. - - Only works for models with gated MLPs. - """ - if self.cfg.gated_mlp: - return torch.stack([cast(GatedMLP, block.mlp).W_gate for block in self.blocks], dim=0) - else: - return None - - @property - def W_out(self) -> Float[torch.Tensor, "n_layers d_mlp d_model"]: - """Stack the MLP output weights across all layers.""" - return torch.stack( - [cast(Union[MLP, GatedMLP], block.mlp).W_out for block in self.blocks], dim=0 - ) - - @property - def b_K(self) -> Float[torch.Tensor, "n_layers n_heads d_head"]: - """Stack the key biases across all layers.""" - return torch.stack([block.attn.b_K for block in self.blocks], dim=0) - - @property - def b_Q(self) -> Float[torch.Tensor, "n_layers n_heads d_head"]: - """Stack the query biases across all layers.""" - return torch.stack([block.attn.b_Q for block in self.blocks], dim=0) - - @property - def b_V(self) -> Float[torch.Tensor, "n_layers n_heads d_head"]: - """Stack the value biases across all layers.""" - return torch.stack([block.attn.b_V for block in self.blocks], dim=0) - - @property - def b_O(self) -> Float[torch.Tensor, "n_layers d_model"]: - """Stack the attn output biases across all layers.""" - return torch.stack([block.attn.b_O for block in self.blocks], dim=0) - - @property - def b_in(self) -> Float[torch.Tensor, "n_layers d_mlp"]: - """Stack the MLP input biases across all layers.""" - return torch.stack( - [cast(Union[MLP, GatedMLP], block.mlp).b_in for block in self.blocks], dim=0 - ) - - @property - def b_out(self) -> Float[torch.Tensor, "n_layers d_model"]: - """Stack the MLP output biases across all layers.""" - return torch.stack( - [cast(Union[MLP, GatedMLP], block.mlp).b_out for block in self.blocks], dim=0 - ) - - @property - def QK(self): - return FactoredMatrix(self.W_Q, self.W_K.transpose(-2, -1)) - - @property - def OV(self): - return FactoredMatrix(self.W_V, self.W_O) - - # Various utility functions - def accumulated_bias( - self, layer: int, mlp_input: bool = False, include_mlp_biases=True - ) -> Float[torch.Tensor, "d_model"]: - """Accumulated Bias. - - Returns the accumulated bias from all layer outputs (ie the b_Os and b_outs), up to the - input of layer L. - - Args: - layer (int): Layer number, in [0, n_layers]. layer==0 means no layers, layer==n_layers - means all layers. - mlp_input (bool): If True, we take the bias up to the input of the MLP - of layer L (ie we include the bias from the attention output of the current layer, - otherwise just biases from previous layers) - include_mlp_biases (bool): Whether to include the biases of MLP layers. Often useful to - have as False if we're expanding attn_out into individual heads, but keeping mlp_out - as is. - - Returns: - bias (torch.Tensor): [d_model], accumulated bias - """ - accumulated_bias = torch.zeros(self.cfg.d_model, device=self.cfg.device) - - for i in range(layer): - block = self.blocks[i] - accumulated_bias += cast(torch.Tensor, block.attn.b_O) - if include_mlp_biases: - accumulated_bias += cast(torch.Tensor, block.mlp.b_out) - if mlp_input: - assert layer < self.cfg.n_layers, "Cannot include attn_bias from beyond the final layer" - block = self.blocks[layer] - accumulated_bias += cast(torch.Tensor, block.attn.b_O) - return accumulated_bias - - def all_composition_scores( - self, mode - ) -> Float[torch.Tensor, "n_layers n_heads n_layers n_heads"]: - """All Composition Scores. - - Returns the Composition scores for all pairs of heads, as a L1, H1, L2, H2 tensor (which is - upper triangular on the first and third axes). - - See - https://transformer-circuits.pub/2021/framework/index.html#:~:text=The%20above%20diagram%20shows%20Q%2D%2C%20K%2D%2C%20and%20V%2DComposition - for three metrics used. - - Args: - mode (str): One of ["Q", "K", "V"], the mode to use for the composition score. - """ - left = self.OV - if mode == "Q": - right = self.QK - elif mode == "K": - right = self.QK.T - elif mode == "V": - right = self.OV - else: - raise ValueError(f"mode must be one of ['Q', 'K', 'V'] not {mode}") - - scores = utils.composition_scores(left, right, broadcast_dims=True) - # Mask scores to be zero for all pairs with the right head in the same layer or earlier - # layer than the left head. - mask = ( - torch.arange(self.cfg.n_layers, device=self.cfg.device)[:, None, None, None] - < torch.arange(self.cfg.n_layers, device=self.cfg.device)[None, None, :, None] - ) - scores = torch.where(mask, scores, torch.zeros_like(scores)) - return scores - - def all_head_labels(self): - """Returns a list of all head names in the model.""" - return [f"L{l}H{h}" for l in range(self.cfg.n_layers) for h in range(self.cfg.n_heads)] - - def load_sample_training_dataset(self, **kwargs): - """Load Sample Training Dataset. - - Helper function to load in a 10K-20K dataset of elements from the model's training data - distribution. - - Wrapper around utils.get_dataset, which identifies the appropriate dataset the pretrained - models. Each dataset has a 'text' field, which contains the relevant info, some have several - meta data fields. - - Kwargs will be passed to utils.get_dataset (e.g. cache_dir to set download location) - - Notes: - - - PT-2's training data is not open source. OpenWebText is a replication (links with - >3 karma on Reddit) - - OPT's training data is not open source, and is a mess of different things that is hard to - replicate. I default to the Pile, which covers some of it, but imperfectly. - - (Some models will have actually been trained on the data supplied here, for some it's from - the validation set). - """ - model_dataset_map = { - "neel": "c4_code", - "neel-solu-old": "pile", - "GPT2LMHeadModel": "openwebtext", - "GPTNeoForCausalLM": "pile", - "GPTNeoXForCausalLM": "pile", - "GPTJForCausalLM": "pile", - "OPTForCausalLM": "pile", - } - if self.cfg.original_architecture in model_dataset_map: - self.dataset = utils.get_dataset( - model_dataset_map[self.cfg.original_architecture], **kwargs - ) - else: - raise ValueError( - f"We do not have an available dataset for the relevant model: {self.cfg.original_architecture}" - ) - return self.dataset - - def sample_datapoint( - self, - tokenize: bool = False, - prepend_bos: Optional[Union[bool, None]] = USE_DEFAULT_VALUE, - padding_side: Optional[Literal["left", "right"]] = USE_DEFAULT_VALUE, - ) -> Union[str, Float[torch.Tensor, "1 pos"]]: - """Sample Data Point from Dataset. - - Helper function to randomly sample a data point from self.dataset, a small dataset from the - data distribution the model was trained on. - - Implicitly calls self.load_sample_training_dataset if it hasn't already been called. Only - works for pretrained models with an associated dataset. But you can manually replace - self.dataset with a dataset of your choice if you want. - - Args: - tokenize (bool): Whether to return tokens (instead of text). Defaults to False. Note - that the returned tokens will be automatically truncated to the model's max context - size. - prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend - the BOS token to the input (applicable when input is a string). Defaults to None, - implying usage of self.cfg.default_prepend_bos (default is True unless specified - otherwise). Pass True or False to override the default. - padding_side (Union[Literal["left", "right"], None], optional): Overrides - self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple - strings of different lengths. - """ - if self.dataset is None: - self.load_sample_training_dataset() - assert self.dataset is not None # keep mypy happy - sample_dataset_size = len(self.dataset) - index = np.random.randint(0, sample_dataset_size) - if not tokenize: - return self.dataset[index]["text"] - else: - return self.to_tokens( - self.dataset[index]["text"], - prepend_bos=prepend_bos, - padding_side=padding_side, - truncate=True, - ) diff --git a/transformer_lens/__init__.py b/transformer_lens/__init__.py index 8da45f2d13..4577ebdd37 100644 --- a/transformer_lens/__init__.py +++ b/transformer_lens/__init__.py @@ -1,25 +1,23 @@ from . import ( - components, conversion_utils, evals, factories, head_detector, hook_points, patching, + supported_models, tools, utilities, ) -from . import loading_from_pretrained as loading -from . import supported_models from .ActivationCache import ActivationCache from .cache.key_value_cache import TransformerLensKeyValueCache from .cache.key_value_cache_entry import TransformerLensKeyValueCacheEntry from .config import TransformerBridgeConfig from .FactoredMatrix import FactoredMatrix -# KEPT infrastructure: HookedRootModule (with HookPoint) survives 4.0 as the -# supported way to hook arbitrary nn.Modules; it is not part of the legacy -# model-class removal below. +# KEPT infrastructure: HookedRootModule (with HookPoint) is the supported way +# to hook arbitrary nn.Modules; it was never part of the legacy model-class +# removal. from .HookedRootModule import HookedRootModule # LIT integration (optional, requires lit-nlp package) @@ -31,33 +29,27 @@ from .SVDInterpreter import SVDInterpreter - -# Legacy names resolved lazily (PEP 562): the deprecated model classes and the -# train shim are deleted in 4.0, and importing transformer_lens must not load -# them eagerly. Each still warns on use via its own module; the deletion PR -# removes entries from this map and nothing else in this file. -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # Static bindings for the lazy names below, so type checkers resolve - # `from transformer_lens import HookedTransformer` to the class, not the - # submodule. Runtime resolution goes through __getattr__. - from . import train - from .BertNextSentencePrediction import BertNextSentencePrediction - from .config import HookedTransformerConfig - from .HookedAudioEncoder import HookedAudioEncoder - from .HookedEncoder import HookedEncoder - from .HookedEncoderDecoder import HookedEncoderDecoder - from .HookedTransformer import HookedTransformer - -_LAZY_LEGACY: dict[str, tuple[str, str | None]] = { - "HookedTransformer": (".HookedTransformer", "HookedTransformer"), - "HookedEncoder": (".HookedEncoder", "HookedEncoder"), - "HookedAudioEncoder": (".HookedAudioEncoder", "HookedAudioEncoder"), - "HookedEncoderDecoder": (".HookedEncoderDecoder", "HookedEncoderDecoder"), - "BertNextSentencePrediction": (".BertNextSentencePrediction", "BertNextSentencePrediction"), - "HookedTransformerConfig": (".config", "HookedTransformerConfig"), - "train": (".train", None), +# Removed in 4.0: directed messages so `from transformer_lens import HookedTransformer` +# (and the other deleted top-level names) fail with a migration pointer instead of a +# bare AttributeError. Submodule-path imports (`from transformer_lens.HookedTransformer +# import ...`) raise ModuleNotFoundError before this hook runs and can't be intercepted here. +_REMOVED_IN_4_0 = { + "HookedTransformer": "Use TransformerBridge.boot_transformers(name), then " + "enable_compatibility_mode() for HookedTransformer-equivalent numerics.", + "HookedEncoder": "Use TransformerBridge.boot_transformers(name) on a BERT model.", + "HookedEncoderDecoder": "Use TransformerBridge.boot_transformers(name) on a T5 model.", + "HookedAudioEncoder": "Use TransformerBridge.boot_transformers(name) on a HuBERT/Wav2Vec2 model.", + "BertNextSentencePrediction": "Use TransformerBridge.boot_transformers(name, " + "model_class=BertForNextSentencePrediction).predict_next_sentence(a, b).", + "HookedTransformerConfig": "Use TransformerBridgeConfig.", + "train": "Use transformer_lens.tools.training (train / TrainConfig).", + "loading": "Model names/aliases moved to transformer_lens.supported_models; " + "config derivation is now internal to TransformerBridge's adapters.", + "loading_from_pretrained": "Config derivation is now internal to TransformerBridge; " + "checkpoint labels live in transformer_lens.tools.model_registry.checkpoints.", + "utils": "Use transformer_lens.utilities (same names).", + "components": "The HookedTransformer component tree was removed; TransformerBridge " + "uses transformer_lens.model_bridge.generalized_components.", } @@ -68,53 +60,25 @@ def __getattr__(name: str): from .model_bridge import TransformerBridge return TransformerBridge - if name in _LAZY_LEGACY: - from importlib import import_module - - module_name, attr = _LAZY_LEGACY[name] - module = import_module(module_name, __name__) - value = module if attr is None else getattr(module, attr) - globals()[name] = value # cache: subsequent access skips __getattr__ - return value + if name in _REMOVED_IN_4_0: + raise AttributeError( + f"{name!r} was removed in TransformerLens 4.0. {_REMOVED_IN_4_0[name]} " + "See docs/source/content/migrating_to_v4.md." + ) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") def __dir__(): - return sorted(set(globals()) | {"TransformerBridge"} | set(_LAZY_LEGACY)) - - -# Five legacy classes share their submodule's name. A submodule-path import -# (``from transformer_lens.HookedTransformer import HookedTransformer``) makes -# the import machinery bind the MODULE onto this package after the fact, and -# ``from transformer_lens import HookedTransformer`` would then return the -# module instead of the class, dependent on import order. The class override -# below de-shadows on every access, so the public name deterministically -# resolves to the class. -import sys as _sys # noqa: E402 -import types as _types # noqa: E402 - -_SHADOWED_CLASS_NAMES = frozenset( - name for name, (_mod, attr) in _LAZY_LEGACY.items() if attr == name -) - - -class _DeShadowingModule(_types.ModuleType): - def __getattribute__(self, name: str): - value = super().__getattribute__(name) - if name in _SHADOWED_CLASS_NAMES and isinstance(value, _types.ModuleType): - value = getattr(value, name) - setattr(self, name, value) - return value - - -_sys.modules[__name__].__class__ = _DeShadowingModule + return sorted(set(globals()) | {"TransformerBridge"}) import os as _os # noqa: E402 # Unconditional: without it, any model whose config writes an integral value for # a float field cannot be loaded at all. See enable_hf_numeric_tower. -from .utilities.hf_utils import enable_hf_numeric_tower as _enable_hf_numeric_tower # noqa: E402 +from .utilities.hf_utils import ( # noqa: E402 + enable_hf_numeric_tower as _enable_hf_numeric_tower, +) _enable_hf_numeric_tower() @@ -124,19 +88,14 @@ def __getattribute__(self, name: str): _enable_hf_retry() __all__ = [ - "HookedTransformerConfig", "TransformerBridge", "TransformerBridgeConfig", "FactoredMatrix", "ActivationCache", - "HookedTransformer", "SVDInterpreter", - "HookedEncoder", - "HookedEncoderDecoder", "HookedRootModule", "TransformerLensKeyValueCache", "TransformerLensKeyValueCacheEntry", - "components", "conversion_utils", "factories", "utilities", diff --git a/transformer_lens/benchmarks/README.md b/transformer_lens/benchmarks/README.md index 2185b09eba..4224df2122 100644 --- a/transformer_lens/benchmarks/README.md +++ b/transformer_lens/benchmarks/README.md @@ -1,6 +1,6 @@ # TransformerBridge Benchmarks -This directory contains a comprehensive benchmark suite for testing TransformerBridge compatibility with HuggingFace models and HookedTransformer. +This directory contains a comprehensive benchmark suite for testing TransformerBridge compatibility with HuggingFace models. ## Overview @@ -25,7 +25,7 @@ results = run_benchmark_suite( model_name="gpt2", device="cpu", use_hf_reference=True, # Compare against HuggingFace model - use_ht_reference=True, # Compare against HookedTransformer + use_hf_reference=True, # Compare against the raw HuggingFace model enable_compatibility_mode=True, verbose=True ) @@ -43,15 +43,12 @@ from transformer_lens.benchmarks import ( benchmark_hook_functionality, benchmark_generation, ) -from transformer_lens import HookedTransformer from transformer_lens.model_bridge import TransformerBridge # Load models bridge = TransformerBridge.boot_transformers("gpt2", device="cpu") bridge.enable_compatibility_mode() -ht = HookedTransformer.from_pretrained("gpt2") - # Run individual benchmarks test_text = "The quick brown fox" @@ -71,7 +68,6 @@ The benchmarks are designed to be used in pytest test suites. Here's how to inte ```python import pytest -from transformer_lens import HookedTransformer from transformer_lens.model_bridge import TransformerBridge from transformer_lens.benchmarks import ( benchmark_loss_equivalence, @@ -84,10 +80,12 @@ class TestTransformerBridgeCompatibility: @pytest.fixture def models(self): """Create models for testing.""" - ht = HookedTransformer.from_pretrained("gpt2") + from transformers import AutoModelForCausalLM + + hf = AutoModelForCausalLM.from_pretrained("gpt2") bridge = TransformerBridge.boot_transformers("gpt2") bridge.enable_compatibility_mode() - return {"ht": ht, "bridge": bridge} + return {"hf": hf, "bridge": bridge} def test_loss_equivalence(self, models): """Test loss computation matches.""" @@ -95,7 +93,7 @@ class TestTransformerBridgeCompatibility: result = benchmark_loss_equivalence( models["bridge"], test_text, - reference_model=models["ht"], + reference_model=models["hf"], atol=1e-3 ) assert result.passed, result.message @@ -106,7 +104,7 @@ class TestTransformerBridgeCompatibility: result = benchmark_logits_equivalence( models["bridge"], test_text, - reference_model=models["ht"], + reference_model=models["hf"], atol=3e-2, rtol=3e-2 ) @@ -118,7 +116,7 @@ class TestTransformerBridgeCompatibility: result = benchmark_hook_functionality( models["bridge"], test_text, - reference_model=models["ht"], + reference_model=models["hf"], atol=2e-3 ) assert result.passed, result.message @@ -181,7 +179,7 @@ The benchmarks use a tiered approach for comparison: - Direct comparison with original HF implementation - Ensures bridge maintains model fidelity -2. **Second Priority**: Compare TransformerBridge → HookedTransformer +2. **Second Priority**: Compare TransformerBridge → the raw HuggingFace model - If HT version exists, compare processed outputs - Ensures compatibility with TransformerLens ecosystem @@ -288,7 +286,7 @@ Failed: 1 (6.2%) - LayerNorm bridging numerical differences - Attention pattern computation differences -- **Performance**: Full hook comparison tests are computationally expensive and only run when a HookedTransformer reference is available. +- **Performance**: Full hook comparison tests are computationally expensive. ## Contributing @@ -303,5 +301,4 @@ When adding new test patterns: ## See Also - [TransformerBridge Documentation](../model_bridge/README.md) -- [HookedTransformer API](../HookedTransformer.py) - [Test Suite](../../tests/) diff --git a/transformer_lens/cache/key_value_cache.py b/transformer_lens/cache/key_value_cache.py index 3ccf81af9a..807790240f 100644 --- a/transformer_lens/cache/key_value_cache.py +++ b/transformer_lens/cache/key_value_cache.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Union, cast +from typing import List, Union import torch from jaxtyping import Int @@ -15,11 +15,6 @@ from .key_value_cache_entry import TransformerLensKeyValueCacheEntry -if TYPE_CHECKING: - from transformer_lens.config.hooked_transformer_config import ( - HookedTransformerConfig, - ) - @dataclass class TransformerLensKeyValueCache: @@ -38,16 +33,14 @@ class TransformerLensKeyValueCache: @classmethod def init_cache( cls, - cfg: Union[TransformerLensConfig, "HookedTransformerConfig"], + cfg: TransformerLensConfig, device: Union[torch.device, str, None], batch_size: int = 1, ): # Determine device for each layer if hasattr(cfg, "n_devices"): # HookedTransformer case: use our multi-GPU logic - device_for_layer = lambda i: get_device_for_block_index( - i, cast("HookedTransformerConfig", cfg), device - ) + device_for_layer = lambda i: get_device_for_block_index(i, cfg, device) else: # Fallback when no model is provided - use single device fallback_device = device if device is not None else cfg.device diff --git a/transformer_lens/components/__init__.py b/transformer_lens/components/__init__.py deleted file mode 100644 index 44c98f6e7f..0000000000 --- a/transformer_lens/components/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Hooked Transformer Components. - -This module contains all the components (e.g. :class:`Attention`, :class:`MLP`, :class:`LayerNorm`) -needed to create many different types of generative language models. They are used by -:class:`transformer_lens.HookedTransformer`. -""" - -# Independent classes -from .abstract_attention import AbstractAttention -from .layer_norm import LayerNorm -from .layer_norm_pre import LayerNormPre -from .pos_embed import PosEmbed -from .rms_norm import RMSNorm -from .rms_norm_pre import RMSNormPre -from .token_typed_embed import TokenTypeEmbed -from .unembed import Unembed - -# Only dependent on independent modules -from .attention import Attention -from .bert_mlm_head import BertMLMHead -from .bert_nsp_head import BertNSPHead -from .bert_pooler import BertPooler -from .embed import Embed -from .grouped_query_attention import GroupedQueryAttention -from .mlps.gated_mlp import GatedMLP -from .mlps.mlp import MLP - -# Interdependent modules -from .bert_block import BertBlock -from .bert_embed import BertEmbed -from .mlps.moe import MoE -from .transformer_block import TransformerBlock -from .t5_attention import T5Attention -from .t5_block import T5Block diff --git a/transformer_lens/components/abstract_attention.py b/transformer_lens/components/abstract_attention.py deleted file mode 100644 index 1c0eb46678..0000000000 --- a/transformer_lens/components/abstract_attention.py +++ /dev/null @@ -1,1050 +0,0 @@ -import math -from abc import ABC -from typing import Dict, Optional, Tuple, Union, cast - -import einops -import torch -import torch.nn as nn -import torch.nn.functional as F -from better_abc import abstract_attribute -from jaxtyping import Float, Int -from torch import Tensor -from transformers.utils.import_utils import is_bitsandbytes_available - -from transformer_lens.cache.key_value_cache_entry import ( - TransformerLensKeyValueCacheEntry, -) -from transformer_lens.components.rms_norm import RMSNorm -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.FactoredMatrix import FactoredMatrix -from transformer_lens.hook_points import HookPoint -from transformer_lens.utilities import get_offset_position_ids -from transformer_lens.utilities.activation_functions import apply_softcap -from transformer_lens.utilities.attention import complex_attn_linear, simple_attn_linear - -if is_bitsandbytes_available(): - import bitsandbytes as bnb - from bitsandbytes.nn.modules import Params4bit - - -class AbstractAttention(ABC, nn.Module): - ROTARY_INITIAL_CACHE_SIZE = 2048 - - alibi: Union[torch.Tensor, None] - q_norm: Optional[RMSNorm] - k_norm: Optional[RMSNorm] - mask: torch.Tensor - IGNORE: torch.Tensor - rotary_sin: torch.Tensor - rotary_cos: torch.Tensor - - def __init__( - self, - cfg: Union[Dict, HookedTransformerConfig], - attn_type: str = "global", - layer_id: Optional[int] = None, - ): - """Abstract Base Class of Attention Blocks, featuring common functionality of both Attention and GroupedQueryAttention blocks. - - Query and Output projections are defined in this class as they are the same for regular and grouped query attention. - Attributes related to Key and Value projections are abstract as their implementations may differ. For example, in GroupedQueryAttention there are less query and key heads than value heads. - To enforce implementation of W_K, W_V, b_K, and b_V by child classes, the better_abc.abstract_attribute class is used. See here for details: https://stackoverflow.com/questions/23831510/abstract-attribute-not-property. - - Args: - cfg (Union[Dict, HookedTransformerConfig]): Config - attn_type (str, optional): "global" or "local", used by GPT-Neo. Local attention means the model can only attend back cfg.window_size tokens (here, 256). Not used by any other model at the moment. Defaults to "global". - layer_id (int, optional): The index of the current layer. Used by the Mistral models (labelled here as stanford-gpt2) to scale down attention scores pre softmax for numerical stability reasons by 1/(layer_id+1). Defaults to None. - """ - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - - if self.cfg.load_in_4bit: - nq = int((self.cfg.d_model * self.cfg.d_head * self.cfg.n_heads) / 2) - self.W_Q: Union[nn.Parameter, "Params4bit"] = Params4bit( - torch.empty(nq, 1, dtype=torch.uint8), requires_grad=False - ) - self.W_O: Union[nn.Parameter, "Params4bit"] = Params4bit( - torch.empty(nq, 1, dtype=torch.uint8), requires_grad=False - ) - else: - self.W_Q = nn.Parameter( - torch.empty( - self.cfg.n_heads, - self.cfg.d_model, - self.cfg.d_head, - dtype=self.cfg.dtype, - ) - ) - self.W_O = nn.Parameter( - torch.empty( - self.cfg.n_heads, - self.cfg.d_head, - self.cfg.d_model, - dtype=self.cfg.dtype, - ) - ) - self.W_K = abstract_attribute() - self.W_V = abstract_attribute() - - self.b_Q = nn.Parameter( - torch.zeros(self.cfg.n_heads, self.cfg.d_head, dtype=self.cfg.dtype) - ) - self.b_K: nn.Parameter = abstract_attribute() - self.b_V: nn.Parameter = abstract_attribute() - self.b_O = nn.Parameter(torch.zeros(self.cfg.d_model, dtype=self.cfg.dtype)) - - if self.cfg.use_qk_norm: - self.q_norm = RMSNorm(self.cfg, length=self.cfg.d_head) - self.k_norm = RMSNorm(self.cfg, length=self.cfg.d_head) - - elif self.cfg.original_architecture in ( - "OlmoeForCausalLM", - "Olmo2ForCausalLM", - "Olmo3ForCausalLM", - ): - # Q/K norms applied on full projected vectors (before head reshape). - # q_norm dim = n_heads * d_head = d_model - self.q_norm: Optional[RMSNorm] = RMSNorm(self.cfg, self.cfg.d_model) - # k_norm dim depends on whether GQA is used: - # OLMo 2 (MHA): n_kv_heads == n_heads, so d_model - # OLMo 3 / OLMoE (GQA): n_kv_heads * d_head - if self.cfg.n_key_value_heads is not None: - k_norm_dim = self.cfg.d_head * self.cfg.n_key_value_heads - else: - k_norm_dim = self.cfg.d_model - self.k_norm: Optional[RMSNorm] = RMSNorm(self.cfg, k_norm_dim) - else: - self.q_norm = None - self.k_norm = None - - self.attn_type = attn_type - if self.attn_type == "local": - if not isinstance(self.cfg.window_size, int): - raise ValueError("Window size must be an integer for local attention") - elif self.attn_type != "global": - raise ValueError(f"Invalid attention type: {self.attn_type}") - - # Encoder-decoder models share one cfg across stacks whose directions - # differ, so a stack that must mask sets this rather than cfg. - self._attention_dir_override: Optional[str] = None - self._ntk_alpha_cached: float = 1.0 - - # Kept as a tiny buffer for state-dict/device compatibility. The actual - # causal mask is built at forward time for the current sequence length. - self.register_buffer("mask", torch.empty((0, 0), dtype=torch.bool)) - self.register_buffer("IGNORE", torch.tensor(-torch.inf)) - - self.layer_id = layer_id - - # attn_scale is a constant that we divide the attention scores by pre-softmax. I'm not entirely sure why it matters, but it's probably a mix of softmax not being scale invariant and numerical stability? - if self.cfg.use_attn_scale: - self.attn_scale = self.cfg.attn_scale # Defaults to sqrt(d_head) - else: - self.attn_scale = 1.0 - if self.cfg.scale_attn_by_inverse_layer_idx: - if self.layer_id is None: # keep mypy happy - raise ValueError("Layer ID must be provided to scale attention scores") - self.attn_scale *= self.layer_id + 1 - - if self.cfg.use_attention_sinks: - # Learned per-head sink logit (GPT-OSS); joins the softmax as an - # extra key column and is dropped afterward. - self.sinks = nn.Parameter(torch.zeros(self.cfg.n_heads, dtype=self.cfg.dtype)) - - self.hook_k = HookPoint() # [batch, pos, head_index, d_head] - self.hook_q = HookPoint() # [batch, pos, head_index, d_head] - self.hook_v = HookPoint() # [batch, pos, head_index, d_head] - self.hook_z = HookPoint() # [batch, pos, head_index, d_head] - self.hook_attn_scores = HookPoint() # [batch, head_index, query_pos, key_pos] - self.hook_pattern = HookPoint() # [batch, head_index, query_pos, key_pos] - self.hook_result = HookPoint() # [batch, pos, head_index, d_model] - - # See HookedTransformerConfig for more details. - if self.cfg.positional_embedding_type == "shortformer": - # This tracks the input to the keys and queries, which is resid_pre + pos_embeds - self.hook_attn_input = HookPoint() # [batch, pos, d_model] - elif self.cfg.positional_embedding_type == "rotary": - # Applies a rotation to each two-element chunk of keys and queries pre dot producting to bake in relative position. See HookedTransformerConfig for details - self.hook_rot_k = HookPoint() - self.hook_rot_q = HookPoint() - if self.cfg.rotary_dim is None: # keep mypy happy - raise ValueError("Rotary dim must be provided for rotary positional embeddings") - rotary_cache_size = min(self.cfg.n_ctx, self.ROTARY_INITIAL_CACHE_SIZE) - sin, cos = self.calculate_sin_cos_rotary( - self.cfg.rotary_dim, - rotary_cache_size, - base=self._rotary_base(), - dtype=self.cfg.dtype, - ) - self.register_buffer("rotary_sin", sin) - self.register_buffer("rotary_cos", cos) - elif self.cfg.positional_embedding_type == "alibi": - # ALiBi bias will be constructed on the first forward pass. - # Note: While computationally efficient, initializing an bias with max n_ctx (16, 1024, 1024) of float32 will occupy ~256MiB of contiguous GPU memory, which may not be optimal for memory usage. - self.alibi = None - - elif self.cfg.positional_embedding_type == "relative_positional_bias": - # will be overwritten by the child T5Attention class - self.has_relative_attention_bias = False - - @property - def OV(self) -> FactoredMatrix: - """ - OV-Circuit, as defined in A Mathematical Framework. Because there's no non-linearity between the value vector and the output of the layer, the output is purely determined by the matrix W_OV = W_V @ W_O, and not W_V or W_O individually. (Mathematically, for a single head, output == pattern @ residual @ W_V @ W_O, see the glossary for more) - - Done in the order W_V, W_O because the paper uses left-multiplying weight matrices, and TransformerLens uses right-multiplying, sorry! - - Returns a FactoredMatrix, with left matrix W_V [head_index, d_model, d_head] and right matrix W_O [head_index, d_head, d_model] - this is a low rank factorisation of the underlying [head_index, d_model, d_model]. FactoredMatrix has helper functions to deal with these large matrices efficiently. To get the OV circuit of a head k, attn.OV[k] works. - """ - return FactoredMatrix(self.W_V, self.W_O) - - @property - def QK(self) -> FactoredMatrix: - """ - QK-Circuit, as defined in A Mathematical Framework. Because there's no non-linearity in the key-query dot product, the output is purely determined by the matrix W_QK = W_Q.T @ W_K, and not W_Q or W_K individually. (Mathematically, for a single head, pattern = destination_residual.T @ W_Q.T @ W_K @ source-residual, see the glossary for more). - - Done in the order Q on the left, K on the right, because the pattern has dimensions [destination_pos, source_pos] - - Returns a FactoredMatrix, with left matrix W_Q [head_index, d_model, d_head] and right matrix W_K.T [head_index, d_head, d_model] - this is a low rank factorisation of the underlying [head_index, d_model, d_model] matrix. FactoredMatrix has helper functions to deal with these large matrices efficiently. To get the QK circuit of a head k, attn.QK[k] works. - """ - W_K_transpose = einops.rearrange( - self.W_K, "head_index d_model d_head -> head_index d_head d_model" - ) - return FactoredMatrix(self.W_Q, W_K_transpose) - - def forward( - self, - query_input: Union[ - Float[torch.Tensor, "batch pos d_model"], - Float[torch.Tensor, "batch pos head_index d_model"], - ], - key_input: Union[ - Float[torch.Tensor, "batch kv_pos d_model"], - Float[torch.Tensor, "batch kv_pos head_index d_model"], - Float[torch.Tensor, "batch kv_pos kv_head_index d_model"], - ], - value_input: Union[ - Float[torch.Tensor, "batch kv_pos d_model"], - Float[torch.Tensor, "batch kv_pos head_index d_model"], - Float[torch.Tensor, "batch kv_pos kv_head_index d_model"], - ], - past_kv_cache_entry: Optional[TransformerLensKeyValueCacheEntry] = None, - additive_attention_mask: Optional[Float[torch.Tensor, "batch 1 1 kv_pos"]] = None, - attention_mask: Optional[Int[torch.Tensor, "batch offset_pos"]] = None, - position_bias: Optional[Float[torch.Tensor, "1 head_index pos kv_pos"]] = None, - ) -> Float[torch.Tensor, "batch pos d_model"]: - """ - shortformer_pos_embed is only used if self.cfg.positional_embedding_type == "shortformer", else defaults to None and is irrelevant. See HookedTransformerConfig for more details - past_kv_cache_entry is an optional entry of past keys and values for this layer, only relevant if generating text. Defaults to None - additive_attention_mask is an optional mask to add to the attention weights. Defaults to None. - attention_mask is the attention mask for padded tokens. Defaults to None. - """ - - q, k, v = self.calculate_qkv_matrices(query_input, key_input, value_input) - - # OLMo-family QK-norm: applied on full projected vectors before head reshape. - if self.cfg.original_architecture in ( - "OlmoeForCausalLM", - "Olmo2ForCausalLM", - "Olmo3ForCausalLM", - ): - assert self.q_norm is not None - assert self.k_norm is not None - q = einops.rearrange( - self.q_norm( - einops.rearrange( - q, - "batch pos head_index d_head -> batch pos (head_index d_head)", - ) - ), - "batch kv_pos (head_index d_head) -> batch kv_pos head_index d_head", - head_index=q.shape[2], - ) - k = einops.rearrange( - self.k_norm( - einops.rearrange( - k, - "batch pos head_index d_head -> batch pos (head_index d_head)", - ) - ), - "batch kv_pos (head_index d_head) -> batch kv_pos head_index d_head", - head_index=k.shape[2], - ) - - # OLMo v1 / OLMoE clamp Q/K/V after projection (and any qk-norm), - # before RoPE and cache append. Out-of-place so hooks stay legal. - if self.cfg.clip_qkv is not None: - q = q.clamp(min=-self.cfg.clip_qkv, max=self.cfg.clip_qkv) - k = k.clamp(min=-self.cfg.clip_qkv, max=self.cfg.clip_qkv) - v = v.clamp(min=-self.cfg.clip_qkv, max=self.cfg.clip_qkv) - - if past_kv_cache_entry is not None: - # Appends the new keys and values to the cached values, and automatically updates the cache - kv_cache_pos_offset = past_kv_cache_entry.past_keys.size(1) - k, v = past_kv_cache_entry.append(k, v) - else: - # Not using a cache - kv_cache_pos_offset = 0 - - if self.cfg.positional_embedding_type == "rotary": - q = self.hook_rot_q(self.apply_rotary(q, kv_cache_pos_offset, attention_mask)) - k = self.hook_rot_k( - self.apply_rotary(k, 0, attention_mask) - ) # keys are cached so no offset - - if self.cfg.use_logn_attn and not self.training: - q = self._apply_logn_scaling(q, kv_cache_pos_offset) - - attn_scores = self.calculate_attention_scores( - q, k - ) # [batch, head_index, query_pos, key_pos] - - if self.cfg.positional_embedding_type == "alibi": - query_ctx = attn_scores.size(-2) - # The key context length is the number of positions in the past - this includes all positions in the cache - key_ctx = attn_scores.size(-1) - - # only recompute when necessary to increase efficiency. - if self.alibi is None or key_ctx > self.alibi.size(-1): - self.alibi = AbstractAttention.create_alibi_bias( - self.cfg.n_heads, key_ctx, self.cfg.device - ) - - # Take the last query_ctx positions so it also works with past_kv_cache - if isinstance(self.alibi, torch.Tensor): - attn_scores += self.alibi[:, -query_ctx:, :key_ctx] - else: - raise TypeError( - f"Expected self.alibi to be a Tensor, but got {type(self.alibi)}" - ) # [batch, head_index, query_pos, key_pos] - elif self.cfg.positional_embedding_type == "relative_positional_bias": - if position_bias is None: - if self.has_relative_attention_bias: - raise ValueError("Positional bias is required for relative_positional_bias") - else: - position_bias = torch.zeros( - 1, - self.cfg.n_heads, - attn_scores.shape[2], - attn_scores.shape[3], - device=attn_scores.device, - ) - - if position_bias is not None: # Add None check - attn_scores += position_bias - if (self._attention_dir_override or self.cfg.attention_dir) == "causal": - # If causal attention, we mask it to only attend backwards. If bidirectional, we don't mask. - attn_scores = self.apply_causal_mask( - attn_scores, kv_cache_pos_offset, attention_mask - ) # [batch, head_index, query_pos, key_pos] - if additive_attention_mask is not None: - attn_scores += additive_attention_mask - - attn_scores = self.hook_attn_scores(attn_scores) - if self.cfg.use_attention_sinks: - # The sink column is appended after hook_attn_scores so hooks keep - # the [batch, head, query_pos, key_pos] shape, and dropped after - # the softmax — every real position's weight is scaled down by the - # sink's share, so pattern rows sum to less than 1. - sink = ( - self.sinks.reshape(1, -1, 1, 1) - .expand(attn_scores.shape[0], -1, attn_scores.shape[-2], -1) - .to(attn_scores.dtype) - ) - combined = torch.cat([attn_scores, sink], dim=-1) - combined = combined - combined.max(dim=-1, keepdim=True).values - pattern = F.softmax(combined, dim=-1)[..., :-1] - else: - pattern = F.softmax(attn_scores, dim=-1) - if not isinstance(pattern, torch.Tensor): - raise TypeError(f"Expected 'pattern' to be a Tensor, got {type(pattern)}") - pattern = torch.where(torch.isnan(pattern), torch.zeros_like(pattern), pattern) - pattern = self.hook_pattern(pattern) # [batch, head_index, query_pos, key_pos] - pattern = pattern.to(device=v.device, dtype=v.dtype) - z = self.calculate_z_scores(v, pattern) # [batch, pos, head_index, d_head] - if not self.cfg.use_attn_result: - if self.cfg.load_in_4bit: - # call bitsandbytes method to dequantize and multiply - W_O_4bit = cast(Params4bit, self.W_O) - out = ( - bnb.matmul_4bit( - z.reshape(z.shape[0], z.shape[1], self.cfg.d_head * self.cfg.n_heads), - W_O_4bit.t(), - bias=None, - quant_state=W_O_4bit.quant_state, - ) - + self.b_O - ) - else: - w = einops.rearrange( - self.W_O, "head_index d_head d_model -> d_model (head_index d_head)" - ).contiguous() - - # Move output projection weights and bias to the same device as z - # so that the final linear operation occurs on the device of the inputs - if w.device != z.device: - w = w.to(z.device) - b_O: Tensor = self.b_O - if b_O.device != z.device: - b_O = b_O.to(z.device) - # Ensure z has the same dtype as weights used in the output projection - if z.dtype != w.dtype: - z = z.to(w.dtype) - - z = z.reshape(z.shape[0], z.shape[1], self.cfg.d_head * self.cfg.n_heads) - - # F.linear is a fused matmul+bias that matches HuggingFace exactly, - # but has a bug on MPS with PyTorch 2.8 (pytorch#161640). - # Fall back to manual matmul on MPS to work around it. - if z.device.type == "mps": - out = torch.matmul(z, w.T) + b_O - else: - out = F.linear(z, w, b_O) - else: - # Explicitly calculate the attention result so it can be accessed by a hook - # This is off by default because it can easily eat through your GPU memory. - if self.cfg.load_in_4bit: - W_O_4bit = cast(Params4bit, self.W_O) - result = self.hook_result( - bnb.matmul_4bit( - z.reshape(z.shape[0], z.shape[1], self.cfg.d_head * self.cfg.n_heads), - W_O_4bit.t(), - bias=None, - quant_state=W_O_4bit.quant_state, - ) - ) - else: - # Add singleton dimensions to make shapes compatible for broadcasting: - w = einops.rearrange( - self.W_O, - "head_index d_head d_model -> 1 1 head_index d_head d_model", - ) - if w.device != z.device: - w = w.to(z.device) - # Ensure z has the same dtype as w before multiplication - if z.dtype != w.dtype: - z = z.to(w.dtype) - z = einops.rearrange( - z, "batch pos head_index d_head -> batch pos head_index d_head 1" - ) - - unhooked_result = (z * w).sum(-2) - - result = self.hook_result(unhooked_result) # [batch, pos, head_index, d_model] - out = ( - einops.reduce(result, "batch position index model->batch position model", "sum") - + self.b_O - ) # [batch, pos, d_model] - return out - - def _apply_qk_norm( - self, x: Float[torch.Tensor, "batch pos head_index d_head"], norm_module: RMSNorm - ) -> Float[torch.Tensor, "batch pos head_index d_head"]: - """Apply QK normalization with proper reshaping. - - Args: - x: Input tensor with shape [batch, pos, head_index, d_head] - norm_module: RMSNorm module to apply - - Returns: - Normalized tensor with same shape as input - """ - # Reshape from [batch, pos, head_index, d_head] to [batch * pos * head_index, d_head] - d_head = x.shape[-1] - x_normed = norm_module(x.reshape(-1, d_head)) - return x_normed.reshape(x.shape) - - def calculate_qkv_matrices( - self, - query_input: Union[ - Float[torch.Tensor, "batch pos d_model"], - Float[torch.Tensor, "batch pos head_index d_model"], - ], - key_input: Union[ - Float[torch.Tensor, "batch kv_pos d_model"], - Float[torch.Tensor, "batch kv_pos head_index d_model"], - ], - value_input: Union[ - Float[torch.Tensor, "batch kv_pos d_model"], - Float[torch.Tensor, "batch kv_pos head_index d_model"], - ], - ) -> Tuple[ - Float[torch.Tensor, "batch pos head_index d_head"], - Float[torch.Tensor, "batch kv_pos head_index d_head"], - Float[torch.Tensor, "batch kv_pos head_index d_head"], - ]: - attn_fn = ( - complex_attn_linear - if self.cfg.use_split_qkv_input or self.cfg.use_attn_in - else simple_attn_linear - ) - if self.cfg.load_in_4bit: - q = self.hook_q(self._project_4bit_qkv(query_input, self.W_Q, self.b_Q)) - else: - q = self.hook_q(attn_fn(query_input, self.W_Q, self.b_Q)) - if self.cfg.load_in_4bit: - k = self.hook_k(self._project_4bit_qkv(key_input, self.W_K, self.b_K)) - else: - k = self.hook_k(attn_fn(key_input, self.W_K, self.b_K)) - - if self.cfg.load_in_4bit: - v = self.hook_v(self._project_4bit_qkv(value_input, self.W_V, self.b_V)) - else: - v = self.hook_v(attn_fn(value_input, self.W_V, self.b_V)) - - if self.cfg.use_qk_norm: - assert self.q_norm is not None - assert self.k_norm is not None - q = self._apply_qk_norm(q, self.q_norm) - k = self._apply_qk_norm(k, self.k_norm) - - return q, k, v - - def _project_4bit_qkv( - self, - input: Union[ - Float[torch.Tensor, "batch pos d_model"], - Float[torch.Tensor, "batch pos head_index d_model"], - ], - weight: Union[nn.Parameter, "Params4bit"], - bias: Float[torch.Tensor, "head_index d_head"], - ) -> Float[torch.Tensor, "batch pos head_index d_head"]: - """Project Q/K/V inputs with a 4-bit weight. - - Split inputs dequantize once and reuse the head-wise projection path. - """ - if not isinstance(weight, Params4bit): - raise ValueError("QKV weights must be Params4bit objects if load_in_4bit is True") - - n_heads, d_head = bias.shape - - if input.ndim == 3: - projected = bnb.matmul_4bit( - input, - weight.t(), - bias=None, - quant_state=weight.quant_state, - ) - return projected.reshape(input.shape[0], input.shape[1], n_heads, d_head) + bias - - if input.ndim == 4: - if input.shape[2] != n_heads: - raise ValueError( - "4-bit split QKV inputs must have one input slice per attention head; " - f"got {input.shape[2]} input heads for {n_heads} projection heads." - ) - dequantized_weight = bnb.functional.dequantize_4bit( - weight.data, quant_state=weight.quant_state - ) - split_weight = einops.rearrange( - dequantized_weight, - "(head_index d_head) d_model -> head_index d_model d_head", - head_index=n_heads, - d_head=d_head, - ) - return complex_attn_linear(input, split_weight, bias) - - raise ValueError( - "4-bit QKV projection input must have shape [batch, pos, d_model] or " - "[batch, pos, head_index, d_model]." - ) - - def _apply_logn_scaling(self, q: torch.Tensor, kv_cache_pos_offset: int) -> torch.Tensor: - """Qwen-1's log-n query scaling past the training length (eval only). - - Thresholds on the training length, never n_ctx — reaching these contexts - requires overriding n_ctx upward, which must not move the threshold. - """ - train_length = self.cfg.train_seq_length or self.cfg.n_ctx - query_length = q.size(1) - key_length = kv_cache_pos_offset + query_length - if key_length <= train_length: - return q - positions = torch.arange( - kv_cache_pos_offset + 1, key_length + 1, device=q.device, dtype=torch.float32 - ) - scale = torch.where( - positions > train_length, - positions.log() / math.log(train_length), - torch.ones((), device=q.device), - ).to(q.dtype) - return q * scale[None, :, None, None] - - def calculate_attention_scores( - self, - q: Float[torch.Tensor, "batch query_pos head_index d_head"], - k: Float[torch.Tensor, "batch key_pos head_index d_head"], - ) -> Float[torch.Tensor, "batch head_index query_pos key_pos"]: - q_ = einops.rearrange( - q, "batch query_pos head_index d_head -> batch head_index query_pos d_head" - ) - k_ = einops.rearrange( - k, "batch key_pos head_index d_head -> batch head_index d_head key_pos" - ) - attn_scores = q_ @ k_ / self.attn_scale - attn_scores = apply_softcap(attn_scores, self.cfg.attn_scores_soft_cap) - return attn_scores - - def calculate_z_scores( - self, - v: Float[torch.Tensor, "batch key_pos head_index d_head"], - pattern: Float[torch.Tensor, "batch head_index query_pos key_pos"], - ) -> Float[torch.Tensor, "batch query_pos head_index d_head"]: - v_ = einops.rearrange( - v, "batch key_pos head_index d_head -> batch head_index key_pos d_head" - ) - pattern_ = einops.rearrange( - pattern, - "batch head_index query_pos key_pos -> batch head_index query_pos key_pos", - ) - z = self.hook_z( - einops.rearrange( - pattern_ @ v_, - "batch head_index query_pos d_head -> batch query_pos head_index d_head", - ) - ) - return z - - def apply_causal_mask( - self, - attn_scores: Float[torch.Tensor, "batch head_index pos pos_plus_past_kv_pos_offset"], - past_kv_pos_offset: int = 0, - attention_mask: Optional[Int[torch.Tensor, "batch offset_pos"]] = None, - ): - # The query context length is the number of positions we take queries from - if not using a past_kv_cache this is just the context length (for the current prompt), but if we're caching it can be different. - query_ctx_length = attn_scores.size(-2) - # The key context length is the number of positions in the past - this includes all positions in the cache - # If not caching, query_ctx_length == key_ctx_length - key_ctx_length = attn_scores.size(-1) - - if query_ctx_length + past_kv_pos_offset != key_ctx_length: - raise ValueError( - f"query_ctx_length {query_ctx_length} + past_kv_pos_offset {past_kv_pos_offset} != key_ctx_length {key_ctx_length} - you likely have a bug." - ) - - mask_device = attention_mask.device if attention_mask is not None else attn_scores.device - final_mask = self._make_causal_mask( - query_ctx_length=query_ctx_length, - key_ctx_length=key_ctx_length, - past_kv_pos_offset=past_kv_pos_offset, - device=mask_device, - ) - if attention_mask is not None: - # Apply a causal mask to the attention scores considering the padding - - # Add singleton dimensions to the attention mask to match the shape of the final mask - attention_mask = einops.rearrange( - attention_mask, "batch offset_pos -> batch 1 1 offset_pos" - ) - - final_mask = final_mask & attention_mask.bool() # [batch, head, pos, offset_pos] - - attn_scores = attn_scores.to(final_mask.device) - ignore = cast(torch.Tensor, self.IGNORE).to(final_mask.device) - return torch.where(final_mask, attn_scores, ignore) - - def _make_causal_mask( - self, - query_ctx_length: int, - key_ctx_length: int, - past_kv_pos_offset: int, - device: torch.device, - ) -> torch.Tensor: - """Create the causal mask for the current attention-score shape.""" - query_positions = torch.arange( - past_kv_pos_offset, - past_kv_pos_offset + query_ctx_length, - device=device, - ) - key_positions = torch.arange(key_ctx_length, device=device) - - final_mask = key_positions[None, :] <= query_positions[:, None] - if self.attn_type == "local": - if not isinstance(self.cfg.window_size, int): - raise ValueError("Window size must be an integer for local attention") - final_mask = final_mask & ( - key_positions[None, :] > query_positions[:, None] - self.cfg.window_size - ) - - return final_mask[None, None, :, :] - - def _rotary_base(self) -> Union[float, int]: - if self.cfg.rotary_base_local is not None and self.attn_type == "local": - return self.cfg.rotary_base_local - return self.cfg.rotary_base - - def calculate_sin_cos_rotary( - self, - rotary_dim: int, - n_ctx: int, - base: Union[float, int] = 10000, - dtype: torch.dtype = torch.float32, - ) -> Tuple[Float[torch.Tensor, "n_ctx rotary_dim"], Float[torch.Tensor, "n_ctx rotary_dim"]]: - """ - Calculate the sine and cosine waves to use in a rotary embedding. See https://blog.eleuther.ai/rotary-embeddings/ for details - - Note: For some inexplicable reason, in GPT-J each ADJACENT pair of elements in k and q are rotated, in GPT-NeoX the pair of elements at k and k+n//2 are rotated (ie folding the full length in half, and then looking at pairs accordingly). I have absolutely no clue why, it should be completely equivalent. - To resolve this, I've coded it to default to the GPT-J mode, but to explicitly check whether it's GPT-NeoX and then do the GPT-NeoX thing if it is. - """ - high_precision = torch.float32 if dtype != torch.float64 else torch.float64 - pos = torch.arange(n_ctx, dtype=high_precision) - dim = torch.arange(rotary_dim // 2, dtype=high_precision) - - use_yarn = self.cfg.use_yarn_rope and not ( - self.cfg.yarn_global_attn_only and self.attn_type == "local" - ) - - # Llama-3.1 uses NTK-by-Parts Rotary Embedding introduced in Section 3.2 in https://arxiv.org/pdf/2309.00071 - # Implementation copied from https://github.com/huggingface/transformers/blob/v4.46.0/src/transformers/modeling_rope_utils.py#L310 - if self.cfg.use_NTK_by_parts_rope: - inv_freq = 1.0 / ( - base ** (torch.arange(0, rotary_dim, 2, dtype=torch.int64).float() / rotary_dim) - ) - factor = self.cfg.NTK_by_parts_factor - low_freq_factor = self.cfg.NTK_by_parts_low_freq_factor - high_freq_factor = self.cfg.NTK_by_parts_high_freq_factor - old_context_len = self.cfg.NTK_original_ctx_len - - low_freq_wavelen = old_context_len / low_freq_factor - high_freq_wavelen = old_context_len / high_freq_factor - - wavelen = 2 * math.pi / inv_freq - inv_freq_llama = torch.where(wavelen > low_freq_wavelen, inv_freq / factor, inv_freq) - smooth_factor = (old_context_len / wavelen - low_freq_factor) / ( - high_freq_factor - low_freq_factor - ) - smoothed_inv_freq = ( - 1 - smooth_factor - ) * inv_freq_llama / factor + smooth_factor * inv_freq_llama - is_medium_freq = ~(wavelen < high_freq_wavelen) * ~(wavelen > low_freq_wavelen) - inv_freq_llama = torch.where(is_medium_freq, smoothed_inv_freq, inv_freq_llama) - freq = 1 / inv_freq_llama - elif use_yarn: - # YARN (Yet Another RoPE extensioN) from https://arxiv.org/abs/2309.00071 - # Implementation follows HuggingFace: transformers/modeling_rope_utils.py - inv_freq = 1.0 / ( - base ** (torch.arange(0, rotary_dim, 2, dtype=high_precision) / rotary_dim) - ) - yarn_factor = self.cfg.yarn_factor - # HF uses original_max_position_embeddings (the pre-extension context length) - # for computing the correction range. - orig_max_pos = self.cfg.yarn_original_max_position_embeddings - beta_fast = self.cfg.yarn_beta_fast - beta_slow = self.cfg.yarn_beta_slow - - def _find_correction_dim(num_rotations: float) -> float: - return (rotary_dim * math.log(orig_max_pos / (num_rotations * 2 * math.pi))) / ( - 2 * math.log(base) - ) - - low = _find_correction_dim(beta_fast) - high = _find_correction_dim(beta_slow) - if self.cfg.yarn_truncate: - low = math.floor(low) - high = math.ceil(high) - low = max(low, 0) - high = min(high, rotary_dim - 1) - - # Linear ramp from 0 to 1 between low and high dims - ramp = torch.arange(rotary_dim // 2, dtype=high_precision) - high_f = float(high) + 0.001 if low == high else float(high) - ramp = torch.clamp((ramp - low) / (high_f - low), 0, 1) - - inv_freq_interp = inv_freq / yarn_factor - # ramp=0 (below low) → extrapolation (original freq), ramp=1 (above high) → interpolation (scaled) - inv_freq = inv_freq_interp * ramp + inv_freq * (1 - ramp) - freq = 1.0 / inv_freq - else: - freq = base ** (dim / (rotary_dim / 2)) - # Apply linear RoPE scaling for global attention layers if configured - # (e.g., Gemma 3 4B uses factor=8.0 for global layers, but not local ones) - scaling_factor = getattr(self.cfg, "rotary_scaling_factor", 1.0) - if scaling_factor != 1.0 and self.attn_type != "local": - freq = freq * scaling_factor - if self.cfg.rotary_adjacent_pairs: - freq = einops.repeat(freq, "d -> (d 2)") - else: - freq = einops.repeat(freq, "d -> (2 d)") - # Create a n_ctx x rotary_dim tensor, where each column is an arithmetic sequence of angles in that frequency - angles = pos[:, None] / freq[None, :] - sin, cos = torch.sin(angles).to(dtype), torch.cos(angles).to(dtype) - # YARN attention_factor scales the embeddings (default 1.0 is a no-op) - if use_yarn and self.cfg.yarn_attention_factor != 1.0: - sin = sin * self.cfg.yarn_attention_factor - cos = cos * self.cfg.yarn_attention_factor - return sin, cos - - def rotate_every_two( - self, x: Float[torch.Tensor, "... rotary_dim"] - ) -> Float[torch.Tensor, "... rotary_dim"]: - """ - Rotary helper function, splits x into blocks of size 2 along the final axis and maps [x0, x1] to [-x1, x0] - - The final axis of x must have even length. - - GPT-NeoX and GPT-J do rotary subtly differently, see calculate_sin_cos_rotary for details. - """ - rot_x = x.clone() - if self.cfg.rotary_adjacent_pairs: - rot_x[..., ::2] = -x[..., 1::2] - rot_x[..., 1::2] = x[..., ::2] - else: - n = x.size(-1) // 2 - rot_x[..., :n] = -x[..., n:] - rot_x[..., n:] = x[..., :n] - - return rot_x - - def apply_rotary( - self, - x: Float[torch.Tensor, "batch pos head_index d_head"], - past_kv_pos_offset: int = 0, - attention_mask: Optional[Int[torch.Tensor, "batch offset_pos"]] = None, - ) -> Float[torch.Tensor, "batch pos head_index d_head"]: - # Only apply rotary to first rotary_dim dimensions (eg, if rotary_dim=64 and d_head=256, only apply to first 1/4 of dimensions) - - if x.device != self.rotary_sin.device: - x = x.to(cast(torch.device, self.rotary_sin.device)) - - x_pos = x.size(1) - if self.cfg.use_dynamic_ntk_rope and not self.training: - self._rescale_rotary_for_ntk(past_kv_pos_offset + x_pos) - - x_rot = x[..., : self.cfg.rotary_dim] - x_pass = x[..., self.cfg.rotary_dim :] - x_flip = self.rotate_every_two(x_rot) - - # Dynamically extend rotary embeddings if needed for long context - max_pos_needed = past_kv_pos_offset + x_pos - if max_pos_needed > self.rotary_cos.shape[0]: - new_size = min( - self.cfg.n_ctx, - max(max_pos_needed, 2 * self.rotary_cos.shape[0]), - ) - self._extend_rotary_embeddings(new_size) - - if attention_mask is None: - rotary_cos = cast(torch.Tensor, self.rotary_cos)[ - None, past_kv_pos_offset : past_kv_pos_offset + x_pos, None, : - ] - rotary_sin = cast(torch.Tensor, self.rotary_sin)[ - None, past_kv_pos_offset : past_kv_pos_offset + x_pos, None, : - ] - x_rotated = x_rot * rotary_cos + x_flip * rotary_sin - else: - offset_position_ids = get_offset_position_ids(past_kv_pos_offset, attention_mask) - offset_position_ids = offset_position_ids.to(cast(torch.device, self.rotary_cos.device)) - mask_rotary_cos = cast(torch.Tensor, self.rotary_cos)[offset_position_ids, None, :] - mask_rotary_sin = cast(torch.Tensor, self.rotary_sin)[offset_position_ids, None, :] - x_rotated = x_rot * mask_rotary_cos + x_flip * mask_rotary_sin - - return torch.cat([x_rotated, x_pass], dim=-1) - - def _ntk_alpha(self, key_length: int, train_length: int) -> float: - """Qwen-1's ``get_ntk_alpha``: 1 up to the training length, then 3, 7, 15, …""" - if key_length <= train_length: - return 1.0 - return float(max(2 ** math.ceil(math.log(key_length / train_length, 2) + 1) - 1, 1)) - - def _rescale_rotary_for_ntk(self, key_length: int) -> None: - """Rebuild the rotary cache on a widened base for long contexts. - - Qwen-1 stretches the base, not the positions, so the table is rebuilt - rather than extended; alpha steps rarely, so this fires rarely. - """ - assert self.cfg.rotary_dim is not None, "rotary_dim must be set for rotary embeddings" - train_length = self.cfg.train_seq_length or self.cfg.n_ctx - alpha = self._ntk_alpha(key_length, train_length) - cached_rows = self.rotary_cos.shape[0] - if alpha == self._ntk_alpha_cached and key_length <= cached_rows: - return - self._ntk_alpha_cached = alpha - base = self._rotary_base() * alpha ** (self.cfg.rotary_dim / (self.cfg.rotary_dim - 2)) - sin, cos = self.calculate_sin_cos_rotary( - self.cfg.rotary_dim, - max(key_length, cached_rows), - base=base, - dtype=self.cfg.dtype, - ) - self.rotary_sin = sin.to(self.rotary_sin.device) - self.rotary_cos = cos.to(self.rotary_cos.device) - - def _extend_rotary_embeddings(self, new_size: int): - """Extend rotary embeddings to support longer contexts dynamically.""" - # Ensure rotary_dim is set - assert self.cfg.rotary_dim is not None, "rotary_dim must be set for rotary embeddings" - - # Calculate new embeddings - sin, cos = self.calculate_sin_cos_rotary( - self.cfg.rotary_dim, - new_size, - base=self._rotary_base(), - dtype=self.cfg.dtype, - ) - - # Update the registered buffers - self.rotary_sin = sin.to(self.rotary_sin.device) - self.rotary_cos = cos.to(self.rotary_cos.device) - - def _extend_mask(self, new_size: int): - """Deprecated no-op kept for external callers.""" - del new_size - - def _load_from_state_dict( - self, - state_dict, - prefix, - local_metadata, - strict, - missing_keys, - unexpected_keys, - error_msgs, - ): - for buffer_name in ("mask", "rotary_sin", "rotary_cos"): - buffer_key = prefix + buffer_name - saved_buffer = state_dict.get(buffer_key) - current_buffer = getattr(self, buffer_name, None) - if ( - isinstance(saved_buffer, torch.Tensor) - and isinstance(current_buffer, torch.Tensor) - and saved_buffer.shape != current_buffer.shape - ): - state_dict = state_dict.copy() - state_dict[buffer_key] = current_buffer - super()._load_from_state_dict( - state_dict, - prefix, - local_metadata, - strict, - missing_keys, - unexpected_keys, - error_msgs, - ) - - @staticmethod - def create_alibi_slope( - n_ctx: int, device: Optional[Union[str, torch.device]] = None - ) -> Float[torch.Tensor, "query key"]: - """Create an ALiBi Slope Matrix. - - Create the slope matrix used in ALiBi, before it is multiplied by the head-specific scalar. - - See :meth:`create_alibi_bias` for the full ALiBi bias calculation. - - Examples: - - >>> AbstractAttention.create_alibi_slope(3) - tensor([[ 0., 0., 0.], - [-1., 0., 0.], - [-2., -1., 0.]]) - - >>> AbstractAttention.create_alibi_slope(4) - tensor([[ 0., 0., 0., 0.], - [-1., 0., 0., 0.], - [-2., -1., 0., 0.], - [-3., -2., -1., 0.]]) - - Args: - n_ctx: The maximum number of tokens in a prompt. - - Returns: - A tensor of shape (n_ctx, n_ctx), where the upper triangle is zero and the lower - triangle is decreasing by a constant slope of 1 (towards the bottom left corner). - """ - # set rows as [[0,1,2...]] - rows = torch.arange(n_ctx, device=device).unsqueeze(0) - - # Set cols as [[0],[1],[2]...] - cols = torch.arange(n_ctx, device=device).unsqueeze(1) - - # Use broadcasting to create the desired lower triangular part of the matrix - slope_matrix = rows - cols - - return slope_matrix.clamp(max=0).to(torch.float32) - - @staticmethod - def create_alibi_multipliers( - n_heads: int, device: Optional[Union[str, torch.device]] = None - ) -> Float[torch.Tensor, "n_heads"]: - """Create the ALiBi Scalar Multipliers for each Head. - - For n heads, the set of multipliers (m) is the geometric sequence that starts at 2^(-8/n), and - uses that same value as its ratio. For example, with 8 heads the values would be [1/(2^1), - 1/(2^2), ... , 1/(2^8)]. With 16 heads the values would be [1/(2^0.5), 1/(2^1), ... , 1/(2^8)]. - - See :meth:`create_alibi_bias` for the full ALiBi bias calculation. - - Examples: - - >>> AbstractAttention.create_alibi_multipliers(8) - tensor([0.5000, 0.2500, 0.1250, 0.0625, 0.0312, 0.0156, 0.0078, 0.0039]) - - >>> AbstractAttention.create_alibi_multipliers(16) - tensor([0.7071, 0.5000, 0.3536, 0.2500, 0.1768, 0.1250, 0.0884, 0.0625, 0.0442, 0.0312, - 0.0221, 0.0156, 0.0110, 0.0078, 0.0055, 0.0039]) - - Args: - n_heads: The number of heads in a layer. - device: The device to create the tensor on. - - Returns: - A tensor of shape (n_heads,) containing the scalar multiplier for each head. - """ - start = 2 ** (-8 / n_heads) - - indices = torch.arange(n_heads, device=device) - - # Compute the multipliers, with the starting value being the same as the ratio - multipliers = start * (start**indices) - - return multipliers - - @staticmethod - def create_alibi_bias( - n_heads: int, n_ctx: int, device: Optional[Union[torch.device, str]] = None - ) -> Float[torch.Tensor, "head_idx query key"]: - """Create the ALiBi Bias for all Heads. - - Calculate the ALiBi bias (https://arxiv.org/pdf/2108.12409.pdf) for all heads in a layer. - - The broad idea behind ALiBi is to remove the positional encoding from the original transformer - model, and instead apply a bias to each attention score. This bias is proportional to the - distance between the query and key (i.e. it encourage paying less attention to more distant - tokens), and is added to the attention scores before the softmax. It is used in models such as - Bloom. - - Examples: - - >>> AbstractAttention.create_alibi_bias(2, 4, torch.device('cpu')) - tensor([[[ 0.0000, 0.0000, 0.0000, 0.0000], - [-0.0625, 0.0000, 0.0000, 0.0000], - [-0.1250, -0.0625, 0.0000, 0.0000], - [-0.1875, -0.1250, -0.0625, 0.0000]], - [[ 0.0000, 0.0000, 0.0000, 0.0000], - [-0.0039, 0.0000, 0.0000, 0.0000], - [-0.0078, -0.0039, 0.0000, 0.0000], - [-0.0117, -0.0078, -0.0039, 0.0000]]]) - - Args: - n_heads: The number of heads in a layer. - n_ctx: The maximum number of tokens in a prompt. - device: The device to create the tensor on. - - Returns: - The ALiBi bias that should be added to the attention scores before the softmax. - """ - slope: Float[torch.Tensor, "query key"] = AbstractAttention.create_alibi_slope( - n_ctx, device - ) - - multipliers: Float[torch.Tensor, "head_idx"] = AbstractAttention.create_alibi_multipliers( - n_heads, device - ) - - # Add singleton dimensions to make shapes compatible for broadcasting: - slope = einops.rearrange(slope, "query key -> 1 query key") - multipliers = einops.rearrange(multipliers, "head_idx -> head_idx 1 1") - - alibi_bias = multipliers * slope - - return alibi_bias diff --git a/transformer_lens/components/attention.py b/transformer_lens/components/attention.py deleted file mode 100644 index 11d94cb4ac..0000000000 --- a/transformer_lens/components/attention.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Hooked Transformer Attention Component. - -This module contains all the component :class:`Attention`. -""" - -from typing import Dict, Optional, Union - -import torch -import torch.nn as nn -from transformers.utils import is_bitsandbytes_available - -from transformer_lens.components import AbstractAttention -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig - -if is_bitsandbytes_available(): - from bitsandbytes.nn.modules import Params4bit - - -class Attention(AbstractAttention): - def __init__( - self, - cfg: Union[Dict, HookedTransformerConfig], - attn_type: str = "global", - layer_id: Optional[int] = None, - ): - """Attention Block - params have shape [head_index, d_model, d_head] (or [head_index, d_head, d_model] for W_O) and multiply on the right. attn_scores refers to query key dot product immediately before attention softmax - - Convention: All attention pattern-style matrices have shape [batch, head_index, query_pos, key_pos] - - Args: - cfg (Union[Dict, HookedTransformerConfig]): Config - attn_type (str, optional): "global" or "local", used by GPT-Neo. Local attention means the model can only attend back cfg.window_size tokens (here, 256). Not used by any other model at the moment. Defaults to "global". - layer_id (int, optional): The index of the current layer. Used by the Mistal models (labelled here as stanford-gpt2) to scale down attention scores pre softmax for numerical stability reasons by 1/(layer_id+1). Defaults to None. - """ - super().__init__(cfg, attn_type, layer_id) - self.cfg = HookedTransformerConfig.unwrap(cfg) - - if self.cfg.load_in_4bit: - # 4-bit quantization convention - nq = int((self.cfg.d_model * self.cfg.d_model) / 2) - self.W_K = Params4bit(torch.empty(nq, 1, dtype=torch.uint8), requires_grad=False) - self.W_V = Params4bit(torch.empty(nq, 1, dtype=torch.uint8), requires_grad=False) - else: - self.W_K = nn.Parameter( - torch.empty( - self.cfg.n_heads, self.cfg.d_model, self.cfg.d_head, dtype=self.cfg.dtype - ) - ) - self.W_V = nn.Parameter( - torch.empty( - self.cfg.n_heads, self.cfg.d_model, self.cfg.d_head, dtype=self.cfg.dtype - ) - ) - self.b_K = nn.Parameter( - torch.zeros(self.cfg.n_heads, self.cfg.d_head, dtype=self.cfg.dtype) - ) - self.b_V = nn.Parameter( - torch.zeros(self.cfg.n_heads, self.cfg.d_head, dtype=self.cfg.dtype) - ) diff --git a/transformer_lens/components/bert_block.py b/transformer_lens/components/bert_block.py deleted file mode 100644 index fec0d052d0..0000000000 --- a/transformer_lens/components/bert_block.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Hooked Transformer Bert Block Component. - -This module contains all the component :class:`BertBlock`. -""" - -from typing import Optional - -import torch -import torch.nn as nn -from jaxtyping import Float - -from transformer_lens.components import Attention, LayerNorm -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.factories.mlp_factory import MLPFactory -from transformer_lens.hook_points import HookPoint -from transformer_lens.utilities import repeat_along_head_dimension - - -class BertBlock(nn.Module): - """ - BERT Block. Similar to the TransformerBlock, except that the LayerNorms are applied after the attention and MLP, rather than before. - """ - - def __init__(self, cfg: HookedTransformerConfig): - super().__init__() - self.cfg = cfg - - self.attn = Attention(cfg) - self.ln1 = LayerNorm(cfg) - self.mlp = MLPFactory.create_mlp(self.cfg) - self.ln2 = LayerNorm(cfg) - - self.hook_q_input = HookPoint() # [batch, pos, n_heads, d_model] - self.hook_k_input = HookPoint() # [batch, pos, n_heads, d_model] - self.hook_v_input = HookPoint() # [batch, pos, n_heads, d_model] - - self.hook_attn_out = HookPoint() # [batch, pos, d_model] - self.hook_mlp_in = HookPoint() # [batch, pos, d_model] - self.hook_mlp_out = HookPoint() # [batch, pos, d_model] - self.hook_resid_pre = HookPoint() # [batch, pos, d_model] - self.hook_resid_mid = HookPoint() # [batch, pos, d_model] - self.hook_resid_post = HookPoint() # [batch, pos, d_model] - self.hook_normalized_resid_post = HookPoint() # [batch, pos, d_model] - - def forward( - self, - resid_pre: Float[torch.Tensor, "batch pos d_model"], - additive_attention_mask: Optional[Float[torch.Tensor, "batch 1 1 pos"]] = None, - ) -> Float[torch.Tensor, "batch pos d_model"]: - resid_pre = self.hook_resid_pre(resid_pre) - - query_input = resid_pre - key_input = resid_pre - value_input = resid_pre - - if self.cfg.use_split_qkv_input: - n_heads = self.cfg.n_heads - query_input = self.hook_q_input(repeat_along_head_dimension(query_input, n_heads)) - key_input = self.hook_k_input(repeat_along_head_dimension(key_input, n_heads)) - value_input = self.hook_v_input(repeat_along_head_dimension(value_input, n_heads)) - - attn_out = self.hook_attn_out( - self.attn( - query_input, - key_input, - value_input, - additive_attention_mask=additive_attention_mask, - ) - ) - resid_mid = self.hook_resid_mid(resid_pre + attn_out) - - mlp_in = resid_mid if not self.cfg.use_hook_mlp_in else self.hook_mlp_in(resid_mid.clone()) - normalized_resid_mid = self.ln1(mlp_in) - mlp_out = self.hook_mlp_out(self.mlp(normalized_resid_mid)) - resid_post = self.hook_resid_post(normalized_resid_mid + mlp_out) - normalized_resid_post = self.hook_normalized_resid_post(self.ln2(resid_post)) - - return normalized_resid_post diff --git a/transformer_lens/components/bert_embed.py b/transformer_lens/components/bert_embed.py deleted file mode 100644 index e0648a7f08..0000000000 --- a/transformer_lens/components/bert_embed.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Hooked Transformer Bert Embed Component. - -This module contains all the component :class:`BertEmbed`. -""" - -from typing import Dict, Optional, Union - -import einops -import torch -import torch.nn as nn -from jaxtyping import Float, Int - -from transformer_lens.components import Embed, LayerNorm, PosEmbed, TokenTypeEmbed -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.hook_points import HookPoint - - -class BertEmbed(nn.Module): - """ - Custom embedding layer for a BERT-like model. This module computes the sum of the token, positional and token-type embeddings and takes the layer norm of the result. - """ - - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - self.embed = Embed(self.cfg) - self.pos_embed = PosEmbed(self.cfg) - self.token_type_embed = TokenTypeEmbed(self.cfg) - self.ln = LayerNorm(self.cfg) - - self.hook_embed = HookPoint() - self.hook_pos_embed = HookPoint() - self.hook_token_type_embed = HookPoint() - - def forward( - self, - input_ids: Int[torch.Tensor, "batch pos"], - token_type_ids: Optional[Int[torch.Tensor, "batch pos"]] = None, - ) -> Float[torch.Tensor, "batch pos d_model"]: - base_index_id = torch.arange(input_ids.shape[1], device=input_ids.device) - index_ids = einops.repeat(base_index_id, "pos -> batch pos", batch=input_ids.shape[0]) - if token_type_ids is None: - token_type_ids = torch.zeros_like(input_ids) - - word_embeddings_out = self.hook_embed(self.embed(input_ids)) - position_embeddings_out = self.hook_pos_embed(self.pos_embed(index_ids)) - token_type_embeddings_out = self.hook_token_type_embed( - self.token_type_embed(token_type_ids) - ) - - embeddings_out = word_embeddings_out + position_embeddings_out + token_type_embeddings_out - layer_norm_out = self.ln(embeddings_out) - return layer_norm_out diff --git a/transformer_lens/components/bert_mlm_head.py b/transformer_lens/components/bert_mlm_head.py deleted file mode 100644 index b229a20b60..0000000000 --- a/transformer_lens/components/bert_mlm_head.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Hooked Encoder Bert MLM Head Component. - -This module contains all the component :class:`BertMLMHead`. -""" - -from typing import Dict, Union - -import torch -import torch.nn as nn -from jaxtyping import Float - -from transformer_lens.components import LayerNorm -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig - - -class BertMLMHead(nn.Module): - """ - Transforms BERT embeddings into logits. The purpose of this module is to predict masked tokens in a sentence. - """ - - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - self.W = nn.Parameter(torch.empty(self.cfg.d_model, self.cfg.d_model, dtype=self.cfg.dtype)) - self.b = nn.Parameter(torch.zeros(self.cfg.d_model, dtype=self.cfg.dtype)) - self.act_fn = nn.GELU() - self.ln = LayerNorm(self.cfg) - - def forward( - self, resid: Float[torch.Tensor, "batch pos d_model"] - ) -> Float[torch.Tensor, "batch pos d_model"]: - resid = torch.matmul(resid, self.W) + self.b - resid = self.act_fn(resid) - resid = self.ln(resid) - return resid diff --git a/transformer_lens/components/bert_nsp_head.py b/transformer_lens/components/bert_nsp_head.py deleted file mode 100644 index 937adc36d5..0000000000 --- a/transformer_lens/components/bert_nsp_head.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Hooked Encoder Bert NSP Head Component. - -This module contains all the component :class:`BertNSPHead`. -""" - -from typing import Dict, Union - -import torch -import torch.nn as nn -from jaxtyping import Float - -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.hook_points import HookPoint - - -class BertNSPHead(nn.Module): - """ - Transforms BERT embeddings into logits. The purpose of this module is to predict whether or not sentence B follows sentence A. - """ - - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - self.W = nn.Parameter(torch.empty(self.cfg.d_model, 2, dtype=self.cfg.dtype)) - self.b = nn.Parameter(torch.zeros(2, dtype=self.cfg.dtype)) - self.hook_nsp_out = HookPoint() - - def forward( - self, resid: Float[torch.Tensor, "batch d_model"] - ) -> Float[torch.Tensor, "batch 2"]: - nsp_logits = torch.matmul(resid, self.W) + self.b - return self.hook_nsp_out(nsp_logits) diff --git a/transformer_lens/components/bert_pooler.py b/transformer_lens/components/bert_pooler.py deleted file mode 100644 index 784d9598d9..0000000000 --- a/transformer_lens/components/bert_pooler.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Hooked Encoder Bert Pooler Component. - -This module contains all the component :class:`BertPooler`. -""" - -from typing import Dict, Union - -import torch -import torch.nn as nn -from jaxtyping import Float - -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.hook_points import HookPoint - - -class BertPooler(nn.Module): - """ - Transforms the [CLS] token representation into a fixed-size sequence embedding. - The purpose of this module is to convert variable-length sequence inputs into a single vector representation suitable for downstream tasks. - (e.g. Next Sentence Prediction) - """ - - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - self.W = nn.Parameter(torch.empty(self.cfg.d_model, self.cfg.d_model, dtype=self.cfg.dtype)) - self.b = nn.Parameter(torch.zeros(self.cfg.d_model, dtype=self.cfg.dtype)) - self.activation = nn.Tanh() - self.hook_pooler_out = HookPoint() - - def forward( - self, resid: Float[torch.Tensor, "batch pos d_model"] - ) -> Float[torch.Tensor, "batch d_model"]: - first_token_tensor = resid[:, 0] - pooled_output = torch.matmul(first_token_tensor, self.W) + self.b - pooled_output = self.hook_pooler_out(self.activation(pooled_output)) - return pooled_output diff --git a/transformer_lens/components/embed.py b/transformer_lens/components/embed.py deleted file mode 100644 index addfff8be3..0000000000 --- a/transformer_lens/components/embed.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Hooked Transformer Embed Component. - -This module contains all the component :class:`Embed`. -""" - -from typing import Dict, Union - -import torch -import torch.nn as nn -from jaxtyping import Float, Int - -from transformer_lens.components import LayerNorm -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig - - -# Embed & Unembed -class Embed(nn.Module): - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - self.W_E: Float[torch.Tensor, "d_vocab d_model"] = nn.Parameter( - torch.empty(self.cfg.d_vocab, self.cfg.d_model, dtype=self.cfg.dtype) - ) - # Some models (e.g. Bloom) need post embedding layer norm - if self.cfg.post_embedding_ln: - self.ln = LayerNorm(self.cfg) - - def forward( - self, tokens: Int[torch.Tensor, "batch pos"] - ) -> Float[torch.Tensor, "batch pos d_model"]: - # If A has shape [a, b] and B has shape [c, d], then A[:, B] has shape [a, c, d] - # B acts as a tensor of indices into the second dimension (so >=0 and <b) - if self.cfg.post_embedding_ln: - return self.ln(self.W_E[tokens, :]) - return self.W_E[tokens, :] diff --git a/transformer_lens/components/grouped_query_attention.py b/transformer_lens/components/grouped_query_attention.py deleted file mode 100644 index 5505429601..0000000000 --- a/transformer_lens/components/grouped_query_attention.py +++ /dev/null @@ -1,204 +0,0 @@ -from typing import Dict, Tuple, Union - -import torch -import torch.nn as nn -from jaxtyping import Float - -from transformer_lens.components import AbstractAttention -from transformer_lens.components.rms_norm import RMSNorm -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.utilities.attention import complex_attn_linear, simple_attn_linear - - -class GroupedQueryAttention(AbstractAttention): - def __init__( - self, - cfg: Union[Dict, HookedTransformerConfig], - attn_type: str = "global", - layer_id: Union[int, None] = None, - ): - """Grouped Query Attention Block - see https://arxiv.org/abs/2305.13245 for details. - Similar to regular attention, W_Q, W_K, and W_V all have shape [head_index, d_model, d_head]. - However, under the hood the key and value weights _W_K and _W_V are stored with shape [n_key_value_heads, d_model, d_head] and are expanded when the corresponding properties' getter is called. - Similarly, during a forward pass, initially K and V are kept in shapes [batch, pos, n_key_value_heads, d_head] and will only be expanded to shapes [batch, pos, n_heads, d_head] - using torch.repeat_interleave when the attention pattern and z-scores are calculated. - - Args: - cfg (Union[Dict, HookedTransformerConfig]): Config - attn_type (str, optional): "global" or "local", used by GPT-Neo. Local attention means the model can only attend back cfg.window_size tokens (here, 256). Not used by any other model at the moment. Defaults to "global". - layer_id (int, optional): The index of the current layer. Used by the Mistal models (labelled here as stanford-gpt2) to scale down attention scores pre softmax for numerical stability reasons by 1/(layer_id+1). Defaults to None. - """ - cfg = HookedTransformerConfig.unwrap(cfg) - assert cfg.n_key_value_heads is not None - super().__init__(cfg, attn_type, layer_id) - self.repeat_kv_heads = cfg.n_heads // cfg.n_key_value_heads - self._W_K = nn.Parameter( - torch.empty( - cfg.n_key_value_heads, - self.cfg.d_model, - self.cfg.d_head, - dtype=cfg.dtype, - ) - ) - self._W_V = nn.Parameter( - torch.empty( - cfg.n_key_value_heads, - self.cfg.d_model, - self.cfg.d_head, - dtype=cfg.dtype, - ) - ) - self._b_K = nn.Parameter( - torch.zeros(cfg.n_key_value_heads, self.cfg.d_head, dtype=cfg.dtype) - ) - self._b_V = nn.Parameter( - torch.zeros(cfg.n_key_value_heads, self.cfg.d_head, dtype=cfg.dtype) - ) - - @property - def W_K(self): - return torch.repeat_interleave(self._W_K, dim=0, repeats=self.repeat_kv_heads) - - @W_K.setter - def W_K(self, value): - self._W_K = value - - @property - def W_V(self): - return torch.repeat_interleave(self._W_V, dim=0, repeats=self.repeat_kv_heads) - - @W_V.setter - def W_V(self, value): - self._W_V = value - - @property - def b_K(self): - return torch.repeat_interleave(self._b_K, dim=0, repeats=self.repeat_kv_heads) - - @b_K.setter - def b_K(self, value): - self._b_K = value - - @property - def b_V(self): - return torch.repeat_interleave(self._b_V, dim=0, repeats=self.repeat_kv_heads) - - @b_V.setter - def b_V(self, value): - self._b_V = value - - def calculate_qkv_matrices( - self, - query_input: Union[ - Float[torch.Tensor, "batch pos d_model"], - Float[torch.Tensor, "batch pos head_index d_model"], - ], - key_input: Union[ - Float[torch.Tensor, "batch pos d_model"], - Float[torch.Tensor, "batch pos kv_head_index d_model"], - ], - value_input: Union[ - Float[torch.Tensor, "batch pos d_model"], - Float[torch.Tensor, "batch pos kv_head_index d_model"], - ], - ) -> Tuple[ - Float[torch.Tensor, "batch pos head_index d_head"], - Float[torch.Tensor, "batch pos kv_head_index d_head"], - Float[torch.Tensor, "batch pos kv_head_index d_head"], - ]: - """Calculate the Q, K, and V matrices for grouped query attention. - This function uses the unexpanded weights _W_K and _W_V to calculate K and V. - - Args: - query_input (Union[Float[torch.Tensor, "batch pos d_model"], Float[torch.Tensor, "batch pos head_index d_model"]]): The input tensor for the query projection. - key_input (Union[Float[torch.Tensor, "batch pos d_model"], Float[torch.Tensor, "batch pos kv_head_index d_model"]]): The input tensor for the key projection. Note that is has as many head dimensions as the GPA block has key-value heads. - value_input (Union[Float[torch.Tensor, "batch pos d_model"], Float[torch.Tensor, "batch pos kv_head_index d_model"]]): The input tensor for the value projection. Note that is has as many head dimensions as the GPA block has key-value heads. - - Returns: - Tuple[Float[torch.Tensor, "batch pos head_index d_head"], Float[torch.Tensor, "batch pos kv_head_index d_head"], Float[torch.Tensor, "batch pos kv_head_index d_head"]]: - A tuple containing the Q, K, and V matrices with the specified shapes. - """ - attn_fn = ( - complex_attn_linear - if self.cfg.use_split_qkv_input or self.cfg.use_attn_in - else simple_attn_linear - ) - - q = self.hook_q( - attn_fn(query_input, self.W_Q, self.b_Q) - ) # [batch, pos, head_index, d_head] - - k = self.hook_k( - attn_fn(key_input, self.W_K, self.b_K) - if self.cfg.ungroup_grouped_query_attention - else attn_fn(key_input, self._W_K, self._b_K) - ) # [batch, pos, head_index, d_head] - v = self.hook_v( - attn_fn(value_input, self.W_V, self.b_V) - if self.cfg.ungroup_grouped_query_attention - else attn_fn(value_input, self._W_V, self._b_V) - ) # [batch, pos, head_index, d_head] - - if self.cfg.use_qk_norm: - assert self.q_norm is not None - assert self.k_norm is not None - q = self._apply_qk_norm(q, self.q_norm) - k = self._apply_qk_norm(k, self.k_norm) - - return q, k, v - - def calculate_attention_scores( - self, - q: Float[torch.Tensor, "batch query_pos head_index d_head"], - k: Float[torch.Tensor, "batch key_pos kv_head_index d_head"], - ) -> Float[torch.Tensor, "batch head_index query_pos key_pos"]: - """Calculate attention scores from Q and the unexpanded K matrix. - K will be expanded from [batch, pos, n_key_value_head, d_head] to [batch, pos, n_query_heads, d_head] using torch.repeat_interleave. - - Args: - q (Float[torch.Tensor, "batch query_pos head_index d_head"]): The Q tensor. - k (Float[torch.Tensor, "batch key_pos kv_head_index d_head"]): The K tensor. - - Returns: - Float[torch.Tensor, "batch head_index query_pos key_pos"]: The attention scores. - """ - if not self.cfg.ungroup_grouped_query_attention: - k = torch.repeat_interleave(k, dim=2, repeats=self.repeat_kv_heads) - return super().calculate_attention_scores(q, k) - - def calculate_z_scores( - self, - v: Float[torch.Tensor, "batch key_pos kv_head_index d_head"], - pattern: Float[torch.Tensor, "batch head_index query_pos key_pos"], - ) -> Float[torch.Tensor, "batch query_pos head_index d_head"]: - """Calculate z scores from the attention pattern and the unexpanded V matrix. - V will be expanded from [batch, pos, n_key_value_head, d_head] to [batch, pos, n_query_heads, d_head] using torch.repeat_interleave. - - Args: - v (Float[torch.Tensor, "batch query_pos head_index d_head"]): The V tensor. - pattern (Float[torch.Tensor, "batch key_pos kv_head_index d_head"]): The attention pattern. - - Returns: - Float[torch.Tensor, "batch head_index query_pos key_pos"]: The z scores. - """ - if not self.cfg.ungroup_grouped_query_attention: - v = torch.repeat_interleave(v, dim=2, repeats=self.repeat_kv_heads) - return super().calculate_z_scores(v, pattern) - - def _apply_qk_norm( - self, x: Float[torch.Tensor, "batch pos head_index d_head"], norm_module: RMSNorm - ) -> Float[torch.Tensor, "batch pos head_index d_head"]: - """Apply QK normalization with proper reshaping. - - Args: - x: Input tensor with shape [batch, pos, head_index, d_head] - norm_module: RMSNorm module to apply - - Returns: - Normalized tensor with same shape as input - """ - # Reshape from [batch, pos, head_index, d_head] to [batch * pos * head_index, d_head] - batch, pos, n_heads, d_head = x.shape - x_reshaped = x.reshape(-1, d_head) - x_normed = norm_module(x_reshaped) - return x_normed.reshape(batch, pos, n_heads, d_head) diff --git a/transformer_lens/components/layer_norm.py b/transformer_lens/components/layer_norm.py deleted file mode 100644 index 061432bfc3..0000000000 --- a/transformer_lens/components/layer_norm.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Hooked Transformer Layer Norm Component. - -This module contains all the component :class:`LayerNorm`. -""" - -from typing import Dict, Optional, Union - -import torch -import torch.nn as nn -from jaxtyping import Float - -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.hook_points import HookPoint - - -class LayerNorm(nn.Module): - def __init__(self, cfg: Union[Dict, HookedTransformerConfig], length: Optional[int] = None): - """ - LayerNorm with optional length parameter - - length (Optional[int]): If the dimension of the LayerNorm. If not provided, assumed to be d_model - """ - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - self.eps = self.cfg.eps - if length is None: - self.length = self.cfg.d_model - else: - self.length = length - - self.w = nn.Parameter(torch.ones(self.length, dtype=self.cfg.dtype)) - self.b = nn.Parameter(torch.zeros(self.length, dtype=self.cfg.dtype)) - - # Adds a hook point for the normalisation scale factor - self.hook_scale = HookPoint() # [batch, pos, 1] - # Hook_normalized is on the LN output - self.hook_normalized = HookPoint() # [batch, pos, length] - - def forward( - self, - x: Union[ - Float[torch.Tensor, "batch pos d_model"], - Float[torch.Tensor, "batch pos head_index d_model"], - ], - ) -> Union[ - Float[torch.Tensor, "batch pos d_model"], - Float[torch.Tensor, "batch pos head_index d_model"], - ]: - if self.cfg.dtype not in [torch.float32, torch.float64]: - x = x.to(torch.float32) - - x = x - x.mean(-1, keepdim=True) # [batch, pos, length] - scale: Float[torch.Tensor, "batch pos 1"] = self.hook_scale( - (x.pow(2).mean(-1, keepdim=True) + self.eps).sqrt() - ) - x = self.hook_normalized(x / scale).to(self.cfg.dtype) # [batch, pos, length] - return x * self.w + self.b diff --git a/transformer_lens/components/layer_norm_pre.py b/transformer_lens/components/layer_norm_pre.py deleted file mode 100644 index e54890da88..0000000000 --- a/transformer_lens/components/layer_norm_pre.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Hooked Transformer Layer Norm Pre Component. - -This module contains all the component :class:`LayerNormPre`. -""" - -from typing import Dict, Union - -import torch -import torch.nn as nn -from jaxtyping import Float - -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.hook_points import HookPoint - - -# LayerNormPre -# I fold the LayerNorm weights and biases into later weights and biases. -# This is just the 'center and normalise' part of LayerNorm -# Centering is equivalent to just deleting one direction of residual space, -# and is equivalent to centering the weight matrices of everything writing to the residual stream -# Normalising is a funkier non-linear operation, that projects the residual stream onto the unit hypersphere -class LayerNormPre(nn.Module): - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - """LayerNormPre - the 'center and normalise' part of LayerNorm. Length is - normally d_model, but is d_mlp for softmax. Not needed as a parameter. This - should only be used in inference mode after folding in LayerNorm weights""" - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - self.eps = self.cfg.eps - - # Adds a hook point for the normalisation scale factor - self.hook_scale = HookPoint() # [batch, pos] - # Hook Normalized captures LN output - here it's a vector with std 1 and mean 0 - self.hook_normalized = HookPoint() # [batch, pos, length] - - def forward( - self, - x: Union[ - Float[torch.Tensor, "batch pos d_model"], - Float[torch.Tensor, "batch pos head_index d_model"], - ], - ) -> Union[ - Float[torch.Tensor, "batch pos d_model"], - Float[torch.Tensor, "batch pos head_index d_model"], - ]: - if self.cfg.dtype not in [torch.float32, torch.float64]: - x = x.to(torch.float32) - - x = x - x.mean(-1, keepdim=True) # [batch, pos, length] - scale: Union[ - Float[torch.Tensor, "batch pos 1"], - Float[torch.Tensor, "batch pos head_index 1"], - ] = self.hook_scale((x.pow(2).mean(-1, keepdim=True) + self.eps).sqrt()) - return self.hook_normalized(x / scale).to(self.cfg.dtype) diff --git a/transformer_lens/components/mlps/can_be_used_as_mlp.py b/transformer_lens/components/mlps/can_be_used_as_mlp.py deleted file mode 100644 index 9c031e6d80..0000000000 --- a/transformer_lens/components/mlps/can_be_used_as_mlp.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Can Be Used as MLP component. - -This module serves as the base for everything within TransformerLens that can be used like an MLP. -This does not necessarily mean that every component extending this class will be an MLP, but -everything extending this class can be used interchangeably for an MLP. -""" - -from typing import Dict, Optional, Union - -import torch -import torch.nn as nn -from jaxtyping import Float - -from transformer_lens.components.layer_norm import LayerNorm -from transformer_lens.components.layer_norm_pre import LayerNormPre -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.factories.activation_function_factory import ( - ActivationFunctionFactory, -) -from transformer_lens.hook_points import HookPoint -from transformer_lens.utilities.activation_functions import ActivationFunction - - -class CanBeUsedAsMLP(nn.Module): - # The actual activation function - act_fn: ActivationFunction - - # The full config object for the model - cfg: HookedTransformerConfig - - # The d mlp value pulled out of the config to make sure it always has a value - d_mlp: int - - # The middle hook point will be None unless it specifically should be used - hook_mid: Optional[HookPoint] # [batch, pos, d_mlp] - - # The layer norm component if the activation function is a layer norm - ln: Optional[nn.Module] - - # MLP weight matrices (Parameter on subclasses; declared here so callers like - # ActivationCache.get_neuron_results get a typed Tensor instead of nn.Module). - W_in: torch.Tensor - W_out: torch.Tensor - - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - """The base init for all MLP like components - - Args: - config (Union[Dict, HookedTransformerConfig]): The config for this instance - - Raises: - ValueError: If there is a misconfiguration - """ - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - if self.cfg.d_mlp is None: - raise ValueError("d_mlp must be set to use an MLP") - - self.d_mlp = self.cfg.d_mlp - - def forward( - self, x: Float[torch.Tensor, "batch pos d_model"] - ) -> Float[torch.Tensor, "batch pos d_model"]: - """The format for all forward functions for any MLP""" - return x - - def select_activation_function(self) -> None: - """This function should be called by all components in their init to get everything needed - for activation functions setup. - - Raises: - ValueError: If the configure activation function is not supported. - """ - - self.act_fn = ActivationFunctionFactory.pick_activation_function(self.cfg) - - if self.cfg.is_layer_norm_activation(): - self.hook_mid = HookPoint() - if self.cfg.normalization_type == "LN": - self.ln = LayerNorm(self.cfg, self.d_mlp) - else: - self.ln = LayerNormPre(self.cfg) diff --git a/transformer_lens/components/mlps/gated_mlp.py b/transformer_lens/components/mlps/gated_mlp.py deleted file mode 100644 index b19f6e3d57..0000000000 --- a/transformer_lens/components/mlps/gated_mlp.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Hooked Transformer Gated MLP Component. - -This module contains all the component :class:`GatedMLP`. -""" - -from typing import Dict, Union - -import torch -import torch.nn as nn -import torch.nn.functional as F -from jaxtyping import Float -from transformers.utils import is_bitsandbytes_available - -from transformer_lens.components.mlps.can_be_used_as_mlp import CanBeUsedAsMLP -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.hook_points import HookPoint - -if is_bitsandbytes_available(): - pass - - -class GatedMLP(CanBeUsedAsMLP): - """ - The equation of a gated MLP: - pre = x @ W_gate - pre_linear = x @ W_in - post = Gelu(pre) * (pre_linear) + b_in - mlp_out = post @ W_out + b_out - - In one equation, mlp_out = (Gelu(x @ W_gate) * (x @ W_in) + b_in) @ W_out + b_out - """ - - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - super().__init__(cfg) - self.select_activation_function() - self.W_in = nn.Parameter(torch.empty(self.cfg.d_model, self.d_mlp, dtype=self.cfg.dtype)) - self.W_out = nn.Parameter(torch.empty(self.d_mlp, self.cfg.d_model, dtype=self.cfg.dtype)) - self.W_gate = nn.Parameter(torch.empty(self.cfg.d_model, self.d_mlp, dtype=self.cfg.dtype)) - - self.b_in = nn.Parameter(torch.zeros(self.d_mlp, dtype=self.cfg.dtype)) - self.b_out = nn.Parameter(torch.zeros(self.cfg.d_model, dtype=self.cfg.dtype)) - - # hook on gate output but before act_fn - self.hook_pre = HookPoint() # [batch, pos, d_mlp] - # hook on the linear component of the input - self.hook_pre_linear = HookPoint() # [batch, pos, d_mlp] - # hook on act_fn(gate_output) * W_in(x) + b_in - self.hook_post = HookPoint() # [batch, pos, d_mlp] - - def forward( - self, x: Float[torch.Tensor, "batch pos d_model"] - ) -> Float[torch.Tensor, "batch pos d_model"]: - # Use F.linear with contiguous transposed weights to match HF's nn.Linear - # memory layout. In bfloat16, matmul accumulation order depends on tensor - # contiguity, so matching HF's layout ensures numerically identical results. - if self.W_gate.device != x.device: - x = x.to(self.W_gate.device) - pre_act = self.hook_pre(F.linear(x, self.W_gate.T.contiguous())) # [batch, pos, d_mlp] - - if ( - self.cfg.is_layer_norm_activation() - and self.hook_mid is not None - and self.ln is not None - ): - mid_act = self.hook_mid(self.act_fn(pre_act)) # [batch, pos, d_mlp] - post_act = self.hook_post(self.ln(mid_act)) - else: - pre_linear = self.hook_pre_linear( - F.linear(x, self.W_in.T.contiguous()) # [batch, pos, d_mlp] - ) - - post_act = self.hook_post( - (self.act_fn(pre_act) * pre_linear) + self.b_in - ) # [batch, pos, d_mlp] - - return F.linear(post_act, self.W_out.T.contiguous(), self.b_out) diff --git a/transformer_lens/components/mlps/gated_mlp_4bit.py b/transformer_lens/components/mlps/gated_mlp_4bit.py deleted file mode 100644 index 1eb489f1ab..0000000000 --- a/transformer_lens/components/mlps/gated_mlp_4bit.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Hooked Transformer Gated MLP Component. - -This module contains all the component :class:`GatedMLP`. -""" - -from typing import Dict, Union - -import torch -import torch.nn as nn -from jaxtyping import Float -from transformers.utils import is_bitsandbytes_available - -from transformer_lens.components.mlps.can_be_used_as_mlp import CanBeUsedAsMLP -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.hook_points import HookPoint - -if is_bitsandbytes_available(): - import bitsandbytes as bnb - from bitsandbytes.nn.modules import Params4bit - - -class GatedMLP4Bit(CanBeUsedAsMLP): - """ - The equation of a gated MLP: - pre = x @ W_gate - pre_linear = x @ W_in - post = Gelu(pre) * (pre_linear) + b_in - mlp_out = post @ W_out + b_out - - In one equation, mlp_out = (Gelu(x @ W_gate) * (x @ W_in) + b_in) @ W_out + b_out - """ - - # Narrow base-class W_in/W_out (declared as torch.Tensor) to bnb's Params4bit - # so .quant_state attribute access type-checks. - W_in: "Params4bit" - W_gate: "Params4bit" - W_out: "Params4bit" - - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - super().__init__(cfg) - self.select_activation_function() - - nq = int((self.cfg.d_model * self.d_mlp) / 2) - self.W_in = Params4bit(torch.empty(nq, 1, dtype=torch.uint8), requires_grad=False) - self.W_gate = Params4bit(torch.empty(nq, 1, dtype=torch.uint8), requires_grad=False) - self.W_out = Params4bit(torch.empty(nq, 1, dtype=torch.uint8), requires_grad=False) - - self.b_in = nn.Parameter(torch.zeros(self.d_mlp, dtype=self.cfg.dtype)) - self.b_out = nn.Parameter(torch.zeros(self.cfg.d_model, dtype=self.cfg.dtype)) - - # hook on gate output but before act_fn - self.hook_pre = HookPoint() # [batch, pos, d_mlp] - # hook on the linear component of the input - self.hook_pre_linear = HookPoint() # [batch, pos, d_mlp] - # hook on act_fn(gate_output) * W_in(x) + b_in - self.hook_post = HookPoint() # [batch, pos, d_mlp] - - def forward( - self, x: Float[torch.Tensor, "batch pos d_model"] - ) -> Float[torch.Tensor, "batch pos d_model"]: - # Technically, all these einsums could be done with a single matmul, but this is more readable. - pre_act = self.hook_pre( - bnb.matmul_4bit(x, self.W_gate.t(), bias=None, quant_state=self.W_gate.quant_state) - ) - - if ( - self.cfg.is_layer_norm_activation() - and self.hook_mid is not None - and self.ln is not None - ): - mid_act = self.hook_mid(self.act_fn(pre_act)) # [batch, pos, d_mlp] - post_act = self.hook_post(self.ln(mid_act)) - else: - pre_linear = self.hook_pre_linear( - bnb.matmul_4bit(x, self.W_in.t(), bias=None, quant_state=self.W_in.quant_state) - ) - - post_act = self.hook_post( - (self.act_fn(pre_act) * pre_linear) + self.b_in - ) # [batch, pos, d_mlp] - - return bnb.matmul_4bit( - post_act, self.W_out.t(), bias=None, quant_state=self.W_out.quant_state - ) diff --git a/transformer_lens/components/mlps/gpt_oss_moe.py b/transformer_lens/components/mlps/gpt_oss_moe.py deleted file mode 100644 index 377f77fb68..0000000000 --- a/transformer_lens/components/mlps/gpt_oss_moe.py +++ /dev/null @@ -1,126 +0,0 @@ -"""GPT-OSS Mixture of Experts implementation for TransformerLens. - -GPT-OSS uses a unique MoE architecture: -- Merged expert weights (gate_up_proj with interleaved gate/up columns) -- Custom GLU activation: gate * sigmoid(gate * 1.702) * (up + 1), with clamping -- Router with bias, softmax applied AFTER top-k selection -- Expert projections have biases -""" - -from typing import Dict, Union - -import torch -import torch.nn as nn -import torch.nn.functional as F -from jaxtyping import Float - -from transformer_lens.components.mlps.can_be_used_as_mlp import CanBeUsedAsMLP -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.hook_points import HookPoint - -GPT_OSS_ALPHA = 1.702 -GPT_OSS_LIMIT = 7.0 - - -class GptOssExpert(nn.Module): - """Single GPT-OSS expert with custom GLU activation. - - The activation differs from standard SiLU: - gate = clamp(x @ W_gate + b_gate, max=7.0) - up = clamp(x @ W_in + b_in, min=-7.0, max=7.0) - glu = gate * sigmoid(gate * 1.702) - out = (up + 1) * glu - result = out @ W_out + b_out - """ - - def __init__(self, cfg: HookedTransformerConfig): - super().__init__() - self.cfg = cfg - assert cfg.d_mlp is not None - - self.W_gate = nn.Linear(cfg.d_model, cfg.d_mlp, bias=True, dtype=cfg.dtype) - self.W_in = nn.Linear(cfg.d_model, cfg.d_mlp, bias=True, dtype=cfg.dtype) - self.W_out = nn.Linear(cfg.d_mlp, cfg.d_model, bias=True, dtype=cfg.dtype) - - self.hook_gate = HookPoint() - self.hook_pre = HookPoint() - self.hook_post = HookPoint() - - def forward(self, x: Float[torch.Tensor, "pos d_model"]) -> Float[torch.Tensor, "pos d_model"]: - gate = self.hook_gate(self.W_gate(x)) - up = self.hook_pre(self.W_in(x)) - - # GPT-OSS custom activation - gate = gate.clamp(max=GPT_OSS_LIMIT) - up = up.clamp(min=-GPT_OSS_LIMIT, max=GPT_OSS_LIMIT) - glu = gate * torch.sigmoid(gate * GPT_OSS_ALPHA) - post = self.hook_post((up + 1) * glu) - - return self.W_out(post) - - -class GptOssMoE(CanBeUsedAsMLP): - """GPT-OSS Mixture of Experts layer. - - Differences from standard TransformerLens MoE (Mixtral): - - Router has bias - - Softmax applied AFTER top-k selection (not before) - - Experts use custom GLU activation (not SiLU) - - Expert projections have biases - """ - - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - super().__init__(cfg) - - assert self.cfg.num_experts is not None - assert self.cfg.experts_per_token is not None - - self.num_experts: int = self.cfg.num_experts - self.experts_per_token: int = self.cfg.experts_per_token - - self.experts = nn.ModuleList([GptOssExpert(self.cfg) for _ in range(self.num_experts)]) - # GPT-OSS router has bias (unlike Mixtral) - self.W_gate = nn.Linear( - self.cfg.d_model, self.cfg.num_experts, bias=True, dtype=self.cfg.dtype - ) - - self.hook_expert_weights = HookPoint() - self.hook_expert_indices = HookPoint() - - def forward( - self, x: Float[torch.Tensor, "batch pos d_model"] - ) -> Float[torch.Tensor, "batch pos d_model"]: - batch, pos, d_model = x.shape - x = x.view(-1, d_model) - - # GPT-OSS routing: softmax AFTER top-k (differs from Mixtral) - gate_logits = self.W_gate(x) - top_values, expert_indices = torch.topk(gate_logits, self.experts_per_token, dim=-1) - # Softmax over just the selected experts - top_weights = F.softmax(top_values, dim=-1, dtype=torch.float) - - # Build full routing weights tensor for hooks (num_tokens, num_experts) - routing_weights = torch.zeros_like(gate_logits, dtype=torch.float) - routing_weights.scatter_(1, expert_indices, top_weights) - - routing_weights = self.hook_expert_weights(routing_weights) - expert_indices = self.hook_expert_indices(expert_indices) - routing_weights = routing_weights.to(x.dtype) - - results = torch.zeros((batch * pos, d_model), dtype=x.dtype, device=x.device) - expert_mask = F.one_hot(expert_indices, num_classes=self.num_experts).permute(2, 1, 0) - - for expert_idx in range(self.num_experts): - expert_layer = self.experts[expert_idx] - idx, top_x = torch.where(expert_mask[expert_idx]) - - if top_x.numel() == 0: - continue - - current_state = x[top_x] - current_hidden_states = ( - expert_layer(current_state) * routing_weights[top_x, expert_idx, None] - ) - results.index_add_(0, top_x, current_hidden_states.to(x.dtype)) - - return results.reshape(batch, pos, d_model) diff --git a/transformer_lens/components/mlps/mlp.py b/transformer_lens/components/mlps/mlp.py deleted file mode 100644 index 9a95f8305c..0000000000 --- a/transformer_lens/components/mlps/mlp.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Hooked Transformer MLP Component. - -This module contains all the component :class:`MLP`. -""" - -from typing import Dict, Union - -import torch -import torch.nn as nn -from jaxtyping import Float - -from transformer_lens.components.mlps.can_be_used_as_mlp import CanBeUsedAsMLP -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.hook_points import HookPoint -from transformer_lens.utilities.addmm import batch_addmm - - -class MLP(CanBeUsedAsMLP): - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - super().__init__(cfg) - self.select_activation_function() - - self.W_in = nn.Parameter(torch.empty(self.cfg.d_model, self.d_mlp, dtype=self.cfg.dtype)) - self.b_in = nn.Parameter(torch.zeros(self.d_mlp, dtype=self.cfg.dtype)) - - self.W_out = nn.Parameter(torch.empty(self.d_mlp, self.cfg.d_model, dtype=self.cfg.dtype)) - self.b_out = nn.Parameter(torch.zeros(self.cfg.d_model, dtype=self.cfg.dtype)) - - self.hook_pre = HookPoint() # [batch, pos, d_mlp] - self.hook_post = HookPoint() # [batch, pos, d_mlp] - - def forward( - self, x: Float[torch.Tensor, "batch pos d_model"] - ) -> Float[torch.Tensor, "batch pos d_model"]: - # This is equivalent to (roughly) W_in @ x + b_in. It's important to - # use a fused addmm to ensure it matches the Huggingface implementation - # exactly. - pre_act = self.hook_pre(batch_addmm(self.b_in, self.W_in, x)) # [batch, pos, d_mlp] - - if ( - self.cfg.is_layer_norm_activation() - and self.hook_mid is not None - and self.ln is not None - ): - mid_act = self.hook_mid(self.act_fn(pre_act)) # [batch, pos, d_mlp] - post_act = self.hook_post(self.ln(mid_act)) - else: - post_act = self.hook_post(self.act_fn(pre_act)) # [batch, pos, d_mlp] - return batch_addmm(self.b_out, self.W_out, post_act) diff --git a/transformer_lens/components/mlps/moe.py b/transformer_lens/components/mlps/moe.py deleted file mode 100644 index 63c61a6e91..0000000000 --- a/transformer_lens/components/mlps/moe.py +++ /dev/null @@ -1,114 +0,0 @@ -from typing import Dict, Union - -import torch -import torch.nn as nn -import torch.nn.functional as F -from jaxtyping import Float - -from transformer_lens.components.mlps.can_be_used_as_mlp import CanBeUsedAsMLP -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.factories.activation_function_factory import ( - ActivationFunctionFactory, -) -from transformer_lens.hook_points import HookPoint - - -class MoEGatedMLP(nn.Module): - """MoEGated MLP - - This MLP matches the implementation for Mixtral on HuggingFace. It is meant to stay within our - MoE, since the format of this MLP is different from the standard MLPs throughout - TransformerLens. - - It may be possible to rework this to follow the same interface as other MLPs, but for the - time being it is being left as is to ensure accuracy. - """ - - def __init__(self, cfg: HookedTransformerConfig): - super().__init__() - self.cfg = cfg - - self.d_mlp = self.cfg.d_mlp - - if self.d_mlp is None: - raise ValueError("d_mlp must be set to use an MLP") - - self.W_in = nn.Linear(self.cfg.d_model, self.d_mlp, bias=False) - self.W_out = nn.Linear(self.d_mlp, self.cfg.d_model, bias=False) - self.W_gate = nn.Linear(self.cfg.d_model, self.d_mlp, bias=False) - - # hook on gate output but before act_fn - self.hook_gate = HookPoint() # [batch, pos, d_mlp] - # hook on the linear component of the input - self.hook_pre = HookPoint() # [batch, pos, d_mlp] - # hook on act_fn(gate_output) * W_in(x) + b_in - self.hook_post = HookPoint() # [batch, pos, d_mlp] - - self.act_fn = ActivationFunctionFactory.pick_activation_function(self.cfg) - - def forward(self, x: Float[torch.Tensor, "pos d_model"]) -> Float[torch.Tensor, "pos d_model"]: - gated_x = self.hook_gate(self.W_gate(x)) - pre_act = self.hook_pre(self.W_in(x)) - post_act = self.hook_post(self.act_fn(gated_x) * pre_act) - return self.W_out(post_act) - - -class MoE(CanBeUsedAsMLP): - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - super().__init__(cfg) - - # Ensure that num_experts and experts_per_token are specified and non-zero - assert self.cfg.num_experts is not None, "num_experts must be specified for MoE layer" - assert self.cfg.experts_per_token, "experts_per_token must be specified for MoE layer" - - self.num_experts: int = self.cfg.num_experts - self.experts_per_token: int = self.cfg.experts_per_token - - assert ( - self.cfg.experts_per_token <= self.cfg.num_experts - ), "experts_per_token must be less than or equal to num_experts" - - self.experts = nn.ModuleList([MoEGatedMLP(self.cfg) for _ in range(self.num_experts)]) - self.W_gate = nn.Linear(self.cfg.d_model, self.cfg.num_experts, bias=False) - - # Hook on the weights of selected experts [batch pos experts_per_token] - self.hook_expert_weights = HookPoint() - # Hook on the indices of selected experts [batch pos experts_per_token] - self.hook_expert_indices = HookPoint() - - def forward( - self, x: Float[torch.Tensor, "batch pos d_model"] - ) -> Float[torch.Tensor, "batch pos d_model"]: - # [batch, pos, d_model] -> [batch, pos, num_experts] - batch, pos, d_model = x.shape - x = x.view(-1, d_model) - gate_logits = self.W_gate(x) - - # choose the top k(=experts_per_token) experts to use - # both are [batch, pos, experts_per_token] - weights = self.hook_expert_weights(F.softmax(gate_logits, dim=1, dtype=torch.float)) - weights, expert_indices = torch.topk(weights, self.experts_per_token, dim=-1) - if self.cfg.norm_topk_prob: - weights /= weights.sum(dim=-1, keepdim=True) - expert_indices = self.hook_expert_indices(expert_indices) - weights = weights.to(x.dtype) - - results = torch.zeros((batch * pos, d_model), dtype=x.dtype, device=x.device) - expert_mask = F.one_hot(expert_indices, num_classes=self.num_experts).permute(2, 1, 0) - for expert_idx in range(self.num_experts): - expert_layer = self.experts[expert_idx] - idx, top_x = torch.where(expert_mask[expert_idx]) - - # Index the correct hidden states and compute the expert hidden state for - # the current expert. We need to make sure to multiply the output hidden - # states by `routing_weights` on the corresponding tokens (top-1 and top-2) - current_state = x[None, top_x].reshape(-1, d_model) - - current_hidden_states = expert_layer(current_state) * weights[top_x, idx, None] - - # However `index_add_` only support torch tensors for indexing so we'll use - # the `top_x` tensor here. - results.index_add_(0, top_x, current_hidden_states.to(x.dtype)) - - results = results.reshape(batch, pos, d_model) - return results diff --git a/transformer_lens/components/pos_embed.py b/transformer_lens/components/pos_embed.py deleted file mode 100644 index d05d284008..0000000000 --- a/transformer_lens/components/pos_embed.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Hooked Transformer POS Embed Component. - -This module contains all the component :class:`PosEmbed`. -""" - -from typing import Dict, Optional, Union - -import einops -import torch -import torch.nn as nn -from jaxtyping import Float, Int - -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.utilities import get_offset_position_ids - - -# Positional Embeddings -class PosEmbed(nn.Module): - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - self.W_pos = nn.Parameter( - torch.empty(self.cfg.n_ctx, self.cfg.d_model, dtype=self.cfg.dtype) - ) - - def forward( - self, - tokens: Int[torch.Tensor, "batch pos"], - past_kv_pos_offset: int = 0, - attention_mask: Optional[Int[torch.Tensor, "batch offset_pos"]] = None, - ) -> Float[torch.Tensor, "batch new_pos d_model"]: - """ - Forward pass for positional embeddings. - - Args: - tokens (Int[torch.Tensor, "batch pos"]): Input tokens. - past_kv_pos_offset (int, optional): The length of tokens in the past_kv_cache. Defaults to 0. - attention_mask (Int[torch.Tensor, "batch pos"], optional): The attention mask for padded tokens. - Defaults to None. - - Returns: - Float[torch.Tensor, "batch pos d_model"]: Absolute position embeddings. - """ - tokens_length = tokens.size(-1) - - if attention_mask is None: - pos_embed = self.W_pos[ - past_kv_pos_offset : tokens_length + past_kv_pos_offset, : - ] # [pos, d_model] - batch_pos_embed = einops.repeat( - pos_embed, "pos d_model -> batch pos d_model", batch=tokens.size(0) - ) - - else: - # Separated from the no padding case for computational efficiency - # (this code is a bit slower than the code above) - - offset_position_ids = get_offset_position_ids(past_kv_pos_offset, attention_mask) - pos_embed = self.W_pos[offset_position_ids] # [batch, pos, d_model] - - # Set the position embeddings to 0 for pad tokens (this is an arbitrary choice) - padding_mask = ~attention_mask.bool() # [batch, tokens_length] - offset_padding_mask = padding_mask[ - :, past_kv_pos_offset : tokens_length + past_kv_pos_offset - ].unsqueeze( - -1 - ) # [batch, pos, 1] - batch_pos_embed = torch.where(offset_padding_mask, 0, pos_embed) - - return batch_pos_embed.clone() diff --git a/transformer_lens/components/rms_norm.py b/transformer_lens/components/rms_norm.py deleted file mode 100644 index dd4238ab40..0000000000 --- a/transformer_lens/components/rms_norm.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Hooked Transformer RMS Norm Component. - -This module contains all the component :class:`RMSNorm`. -""" - -from typing import Dict, Optional, Union - -import torch -import torch.nn as nn -from jaxtyping import Float - -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.hook_points import HookPoint - -# RMSNorm operates on the last dimension and supports both 2D and 3D inputs. -# The 2D case arises when callers (e.g. QK normalization) reshape before normalizing. -RMSNormInput = Union[ - Float[torch.Tensor, "batch pos length"], - Float[torch.Tensor, "batch_pos length"], -] - - -class RMSNorm(nn.Module): - def __init__(self, cfg: Union[Dict, HookedTransformerConfig], length: Optional[int] = None): - """ - RMSNorm - LayerNorm without the centering and bias (RMS = Root Mean Square) - - length (Optional[int]): If the dimension of the RMSNorm. If not provided, assumed to be d_model - """ - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - self.eps = self.cfg.eps - if length is None: - self.length = self.cfg.d_model - else: - self.length = length - - self.w = nn.Parameter(torch.ones(self.length, dtype=self.cfg.dtype)) - - # Adds a hook point for the normalisation scale factor - self.hook_scale = HookPoint() # [batch, pos, 1] - self.hook_normalized = HookPoint() # [batch, pos, length] - - def forward(self, x: RMSNormInput) -> RMSNormInput: - if self.cfg.dtype not in [torch.float32, torch.float64]: - x = x.to(torch.float32) - scale: Float[torch.Tensor, "batch pos 1"] = self.hook_scale( - (x.pow(2).mean(-1, keepdim=True) + self.eps).sqrt() - ) - x = self.hook_normalized(x / scale).to(self.cfg.dtype) # [batch, pos, length] - - if x.device != self.w.device: - self.to(x.device) - - return x * self.w diff --git a/transformer_lens/components/rms_norm_pre.py b/transformer_lens/components/rms_norm_pre.py deleted file mode 100644 index 8742f2444a..0000000000 --- a/transformer_lens/components/rms_norm_pre.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Hooked Transformer RMS Norm Pre Component. - -This module contains all the component :class:`RMSNormPre`. -""" - -from typing import Dict, Union - -import torch -import torch.nn as nn -from jaxtyping import Float - -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.hook_points import HookPoint - - -class RMSNormPre(nn.Module): - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - """RMSNormPre - LayerNormPre without the centering and bias (RMS = Root Mean Square)""" - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - self.eps = self.cfg.eps - - # Adds a hook point for the normalisation scale factor - self.hook_scale = HookPoint() # [batch, pos] - self.hook_normalized = HookPoint() # [batch, pos, length] - - def forward( - self, x: Float[torch.Tensor, "batch pos length"] - ) -> Float[torch.Tensor, "batch pos length"]: - if self.cfg.dtype not in [torch.float32, torch.float64]: - x = x.to(torch.float32) - - scale: Float[torch.Tensor, "batch pos 1"] = self.hook_scale( - (x.pow(2).mean(-1, keepdim=True) + self.eps).sqrt() - ) - return self.hook_normalized(x / scale).to(self.cfg.dtype) # [batch, pos, length] diff --git a/transformer_lens/components/t5_attention.py b/transformer_lens/components/t5_attention.py deleted file mode 100644 index e4736dde96..0000000000 --- a/transformer_lens/components/t5_attention.py +++ /dev/null @@ -1,148 +0,0 @@ -import math -from typing import Dict, Optional, Union - -import torch -import torch.nn as nn -from jaxtyping import Float, Int - -from transformer_lens.components.abstract_attention import AbstractAttention -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.hook_points import HookPoint - - -class T5Attention(AbstractAttention): - r""" - T5 attention - with relative attention bias and cross-attention support - This realisation expects you to precompute relative positional bias, and then feed it to forward - like - ```python - attn = T5Attention(cfg, has_relative_attention_bias=True) - positional_bias = attn.compute_relative_attention_bias(query_len, key_len, device=device) - result = attn(query, key, value, position_bias=positional_bias) - ``` - """ - - def __init__( - self, - cfg: Union[Dict, HookedTransformerConfig], - has_relative_attention_bias: bool = False, - attn_type: str = "global", - layer_id: Optional[int] = None, - is_decoder: bool = False, - ): - super().__init__(cfg, attn_type, layer_id) - if isinstance(cfg, Dict): - cfg = HookedTransformerConfig.from_dict(cfg) - self.cfg = cfg - self.has_relative_attention_bias: bool = has_relative_attention_bias - # Decoder buckets are unidirectional: every key sits at a non-positive - # offset, so bidirectional bucketing halves the usable resolution. - self.is_decoder: bool = is_decoder - if is_decoder: - # T5's cfg carries attention_dir="bidirectional" for the encoder; - # decoder self-attention must still mask the future. - self._attention_dir_override = "causal" - - if self.has_relative_attention_bias: - if ( - cfg.relative_attention_num_buckets is None - or cfg.relative_attention_max_distance is None - ): - raise ValueError( - "You need to specify relative_attention_num_buckets and relative_attention_max_distance in config to use relative attention bias" - ) - - self.relative_attention_num_buckets = cfg.relative_attention_num_buckets - self.relative_attention_max_distance = cfg.relative_attention_max_distance - self.rel_pos_bias = nn.Embedding(self.relative_attention_num_buckets, self.cfg.n_heads) - self.rel_pos_hook = HookPoint() - - self.W_K = nn.Parameter( - torch.empty(self.cfg.n_heads, self.cfg.d_model, self.cfg.d_head, dtype=cfg.dtype) - ) - self.W_V = nn.Parameter( - torch.empty(self.cfg.n_heads, self.cfg.d_model, self.cfg.d_head, dtype=cfg.dtype) - ) - self.b_K = nn.Parameter(torch.zeros(self.cfg.n_heads, self.cfg.d_head, dtype=cfg.dtype)) - self.b_V = nn.Parameter(torch.zeros(self.cfg.n_heads, self.cfg.d_head, dtype=cfg.dtype)) - - @staticmethod - def _relative_position_bucket( - relative_position: Int[torch.Tensor, "query_pos kv_pos"], - bidirectional=True, - num_buckets=32, - max_distance=128, - ) -> Int[torch.Tensor, "query_pos kv_pos"]: - """ - added from - https://github.com/huggingface/transformers/blob/e0c3cee17085914bbe505c159beeb8ae39bc37dd/src/transformers/models/t5/modeling_t5.py#L382 - which is adapted from - https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593 - - - Translate relative position to a bucket number for relative attention. The relative position is defined as - memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to - position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for - small absolute relative_position and larger buckets for larger absolute relative_positions. All relative - positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket. - This should allow for more graceful generalization to longer sequences than the model has been trained on - - Args: - relative_position: an int32 Tensor - bidirectional: a boolean - whether the attention is bidirectional - num_buckets: an integer - max_distance: an integer - - Returns: - a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets) - """ - relative_buckets = torch.zeros_like(relative_position) - - if bidirectional: - num_buckets //= 2 - relative_buckets += (relative_position > 0).to(torch.long) * num_buckets - relative_position = torch.abs(relative_position) - else: - relative_position = -torch.min(relative_position, torch.zeros_like(relative_position)) - # now relative_position is in the range [0, inf) - - # half of the buckets are for exact increments in positions - max_exact = num_buckets // 2 - is_small = relative_position < max_exact - - # The other half of the buckets are for logarithmically bigger bins in positions up to max_distance - relative_position_if_large = max_exact + ( - torch.log(relative_position.float() / max_exact) - / math.log(max_distance / max_exact) - * (num_buckets - max_exact) - ).to(torch.long) - relative_position_if_large = torch.min( - relative_position_if_large, - torch.full_like(relative_position_if_large, num_buckets - 1), - ) - - relative_buckets += torch.where(is_small, relative_position, relative_position_if_large) - return relative_buckets - - def compute_relative_attention_bias( - self, query_length: int, key_length: int, device=None - ) -> Float[torch.Tensor, "1 head_index pos kv_pos"]: - """Compute binned relative position bias""" - if device is None: - device = self.rel_pos_bias.weight.device - context_position = torch.arange(query_length, dtype=torch.long, device=device)[:, None] - memory_position = torch.arange(key_length, dtype=torch.long, device=device)[None, :] - relative_position = memory_position - context_position # shape (query_length, key_length) - relative_position_bucket = self._relative_position_bucket( - relative_position, # shape (query_length, key_length) - bidirectional=not self.is_decoder, - num_buckets=self.relative_attention_num_buckets, - max_distance=self.relative_attention_max_distance, - ) - values = self.rel_pos_bias( - relative_position_bucket - ) # shape (query_length, key_length, num_heads) - values = values.permute([2, 0, 1]).unsqueeze( - 0 - ) # shape (1, num_heads, query_length, key_length) - return values diff --git a/transformer_lens/components/t5_block.py b/transformer_lens/components/t5_block.py deleted file mode 100644 index 7b2b33745f..0000000000 --- a/transformer_lens/components/t5_block.py +++ /dev/null @@ -1,158 +0,0 @@ -from typing import Optional - -import torch -import torch.nn as nn -from jaxtyping import Float - -from transformer_lens.cache.key_value_cache_entry import ( - TransformerLensKeyValueCacheEntry, -) -from transformer_lens.components import RMSNorm, T5Attention -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.factories.mlp_factory import MLPFactory -from transformer_lens.hook_points import HookPoint -from transformer_lens.utilities import repeat_along_head_dimension - - -class T5Block(nn.Module): - """ - T5 decoder Block. Uses T5Layernorm, and T5attention instead of usual ones. - Also uses cross attention if is_decoder is True. - """ - - def __init__(self, cfg: HookedTransformerConfig, block_index: int, is_decoder: bool): - super().__init__() - self.cfg = cfg - self.is_decoder = is_decoder - - self.ln1 = RMSNorm(cfg) - self.attn = T5Attention( - cfg, has_relative_attention_bias=block_index == 0, is_decoder=is_decoder - ) - self.ln2 = RMSNorm(cfg) - if self.is_decoder: - self.cross_attn = T5Attention(cfg) - self.ln3 = RMSNorm(cfg) - self.mlp = MLPFactory.create_mlp(self.cfg) # [batch, pos, n_heads] - - self.hook_q_input = HookPoint() # [batch, pos, n_heads, d_model] - self.hook_k_input = HookPoint() # [batch, pos, n_heads, d_model] - self.hook_v_input = HookPoint() # [batch, pos, n_heads, d_model] - - self.hook_attn_in = HookPoint() # [batch, pos, d_model] - self.hook_attn_out = HookPoint() # [batch, pos, d_model] - if self.is_decoder: - self.hook_cross_attn_in = HookPoint() # [batch, pos, d_model] - self.hook_cross_attn_out = HookPoint() # [batch, pos, d_model] - self.hook_resid_mid_cross = HookPoint() # [batch, pos, d_model] - - self.hook_mlp_in = HookPoint() # [batch, pos, d_model] - self.hook_mlp_out = HookPoint() # [batch, pos, d_model] - self.hook_resid_pre = HookPoint() # [batch, pos, d_model] - self.hook_resid_mid = HookPoint() # [batch, pos, d_model] - self.hook_resid_post = HookPoint() # [batch, pos, d_model] - - def forward( - self, - resid_pre: Float[torch.Tensor, "batch pos d_model"], - additive_attention_mask: Optional[Float[torch.Tensor, "batch 1 1 pos"]] = None, - encoder_additive_attention_mask: Optional[ - Float[torch.Tensor, "batch 1 1 encoder_pos"] - ] = None, - position_bias: Optional[Float[torch.Tensor, "1 head_index pos kv_pos"]] = None, - encoder_hidden_states: Optional[Float[torch.Tensor, "batch encoder_pos d_model"]] = None, - past_kv_cache_entry: Optional[TransformerLensKeyValueCacheEntry] = None, - ) -> Float[torch.Tensor, "batch pos d_model"]: - """A single Transformer block. - - Args: - resid_pre (torch.Tensor): The residual stream - shape [batch, pos, d_model] - encoder_hidden_states (torch.Tensor): The hidden states of the encoder for cross attention - shape [batch, encoder_pos, d_model] - cache (TransformerLensKeyValueCache): A cache of previous keys and values, used only when generating text. Defaults to None. - attention_mask (torch.Tensor, optional): The attention mask for padded tokens. Defaults to None. - - Returns: - Float[torch.Tensor, "batch pos d_model"]: The block output residual stream. - """ - resid_pre = self.hook_resid_pre(resid_pre) # [batch, pos, d_model] - - attn_in = resid_pre - - if self.cfg.use_attn_in: - attn_in = self.hook_attn_in( - repeat_along_head_dimension(resid_pre, n_heads=self.cfg.n_heads) - ) - - if self.cfg.use_split_qkv_input: - n_kv_heads = ( - self.cfg.n_key_value_heads - if self.cfg.n_key_value_heads is not None - else self.cfg.n_heads - ) - query_input = self.hook_q_input( - repeat_along_head_dimension(resid_pre, n_heads=self.cfg.n_heads) - ) - key_input = self.hook_k_input( - repeat_along_head_dimension(resid_pre, n_heads=n_kv_heads) - ) - value_input = self.hook_v_input( - repeat_along_head_dimension(resid_pre, n_heads=n_kv_heads) - ) - else: - query_input = attn_in - key_input = attn_in - value_input = attn_in - - attn_out = self.hook_attn_out( - # ln1 applied per Q/K/V input so split-QKV hooks stay independent - self.attn( - query_input=self.ln1(query_input), - key_input=self.ln1(key_input), - value_input=self.ln1(value_input), - past_kv_cache_entry=past_kv_cache_entry, - additive_attention_mask=additive_attention_mask, - position_bias=position_bias, - ) - ) - - # [batch, pos, d_model] - - resid_mid = self.hook_resid_mid(resid_pre + attn_out) # [batch, pos, d_model] - - if self.is_decoder: - cross_attn_in = ( - resid_mid - if not self.cfg.use_attn_in - else self.hook_cross_attn_in(resid_mid.clone()) - ) - - if encoder_hidden_states is None: - raise ValueError("Encoder hidden states must be provided for cross attention!") - - cross_attn_out = self.hook_cross_attn_out( - self.cross_attn( - query_input=self.ln2(cross_attn_in), - key_input=encoder_hidden_states, - value_input=encoder_hidden_states, - additive_attention_mask=encoder_additive_attention_mask, - ) - ) - resid_mid_cross = self.hook_resid_mid_cross(resid_mid + cross_attn_out) - - mlp_in = ( - resid_mid_cross - if not self.cfg.use_hook_mlp_in - else self.hook_mlp_in(resid_mid_cross.clone()) - ) - - normalized_resid_mid = self.ln3(mlp_in) - else: - mlp_in = ( - resid_mid if not self.cfg.use_hook_mlp_in else self.hook_mlp_in(resid_mid.clone()) - ) - normalized_resid_mid = self.ln2(mlp_in) - - mlp_out = self.hook_mlp_out(self.mlp(normalized_resid_mid)) # [batch, pos, d_model] - resid_post = self.hook_resid_post(mlp_in + mlp_out) # [batch, pos, d_model] - - return resid_post diff --git a/transformer_lens/components/token_typed_embed.py b/transformer_lens/components/token_typed_embed.py deleted file mode 100644 index 1663a58d94..0000000000 --- a/transformer_lens/components/token_typed_embed.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Hooked Transformer Token Typed Embed Component. - -This module contains all the component :class:`TokenTypeEmbed`. -""" - -from typing import Dict, Union - -import torch -import torch.nn as nn -from jaxtyping import Int - -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig - - -class TokenTypeEmbed(nn.Module): - """ - The token-type embed is a binary ids indicating whether a token belongs to sequence A or B. For example, for two sentences: "[CLS] Sentence A [SEP] Sentence B [SEP]", token_type_ids would be [0, 0, ..., 0, 1, ..., 1, 1]. `0` represents tokens from Sentence A, `1` from Sentence B. If not provided, BERT assumes a single sequence input. Typically, shape is (batch_size, sequence_length). - - See the BERT paper for more information: https://arxiv.org/pdf/1810.04805.pdf - """ - - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - self.W_token_type = nn.Parameter(torch.empty(2, self.cfg.d_model, dtype=self.cfg.dtype)) - - def forward(self, token_type_ids: Int[torch.Tensor, "batch pos"]): - return self.W_token_type[token_type_ids, :] diff --git a/transformer_lens/components/transformer_block.py b/transformer_lens/components/transformer_block.py deleted file mode 100644 index e1ebda8428..0000000000 --- a/transformer_lens/components/transformer_block.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Hooked Transformer Transformer Block Component. - -This module contains all the component :class:`TransformerBlock`. -""" - -from typing import Callable, Dict, Optional, Union - -import torch -import torch.nn as nn -from jaxtyping import Float, Int - -from transformer_lens.cache.key_value_cache_entry import ( - TransformerLensKeyValueCacheEntry, -) -from transformer_lens.components import ( - Attention, - GroupedQueryAttention, - LayerNorm, - LayerNormPre, - RMSNorm, - RMSNormPre, -) -from transformer_lens.components.mlps.can_be_used_as_mlp import CanBeUsedAsMLP -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.factories.mlp_factory import MLPFactory -from transformer_lens.hook_points import HookPoint -from transformer_lens.utilities import repeat_along_head_dimension -from transformer_lens.utilities.architectures import POST_NORM_ARCHITECTURES - - -class TransformerBlock(nn.Module): - ln1: nn.Module - ln2: nn.Module - mlp: CanBeUsedAsMLP - - def __init__(self, cfg: Union[Dict, HookedTransformerConfig], block_index): - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - normalization_layer: Callable # type: ignore - normalization_layer_after: Callable # type: ignore - - self.normalization_type = self.cfg.normalization_type - - if self.normalization_type == "LN": - normalization_layer = LayerNorm - elif self.normalization_type == "LNPre": - # We've folded in LayerNorm weights, so just need the center + scale parts - normalization_layer = LayerNormPre - elif self.normalization_type == "RMS": - normalization_layer = RMSNorm - elif self.normalization_type == "RMSPre": - normalization_layer = RMSNormPre - elif self.normalization_type is None: - # This should just be the identity. - # We need to make this a lambda so we can call it on the config, just like the others - normalization_layer = lambda cfg: nn.Identity() - else: - raise ValueError(f"Invalid normalization_type passed in: {self.normalization_type}") - - if self.cfg.use_normalization_before_and_after: - # If we use LN before and after, we do *not* fold in the weights to the LN - # after, though we can fold for the one before. - if self.normalization_type is None: - normalization_layer_after = lambda cfg: nn.Identity() - elif self.normalization_type.startswith("RMS"): - normalization_layer_after = RMSNorm - elif self.normalization_type.startswith("LayerNorm"): - normalization_layer_after = LayerNorm - - self.ln1 = normalization_layer(cfg) - if self.cfg.use_normalization_before_and_after: - self.ln1_post = normalization_layer_after(cfg) - if not self.cfg.attn_only: - self.ln2 = normalization_layer(cfg) - if self.cfg.use_normalization_before_and_after: - self.ln2_post = normalization_layer_after(cfg) - - attention = Attention if self.cfg.n_key_value_heads is None else GroupedQueryAttention - if not self.cfg.use_local_attn: - self.attn = attention(self.cfg, "global", block_index) - else: - if self.cfg.attn_types is None: - raise ValueError("attn_types must be set when using local attention") - attn_type = self.cfg.attn_types[block_index] - self.attn = attention(self.cfg, attn_type, block_index) - if not self.cfg.attn_only: - self.mlp = MLPFactory.create_mlp(self.cfg) - - self.hook_attn_in = HookPoint() # [batch, pos, n_heads, d_model] - self.hook_q_input = HookPoint() # [batch, pos, n_heads, d_model] - self.hook_k_input = HookPoint() # [batch, pos, n_heads, d_model] - self.hook_v_input = HookPoint() # [batch, pos, n_heads, d_model] - self.hook_mlp_in = HookPoint() # [batch, pos, d_model] - - self.hook_attn_out = HookPoint() # [batch, pos, d_model] - self.hook_mlp_out = HookPoint() # [batch, pos, d_model] - - self.hook_resid_pre = HookPoint() # [batch, pos, d_model] - if not self.cfg.attn_only and not self.cfg.parallel_attn_mlp: - self.hook_resid_mid = HookPoint() # [batch, pos, d_model] - self.hook_resid_post = HookPoint() # [batch, pos, d_model] - - def forward( - self, - resid_pre: Float[torch.Tensor, "batch pos d_model"], - shortformer_pos_embed: Optional[Float[torch.Tensor, "batch pos d_model"]] = None, - past_kv_cache_entry: Optional[TransformerLensKeyValueCacheEntry] = None, - attention_mask: Optional[Int[torch.Tensor, "batch offset_pos"]] = None, - ) -> Float[torch.Tensor, "batch pos d_model"]: - """A single Transformer block. - - Args: - resid_pre (torch.Tensor): The residual stream - shape [batch, pos, d_model] - cache (TransformerLensKeyValueCache): A cache of previous keys and values, used only when generating text. Defaults to None. - shortformer_pos_embed (torch.Tensor, optional): Only used for positional_embeddings_type == "shortformer". The positional embeddings. See HookedTransformerConfig for details. Defaults to None. - attention_mask (torch.Tensor, optional): The attention mask for padded tokens. Defaults to None. - - Returns: - Float[torch.Tensor, "batch pos d_model"]: Our resulting tensor - """ - resid_pre = self.hook_resid_pre(resid_pre) # [batch, pos, d_model] - - if self.cfg.use_attn_in or self.cfg.use_split_qkv_input: - # We're adding a head dimension - if shortformer_pos_embed is not None: - shortformer_pos_embed = repeat_along_head_dimension( - shortformer_pos_embed, n_heads=self.cfg.n_heads - ) - else: - attn_in = resid_pre - - if self.cfg.use_attn_in: - attn_in = self.hook_attn_in( - repeat_along_head_dimension(resid_pre, n_heads=self.cfg.n_heads) - ) - - if self.cfg.use_split_qkv_input: - n_kv_heads = ( - self.cfg.n_key_value_heads - if self.cfg.n_key_value_heads is not None - and not self.cfg.ungroup_grouped_query_attention - else self.cfg.n_heads - ) - query_input = self.hook_q_input( - repeat_along_head_dimension(resid_pre, n_heads=self.cfg.n_heads) - ) - key_input = self.hook_k_input( - repeat_along_head_dimension(resid_pre, n_heads=n_kv_heads) - ) - value_input = self.hook_v_input( - repeat_along_head_dimension(resid_pre, n_heads=n_kv_heads) - ) - else: - query_input = attn_in - key_input = attn_in - value_input = attn_in - - if self.cfg.original_architecture in POST_NORM_ARCHITECTURES: - attn_out = self.attn( - query_input=query_input, - key_input=key_input, - value_input=value_input, - past_kv_cache_entry=past_kv_cache_entry, - attention_mask=attention_mask, - ) - else: - attn_out = ( - # hook the residual stream states that are used to calculate the - # queries, keys and values, independently. - # Then take the layer norm of these inputs, and pass these to the attention module. - self.attn( - query_input=self.ln1(query_input) - + (0.0 if shortformer_pos_embed is None else shortformer_pos_embed), - key_input=self.ln1(key_input) - + (0.0 if shortformer_pos_embed is None else shortformer_pos_embed), - value_input=self.ln1(value_input), - past_kv_cache_entry=past_kv_cache_entry, - attention_mask=attention_mask, - ) - ) # [batch, pos, d_model] - if self.cfg.use_normalization_before_and_after: - # If we use LayerNorm both before and after, then apply the second LN after the layer - # and before the hook. We do it before the hook so hook_attn_out captures "that which - # is added to the residual stream" - attn_out = self.ln1_post(attn_out) - if self.cfg.original_architecture in POST_NORM_ARCHITECTURES: - # OLMo 2/3 post-norm: ln1 applies before the residual add, so it must - # precede the hook for hook_attn_out to capture the additive contribution. - attn_out = self.ln1(attn_out) - attn_out = self.hook_attn_out(attn_out) - - if resid_pre.device != attn_out.device: - resid_pre = resid_pre.to(attn_out.device) - - if not self.cfg.attn_only and not self.cfg.parallel_attn_mlp: - resid_mid = self.hook_resid_mid(resid_pre + attn_out) # [batch, pos, d_model] - mlp_in = ( - resid_mid if not self.cfg.use_hook_mlp_in else self.hook_mlp_in(resid_mid.clone()) - ) - if self.cfg.original_architecture in POST_NORM_ARCHITECTURES: - # Post-norm: apply_mlp applies ln2 before hook_mlp_out internally. - mlp_out = self.apply_mlp(mlp_in) - else: - normalized_resid_mid = self.ln2(mlp_in) - mlp_out = self.apply_mlp(normalized_resid_mid) - resid_post = self.hook_resid_post(resid_mid + mlp_out) # [batch, pos, d_model] - elif self.cfg.parallel_attn_mlp: - # Dumb thing done by GPT-J, both MLP and Attn read from resid_pre and write to resid_post, no resid_mid used. - # In GPT-J, LN1 and LN2 are tied, in GPT-NeoX they aren't. - normalized_resid_pre_2 = self.ln2( - resid_pre if not self.cfg.use_hook_mlp_in else self.hook_mlp_in(resid_pre.clone()) - ) - mlp_out = self.apply_mlp(normalized_resid_pre_2) - resid_post = self.hook_resid_post( - resid_pre + attn_out + mlp_out - ) # [batch, pos, d_model] - else: - resid_post = self.hook_resid_post(resid_pre + attn_out) # [batch, pos, d_model] - return resid_post - - def apply_mlp( - self, normalized_resid: Float[torch.Tensor, "batch pos d_model"] - ) -> Float[torch.Tensor, "batch pos d_model"]: - """Centralized point where the MLP is applied to the forward pass - - Returns: - Float[torch.Tensor, "batch pos d_model"]: Our resulting tensor - """ - mlp_out = self.mlp(normalized_resid) # [batch, pos, d_model] - if self.cfg.use_normalization_before_and_after: - mlp_out = self.ln2_post(mlp_out) - if self.cfg.original_architecture in POST_NORM_ARCHITECTURES: - # OLMo 2/3 post-norm: ln2 applies before the residual add, so it must - # precede the hook for hook_mlp_out to capture the additive contribution. - mlp_out = self.ln2(mlp_out) - return self.hook_mlp_out(mlp_out) diff --git a/transformer_lens/components/unembed.py b/transformer_lens/components/unembed.py deleted file mode 100644 index 4ffa3df492..0000000000 --- a/transformer_lens/components/unembed.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Hooked Transformer Unembed Component. - -This module contains all the component :class:`Unembed`. -""" - -from typing import Dict, Union - -import torch -import torch.nn as nn -import torch.nn.functional as F -from jaxtyping import Float - -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.hook_points import HookPoint - - -class Unembed(nn.Module): - def __init__(self, cfg: Union[Dict, HookedTransformerConfig]): - super().__init__() - self.cfg = HookedTransformerConfig.unwrap(cfg) - # Note that there's a separate variable for d_vocab_out and d_vocab (the input vocab size). For language tasks these are always the same, but for algorithmic tasks we may want them to be different. - self.W_U: Float[torch.Tensor, "d_model d_vocab_out"] = nn.Parameter( - torch.empty(self.cfg.d_model, self.cfg.d_vocab_out, dtype=self.cfg.dtype) - ) - self.b_U: Float[torch.Tensor, "d_vocab_out"] = nn.Parameter( - torch.zeros(self.cfg.d_vocab_out, dtype=self.cfg.dtype) - ) - - # Add hooks for compatibility with HookedTransformer - self.hook_in = HookPoint() - self.hook_out = HookPoint() - - def forward( - self, residual: Float[torch.Tensor, "batch pos d_model"] - ) -> Float[torch.Tensor, "batch pos d_vocab_out"]: - residual = self.hook_in(residual) - # Use F.linear with contiguous transposed weight to match HF's nn.Linear - # memory layout, ensuring identical bfloat16 matmul accumulation order. - result = F.linear(residual, self.W_U.T.contiguous(), self.b_U) - return self.hook_out(result) diff --git a/transformer_lens/config/AGENTS.md b/transformer_lens/config/AGENTS.md index acfa2b986e..394a88915d 100644 --- a/transformer_lens/config/AGENTS.md +++ b/transformer_lens/config/AGENTS.md @@ -1,6 +1,6 @@ # Config — AGENTS.md -The config dataclasses that drive both `HookedTransformer` and `TransformerBridge`. Read [the root AGENTS.md](../../AGENTS.md) for project-wide rules. +The config dataclasses that drive `TransformerBridge`. Read [the root AGENTS.md](../../AGENTS.md) for project-wide rules. ## File map @@ -8,7 +8,6 @@ The config dataclasses that drive both `HookedTransformer` and `TransformerBridg |---|---|---| | [`transformer_lens_config.py`](transformer_lens_config.py) | `TransformerLensConfig` | Minimal base — only fields actually used by the system | | [`transformer_bridge_config.py`](transformer_bridge_config.py) | `TransformerBridgeConfig(TransformerLensConfig)` | The Bridge config; what every Bridge adapter receives as `cfg` | -| [`hooked_transformer_config.py`](hooked_transformer_config.py) | `HookedTransformerConfig` | Legacy HT-only config (deprecated; see [AGENTS.md §2](../../AGENTS.md#2-two-systems-live-in-this-repo)) | ## Adding a new HF-config attr to `TransformerBridgeConfig` — decision tree diff --git a/transformer_lens/config/__init__.py b/transformer_lens/config/__init__.py index 3d6142ebb1..82b75454b9 100644 --- a/transformer_lens/config/__init__.py +++ b/transformer_lens/config/__init__.py @@ -1,7 +1,7 @@ """Configuration classes for TransformerLens.""" -from .hooked_transformer_config import HookedTransformerConfig + from .transformer_bridge_config import TransformerBridgeConfig from .transformer_lens_config import TransformerLensConfig -__all__ = ["HookedTransformerConfig", "TransformerBridgeConfig", "TransformerLensConfig"] +__all__ = ["TransformerBridgeConfig", "TransformerLensConfig"] diff --git a/transformer_lens/config/hooked_transformer_config.py b/transformer_lens/config/hooked_transformer_config.py deleted file mode 100644 index 43bc9136e3..0000000000 --- a/transformer_lens/config/hooked_transformer_config.py +++ /dev/null @@ -1,442 +0,0 @@ -"""Hooked Transformer Config. - -Module with a dataclass for storing the configuration of a -:class:`transformer_lens.HookedTransformer` model. -""" - -from __future__ import annotations - -import pprint -import random -from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Union - -import numpy as np -import torch - -from transformer_lens.utilities.activation_functions import ( - SOFTCAP_DISABLED, - SUPPORTED_ACTIVATIONS, -) -from transformer_lens.utilities.devices import get_device - -from .transformer_lens_config import TransformerLensConfig - - -@dataclass -class HookedTransformerConfig(TransformerLensConfig): - """ - Configuration class to store the configuration of a HookedTransformer model. - - See further_comments.md for more details on the more complex arguments. - - Args: - d_model (int): The dimensionality of the embeddings. - d_head (int): The dimensionality of each attention head. - n_layers (int): The number of transformer blocks (one block = one attn layer AND one MLP layer). - n_ctx (int): The maximum sequence length. - n_heads (int): The number of attention heads. If not - specified, will be set to d_model // d_head. (This is represented by a default value of -1) - d_mlp (int, *optional*): The dimensionality of the feedforward mlp - network. Defaults to 4 * d_model, and in an attn-only model is None. - d_vocab (int): The size of the vocabulary. Defaults to -1, which means not set. If not set, will be - automatically set from the tokenizer's vocab size. - act_fn (str, *optional*): The activation function to use. Always - lowercase. Supports ['relu', 'gelu', 'silu', 'gelu_new', 'solu_ln', - 'gelu_fast']. Must be set unless using an attn-only model. - eps (float): The epsilon value to use for layer normalization. Defaults - to 1e-5 - use_attn_result (bool): whether to explicitly calculate the amount - each head adds to the residual stream (with a hook) and THEN add it - up, vs just calculating the sum. This can be very memory intensive - for large models, so defaults to False - use_split_qkv_input (bool): whether to explicitly calculate the input of - each head separately, with a hook. Defaults to false to save memory. - use_hook_mlp_in (bool): whether to use a hook to get the input to the - MLP layer. Defaults to false to save memory. - use_attn_in (bool): whether to explicitly calculate the input of each - attention head separately, with a hook. Defaults to false to save memory - use_attn_scale (bool): whether to scale the attention weights by - 1/sqrt(d_head) - ungroup_grouped_query_attention (bool): whether to ungroup key and value heads, for models that use - grouped query attention. - attn_scale (float): The amount to divide attention scores by (if applicable). Defaults to - sqrt(d_head) - model_name (str): the name of the model, used to load - weights from HuggingFace or initialized to "custom" if not passed - original_architecture (str, *optional*): the family of the model, used - to help load - weights from HuggingFace or initialized to "custom" if not passed - from_checkpoint (bool): Whether the model weights were - loaded from a checkpoint (only applies to pretrained models) - checkpoint_index (int, *optional*): The index of the - checkpoint loaded (only applies to pretrained models). - checkpoint_label_type (str, *optional*): Whether - checkpoints are labelled by the number of steps or number of tokens. - checkpoint_value (int, *optional*): The value of the - checkpoint label (whether of steps or tokens). - tokenizer_name (str, *optional*): the full name of the model, passed into - HuggingFace to access the tokenizer. Only used when passing in - custom config, if loading from pretrained then this is not needed. - use_local_attn (bool): whether to use local attention - ie each - destination token can only attend to source tokens a certain distance back. - window_size (int, *optional*): the size of the window for local - attention - attn_types (List[str], *optional*): the types of attention to use for - local attention - init_mode (str): the initialization mode to use for the - weights. Only relevant for custom models, ignored for pre-trained. - We now support 'gpt2', 'xavier_uniform', 'xavier_normal', 'kaiming_uniform', - 'kaiming_normal'. MuP support to come. Defaults to 'gpt2'. - normalization_type (str, *optional*): the type of normalization to use. - Options are None (no normalization), 'LN' (use LayerNorm, including weights - & biases) and 'LNPre' (use LayerNorm, but no weights or biases), 'RMS' - (use RMSNorm, including weights) and 'RMSPre' (use RMSNorm, but no weights or biases). - Defaults to LN - device(str): The device to use for the model. Defaults to 'cuda' if - available, else 'cpu'. Must be 'cuda' if `n_devices` > 1. - n_devices (int): The number of devices to use for the model. Defaults to 1. Layers are loaded - to support "pipeline parallelism", where each device is responsible for a subset of the layers. - attention_dir (str): Whether to use causal (aka unidirectional aka GPT-2 - style) or bidirectional attention. Options are 'causal' and - 'bidirectional'. Defaults to 'causal' - attn_only (bool): Whether to only use attention layers, no feedforward - layers. Defaults to False - seed (int, *optional*): The seed to use for the model. - Used to set sources of randomness (Python, PyTorch and NumPy) and to initialize weights. - Defaults to None. We recommend setting a seed, so your experiments are reproducible. - initializer_range (float): The standard deviation of the normal used to - initialise the weights, initialized to 0.8 / sqrt(d_model). If init_mode is - 'xavier_uniform' or 'xavier_normal', this value is instead treated as the `gain` parameter for the weight - initialisation (a constant factor to scale the weights by). Defaults to -1.0, which means not set. - init_weights (bool): Whether to initialize the weights. Defaults to - True. If False, does not initialize weights. - scale_attn_by_inverse_layer_idx (bool): Whether to scale the attention - weights by 1/(layer_id+1), used by Mistral (Stanford) models for numerical stability when - training in FP16. Defaults to False. - positional_embedding_type (str): The positional embedding used. Options - are 'standard' (ie GPT-2 style, absolute, randomly initialized learned positional - embeddings, directly added to the residual stream), 'rotary' - (described here: https://blog.eleuther.ai/rotary-embeddings/ ) and - 'shortformer' (GPT-2 style absolute & learned, but rather than being - added to the residual stream they're only added to the inputs to the - keys and the queries (ie key = W_K(res_stream + pos_embed), but - values and MLPs don't get any positional info)). Sinusoidal are not - currently supported. Defaults to 'standard'. - final_rms (bool): Whether to replace the final normalization (just - before the unembed) with RMSNorm (ie no centering or bias, just - scaling + weights). Only included because of a dumb bug in my - original SoLU code. Defaults to False. - d_vocab_out (int, *optional*): The size of the output vocabulary. Defaults to -1, which means not set. If not - set, will be equal to d_vocab. Mainly useful for algorithmic tasks - where the input and output vocabularies may be different. - parallel_attn_mlp (bool): Whether to parallelize the attention and MLP - layers - a weird cursed thing done by GPT-J. Means that - mlp_out=MLP(ln1(resid_pre)) and resid_post=resid_pre+attn_out+mlp_out. Defaults to False. - rotary_dim (int, *optional*): The dimensionality of the rotary - embeddings, may be d_head in which case only the first rotary_dim - dimensions of each head are rotated. Defaults to None, if - positional_embedding_type=="rotary" post-init then sets it to d_head, i.e. "rotate all - dimensions of the query and key". - n_params (int, *optional*): The number of "hidden weight" parameters - in the model, **excluding** embeddings, unembedding, biases, and - layer norms. Counts only the attention projections (W_Q, W_K, W_V, - W_O) and MLP weights (W_in, W_out, plus W_gate when ``gated_mlp=True``). - This matches the convention from the - `scaling laws paper <https://arxiv.org/pdf/2001.08361.pdf>`_, - which found this to be the most meaningful number for predicting - performance. **Note:** this is NOT the same as - ``sum(p.numel() for p in model.parameters())`` — that would - include embeddings and biases and yield a larger number. Use the - ``sum(p.numel() ...)`` form if you want the total parameter count - (e.g. for memory-budget calculations). Automatically calculated; - not intended to be set by the user. - use_hook_tokens (bool): Will add a hook point on the token input to - HookedTransformer.forward, which lets you cache or intervene on the tokens. - Defaults to False. - gated_mlp (bool): If True, the MLP layer uses a gated formulation - (SwiGLU/GeGLU-style): ``mlp_out = W_out @ (act_fn(W_gate @ x) * (W_in @ x))``, - with an extra ``W_gate`` weight matrix alongside ``W_in`` and ``W_out``. Used by - LLaMA, Mistral, Gemma, Qwen and similar families. When False (default), the MLP - is the plain ``mlp_out = W_out @ act_fn(W_in @ x)`` form. ``loading_from_pretrained`` - sets this automatically per architecture; only set manually for a custom config. - default_prepend_bos (bool, optional): Default behavior of whether to prepend the BOS token when the - methods of HookedTransformer process input text to tokenize (only when input is a string). - Defaults to True - even for models not explicitly trained with this, heads often use the - first position as a resting position and accordingly lose information from the first token, - so this empirically seems to give better results. To change the default behavior to False, pass in - default_prepend_bos=False. Note that you can also locally override the default behavior by passing - in prepend_bos=True/False when you call a method that processes the input string. - dtype (torch.dtype, *optional*): The model's dtype. Defaults to torch.float32. - tokenizer_prepends_bos (bool, *optional*): This flag is set by set_tokenizer. It is set to True only - when the tokenizer automatically prepends the BOS token if initialized with add_bos_token=True. - We need this information to dynamically control bos prepending. - load_in_4bit(bool): If this flag is set, then it's assumed that parameters are 4-bit quantized - with bitsandbytes. Currently only supported for Llama. - quantization_method (str, *optional*): the ``quant_method`` declared by the checkpoint's - HF config ("mxfp4", "bitsandbytes", "gptq", ...), captured while that config is already - in hand so later load steps need not refetch it. None when unquantized, and also when - the config was never fetched (the llama/gemma name-based branches of - ``convert_hf_model_config`` infer the architecture from the model name instead). - n_key_value_heads (int, *optional*): The number of groups of heads that use the same key and value matrix. - Only for models that use Grouped Query Attention. - post_embedding_ln (bool): Whether to apply layer normalization after embedding the tokens. Defaults - to False. - num_experts (int, *optional*): The number of experts to use in the MoE layer. If set, experts_per_token - must also be set. Set to None if not using MoE. - experts_per_token (int, *optional*): The number of experts to use for each pass in the MoE layer. If set, - num_experts must also be set. Set to None if not using MoE. - relative_attention_max_distance (int, *optional*): The maximum distance between tokens for relative - attention. If set, relative_attention_num_buckets must also be set.Only used in EncoderDecoder models, like T5. - relative_attention_num_buckets (int, *optional*): The number of buckets to use for relative attention. - If set, relative_attention_max_distance must also be set.Only used in EncoderDecoder models, like T5. - decoder_start_token_id (int, *optional*): The start token id for the decoder. Only used in EncoderDecoder models, like T5. - tie_word_embeddings (bool): Whether to tie the word embeddings and the output layer weights. Defaults to False. Only used in EncoderDecoder (T5) by now. - use_normalization_before_and_after (bool): Whether to apply normalization (LN/RMS/etc) - to both the input of an attn/MLP block *and* the output (before adding back to the - residual stream). Currently only used in Gemma-2. Defaults to False. - attn_scores_soft_cap (float): An optional softcap for attention scores pre-softmax. If - used, it will map attn_scores -> soft_cap * tanh(attn_scores / soft_cap). As tanh's - output is in [-1, 1], this maps attn_scores to [-soft_cap, soft_cap], with little - effect on small values, but squashing large values into that interval. Currently only - used in Gemma-2. Defaults to -1.0, which means not set. - output_logits_soft_cap (float): An optional softcap for output logits, currently only used - in Gemma-2 (see attn_scores_soft_cap for details). Defaults to -1.0, which means not - set. - use_NTK_by_parts_rope (bool): Whether to apply the "NTK-by-parts" method when using Rotary - Positional Embedding. This method adjusts the interpolation based on frequency factors - for different parts of the hidden dimensions. See Section 3.2 in - https://arxiv.org/pdf/2309.00071 for details. Defaults to False. - NTK_by_parts_low_freq_factor (float): The threshold applied to low-frequency hidden - dimensions during interpolation when using the "NTK-by-parts" method. Defaults to 1.0. - NTK_by_parts_high_freq_factor (float): The threshold applied to high-frequency hidden - dimensions during interpolation in the "NTK-by-parts" method. Defaults to 4.0. - NTK_by_parts_factor (float): The overall factor used in the "NTK-by-parts" method that - affects the rate of change between low and high-frequency interpolation strategies. - Defaults to 8.0. - use_yarn_rope (bool): Whether to apply YARN (Yet Another RoPE extensioN) scaling to - rotary positional embeddings. YARN blends interpolated and extrapolated frequencies - per dimension using correction ranges. See https://arxiv.org/abs/2309.00071 for - details. Used by OLMo 3. Defaults to False. - yarn_factor (float): The interpolation factor for YARN RoPE scaling. Defaults to 1.0. - yarn_attention_factor (float): Multiplicative scaling applied to sin/cos embeddings in - YARN. Defaults to 1.0. - yarn_beta_fast (float): Upper rotation threshold for YARN correction range. Defaults to 32. - yarn_beta_slow (float): Lower rotation threshold for YARN correction range. Defaults to 1. - yarn_truncate (bool): Whether to floor/ceil the YARN correction-range bounds - (HF's `truncate`). GPT-OSS ships truncate=False. Defaults to True. - yarn_global_attn_only (bool): Whether YARN applies only to global-attention - layers, with sliding/local layers keeping plain rope (Olmo-3's - per-layer-type rope). Defaults to False. - use_attention_sinks (bool): Whether attention carries a learned per-head sink - logit (GPT-OSS) that joins the softmax as an extra key column and is - dropped afterward. Defaults to False. - yarn_original_max_position_embeddings (int): The original max position embeddings before - YARN extension. Defaults to 4096. - use_qk_norm (bool): Whether to apply RMSNorm to the query and key projections before - computing attention scores. Used by Gemma 3 models. Defaults to False. - rotary_base_local (float, *optional*): The base for rotary positional embeddings in local - attention layers. Used by models with hybrid local/global attention (e.g., Gemma 3) - which use different RoPE bases for local (10k) and global (1M) attention. Defaults - to None, which means the standard rotary_base is used for all layers. - norm_topk_prob (bool): Whether to normalize the top-k probabilities in the MoE layer. - use_logn_attn (bool): Qwen-1's log-n attention: scale queries by - log_{train_len}(position) past the training length (eval only). - train_seq_length (int, *optional*): the length the model was trained at - (Qwen-1's ``seq_length``). Both log-n scaling and dynamic-NTK RoPE - threshold on it, and it stays fixed when n_ctx is overridden upward. - use_dynamic_ntk_rope (bool): Qwen-1's dynamic NTK: rescale the rotary - base by ``alpha ** (rotary_dim / (rotary_dim - 2))`` once the key - length exceeds the training length (eval only). - clip_qkv (float, *optional*): Clamp Q/K/V activations to [-clip_qkv, clip_qkv] after - projection (and any qk-norm), as OLMo v1 and OLMoE do. Defaults to None (no clamp). - """ - - model_name: str = "custom" - act_fn: str = "relu" - eps: float = 1e-5 - use_attn_scale: bool = True - attn_scale: float = -1.0 - use_hook_mlp_in: bool = False - use_attn_in: bool = False - use_qk_norm: bool = False - clip_qkv: Optional[float] = None - use_logn_attn: bool = False - train_seq_length: Optional[int] = None - use_dynamic_ntk_rope: bool = False - use_local_attn: bool = False - ungroup_grouped_query_attention: bool = False - original_architecture: Optional[str] = None - from_checkpoint: bool = False - checkpoint_index: Optional[int] = None - checkpoint_label_type: Optional[str] = None - checkpoint_value: Optional[int] = None - tokenizer_name: Optional[str] = None - window_size: Optional[int] = None - attn_types: Optional[List] = None - init_mode: str = "gpt2" - normalization_type: Optional[str] = "LN" - n_devices: int = 1 - attention_dir: str = "causal" - attn_only: bool = False - seed: Optional[int] = None - initializer_range: float = -1.0 - init_weights: bool = True - scale_attn_by_inverse_layer_idx: bool = False - final_rms: bool = False - d_vocab_out: int = -1 - parallel_attn_mlp: bool = False - rotary_dim: Optional[int] = None - n_params: Optional[int] = None - use_hook_tokens: bool = False - gated_mlp: bool = False - dtype: torch.dtype = torch.float32 - tokenizer_prepends_bos: Optional[bool] = None - post_embedding_ln: bool = False - rotary_base: Union[float, int] = 10000 - rotary_base_local: Optional[ - Union[float, int] - ] = None # For models with different RoPE bases per attention type (e.g., Gemma 3) - rotary_scaling_factor: float = ( - 1.0 # Linear RoPE scaling factor for global attention (e.g., 8.0 for Gemma 3 4B) - ) - trust_remote_code: bool = False - rotary_adjacent_pairs: bool = False - load_in_4bit: bool = False - quantization_method: Optional[str] = None - num_experts: Optional[int] = None - experts_per_token: Optional[int] = None - relative_attention_max_distance: Optional[int] = None - relative_attention_num_buckets: Optional[int] = None - decoder_start_token_id: Optional[int] = None - tie_word_embeddings: bool = False - use_normalization_before_and_after: bool = False - attn_scores_soft_cap: float = SOFTCAP_DISABLED - output_logits_soft_cap: float = SOFTCAP_DISABLED - use_NTK_by_parts_rope: bool = False - NTK_by_parts_low_freq_factor: float = 1.0 - NTK_by_parts_high_freq_factor: float = 4.0 - NTK_by_parts_factor: float = 8.0 - NTK_original_ctx_len: int = 8192 - use_yarn_rope: bool = False - yarn_factor: float = 1.0 - yarn_attention_factor: float = 1.0 - yarn_beta_fast: float = 32.0 - yarn_beta_slow: float = 1.0 - yarn_original_max_position_embeddings: int = 4096 - # HF yarn's `truncate` option: floor/ceil the correction range bounds. - # GPT-OSS ships truncate=False, keeping the bounds fractional. - yarn_truncate: bool = True - # Per-layer-type rope (Olmo-3): YARN applies only on global-attention - # layers; sliding/local layers keep plain rope. - yarn_global_attn_only: bool = False - # GPT-OSS: learned per-head sink logit that joins the attention softmax as - # an extra key column and is dropped afterward, so real positions share - # probability mass with the sink. - use_attention_sinks: bool = False - norm_topk_prob: bool = False - - def __post_init__(self): - super().__post_init__() - - if self.seed is not None: - self.set_seed_everywhere(self.seed) - if self.use_local_attn: - assert self.window_size is not None, "window_size must be specified for local attention" - assert self.attn_types is not None, "attn_types must be specified for local attention" - if not self.attn_only: - assert self.act_fn is not None, "act_fn must be specified for non-attn-only models" - assert ( - self.act_fn in SUPPORTED_ACTIVATIONS - ), f"act_fn={self.act_fn} must be one of {SUPPORTED_ACTIVATIONS}" - if self.initializer_range < 0 and self.init_mode == "gpt2": - # Roughly copy the GPT-2 value, but proportional to sqrt(1/d_model) - self.initializer_range = 0.8 / np.sqrt(self.d_model) - if self.initializer_range < 0 and self.init_mode != "gpt2": - # This is the gain parameter for the weight initialisation - self.initializer_range = 1.0 - - if self.d_vocab_out == -1: - # d_vocab_out defaults to d_vocab, unless there's an algorithmic task - # If d_vocab is not set, it'll be inferred from tokenizer_name or from a tokenizer - # explicitly passed to HookedTransformer initialisation. - self.d_vocab_out = self.d_vocab - - if self.positional_embedding_type == "rotary" and self.rotary_dim is None: - self.rotary_dim = self.d_head - - if self.num_experts is not None: - assert ( - self.experts_per_token is not None - ), "experts_per_token must be set if num_experts is set" - if self.experts_per_token is not None: - assert ( - self.num_experts is not None - ), "num_experts must be set if experts_per_token is set" - - # Attention params (W_Q, W_K, W_V, W_O), ignoring biases/LN - self.n_params = self.n_layers * ((self.d_model * self.d_head * self.n_heads * 4)) - if not self.attn_only: - assert self.d_mlp is not None # mypy - # MLP params (W_in, W_out), ignoring biases/LN - mlp_params_per_layer = self.d_model * self.d_mlp * (2 + self.gated_mlp) - - if self.num_experts: - # Scale by num_experts and add gate params - mlp_params_per_layer = (mlp_params_per_layer + self.d_model) * self.num_experts - self.n_params += self.n_layers * mlp_params_per_layer - - if self.device is None: - self.device = get_device() - else: - from transformer_lens.utilities import warn_if_mps - - warn_if_mps(self.device) - - if self.n_devices > 1: - assert ( - torch.cuda.device_count() >= self.n_devices - ), f"Not enough CUDA devices to support n_devices {self.n_devices}" - - if self.use_attn_scale and self.attn_scale == -1.0: - self.attn_scale = np.sqrt(self.d_head) - - assert self.default_prepend_bos in [ - True, - False, - ], f"default_prepend_bos must be either True or False, but {self.default_prepend_bos} is given" - - @classmethod - def unwrap(cls, config: Union[Dict, "TransformerLensConfig"]) -> HookedTransformerConfig: - """ - Convenience function to avoid duplicate code from a common way config is passed to various components - """ - if isinstance(config, Dict): - return cls.from_dict(config) - elif isinstance(config, cls): - return config - else: - # Convert from TransformerLensConfig to HookedTransformerConfig - return cls.from_dict(config.to_dict()) - - @classmethod - def from_dict(cls, config_dict: Dict[str, Any]) -> HookedTransformerConfig: - """ - Instantiates a `HookedTransformerConfig` from a Python dictionary of - parameters. - """ - return cls(**config_dict) - - def to_dict(self): - return self.__dict__ - - def __repr__(self): - return "HookedTransformerConfig:\n" + pprint.pformat(self.to_dict()) - - def set_seed_everywhere(self, seed: int): - torch.manual_seed(seed) - random.seed(seed) - np.random.seed(seed) - - def is_layer_norm_activation(self) -> bool: - return self.act_fn is not None and self.act_fn.endswith("_ln") diff --git a/transformer_lens/conversion_utils/hook_conversion_utils.py b/transformer_lens/conversion_utils/hook_conversion_utils.py index 2712c7f242..a5dab16afa 100644 --- a/transformer_lens/conversion_utils/hook_conversion_utils.py +++ b/transformer_lens/conversion_utils/hook_conversion_utils.py @@ -54,17 +54,3 @@ def model_info_cfg(cfg): # TODO: WeightConversionFactory import needs to be updated or removed print(f"Hook conversion details for architecture {cfg.original_architecture}:") print("Hook conversion factory not yet implemented") - - -def model_info(model_name): - """ - Displays the weight conversion from HuggingFace to TransformerLens for a given model name. - - Args: - model_name (str): Name of the pretrained model to analyze - (e.g., 'gpt2', 'bert-base-uncased', etc.) - """ - from transformer_lens.loading_from_pretrained import get_pretrained_model_config - - cfg = get_pretrained_model_config(model_name) - model_info_cfg(cfg) diff --git a/transformer_lens/factories/mlp_factory.py b/transformer_lens/factories/mlp_factory.py deleted file mode 100644 index 6947ccf43d..0000000000 --- a/transformer_lens/factories/mlp_factory.py +++ /dev/null @@ -1,25 +0,0 @@ -"""MLP Factory - -Centralized location for creating any MLP needed within TransformerLens -""" - -from transformer_lens.components.mlps.can_be_used_as_mlp import CanBeUsedAsMLP -from transformer_lens.components.mlps.gated_mlp import GatedMLP -from transformer_lens.components.mlps.gated_mlp_4bit import GatedMLP4Bit -from transformer_lens.components.mlps.gpt_oss_moe import GptOssMoE -from transformer_lens.components.mlps.mlp import MLP -from transformer_lens.components.mlps.moe import MoE -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig - - -class MLPFactory: - @staticmethod - def create_mlp(cfg: HookedTransformerConfig) -> CanBeUsedAsMLP: - if cfg.num_experts: - if cfg.original_architecture == "GptOssForCausalLM": - return GptOssMoE(cfg) - return MoE(cfg) - elif cfg.gated_mlp: - return GatedMLP(cfg) if not cfg.load_in_4bit else GatedMLP4Bit(cfg) - else: - return MLP(cfg) diff --git a/transformer_lens/hook_points.py b/transformer_lens/hook_points.py index b7559c0304..0be04ba431 100644 --- a/transformer_lens/hook_points.py +++ b/transformer_lens/hook_points.py @@ -432,20 +432,15 @@ def list_hooks( return out -# HookedRootModule moved to transformer_lens.HookedRootModule (3.0). Import it from -# its dedicated module. Importing from here is deprecated and will trigger a warning. +# HookedRootModule moved to transformer_lens.HookedRootModule in 3.0; the +# deprecated re-export from here was removed in 4.0 as promised. The class itself +# is kept — import it from transformer_lens or transformer_lens.HookedRootModule. def __getattr__(name: str): if name == "HookedRootModule": - import warnings - - from transformer_lens.HookedRootModule import HookedRootModule - - warnings.warn( - "Importing HookedRootModule from transformer_lens.hook_points is " - "deprecated and will be removed in TransformerLens 4.0. Import it from " - "transformer_lens (preferred) or transformer_lens.HookedRootModule instead.", - DeprecationWarning, - stacklevel=2, + raise AttributeError( + "Importing HookedRootModule from transformer_lens.hook_points was " + "removed in TransformerLens 4.0. Import it from transformer_lens " + "(preferred) or transformer_lens.HookedRootModule instead — the class " + "itself is kept." ) - return HookedRootModule raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/transformer_lens/lit/README.md b/transformer_lens/lit/README.md index e036700ea2..31c0244e3e 100644 --- a/transformer_lens/lit/README.md +++ b/transformer_lens/lit/README.md @@ -30,7 +30,7 @@ pip install transformer-lens lit-nlp ### In a Jupyter/Colab Notebook ```python -from transformer_lens import HookedTransformer +from transformer_lens.model_bridge import TransformerBridge from transformer_lens.lit import ( HookedTransformerLIT, HookedTransformerLITConfig, @@ -38,8 +38,8 @@ from transformer_lens.lit import ( LITWidget, ) -# Load model -model = HookedTransformer.from_pretrained("gpt2-small") +# Load model (the LIT wrapper accepts any TransformerLens model, e.g. TransformerBridge) +model = TransformerBridge.boot_transformers("gpt2-small") # Create LIT wrapper config = HookedTransformerLITConfig( @@ -66,14 +66,14 @@ widget.render() ### As a Standalone Server ```python -from transformer_lens import HookedTransformer +from transformer_lens.model_bridge import TransformerBridge from transformer_lens.lit import ( HookedTransformerLIT, SimpleTextDataset, serve, ) -model = HookedTransformer.from_pretrained("gpt2-small") +model = TransformerBridge.boot_transformers("gpt2-small") lit_model = HookedTransformerLIT(model) serve( diff --git a/transformer_lens/lit/__init__.py b/transformer_lens/lit/__init__.py index 80cabf1080..9b769d9d31 100644 --- a/transformer_lens/lit/__init__.py +++ b/transformer_lens/lit/__init__.py @@ -63,7 +63,6 @@ __all__ = [ # Model wrappers "HookedTransformerLIT", - "HookedTransformerLITBatched", "HookedTransformerLITConfig", "TransformerLensLIT", "TransformerLensLITBatched", diff --git a/transformer_lens/loading_from_pretrained.py b/transformer_lens/loading_from_pretrained.py deleted file mode 100644 index 72b0335016..0000000000 --- a/transformer_lens/loading_from_pretrained.py +++ /dev/null @@ -1,2316 +0,0 @@ -"""Loading Pretrained Models Utilities. - -This module contains functions for loading pretrained models from the Hugging Face Hub. -""" - -from __future__ import annotations - -import dataclasses -import logging -import math -import os -import re -from pathlib import Path -from typing import Any - -import torch -from huggingface_hub import HfApi -from transformers import ( - AutoConfig, - AutoModel, - AutoModelForCausalLM, - BertForPreTraining, - HubertModel, - T5ForConditionalGeneration, - Wav2Vec2Model, -) -from transformers.utils.quantization_config import Mxfp4Config - -import transformer_lens.utilities as utils -from transformer_lens.config.hooked_transformer_config import HookedTransformerConfig -from transformer_lens.pretrained.weight_conversions import ( - convert_apertus_weights, - convert_bert_weights, - convert_bloom_weights, - convert_coder_weights, - convert_gemma_weights, - convert_gpt2_weights, - convert_gpt_oss_weights, - convert_gptj_weights, - convert_hubert_weights, - convert_llama_weights, - convert_mingpt_weights, - convert_mistral_weights, - convert_mixtral_weights, - convert_neel_solu_old_weights, - convert_neo_weights, - convert_neox_weights, - convert_olmo2_weights, - convert_olmo3_weights, - convert_olmo_weights, - convert_olmoe_weights, - convert_opt_weights, - convert_phi3_weights, - convert_phi_weights, - convert_qwen2_weights, - convert_qwen3_weights, - convert_qwen_weights, - convert_t5_weights, -) -from transformer_lens.supported_models import MODEL_ALIASES, OFFICIAL_MODEL_NAMES -from transformer_lens.tools.model_registry.checkpoints import ( - PYTHIA_CHECKPOINTS, - PYTHIA_V0_CHECKPOINTS, - STANFORD_CRFM_CHECKPOINTS, -) -from transformer_lens.utilities.architectures import POST_NORM_ARCHITECTURES -from transformer_lens.utilities.heterogeneous_config import het_safe_view -from transformer_lens.utilities.hf_utils import get_rotary_pct_from_config -from transformer_lens.utilities.quantization import ( - quantization_method, - unreadable_weight_reason, -) - -NON_HF_HOSTED_MODEL_NAMES = [ - "llama-7b-hf", - "llama-13b-hf", - "llama-30b-hf", - "llama-65b-hf", -] -"""Official model names for models not hosted on HuggingFace.""" - -NEED_REMOTE_CODE_MODELS = ( - "bigcode/santacoder", - "Qwen/Qwen-", - "Qwen/Qwen3-", - "microsoft/phi-2", - "microsoft/phi-4", - "apple/OpenELM", - "openai/gpt-oss-", - "swiss-ai/Apertus-", -) - - -def _get_rope_theta(hf_config: Any, default: float = 10000.0) -> float | int: - """Extract rope_theta from a HuggingFace config, handling both old and new formats. - - In transformers v5+, rope_theta moved from a top-level attribute to - hf_config.rope_parameters['rope_theta']. - """ - # Try direct attribute first (transformers < 5.0) - rope_theta = getattr(hf_config, "rope_theta", None) - if rope_theta is not None: - return rope_theta - # Try rope_parameters dict (transformers >= 5.0) - rope_params = getattr(hf_config, "rope_parameters", None) - if rope_params is not None and isinstance(rope_params, dict): - return rope_params.get("rope_theta", default) - return default - - -def _apply_llama3_rope_scaling(cfg_dict: dict[str, Any], hf_config: Any) -> None: - """Populate the NTK-by-parts fields when a config requests llama3 rope scaling.""" - rope_scaling = getattr(hf_config, "rope_scaling", None) - if not rope_scaling: - return - rope_type = (rope_scaling.get("type") or rope_scaling.get("rope_type") or "").lower() - if rope_type != "llama3": - return - cfg_dict["use_NTK_by_parts_rope"] = True - cfg_dict["NTK_original_ctx_len"] = rope_scaling.get( - "original_max_position_embeddings", hf_config.max_position_embeddings - ) - cfg_dict["NTK_by_parts_low_freq_factor"] = rope_scaling.get("low_freq_factor", 1.0) - cfg_dict["NTK_by_parts_high_freq_factor"] = rope_scaling.get("high_freq_factor", 4.0) - cfg_dict["NTK_by_parts_factor"] = rope_scaling.get("factor", 1.0) - - -def make_model_alias_map() -> dict[str, str]: - """ - Converts OFFICIAL_MODEL_NAMES (the list of actual model names on - HuggingFace) and MODEL_ALIASES (a dictionary mapping official model names to - aliases) into a dictionary mapping all aliases to the official model name. - """ - model_alias_map = {} - for official_model_name in OFFICIAL_MODEL_NAMES: - aliases = MODEL_ALIASES.get(official_model_name, []) - for alias in aliases: - model_alias_map[alias.lower()] = official_model_name - model_alias_map[official_model_name.lower()] = official_model_name - return model_alias_map - - -def get_official_model_name(model_name: str) -> str: - """ - Returns the official model name for a given model name (or alias). - """ - model_alias_map = make_model_alias_map() - official_model_name = model_alias_map.get(model_name.lower()) - if official_model_name is None: - raise ValueError( - f"{model_name} not found. Valid official model names (excl aliases): {OFFICIAL_MODEL_NAMES}" - ) - return official_model_name - - -def convert_hf_model_config(model_name: str, **kwargs: Any) -> dict[str, Any]: - """ - Returns the model config for a HuggingFace model, converted to a dictionary - in the HookedTransformerConfig format. - - Takes the official_model_name as an input. - """ - # In case the user passed in an alias - if (Path(model_name) / "config.json").exists(): - logging.info("Loading model config from local directory") - official_model_name = model_name - else: - official_model_name = get_official_model_name(model_name) - - # Load HuggingFace model config. Stays None on the name-based branches - # below, which infer the architecture from the model name and never fetch. - hf_config: Any = None - if "llama" in official_model_name.lower(): - architecture = "LlamaForCausalLM" - elif "gemma-3" in official_model_name.lower() or "medgemma" in official_model_name.lower(): - # Gemma 3: 270M and 1B are text-only (CausalLM), 4B+ are multimodal (ConditionalGeneration) - # Exception: medgemma-27b-text-it is text-only - if "270m" in official_model_name.lower() or "1b" in official_model_name.lower(): - architecture = "Gemma3ForCausalLM" - elif "medgemma-27b-text" in official_model_name.lower(): - # medgemma-27b-text-it is text-only variant - architecture = "Gemma3ForCausalLM" - else: - # 4B, 12B, 27B and medgemma are multimodal - architecture = "Gemma3ForConditionalGeneration" - elif "gemma-2-" in official_model_name.lower(): - architecture = "Gemma2ForCausalLM" - elif "gemma" in official_model_name.lower(): - architecture = "GemmaForCausalLM" - else: - huggingface_token = os.environ.get("HF_TOKEN", "") - hf_config = AutoConfig.from_pretrained( - official_model_name, - token=huggingface_token if len(huggingface_token) > 0 else None, - **kwargs, - ) - # het view: transformers>=5.15 per-layer fields raise (not - # AttributeError) on global reads, so every hasattr/getattr probe in - # the branch chain below is a crash site without it. - hf_config = het_safe_view(hf_config) - architecture = hf_config.architectures[0] - - cfg_dict: dict[str, Any] - if official_model_name.startswith( - ("llama-7b", "meta-llama/Llama-2-7b") - ): # same architecture for LLaMA and Llama-2 - cfg_dict = { - "d_model": 4096, - "d_head": 4096 // 32, - "n_heads": 32, - "d_mlp": 11008, - "n_layers": 32, - "n_ctx": 2048 if official_model_name.startswith("llama-7b") else 4096, - "eps": 1e-6 if official_model_name.startswith("llama-7b") else 1e-5, - "d_vocab": 32000, - "act_fn": "silu", - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_adjacent_pairs": False, - "rotary_dim": 4096 // 32, - "final_rms": True, - "gated_mlp": True, - } - elif official_model_name.startswith("codellama"): # same architecture CodeLlama and Llama-2 - cfg_dict = { - "d_model": 4096, - "d_head": 4096 // 32, - "n_heads": 32, - "d_mlp": 11008, - "n_layers": 32, - "n_ctx": 4096, - "eps": 1e-5, - "d_vocab": 32016, - "act_fn": "silu", - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_dim": 4096 // 32, - "final_rms": True, - "gated_mlp": True, - "rotary_base": 1000000, - } - if "python" in official_model_name.lower(): - # The vocab size of python version of CodeLlama-7b is 32000 - cfg_dict["d_vocab"] = 32000 - elif official_model_name.startswith( - ("llama-13b", "meta-llama/Llama-2-13b") - ): # same architecture for LLaMA and Llama-2 - cfg_dict = { - "d_model": 5120, - "d_head": 5120 // 40, - "n_heads": 40, - "d_mlp": 13824, - "n_layers": 40, - "n_ctx": 2048 if official_model_name.startswith("llama-13b") else 4096, - "eps": 1e-6 if official_model_name.startswith("llama-13b") else 1e-5, - "d_vocab": 32000, - "act_fn": "silu", - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_adjacent_pairs": False, - "rotary_dim": 5120 // 40, - "final_rms": True, - "gated_mlp": True, - } - elif "llama-30b" in official_model_name: - cfg_dict = { - "d_model": 6656, - "d_head": 6656 // 52, - "n_heads": 52, - "d_mlp": 17920, - "n_layers": 60, - "n_ctx": 2048, - "eps": 1e-6, - "d_vocab": 32000, - "act_fn": "silu", - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_adjacent_pairs": False, - "rotary_dim": 6656 // 52, - "final_rms": True, - "gated_mlp": True, - } - elif "llama-65b" in official_model_name: - cfg_dict = { - "d_model": 8192, - "d_head": 8192 // 64, - "n_heads": 64, - "d_mlp": 22016, - "n_layers": 80, - "n_ctx": 2048, - "eps": 1e-6, - "d_vocab": 32000, - "act_fn": "silu", - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_dim": 8192 // 64, - "rotary_adjacent_pairs": False, - "final_rms": True, - "gated_mlp": True, - } - elif "Llama-2-70b" in official_model_name: - cfg_dict = { - "d_model": 8192, - "d_head": 128, - "n_heads": 64, - "d_mlp": 28672, - "n_layers": 80, - "n_ctx": 4096, - "eps": 1e-5, - "d_vocab": 32000, - "act_fn": "silu", - "n_key_value_heads": 8, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_adjacent_pairs": False, - "rotary_dim": 128, - "final_rms": True, - "gated_mlp": True, - } - elif "Meta-Llama-3-8B" in official_model_name: - cfg_dict = { - "d_model": 4096, - "d_head": 128, - "n_heads": 32, - "d_mlp": 14336, - "n_layers": 32, - "n_ctx": 8192, - "eps": 1e-5, - "d_vocab": 128256, - "act_fn": "silu", - "n_key_value_heads": 8, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_adjacent_pairs": False, - "rotary_dim": 128, - "final_rms": True, - "gated_mlp": True, - "rotary_base": 500000.0, - } - elif "Meta-Llama-3-70B" in official_model_name: - cfg_dict = { - "d_model": 8192, - "d_head": 128, - "n_heads": 64, - "d_mlp": 28672, - "n_layers": 80, - "n_ctx": 8192, - "eps": 1e-5, - "d_vocab": 128256, - "act_fn": "silu", - "n_key_value_heads": 8, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_adjacent_pairs": False, - "rotary_dim": 128, - "final_rms": True, - "gated_mlp": True, - "rotary_base": 500000.0, - } - elif "Llama-3.2-1B" in official_model_name: - cfg_dict = { - "d_model": 2048, - "d_head": 64, - "n_heads": 32, - "d_mlp": 8192, - "n_layers": 16, - "n_ctx": 2048, # capped due to memory issues - "eps": 1e-5, - "d_vocab": 128256, - "act_fn": "silu", - "n_key_value_heads": 8, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_adjacent_pairs": False, - "rotary_dim": 64, - "final_rms": True, - "gated_mlp": True, - "rotary_base": 500000.0, - "use_NTK_by_parts_rope": True, - "NTK_by_parts_low_freq_factor": 1.0, - "NTK_by_parts_high_freq_factor": 4.0, - "NTK_by_parts_factor": 32.0, - "NTK_original_ctx_len": 8192, - } - elif "Llama-3.2-3B" in official_model_name: - cfg_dict = { - "d_model": 3072, - "d_head": 128, - "n_heads": 24, - "d_mlp": 8192, - "n_layers": 28, - "n_ctx": 2048, # capped due to memory issues - "eps": 1e-5, - "d_vocab": 128256, - "act_fn": "silu", - "n_key_value_heads": 8, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_adjacent_pairs": False, - "rotary_dim": 128, - "final_rms": True, - "gated_mlp": True, - "rotary_base": 500000.0, - "use_NTK_by_parts_rope": True, - "NTK_by_parts_low_freq_factor": 1.0, - "NTK_by_parts_high_freq_factor": 4.0, - "NTK_by_parts_factor": 32.0, - "NTK_original_ctx_len": 8192, - } - elif "Llama-3.3-70B" in official_model_name: - cfg_dict = { - "d_model": 8192, - "d_head": 128, - "n_heads": 64, - "d_mlp": 28672, - "n_layers": 80, - "n_ctx": 2048, # capped due to memory issues - "eps": 1e-5, - "d_vocab": 128256, - "act_fn": "silu", - "n_key_value_heads": 8, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_adjacent_pairs": False, - "rotary_dim": 128, - "final_rms": True, - "gated_mlp": True, - "rotary_base": 500000.0, - "use_NTK_by_parts_rope": True, - "NTK_by_parts_low_freq_factor": 1.0, - "NTK_by_parts_high_freq_factor": 4.0, - "NTK_by_parts_factor": 8.0, - "NTK_original_ctx_len": 8192, - } - elif "Llama-3.1-8B" in official_model_name: - cfg_dict = { - "d_model": 4096, - "d_head": 128, - "n_heads": 32, - "d_mlp": 14336, - "n_layers": 32, - "n_ctx": 2048, # capped due to memory issues - "eps": 1e-5, - "d_vocab": 128256, - "act_fn": "silu", - "n_key_value_heads": 8, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_adjacent_pairs": False, - "rotary_dim": 128, - "final_rms": True, - "gated_mlp": True, - "rotary_base": 500000.0, - "use_NTK_by_parts_rope": True, - "NTK_by_parts_low_freq_factor": 1.0, - "NTK_by_parts_high_freq_factor": 4.0, - "NTK_by_parts_factor": 8.0, - "NTK_original_ctx_len": 8192, - } - elif "Llama-3.1-70B" in official_model_name: - cfg_dict = { - "d_model": 8192, - "d_head": 128, - "n_heads": 64, - "d_mlp": 28672, - "n_layers": 80, - "n_ctx": 2048, # capped due to memory issues - "eps": 1e-5, - "d_vocab": 128256, - "act_fn": "silu", - "n_key_value_heads": 8, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_adjacent_pairs": False, - "rotary_dim": 128, - "final_rms": True, - "gated_mlp": True, - "rotary_base": 500000.0, - "use_NTK_by_parts_rope": True, - "NTK_by_parts_low_freq_factor": 1.0, - "NTK_by_parts_high_freq_factor": 4.0, - "NTK_by_parts_factor": 8.0, - "NTK_original_ctx_len": 8192, - } - elif architecture == "GPTNeoForCausalLM": - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_heads, - "n_heads": hf_config.num_heads, - "d_mlp": hf_config.hidden_size * 4, - "n_layers": hf_config.num_layers, - "n_ctx": hf_config.max_position_embeddings, - "eps": hf_config.layer_norm_epsilon, - "d_vocab": hf_config.vocab_size, - "attn_types": hf_config.attention_layers, - "act_fn": hf_config.activation_function, - "use_attn_scale": False, - "use_local_attn": True, - "window_size": hf_config.window_size, - "scale_attn_by_inverse_layer_idx": False, - "normalization_type": "LN", - } - elif architecture == "GPT2LMHeadModel": - cfg_dict = { - "d_model": hf_config.n_embd, - "d_head": hf_config.n_embd // hf_config.n_head, - "n_heads": hf_config.n_head, - "d_mlp": hf_config.n_embd * 4, - "n_layers": hf_config.n_layer, - "n_ctx": hf_config.n_ctx, - "eps": hf_config.layer_norm_epsilon, - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.activation_function, - "use_attn_scale": getattr(hf_config, "scale_attn_weights", True), - "use_local_attn": False, - "scale_attn_by_inverse_layer_idx": hf_config.scale_attn_by_inverse_layer_idx, - "normalization_type": "LN", - } - elif architecture == "OPTForCausalLM": - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "d_mlp": hf_config.ffn_dim, - "n_layers": hf_config.num_hidden_layers, - "n_ctx": hf_config.max_position_embeddings, - "eps": 1e-5, - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.activation_function, - "use_attn_scale": True, - "use_local_attn": False, - "scale_attn_by_inverse_layer_idx": False, - "normalization_type": "LN", - } - elif architecture == "GPTJForCausalLM": - cfg_dict = { - "d_model": hf_config.n_embd, - "d_head": hf_config.n_embd // hf_config.n_head, - "n_heads": hf_config.n_head, - "d_mlp": 4 * hf_config.n_embd, - "n_layers": hf_config.n_layer, - "n_ctx": hf_config.n_positions, - "eps": 1e-5, - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.activation_function, - "use_attn_scale": True, - "use_local_attn": False, - "scale_attn_by_inverse_layer_idx": False, - "parallel_attn_mlp": True, - "positional_embedding_type": "rotary", - "rotary_dim": hf_config.rotary_dim, - "rotary_adjacent_pairs": True, - "normalization_type": "LN", - } - elif architecture == "GPTNeoXForCausalLM": - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - "n_ctx": hf_config.max_position_embeddings, - "eps": hf_config.layer_norm_eps, - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.hidden_act, - "use_attn_scale": True, - "use_local_attn": False, - "scale_attn_by_inverse_layer_idx": False, - # GPTNeoX ships sequential variants too (use_parallel_residual=False). - "parallel_attn_mlp": getattr(hf_config, "use_parallel_residual", True), - "positional_embedding_type": "rotary", - "rotary_adjacent_pairs": False, - "normalization_type": "LN", - "default_prepend_bos": False, - } - rotary_pct = get_rotary_pct_from_config(hf_config) - cfg_dict["rotary_dim"] = round(rotary_pct * cfg_dict["d_head"]) - elif architecture == "HubertModel": - # Basic transformer configuration - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - # HuBERT operates on audio frames, not tokens — n_ctx is flexible - "n_ctx": getattr(hf_config, "max_position_embeddings", 8192), - "eps": hf_config.layer_norm_eps, - "act_fn": getattr(hf_config, "hidden_act", "gelu"), - "attention_dir": "bidirectional", - "d_vocab": -1, # no text vocabulary - } - elif "wav2vec2-base" in official_model_name or "wav2vec2-large" in official_model_name: - # Basic transformer configuration - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - # HuBERT operates on audio frames, not tokens — n_ctx is flexible - "n_ctx": getattr(hf_config, "max_position_embeddings", 8192), - "eps": hf_config.layer_norm_eps, - "act_fn": getattr(hf_config, "hidden_act", "gelu"), - "attention_dir": "bidirectional", - "d_vocab": -1, # no text vocabulary - } - elif architecture == "HubertForCTC": - # Basic transformer configuration - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - "n_ctx": getattr(hf_config, "max_position_embeddings", 8192), - "eps": hf_config.layer_norm_eps, - "act_fn": getattr(hf_config, "hidden_act", "gelu"), - "attention_dir": "bidirectional", - # For CTC models: - "d_vocab": hf_config.vocab_size, # text vocab from tokenizer - } - elif architecture == "BertForMaskedLM": - # All supported Bert architectures have the same config, - # so we can use the BertForMaskedLM config for all of them - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - "n_ctx": hf_config.max_position_embeddings, - "eps": hf_config.layer_norm_eps, - "d_vocab": hf_config.vocab_size, - "act_fn": "gelu", - "attention_dir": "bidirectional", - } - elif architecture == "MistralForCausalLM": - use_local_attn = True if hf_config.sliding_window else False - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": ( - hf_config.head_dim - if hasattr(hf_config, "head_dim") - and hf_config.head_dim is not None - and hf_config.head_dim > 0 - else hf_config.hidden_size // hf_config.num_attention_heads - ), - "n_heads": hf_config.num_attention_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - "n_ctx": 2048, # Capped due to memory issues - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.hidden_act, - "window_size": hf_config.sliding_window, # None if no sliding window was used - "attn_types": ["local"] * hf_config.num_hidden_layers if use_local_attn else None, - "eps": hf_config.rms_norm_eps, - "rotary_base": _get_rope_theta(hf_config), - "n_key_value_heads": hf_config.num_key_value_heads, - "use_local_attn": use_local_attn, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "gated_mlp": True, - } - elif architecture == "MixtralForCausalLM": - cfg_dict = { - "dtype": torch.bfloat16, - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - "n_ctx": hf_config.max_position_embeddings, # Capped due to memory issues - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.hidden_act, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_base": _get_rope_theta(hf_config), - # None on the released 8x7B, but a variant that sets it must window. - "window_size": hf_config.sliding_window, - "attn_types": ( - ["local" if hf_config.sliding_window else "global"] * hf_config.num_hidden_layers - ), - "eps": hf_config.rms_norm_eps, - "n_key_value_heads": hf_config.num_key_value_heads, - "gated_mlp": True, - "use_local_attn": bool(hf_config.sliding_window), - "rotary_dim": hf_config.hidden_size // hf_config.num_attention_heads, - "num_experts": hf_config.num_local_experts, - "experts_per_token": hf_config.num_experts_per_tok, - # MixtralTopKRouter renormalizes the top-k weights unconditionally - # (modeling_mixtral.py: `router_top_value /= router_top_value.sum(...)`), - # and MixtralConfig has no norm_topk_prob field to read it from — so - # this is pinned to HF's behavior rather than sourced from the config. - # TL's MoE skips the renormalization unless this is set, which would - # leave routing weights unnormalized and the outputs silently wrong. - "norm_topk_prob": True, - } - elif architecture == "GptOssForCausalLM": - cfg_dict = { - "dtype": torch.bfloat16, - "d_model": hf_config.hidden_size, - "d_head": hf_config.head_dim, - "n_heads": hf_config.num_attention_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - "n_ctx": hf_config.max_position_embeddings, - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.hidden_act, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_base": _get_rope_theta(hf_config), - "eps": hf_config.rms_norm_eps, - "n_key_value_heads": hf_config.num_key_value_heads, - "gated_mlp": True, - "final_rms": True, - "rotary_dim": hf_config.head_dim, - "num_experts": hf_config.num_local_experts, - "experts_per_token": hf_config.num_experts_per_tok, - "use_attention_sinks": True, - # Alternating sliding_attention / full_attention layers; HT's local - # attention mask (last window_size keys) matches HF's sliding window. - "use_local_attn": True, - "window_size": hf_config.sliding_window, - "attn_types": [ - "local" if layer_type == "sliding_attention" else "global" - for layer_type in hf_config.layer_types - ], - } - rope_params = getattr(hf_config, "rope_parameters", None) or {} - if rope_params.get("rope_type") == "yarn": - yarn_factor = rope_params["factor"] - attention_factor = rope_params.get("attention_factor") - if attention_factor is None: - # HF's default: get_mscale(factor) = 0.1 * ln(factor) + 1 - attention_factor = 0.1 * math.log(yarn_factor) + 1.0 if yarn_factor > 1 else 1.0 - cfg_dict.update( - { - "use_yarn_rope": True, - "yarn_factor": float(yarn_factor), - "yarn_attention_factor": float(attention_factor), - "yarn_beta_fast": float(rope_params.get("beta_fast") or 32.0), - "yarn_beta_slow": float(rope_params.get("beta_slow") or 1.0), - "yarn_original_max_position_embeddings": rope_params[ - "original_max_position_embeddings" - ], - "yarn_truncate": rope_params.get("truncate", True), - } - ) - elif architecture == "BloomForCausalLM": - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.n_head, - "n_heads": hf_config.n_head, - "d_mlp": hf_config.hidden_size * 4, - "n_layers": hf_config.n_layer, - "n_ctx": 2048, # Capped due to HF Tokenizer Constraints - "d_vocab": hf_config.vocab_size, - "act_fn": "gelu_fast", - "eps": hf_config.layer_norm_epsilon, - "normalization_type": "LN", - "post_embedding_ln": True, - "positional_embedding_type": "alibi", - "default_prepend_bos": False, - } - elif architecture == "GPT2LMHeadCustomModel": - # santacoder - cfg_dict = { - "d_model": hf_config.n_embd, - "d_head": hf_config.n_embd // hf_config.n_head, - "n_heads": hf_config.n_head, - "d_mlp": hf_config.n_embd * 4, - "n_layers": hf_config.n_layer, - "n_ctx": hf_config.n_positions, - "eps": hf_config.layer_norm_epsilon, - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.activation_function, - "use_attn_scale": True, - "use_local_attn": False, - "trust_remote_code": "santacoder" - in official_model_name, # Only santacoder needs trust_remote_code - "scale_attn_by_inverse_layer_idx": hf_config.scale_attn_by_inverse_layer_idx, - "normalization_type": "LN", - } - elif architecture == "LlamaForCausalLM": - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - "n_ctx": hf_config.max_position_embeddings, - "eps": hf_config.rms_norm_eps, - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.hidden_act, - "n_key_value_heads": ( - hf_config.num_key_value_heads - if hf_config.num_key_value_heads != hf_config.num_attention_heads - else None - ), - # This is done because the current implementation of GQA will use Grouped-Query Attention if - # n_key_value_heads is not None, but hf_config.num_key_value_heads is sometimes specified as - # the same as hf_config.num_attention_heads, in which case GQA should not be used. - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_adjacent_pairs": False, - "rotary_dim": hf_config.hidden_size // hf_config.num_attention_heads, - # Llama-arch checkpoints without a name-matched branch above (Yi ships - # 5e6) reach here; the config default would silently be 10000. - "rotary_base": _get_rope_theta(hf_config), - "final_rms": True, - "gated_mlp": True, - } - _apply_llama3_rope_scaling(cfg_dict, hf_config) - elif architecture == "QWenLMHeadModel": - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "d_mlp": hf_config.intermediate_size // 2, - "n_layers": hf_config.num_hidden_layers, - # QWenLMHeadModel uses seq_length in its remote-code attention/rotary logic. - "n_ctx": hf_config.seq_length, - "use_logn_attn": getattr(hf_config, "use_logn_attn", False), - "use_dynamic_ntk_rope": getattr(hf_config, "use_dynamic_ntk", False), - "train_seq_length": hf_config.seq_length, - "eps": hf_config.layer_norm_epsilon, - "d_vocab": hf_config.vocab_size, - "act_fn": "silu", - "use_attn_scale": hf_config.scale_attn_weights, - "initializer_range": hf_config.initializer_range, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_dim": hf_config.kv_channels, - "rotary_adjacent_pairs": False, - "tokenizer_prepends_bos": True, - "trust_remote_code": True, - "final_rms": True, - "gated_mlp": True, - "default_prepend_bos": False, - } - elif architecture == "Qwen2ForCausalLM": - # Note that Qwen1.5 models have architecture type Qwen2ForCausalLM. - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "n_key_value_heads": hf_config.num_key_value_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - "n_ctx": hf_config.max_position_embeddings, - "eps": hf_config.rms_norm_eps, - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.hidden_act, - "use_attn_scale": True, - "initializer_range": hf_config.initializer_range, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_base": int(_get_rope_theta(hf_config)), - "rotary_adjacent_pairs": False, - "rotary_dim": hf_config.hidden_size // hf_config.num_attention_heads, - "tokenizer_prepends_bos": True, - "final_rms": True, - "gated_mlp": True, - "default_prepend_bos": False, - } - elif architecture == "Qwen3ForCausalLM": - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.head_dim - if hasattr(hf_config, "head_dim") - and hf_config.head_dim is not None - and hf_config.head_dim > 0 - else hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "n_key_value_heads": ( - hf_config.num_key_value_heads - if hf_config.num_key_value_heads != hf_config.num_attention_heads - else None - ), - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - "n_ctx": 2048, - "eps": hf_config.rms_norm_eps, - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.hidden_act, - "use_attn_scale": True, - "initializer_range": hf_config.initializer_range, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_base": int(_get_rope_theta(hf_config)), - "rotary_adjacent_pairs": False, - "rotary_dim": hf_config.head_dim - if hasattr(hf_config, "head_dim") and hf_config.head_dim > 0 - else hf_config.hidden_size // hf_config.num_attention_heads, - "tokenizer_prepends_bos": True, - "final_rms": True, - "gated_mlp": True, - "default_prepend_bos": False, - "use_qk_norm": True, - "trust_remote_code": True, - } - elif architecture == "PhiForCausalLM": - # Architecture for microsoft/phi models - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - "n_ctx": hf_config.max_position_embeddings, - "eps": hf_config.layer_norm_eps, - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.hidden_act, - "initializer_range": hf_config.initializer_range, - "normalization_type": "LN", - "positional_embedding_type": "rotary", - "trust_remote_code": True, - "rotary_base": _get_rope_theta(hf_config), - "use_attn_scale": True, - "parallel_attn_mlp": True, - "default_prepend_bos": False, - } - partial_rotary_factor = hf_config.partial_rotary_factor - cfg_dict["rotary_dim"] = round(partial_rotary_factor * cfg_dict["d_head"]) - elif architecture == "Phi3ForCausalLM": - # Architecture for microsoft/phi3 models - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - "n_key_value_heads": ( - hf_config.num_key_value_heads - if hf_config.num_key_value_heads != hf_config.num_attention_heads - else None - ), - "n_ctx": hf_config.max_position_embeddings, - "eps": hf_config.rms_norm_eps, - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.hidden_act, - "initializer_range": hf_config.initializer_range, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_base": _get_rope_theta(hf_config), - "use_attn_scale": True, - "gated_mlp": True, - "parallel_attn_mlp": False, - "rotary_dim": hf_config.hidden_size // hf_config.num_attention_heads, - # Phi-3-mini-4k ships sliding_window=2047 inside its 4096 n_ctx, and - # HF windows every layer (no layer_types). - "window_size": getattr(hf_config, "sliding_window", None), - "use_local_attn": bool(getattr(hf_config, "sliding_window", None)), - "attn_types": ( - ["local"] * hf_config.num_hidden_layers - if getattr(hf_config, "sliding_window", None) - else None - ), - } - elif architecture == "ApertusForCausalLM": - n_heads = hf_config.num_attention_heads - d_head = hf_config.hidden_size // n_heads - num_kv_heads = getattr(hf_config, "num_key_value_heads", n_heads) - n_kv_heads = num_kv_heads if num_kv_heads != n_heads else None - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": d_head, - "n_heads": n_heads, - "n_key_value_heads": n_kv_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - "n_ctx": hf_config.max_position_embeddings, - "eps": hf_config.rms_norm_eps, - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.hidden_act, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_dim": d_head, - "rotary_base": _get_rope_theta(hf_config), - "gated_mlp": False, - "final_rms": True, - "use_qk_norm": True, # HF applies q_norm/k_norm unconditionally; no config gate exists - } - rope_scaling = getattr(hf_config, "rope_scaling", None) - if rope_scaling: - rope_type = (rope_scaling.get("type") or rope_scaling.get("rope_type") or "").lower() - else: - rope_type = "" - if rope_type == "llama3": - assert rope_scaling is not None - cfg_dict["use_NTK_by_parts_rope"] = True - cfg_dict["NTK_original_ctx_len"] = rope_scaling.get( - "original_max_position_embeddings", hf_config.max_position_embeddings - ) - cfg_dict["NTK_by_parts_low_freq_factor"] = rope_scaling.get("low_freq_factor", 1.0) - cfg_dict["NTK_by_parts_high_freq_factor"] = rope_scaling.get("high_freq_factor", 4.0) - cfg_dict["NTK_by_parts_factor"] = rope_scaling.get("factor", 1.0) - - elif official_model_name.startswith("google/gemma-2b"): - # Architecture for Gemma 2b and Gemma 2b Instruct models - cfg_dict = { - "d_model": 2048, - "d_head": 256, - "n_heads": 8, - "d_mlp": 16384, - "n_layers": 18, - "n_ctx": 8192, - "eps": 1e-06, - "d_vocab": 256000, - "act_fn": "gelu", - "initializer_range": 0.02, - "normalization_type": "RMS", - "rotary_base": 10000, - "rotary_dim": 256, - "positional_embedding_type": "rotary", - "use_attn_scale": True, - "n_key_value_heads": 1, - "gated_mlp": True, - "final_rms": True, - } - elif official_model_name.startswith("google/gemma-7b"): - # Architecture for Gemma 7b and Gemma 7b Instruct models - cfg_dict = { - "d_model": 3072, - "d_head": 256, - "n_heads": 16, - "d_mlp": 24576, - "n_layers": 28, - "n_ctx": 8192, - "eps": 1e-06, - "d_vocab": 256000, - "act_fn": "gelu", - "initializer_range": 0.02, - "normalization_type": "RMS", - "rotary_base": 10000.0, - "rotary_dim": 256, - "positional_embedding_type": "rotary", - "use_attn_scale": True, - "n_key_value_heads": 16, - "gated_mlp": True, - "final_rms": True, - } - elif official_model_name.startswith("google/gemma-2-2b"): - # Architecture for Gemma-2 2b and Gemma-2 2b Instruct models - cfg_dict = { - "d_model": 2304, - "d_head": 256, - "n_heads": 8, - "d_mlp": 9216, - "n_layers": 26, - "n_ctx": 8192, - "eps": 1e-06, - "d_vocab": 256000, - "act_fn": "gelu_pytorch_tanh", - "initializer_range": 0.02, - "normalization_type": "RMS", - "rotary_base": 10000.0, - "positional_embedding_type": "rotary", - "use_attn_scale": True, - "n_key_value_heads": 4, - "window_size": 4096, - "use_local_attn": True, - "attn_types": ["local", "global"] * 13, # HF makes even layers sliding - "attn_scores_soft_cap": 50.0, - "output_logits_soft_cap": 30.0, - "gated_mlp": True, - "final_rms": True, - "use_normalization_before_and_after": True, - } - elif official_model_name.startswith("google/gemma-2-9b"): - # Architecture for Gemma-2 9b and Gemma-2 9b Instruct models - cfg_dict = { - "d_model": 3584, - "d_head": 256, - "n_heads": 16, - "d_mlp": 14336, - "n_layers": 42, - "n_ctx": 8192, - "eps": 1e-06, - "d_vocab": 256000, - "act_fn": "gelu_pytorch_tanh", - "initializer_range": 0.02, - "normalization_type": "RMS", - "rotary_base": 10000.0, - "positional_embedding_type": "rotary", - "use_attn_scale": True, - "n_key_value_heads": 8, - "window_size": 4096, - "use_local_attn": True, - "attn_types": ["local", "global"] * 21, # HF makes even layers sliding - "attn_scores_soft_cap": 50.0, - "output_logits_soft_cap": 30.0, - "gated_mlp": True, - "final_rms": True, - "use_normalization_before_and_after": True, - } - elif official_model_name.startswith("google/gemma-2-27b"): - # Architecture for Gemma-2 27b and Gemma-2 27b Instruct models - cfg_dict = { - "d_model": 4608, - "d_head": 128, - "n_heads": 32, - "d_mlp": 36864, - "n_layers": 46, - "n_ctx": 8192, - "eps": 1e-06, - "d_vocab": 256000, - "act_fn": "gelu_pytorch_tanh", - "initializer_range": 0.02, - "normalization_type": "RMS", - "rotary_base": 10000.0, - "positional_embedding_type": "rotary", - "use_attn_scale": True, - "attn_scale": 12.0, - "n_key_value_heads": 16, - "window_size": 4096, - "use_local_attn": True, - "attn_types": ["local", "global"] * 23, # HF makes even layers sliding - "attn_scores_soft_cap": 50.0, - "output_logits_soft_cap": 30.0, - "gated_mlp": True, - "final_rms": True, - "use_normalization_before_and_after": True, - } - elif official_model_name.startswith("google/gemma-3-270m"): - # Architecture for Gemma-3 270m and Gemma-3 270m Instruct models - cfg_dict = { - "d_model": 640, - "d_head": 256, - "n_heads": 4, - "d_mlp": 2048, - "n_layers": 18, - "n_ctx": 8192, # Safe default (model supports up to 32K). Override: cfg_kwargs={"n_ctx": 32768} - "eps": 1e-06, - "d_vocab": 262144, - "act_fn": "gelu_pytorch_tanh", - "initializer_range": 0.02, - "normalization_type": "RMS", - "rotary_base": 1000000, # Global attention layers - "rotary_base_local": 10000, # Local attention layers (per Gemma 3 paper) - "positional_embedding_type": "rotary", - "use_attn_scale": True, - "n_key_value_heads": 1, - "gated_mlp": True, - "final_rms": True, - "use_normalization_before_and_after": True, - "use_qk_norm": True, - "window_size": 512, - "use_local_attn": True, - "attn_types": [ - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - ], - } - elif official_model_name.startswith("google/gemma-3-1b"): - # Architecture for Gemma-3 1b-pt and Gemma-3 1b-it models - cfg_dict = { - "d_model": 1152, - "d_head": 256, - "n_heads": 4, - "d_mlp": 6912, - "n_layers": 26, - "n_ctx": 8192, # Safe default (model supports up to 32K). Override: cfg_kwargs={"n_ctx": 32768} - "eps": 1e-06, - "d_vocab": 262144, - "act_fn": "gelu_pytorch_tanh", - "initializer_range": 0.02, - "normalization_type": "RMS", - "rotary_base": 1000000, # Global attention layers - "rotary_base_local": 10000, # Local attention layers (per Gemma 3 paper) - "positional_embedding_type": "rotary", - "use_attn_scale": True, - "n_key_value_heads": 1, - "gated_mlp": True, - "final_rms": True, - "use_normalization_before_and_after": True, - "use_qk_norm": True, - "window_size": 512, - "use_local_attn": True, - "attn_types": [ - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - ], - } - elif official_model_name.startswith("google/gemma-3-4b") or official_model_name.startswith( - "google/medgemma-4b" - ): - # Architecture for Gemma-3 4b and MedGemma 4b models (multimodal, text-only extraction) - cfg_dict = { - "d_model": 2560, - "d_head": 256, - "n_heads": 8, - "d_mlp": 10240, - "n_layers": 34, - "n_ctx": 8192, # Safe default (model supports up to 128K). Override: cfg_kwargs={"n_ctx": 131072} - "eps": 1e-06, - "d_vocab": 262208, - "act_fn": "gelu_pytorch_tanh", - "initializer_range": 0.02, - "normalization_type": "RMS", - "rotary_base": 1000000, # Global attention layers - "rotary_base_local": 10000, # Local attention layers (per Gemma 3 paper) - "rotary_scaling_factor": 8.0, # Linear RoPE scaling for global layers - "positional_embedding_type": "rotary", - "use_attn_scale": True, - "n_key_value_heads": 4, - "gated_mlp": True, - "final_rms": True, - "use_normalization_before_and_after": True, - "use_qk_norm": True, - "window_size": 1024, - "use_local_attn": True, - "attn_types": [ - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - ], - } - elif official_model_name.startswith("google/gemma-3-12b"): - # Architecture for Gemma-3 12b models (multimodal, text-only extraction) - cfg_dict = { - "d_model": 3840, - "d_head": 256, - "n_heads": 16, - "d_mlp": 15360, - "n_layers": 48, - "n_ctx": 8192, # Safe default (model supports up to 128K). Override: cfg_kwargs={"n_ctx": 131072} - "eps": 1e-06, - "d_vocab": 262208, - "act_fn": "gelu_pytorch_tanh", - "initializer_range": 0.02, - "normalization_type": "RMS", - "rotary_base": 1000000, # Global attention layers - "rotary_base_local": 10000, # Local attention layers (per Gemma 3 paper) - "rotary_scaling_factor": 8.0, # Linear RoPE scaling for global layers - "positional_embedding_type": "rotary", - "use_attn_scale": True, - "n_key_value_heads": 8, - "gated_mlp": True, - "final_rms": True, - "use_normalization_before_and_after": True, - "use_qk_norm": True, - "window_size": 1024, - "use_local_attn": True, - "attn_types": [ - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - ], - } - elif official_model_name.startswith("google/gemma-3-27b") or official_model_name.startswith( - "google/medgemma-27b" - ): - # Architecture for Gemma-3 27b and MedGemma 27b models (multimodal/text-only extraction) - # Note: medgemma-27b-text-it uses Gemma3ForCausalLM (text-only), others use Gemma3ForConditionalGeneration - cfg_dict = { - "d_model": 5376, - "d_head": 128, - "n_heads": 32, - "d_mlp": 21504, - "n_layers": 62, - "n_ctx": 8192, # Safe default (model supports up to 128K). Override: cfg_kwargs={"n_ctx": 131072} - "eps": 1e-06, - "d_vocab": ( - 262144 if official_model_name == "google/medgemma-27b-text-it" else 262208 - ), # text-only variant uses 262144 - "act_fn": "gelu_pytorch_tanh", - "initializer_range": 0.02, - "normalization_type": "RMS", - "rotary_base": 1000000, # Global attention layers - "rotary_base_local": 10000, # Local attention layers (per Gemma 3 paper) - "rotary_scaling_factor": 8.0, # Linear RoPE scaling for global layers - "positional_embedding_type": "rotary", - "use_attn_scale": True, - "n_key_value_heads": 16, - "gated_mlp": True, - "final_rms": True, - "use_normalization_before_and_after": True, - "use_qk_norm": True, - "window_size": 1024, - "use_local_attn": True, - "attn_types": [ - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - "local", - "local", - "local", - "global", - "local", - "local", - ], - } - elif official_model_name.startswith("allenai/OLMo-1B") and official_model_name.endswith("hf"): - cfg_dict = { - "d_model": 2048, - "d_head": 128, - "n_heads": 16, - "d_mlp": 8192, - "n_layers": 16, - "n_ctx": 2048, - "eps": 1e-05, - "d_vocab": 50304, - "act_fn": "silu", - "initializer_range": 0.02, - "normalization_type": "LN", - "rotary_base": 10000.0, - "attn_types": ["global"] * 16, - "positional_embedding_type": "rotary", - "gated_mlp": True, - "clip_qkv": getattr(hf_config, "clip_qkv", None), - } - elif official_model_name.startswith("allenai/OLMo-7B") and official_model_name.endswith("hf"): - cfg_dict = { - "d_model": 4096, - "d_head": 128, - "n_heads": 32, - "d_mlp": 11008, - "n_layers": 32, - "n_ctx": 2048, - "eps": 1e-05, - "d_vocab": 50304, - "act_fn": "silu", - "initializer_range": 0.02, - "normalization_type": "LN", - "rotary_base": 10000.0, - "attn_types": ["global"] * 32, - "positional_embedding_type": "rotary", - "gated_mlp": True, - "clip_qkv": getattr(hf_config, "clip_qkv", None), - } - elif official_model_name.startswith("allenai/OLMo-2-0425-1B"): - cfg_dict = { - "d_model": 2048, - "d_head": 128, - "n_heads": 16, - "d_mlp": 8192, - "n_layers": 16, - "n_ctx": 4096, - "eps": 1e-06, - "d_vocab": 100352, - "act_fn": "silu", - "initializer_range": 0.02, - "normalization_type": "RMS", - "rotary_base": 500000.0, - "attn_types": ["global"] * 16, - "positional_embedding_type": "rotary", - "gated_mlp": True, - } - elif official_model_name.startswith("allenai/OLMo-2-1124-7B"): - cfg_dict = { - "d_model": 4096, - "d_head": 128, - "n_heads": 32, - "d_mlp": 11008, - "n_layers": 32, - "n_ctx": 4096, - "eps": 1e-06, - "d_vocab": 100352, - "act_fn": "silu", - "initializer_range": 0.02, - "normalization_type": "RMS", - "rotary_base": 500000.0, - "attn_types": ["global"] * 32, - "positional_embedding_type": "rotary", - "gated_mlp": True, - } - elif architecture == "Olmo3ForCausalLM": - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "n_key_value_heads": hf_config.num_key_value_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - "n_ctx": hf_config.max_position_embeddings, - "eps": hf_config.rms_norm_eps, - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.hidden_act, - "initializer_range": hf_config.initializer_range, - "normalization_type": "RMS", - "positional_embedding_type": "rotary", - "rotary_base": _get_rope_theta(hf_config, default=500000.0), - "gated_mlp": True, - "tie_word_embeddings": hf_config.tie_word_embeddings, - } - # OLMo 3 uses per-layer-type rope (transformers 5.x): plain rope on - # sliding_attention layers, YARN on full_attention layers. - rope_params = getattr(hf_config, "rope_parameters", None) - rope_scaling = getattr(hf_config, "rope_scaling", None) - if isinstance(rope_params, dict) and "full_attention" in rope_params: - full_rope = rope_params["full_attention"] or {} - sliding_rope = rope_params.get("sliding_attention") or {} - cfg_dict["rotary_base"] = full_rope.get("rope_theta", 500000.0) - sliding_theta = sliding_rope.get("rope_theta") - if sliding_theta is not None and sliding_theta != cfg_dict["rotary_base"]: - cfg_dict["rotary_base_local"] = sliding_theta - if full_rope.get("rope_type") == "yarn": - cfg_dict["use_yarn_rope"] = True - cfg_dict["yarn_global_attn_only"] = True - cfg_dict["yarn_factor"] = float(full_rope.get("factor", 8.0)) - cfg_dict["yarn_attention_factor"] = float(full_rope.get("attention_factor", 1.0)) - cfg_dict["yarn_beta_fast"] = float(full_rope.get("beta_fast") or 32.0) - cfg_dict["yarn_beta_slow"] = float(full_rope.get("beta_slow") or 1.0) - cfg_dict["yarn_original_max_position_embeddings"] = full_rope.get( - "original_max_position_embeddings", 4096 - ) - cfg_dict["yarn_truncate"] = full_rope.get("truncate", True) - elif rope_scaling and rope_scaling.get("rope_type") == "yarn": - # transformers < 5.0 flat rope_scaling dict - cfg_dict["use_yarn_rope"] = True - cfg_dict["yarn_factor"] = rope_scaling.get("factor", 8.0) - cfg_dict["yarn_attention_factor"] = rope_scaling.get("attention_factor", 1.0) - cfg_dict["yarn_beta_fast"] = rope_scaling.get("beta_fast", 32.0) - cfg_dict["yarn_beta_slow"] = rope_scaling.get("beta_slow", 1.0) - cfg_dict["yarn_original_max_position_embeddings"] = rope_scaling.get( - "original_max_position_embeddings", 4096 - ) - layer_types = getattr(hf_config, "layer_types", None) - if layer_types: - cfg_dict["attn_types"] = [ - "local" if t == "sliding_attention" else "global" for t in layer_types - ] - # Without use_local_attn, attn_types is inert and every layer runs - # global attention with no sliding mask. - if "sliding_attention" in layer_types and hf_config.sliding_window: - cfg_dict["use_local_attn"] = True - cfg_dict["window_size"] = hf_config.sliding_window - else: - cfg_dict["attn_types"] = ["global"] * hf_config.num_hidden_layers - elif architecture == "OlmoeForCausalLM": - cfg_dict = { - "d_model": hf_config.hidden_size, - "d_head": hf_config.hidden_size // hf_config.num_attention_heads, - "n_heads": hf_config.num_attention_heads, - "d_mlp": hf_config.intermediate_size, - "n_layers": hf_config.num_hidden_layers, - "n_ctx": hf_config.max_position_embeddings, - "eps": hf_config.rms_norm_eps, - "d_vocab": hf_config.vocab_size, - "act_fn": hf_config.hidden_act, - "num_experts": hf_config.num_experts, - "experts_per_token": hf_config.num_experts_per_tok, - "norm_topk_prob": hf_config.norm_topk_prob, - "n_key_value_heads": hf_config.num_key_value_heads, - "rotary_base": _get_rope_theta(hf_config), - "tie_word_embeddings": hf_config.tie_word_embeddings, - "initializer_range": hf_config.initializer_range, - "positional_embedding_type": "rotary", - "rotary_dim": hf_config.hidden_size // hf_config.num_attention_heads, - "gated_mlp": True, - "normalization_type": "RMS", - "clip_qkv": getattr(hf_config, "clip_qkv", None), - } - elif architecture == "T5ForConditionalGeneration": - cfg_dict = { - "d_model": hf_config.d_model, - "d_head": hf_config.d_kv, - "n_heads": hf_config.num_heads, - "d_mlp": hf_config.d_ff, - "d_vocab": hf_config.vocab_size, - "n_layers": hf_config.num_layers, - "n_ctx": getattr(hf_config, "max_length", None) or hf_config.n_positions, - "eps": hf_config.layer_norm_epsilon, - "act_fn": hf_config.feed_forward_proj, - "positional_embedding_type": "relative_positional_bias", - "relative_attention_max_distance": hf_config.relative_attention_max_distance, - "relative_attention_num_buckets": hf_config.relative_attention_num_buckets, - "decoder_start_token_id": hf_config.decoder_start_token_id, - "attention_dir": "bidirectional", - "use_attn_scale": False, - "tie_word_embeddings": hf_config.tie_word_embeddings, - } - else: - raise NotImplementedError(f"{architecture} is not currently supported.") - # All of these models use LayerNorm - cfg_dict["original_architecture"] = architecture - # Carried on the cfg so the loader can act on the quantization without a - # second AutoConfig fetch (which would be a Hub round trip per load). - cfg_dict["quantization_method"] = quantization_method(hf_config) - # The name such that AutoTokenizer.from_pretrained works - cfg_dict["tokenizer_name"] = official_model_name - if kwargs.get("trust_remote_code", False): - cfg_dict["trust_remote_code"] = True - # TinyStories models were trained with seq_len=512, but the HuggingFace config - # reports max_position_embeddings=2048. Override n_ctx so the positional embedding - # weights are trimmed during weight conversion. - # See: https://github.com/TransformerLensOrg/TransformerLens/issues/492 - if official_model_name.startswith("roneneldan/TinyStories"): - cfg_dict["n_ctx"] = 512 - return cfg_dict - - -def convert_neel_model_config(official_model_name: str, **kwargs: Any) -> dict[str, Any]: - """ - Loads the config for a model trained by me (NeelNanda), converted to a dictionary - in the HookedTransformerConfig format. - - AutoConfig is not supported, because these models are in the HookedTransformer format, so we directly download and load the json. - """ - official_model_name = get_official_model_name(official_model_name) - cfg_json: dict = utils.download_file_from_hf(official_model_name, "config.json", **kwargs) - cfg_arch = cfg_json.get( - "architecture", "neel" if "_old" not in official_model_name else "neel-solu-old" - ) - cfg_dict = { - "d_model": cfg_json["d_model"], - "n_layers": cfg_json["n_layers"], - "d_mlp": cfg_json["d_mlp"], - "d_head": cfg_json["d_head"], - "n_heads": cfg_json["n_heads"], - "n_ctx": cfg_json["n_ctx"], - "d_vocab": cfg_json["d_vocab"], - "tokenizer_name": cfg_json.get("tokenizer_name", None), - "act_fn": cfg_json["act_fn"], - "attn_only": cfg_json["attn_only"], - "final_rms": cfg_json.get("final_rms", False), - "original_architecture": cfg_arch, - } - if "normalization" in cfg_json: - cfg_dict["normalization_type"] = cfg_json["normalization"] - else: - cfg_dict["normalization_type"] = cfg_json["normalization_type"] - if "shortformer_pos" in cfg_json: - cfg_dict["positional_embedding_type"] = ( - "shortformer" if cfg_json["shortformer_pos"] else "standard" - ) - else: - cfg_dict["positional_embedding_type"] = "standard" - return cfg_dict - - -def get_pretrained_model_config( - model_name: str, - hf_cfg: dict[str, Any] | None = None, - checkpoint_index: int | None = None, - checkpoint_value: int | None = None, - fold_ln: bool = False, - device: str | torch.device | None = None, - n_devices: int = 1, - default_prepend_bos: bool | None = None, - dtype: torch.dtype = torch.float32, - first_n_layers: int | None = None, - n_ctx: int | None = None, - **kwargs: Any, -) -> HookedTransformerConfig: - """Returns the pretrained model config as an HookedTransformerConfig object. - - There are two types of pretrained models: HuggingFace models (where - AutoModel and AutoConfig work), and models trained by me (NeelNanda) which - aren't as integrated with HuggingFace infrastructure. - - Args: - model_name: The name of the model. This can be either the official - HuggingFace model name, or the name of a model trained by me - (NeelNanda). - hf_cfg (dict, optional): Config of a loaded pretrained HF model, - converted to a dictionary. - checkpoint_index (int, optional): If loading from a - checkpoint, the index of the checkpoint to load. Defaults to None. - checkpoint_value (int, optional): If loading from a checkpoint, the - value of - the checkpoint to load, ie the step or token number (each model has - checkpoints labelled with exactly one of these). Defaults to None. - fold_ln (bool, optional): Whether to fold the layer norm into the - subsequent linear layers (see HookedTransformer.fold_layer_norm for - details). Defaults to False. - device (str, optional): The device to load the model onto. By - default will load to CUDA if available, else CPU. - n_devices (int, optional): The number of devices to split the model across. Defaults to 1. - default_prepend_bos (bool, optional): Default behavior of whether to prepend the BOS token when the - methods of HookedTransformer process input text to tokenize (only when input is a string). - Resolution order for default_prepend_bos: - 1. If user passes value explicitly, use that value - 2. Model-specific default from cfg_dict if it exists (e.g. for bloom models it's False) - 3. Global default (True) - - Even for models not explicitly trained with the BOS token, heads often use the - first position as a resting position and accordingly lose information from the first token, - so this empirically seems to give better results. Note that you can also locally override the default behavior - by passing in prepend_bos=True/False when you call a method that processes the input string. - dtype (torch.dtype, optional): The dtype to load the TransformerLens model in. - kwargs: Other optional arguments passed to HuggingFace's from_pretrained. - Also given to other HuggingFace functions when compatible. - - """ - if Path(model_name).exists(): - # If the model_name is a path, it's a local model - cfg_dict = convert_hf_model_config(model_name, **kwargs) - official_model_name = model_name - else: - official_model_name = get_official_model_name(model_name) - if ( - official_model_name.startswith("NeelNanda") - or official_model_name.startswith("ArthurConmy") - or official_model_name.startswith("Baidicoot") - ): - cfg_dict = convert_neel_model_config(official_model_name, **kwargs) - else: - if official_model_name.startswith(NEED_REMOTE_CODE_MODELS) and not kwargs.get( - "trust_remote_code", False - ): - logging.warning( - f"Loading model {official_model_name} requires setting trust_remote_code=True" - ) - kwargs["trust_remote_code"] = True - cfg_dict = convert_hf_model_config(official_model_name, **kwargs) - # Processing common to both model types - # Remove any prefix, saying the organization who made a model. - cfg_dict["model_name"] = official_model_name.split("/")[-1] - # Don't need to initialize weights, we're loading from pretrained - cfg_dict["init_weights"] = False - - if ( - "positional_embedding_type" in cfg_dict - and cfg_dict["positional_embedding_type"] == "shortformer" - and fold_ln - ): - logging.warning( - "You tried to specify fold_ln=True for a shortformer model, but this can't be done! Setting fold_ln=False instead." - ) - fold_ln = False - - # Post-norm blocks normalize the sublayer output, so folding the norm weights - # into adjacent linear layers is not mathematically valid. - architecture = cfg_dict.get("original_architecture") - if architecture in POST_NORM_ARCHITECTURES and fold_ln: - logging.warning( - f"fold_ln=True is incompatible with {architecture}'s post-norm architecture. " - "Setting fold_ln=False." - ) - fold_ln = False - - if device is not None: - cfg_dict["device"] = device - - cfg_dict["dtype"] = dtype - - if fold_ln: - if cfg_dict["normalization_type"] in ["LN", "LNPre"]: - cfg_dict["normalization_type"] = "LNPre" - elif cfg_dict["normalization_type"] in ["RMS", "RMSPre"]: - cfg_dict["normalization_type"] = "RMSPre" - else: - logging.warning("Cannot fold in layer norm, normalization_type is not LN.") - - if checkpoint_index is not None or checkpoint_value is not None: - checkpoint_labels, checkpoint_label_type = get_checkpoint_labels( - official_model_name, - **kwargs, - ) - cfg_dict["from_checkpoint"] = True - cfg_dict["checkpoint_label_type"] = checkpoint_label_type - if checkpoint_index is not None: - cfg_dict["checkpoint_index"] = checkpoint_index - cfg_dict["checkpoint_value"] = checkpoint_labels[checkpoint_index] - elif checkpoint_value is not None: - assert ( - checkpoint_value in checkpoint_labels - ), f"Checkpoint value {checkpoint_value} is not in list of available checkpoints" - cfg_dict["checkpoint_value"] = checkpoint_value - cfg_dict["checkpoint_index"] = checkpoint_labels.index(checkpoint_value) - else: - cfg_dict["from_checkpoint"] = False - - cfg_dict["device"] = device - cfg_dict["n_devices"] = n_devices - - if default_prepend_bos is not None: - # User explicitly set prepend_bos behavior, override config/default value - cfg_dict["default_prepend_bos"] = default_prepend_bos - elif "default_prepend_bos" not in cfg_dict: - # No config value or user override, set default value (True) - cfg_dict["default_prepend_bos"] = True - - if hf_cfg is not None: - cfg_dict["load_in_4bit"] = hf_cfg.get("quantization_config", {}).get("load_in_4bit", False) - # A user-supplied hf_model is the more authoritative source: it says how - # the weights in hand are actually stored, not how the Hub repo declares - # them. .get, not []: convert_neel_model_config builds cfg_dict without - # ever seeing an HF config, so the key need not be there. - cfg_dict["quantization_method"] = quantization_method(hf_cfg) or cfg_dict.get( - "quantization_method" - ) - cfg_dict["d_vocab"] = hf_cfg.get("vocab_size", cfg_dict["d_vocab"]) - if cfg_dict["original_architecture"] == "Qwen2ForCausalLM": - rope_params = hf_cfg.get("rope_parameters", {}) or {} - cfg_dict["rotary_base"] = hf_cfg.get( - "rope_theta", rope_params.get("rope_theta", cfg_dict["rotary_base"]) - ) - if first_n_layers is not None: - cfg_dict["n_layers"] = first_n_layers - - if n_ctx is not None: - default_n_ctx = cfg_dict.get("n_ctx") - if default_n_ctx is not None and n_ctx > default_n_ctx: - logging.warning( - f"You are setting n_ctx={n_ctx} which is larger than this model's " - f"default context length of {default_n_ctx}. The model was not " - f"trained on sequences this long and may produce unreliable results. " - f"Ensure you have sufficient memory for this context length." - ) - cfg_dict["n_ctx"] = n_ctx - - cfg = HookedTransformerConfig.from_dict(cfg_dict) - return cfg - - -def get_num_params_of_pretrained(model_name: str) -> int: - """ - Returns the number of parameters of a pretrained model, used to filter to only run code for sufficiently small models. - """ - cfg = get_pretrained_model_config(model_name) - if cfg.n_params is None: - raise ValueError(f"n_params not calculated for model {model_name}") - return cfg.n_params - - -# %% Load checkpointed model state dicts -# Checkpoint schedules (STANFORD_CRFM_CHECKPOINTS, PYTHIA_CHECKPOINTS, -# PYTHIA_V0_CHECKPOINTS) are imported from tools/model_registry/checkpoints.py, -# their canonical home. - - -def get_checkpoint_labels(model_name: str, **kwargs: Any) -> tuple[list[int], str]: - """Returns the checkpoint labels for a given model, and the label_type - (step or token). Raises an error for models that are not checkpointed.""" - official_model_name = get_official_model_name(model_name) - if official_model_name.startswith("stanford-crfm/"): - return STANFORD_CRFM_CHECKPOINTS, "step" - elif official_model_name.startswith("EleutherAI/pythia"): - if "v0" in official_model_name: - return PYTHIA_V0_CHECKPOINTS, "step" - else: - logging.warning( - "Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models." - ) - return PYTHIA_CHECKPOINTS, "step" - elif official_model_name.startswith("NeelNanda/"): - api = HfApi() - files_list = api.list_repo_files( - official_model_name, - **utils.select_compatible_kwargs(kwargs, api.list_repo_files), - ) - labels = [] - for file_name in files_list: - match = re.match(r"checkpoints/.*_(\d*)\.pth", file_name) - if match: - labels.append(int(match.group(1))) - if labels[-1] > 1e9: - label_type = "token" - else: - label_type = "step" - return labels, label_type - else: - raise ValueError(f"Model {official_model_name} is not checkpointed.") - - -# %% Loading state dicts -def _mxfp4_dequantize_config(cfg: HookedTransformerConfig) -> Any | None: - """Return ``Mxfp4Config(dequantize=True)`` for packed-MXFP4 checkpoints. - - Reads the method captured on ``cfg`` (no refetch). Blind spot by - construction: llama/gemma names never fetch a config, so their - quantization_method is always None there — such checkpoints are refused by - ``_refuse_unsupported_quantization`` rather than auto-dequantized. - """ - if cfg.quantization_method != "mxfp4": - return None - return Mxfp4Config(dequantize=True) - - -def _refuse_unsupported_quantization(cfg: HookedTransformerConfig, hf_model: Any) -> None: - """Refuse a quantized checkpoint before the weight converters read it. - - Same-shape int8/FP8 converts silently AND survives load_state_dict (cast - to fp32), so refusal must happen here. Reads the LOADED model's config — - cfg.quantization_method is structurally None for name-based llama/gemma — - and refuses on stored weights, not the declaration, so dequantized loads - that still advertise a quant_method keep working. - """ - if hf_model is None: - return - method = quantization_method(getattr(hf_model, "config", None)) - if method is None: - return - # The one supported quantized HookedTransformer flow (weight conversion and - # abstract_attention's matmul_4bit both handle it). - if cfg.load_in_4bit and method == "bitsandbytes": - return - # Refuse on the stored weights, not the declaration: a checkpoint loaded - # with dequantize=True still advertises its original quant_method while - # holding perfectly readable bf16 tensors. Meta params are skipped — an - # offloaded load is a different problem with a different message. - offender = next( - ( - (name, unreadable_weight_reason(param)) - for name, param in hf_model.named_parameters() - if param.device.type != "meta" and unreadable_weight_reason(param) is not None - ), - None, - ) - if offender is None: - return - name, reason = offender - raise NotImplementedError( - f"HookedTransformer cannot convert this {method!r}-quantized checkpoint: " - f"{name} cannot be read because {reason}. The weight converters read " - "weights directly, so packed or scale-separated storage silently " - "produces wrong values. Load the model dequantized, or use " - "TransformerBridge for a quantized forward pass." - ) - - -def get_pretrained_state_dict( - official_model_name: str, - cfg: HookedTransformerConfig, - hf_model: Any | None = None, - dtype: torch.dtype = torch.float32, - **kwargs: Any, -) -> dict[str, torch.Tensor]: - """ - Loads in the model weights for a pretrained model, and processes them to - have the HookedTransformer parameter names and shapes. Supports checkpointed - models (and expects the checkpoint info to be stored in the config object) - - hf_model: Optionally, a HuggingFace model object. If provided, we will use - these weights rather than reloading the model. - dtype: The dtype to load the HuggingFace model in. - kwargs: Other optional arguments passed to HuggingFace's from_pretrained. - Also given to other HuggingFace functions when compatible. - """ - if "torch_dtype" in kwargs: - dtype = kwargs["torch_dtype"] - del kwargs["torch_dtype"] - if Path(official_model_name).exists(): - official_model_name = str(Path(official_model_name).resolve()) - logging.info(f"Loading model from local path {official_model_name}") - else: - official_model_name = get_official_model_name(official_model_name) - if official_model_name.startswith(NEED_REMOTE_CODE_MODELS) and not kwargs.get( - "trust_remote_code", False - ): - logging.warning( - f"Loading model {official_model_name} state dict requires setting trust_remote_code=True" - ) - kwargs["trust_remote_code"] = True - if ( - official_model_name.startswith("NeelNanda") - or official_model_name.startswith("ArthurConmy") - or official_model_name.startswith("Baidicoot") - ): - api = HfApi() - repo_files = api.list_repo_files( - official_model_name, - **utils.select_compatible_kwargs(kwargs, api.list_repo_files), - ) - if cfg.from_checkpoint: - file_name = list( - filter(lambda x: x.endswith(f"{cfg.checkpoint_value}.pth"), repo_files) - )[0] - else: - file_name = list(filter(lambda x: x.endswith("final.pth"), repo_files))[0] - state_dict = utils.download_file_from_hf(official_model_name, file_name, **kwargs) - - state_dict = {k: v.to(dtype) for k, v in state_dict.items()} - - if cfg.original_architecture == "neel-solu-old": - state_dict = convert_neel_solu_old_weights(state_dict, cfg) - elif cfg.original_architecture == "mingpt": - state_dict = convert_mingpt_weights(state_dict, cfg) - return state_dict - else: - if cfg.from_checkpoint: - huggingface_token = os.environ.get("HF_TOKEN", "") - if official_model_name.startswith("stanford-crfm"): - hf_model = AutoModelForCausalLM.from_pretrained( - official_model_name, - revision=f"checkpoint-{cfg.checkpoint_value}", - dtype=dtype, - token=huggingface_token if len(huggingface_token) > 0 else None, - **kwargs, - ) - elif official_model_name.startswith("EleutherAI/pythia"): - hf_model = AutoModelForCausalLM.from_pretrained( - official_model_name, - revision=f"step{cfg.checkpoint_value}", - dtype=dtype, - token=huggingface_token, - **kwargs, - ) - else: - raise ValueError(f"Checkpoints for model {official_model_name} are not supported") - elif hf_model is None: - huggingface_token = os.environ.get("HF_TOKEN", "") - if official_model_name in NON_HF_HOSTED_MODEL_NAMES: - raise NotImplementedError("Model not hosted on HuggingFace, must pass in hf_model") - elif "hubert" in official_model_name: - hf_model = HubertModel.from_pretrained( - official_model_name, - dtype=dtype, - token=huggingface_token if len(huggingface_token) > 0 else None, - **kwargs, - ) - elif "wav2vec2" in official_model_name: - hf_model = Wav2Vec2Model.from_pretrained( - official_model_name, - dtype=dtype, - token=huggingface_token if len(huggingface_token) > 0 else None, - **kwargs, - ) - elif "bert" in official_model_name: - hf_model = BertForPreTraining.from_pretrained( - official_model_name, - dtype=dtype, - token=huggingface_token if len(huggingface_token) > 0 else None, - **kwargs, - ) - elif "t5" in official_model_name: - hf_model = T5ForConditionalGeneration.from_pretrained( - official_model_name, - dtype=dtype, - token=huggingface_token if len(huggingface_token) > 0 else None, - **kwargs, - ) - elif cfg.original_architecture == "Gemma3ForConditionalGeneration": - # Multimodal Gemma 3 models - use AutoModel - hf_model = AutoModel.from_pretrained( - official_model_name, - dtype=dtype, - token=huggingface_token if len(huggingface_token) > 0 else None, - **kwargs, - ) - else: - if "quantization_config" not in kwargs: - mxfp4_dequantize = _mxfp4_dequantize_config(cfg) - if mxfp4_dequantize is not None: - kwargs = {**kwargs, "quantization_config": mxfp4_dequantize} - # Older models may lack pad_token_id (required in newer transformers) - try: - hf_model = AutoModelForCausalLM.from_pretrained( - official_model_name, - dtype=dtype, - token=huggingface_token if len(huggingface_token) > 0 else None, - **kwargs, - ) - except AttributeError as e: - if "pad_token_id" in str(e): - hf_config = AutoConfig.from_pretrained( - official_model_name, - token=huggingface_token if len(huggingface_token) > 0 else None, - ) - hf_config.pad_token_id = getattr(hf_config, "pad_token_id", None) - hf_model = AutoModelForCausalLM.from_pretrained( - official_model_name, - config=hf_config, - dtype=dtype, - token=huggingface_token if len(huggingface_token) > 0 else None, - **kwargs, - ) - else: - raise - - # Load model weights, and fold in layer norm weights - if hf_model is not None: - for param in hf_model.parameters(): - param.requires_grad = False - - _refuse_unsupported_quantization(cfg, hf_model) - - if cfg.original_architecture == "GPT2LMHeadModel": - state_dict = convert_gpt2_weights(hf_model, cfg) - elif cfg.original_architecture == "GPTNeoForCausalLM": - state_dict = convert_neo_weights(hf_model, cfg) - elif cfg.original_architecture == "OPTForCausalLM": - state_dict = convert_opt_weights(hf_model, cfg) - elif cfg.original_architecture == "GPTJForCausalLM": - state_dict = convert_gptj_weights(hf_model, cfg) - elif cfg.original_architecture == "GPTNeoXForCausalLM": - state_dict = convert_neox_weights(hf_model, cfg) - elif cfg.original_architecture == "LlamaForCausalLM": - state_dict = convert_llama_weights(hf_model, cfg) - elif cfg.original_architecture == "HubertModel": - state_dict = convert_hubert_weights(hf_model, cfg) - elif ( - cfg.original_architecture == "Wav2Vec2Model" - or cfg.original_architecture == "Wav2Vec2ForPreTraining" - ): - state_dict = convert_hubert_weights(hf_model, cfg) - elif cfg.original_architecture == "HubertForCTC": - state_dict = convert_hubert_weights(hf_model, cfg) - elif cfg.original_architecture == "BertForMaskedLM": - state_dict = convert_bert_weights(hf_model, cfg) - elif cfg.original_architecture == "T5ForConditionalGeneration": - state_dict = convert_t5_weights(hf_model, cfg) - elif cfg.original_architecture == "MistralForCausalLM": - state_dict = convert_mistral_weights(hf_model, cfg) - elif cfg.original_architecture == "MixtralForCausalLM": - state_dict = convert_mixtral_weights(hf_model, cfg) - elif cfg.original_architecture == "GptOssForCausalLM": - state_dict = convert_gpt_oss_weights(hf_model, cfg) - elif cfg.original_architecture == "BloomForCausalLM": - state_dict = convert_bloom_weights(hf_model, cfg) - elif cfg.original_architecture == "GPT2LMHeadCustomModel": - state_dict = convert_coder_weights(hf_model, cfg) - elif cfg.original_architecture == "QWenLMHeadModel": - state_dict = convert_qwen_weights(hf_model, cfg) - elif cfg.original_architecture == "Qwen2ForCausalLM": - state_dict = convert_qwen2_weights(hf_model, cfg) - elif cfg.original_architecture == "Qwen3ForCausalLM": - state_dict = convert_qwen3_weights(hf_model, cfg) - elif cfg.original_architecture == "PhiForCausalLM": - state_dict = convert_phi_weights(hf_model, cfg) - elif cfg.original_architecture == "Phi3ForCausalLM": - state_dict = convert_phi3_weights(hf_model, cfg) - elif cfg.original_architecture == "GemmaForCausalLM": - state_dict = convert_gemma_weights(hf_model, cfg) - elif cfg.original_architecture == "Gemma2ForCausalLM": - state_dict = convert_gemma_weights(hf_model, cfg) - elif cfg.original_architecture == "ApertusForCausalLM": - state_dict = convert_apertus_weights(hf_model, cfg) - elif cfg.original_architecture == "Gemma3ForCausalLM": - state_dict = convert_gemma_weights(hf_model, cfg) - elif cfg.original_architecture == "Gemma3ForConditionalGeneration": - state_dict = convert_gemma_weights(hf_model, cfg) - elif cfg.original_architecture == "OlmoForCausalLM": - state_dict = convert_olmo_weights(hf_model, cfg) - elif cfg.original_architecture == "Olmo2ForCausalLM": - state_dict = convert_olmo2_weights(hf_model, cfg) - elif cfg.original_architecture == "OlmoeForCausalLM": - state_dict = convert_olmoe_weights(hf_model, cfg) - elif cfg.original_architecture == "Olmo3ForCausalLM": - state_dict = convert_olmo3_weights(hf_model, cfg) - else: - raise ValueError( - f"Loading weights from the architecture is not currently supported: {cfg.original_architecture}, generated from model name {cfg.model_name}. Feel free to open an issue on GitHub to request this feature." - ) - - return state_dict - - -def fill_missing_keys( - model: torch.nn.Module, state_dict: dict[str, torch.Tensor] -) -> dict[str, torch.Tensor]: - """Takes in a state dict from a pretrained model, and fills in any missing keys with the default initialization. - - This function is assumed to be run before weights are initialized. - - Args: - model: The model to fill missing keys for - state_dict: State dict from a pretrained model - - Returns: - dict: State dict with missing keys filled in - """ - default_state_dict = model.state_dict() - missing_keys = set(default_state_dict.keys()) - set(state_dict.keys()) - # A missing attention weight matrix means a converter/component naming - # mismatch (e.g. W_K written where GroupedQueryAttention expects _W_K). - # Filling it with an empty tensor silently zeroes the sublayer while every - # downstream number still looks plausible — fail loudly instead. - # W_in/W_gate/W_out join the attention set: zero-filling an MLP matrix is - # the same silent-sublayer-death, just on the other branch. - fail_loud_weight_names = { - "W_Q", - "W_K", - "W_V", - "W_O", - "_W_K", - "_W_V", - "W_in", - "W_gate", - "W_out", - } - missing_fail_loud = sorted( - key - for key in missing_keys - if "hf_model" not in key and key.rsplit(".", 1)[-1] in fail_loud_weight_names - ) - if missing_fail_loud: - raise ValueError( - f"Pretrained state dict is missing weight matrices the model " - f"expects: {missing_fail_loud}. This usually means the weight " - f"converter and the instantiated module disagree on parameter " - f"naming (e.g. GQA's underscore-prefixed _W_K/_W_V vs W_K/W_V). Refusing " - f"to zero-fill them, which would silently produce wrong outputs." - ) - # Norm weights fill with DEFAULTS (w=1, b=0), which is frequently correct - # (models without biases) but silently wrong when the checkpoint really has - # them — so the fill is named, not silent. - norm_key_names = {"w", "b"} - for key in missing_keys: - if "hf_model" in key: - # Skip keys that are from the HuggingFace model, if loading from HF. - continue - leaf = key.rsplit(".", 1)[-1] - if "W_" in key: - logging.warning( - "Missing key for a weight matrix in pretrained, filled in with an empty tensor: {}".format( - key - ) - ) - elif leaf in norm_key_names and ("ln" in key or "norm" in key): - logging.warning( - "Missing normalization key in pretrained, filled with its default " - "(identity norm): {}".format(key) - ) - state_dict[key] = default_state_dict[key] - return state_dict - - -@dataclasses.dataclass -class Config: - d_model: int = 768 - debug: bool = True - layer_norm_eps: float = 1e-5 - d_vocab: int = 50257 - init_range: float = 0.02 - n_ctx: int = 1024 - d_head: int = 64 - d_mlp: int = 3072 - n_heads: int = 12 - n_layers: int = 12 - - -def get_basic_config(model_name: str, **kwargs: Any) -> Config: - """Returns the configuration parameters of the model as a basic Config dataclass.""" - return Config( - **{ - k: v - for k, v in get_pretrained_model_config(model_name, **kwargs).to_dict().items() - if k - in [ - "d_model", - "debug", - "layer_norm_eps", - "d_vocab", - "init_range", - "n_ctx", - "d_head", - "d_mlp", - "n_heads", - "n_layers", - ] - } - ) diff --git a/transformer_lens/model_bridge/remote_bridge.py b/transformer_lens/model_bridge/remote_bridge.py index 8aef5e0cad..9dd8e7c929 100644 --- a/transformer_lens/model_bridge/remote_bridge.py +++ b/transformer_lens/model_bridge/remote_bridge.py @@ -213,7 +213,7 @@ def to_tokens(self, input: Any, prepend_bos: bool | None = None, truncate: bool ``boot_inspect(m).run_with_cache("text")`` matches ``boot_transformers(m)`` on the same string — a bare ``encode`` (no BOS) would silently diverge. """ - from transformer_lens import utils + from transformer_lens import utilities as utils assert self.tokenizer is not None, "Tokenizer must be set." if prepend_bos is None: diff --git a/transformer_lens/model_bridge/sources/transformers/source.py b/transformer_lens/model_bridge/sources/transformers/source.py index 18316f9b55..d0adb033ea 100644 --- a/transformer_lens/model_bridge/sources/transformers/source.py +++ b/transformer_lens/model_bridge/sources/transformers/source.py @@ -120,7 +120,7 @@ def boot( official_name = resolve_model_alias(model_name) if official_name is not None: logging.warning( - f"DEPRECATED: You are using a deprecated, model_name alias '{model_name}'. TransformerLens will now load the official transformers model name, '{official_name}' instead.\n Please update your code to use the official name by changing model_name from '{model_name}' to '{official_name}'.\nSince TransformerLens v3, all model names should be the official transformers model names.\nThe aliases will be removed in the next version of TransformerLens, so please do the update now." + f"DEPRECATED: You are using a deprecated, model_name alias '{model_name}'. TransformerLens will now load the official transformers model name, '{official_name}' instead.\n Please update your code to use the official name by changing model_name from '{model_name}' to '{official_name}'.\nSince TransformerLens v3, all model names should be the official transformers model names.\nThe aliases may be removed in a future version of TransformerLens, so please do the update now." ) model_name = official_name if checkpoint_index is not None or checkpoint_value is not None: diff --git a/transformer_lens/pretrained/__init__.py b/transformer_lens/pretrained/__init__.py deleted file mode 100644 index c67f467453..0000000000 --- a/transformer_lens/pretrained/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Pretrained model utilities and weight conversions.""" diff --git a/transformer_lens/pretrained/weight_conversions/__init__.py b/transformer_lens/pretrained/weight_conversions/__init__.py deleted file mode 100644 index f2df6c917d..0000000000 --- a/transformer_lens/pretrained/weight_conversions/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -from .bert import convert_bert_weights -from .bloom import convert_bloom_weights -from .coder import convert_coder_weights -from .gemma import convert_gemma_weights -from .gpt2 import convert_gpt2_weights -from .gptj import convert_gptj_weights -from .llama import convert_llama_weights -from .mingpt import convert_mingpt_weights -from .mistral import convert_mistral_weights -from .mixtral import convert_mixtral_weights -from .nanogpt import convert_nanogpt_weights -from .neel_solu_old import convert_neel_solu_old_weights -from .neo import convert_neo_weights -from .neox import convert_neox_weights -from .olmo import convert_olmo_weights -from .olmo2 import convert_olmo2_weights -from .olmo3 import convert_olmo3_weights -from .olmoe import convert_olmoe_weights -from .opt import convert_opt_weights -from .phi import convert_phi_weights -from .phi3 import convert_phi3_weights -from .qwen import convert_qwen_weights -from .qwen2 import convert_qwen2_weights -from .qwen3 import convert_qwen3_weights -from .t5 import convert_t5_weights -from .hubert import convert_hubert_weights -from .apertus import convert_apertus_weights -from .openai import convert_gpt_oss_weights diff --git a/transformer_lens/pretrained/weight_conversions/apertus.py b/transformer_lens/pretrained/weight_conversions/apertus.py deleted file mode 100644 index c9211f975e..0000000000 --- a/transformer_lens/pretrained/weight_conversions/apertus.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Apertus weight conversion. - -Converts Apertus (Swiss AI) weights to HookedTransformer format. Apertus is -structurally similar to Llama but uses non-gated MLP with XIeLU activation, -and different layer norm names (attention_layernorm / feedforward_layernorm). -""" - -import logging -from typing import cast - -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - -logger = logging.getLogger(__name__) - -logger = logging.getLogger(__name__) - - -def convert_apertus_weights(apertus, cfg: TransformerLensConfig): - state_dict = {} - - state_dict["embed.W_E"] = apertus.model.embed_tokens.weight - - using_gqa = cfg.n_key_value_heads is not None - gqa_uscore = "_" if using_gqa else "" - n_kv_heads = cast(int, cfg.n_key_value_heads if using_gqa else cfg.n_heads) - - assert cfg.d_mlp is not None # keep mypy happy - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = apertus.model.layers[l].attention_layernorm.weight - - W_Q = apertus.model.layers[l].self_attn.q_proj.weight - W_K = apertus.model.layers[l].self_attn.k_proj.weight - W_V = apertus.model.layers[l].self_attn.v_proj.weight - - if not getattr(cfg, "load_in_4bit", False): - W_Q = einops.rearrange(W_Q, "(n h) m->n m h", n=cfg.n_heads) - W_K = einops.rearrange(W_K, "(n h) m->n m h", n=n_kv_heads) - W_V = einops.rearrange(W_V, "(n h) m->n m h", n=n_kv_heads) - - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.{gqa_uscore}W_K"] = W_K - state_dict[f"blocks.{l}.attn.{gqa_uscore}W_V"] = W_V - - # QK normalization weights - if getattr(cfg, "use_qk_norm", False): - state_dict[f"blocks.{l}.attn.q_norm.w"] = apertus.model.layers[ - l - ].self_attn.q_norm.weight - state_dict[f"blocks.{l}.attn.k_norm.w"] = apertus.model.layers[ - l - ].self_attn.k_norm.weight - - state_dict[f"blocks.{l}.attn.b_Q"] = torch.zeros( - cfg.n_heads, cfg.d_head, dtype=cfg.dtype, device=cfg.device - ) - state_dict[f"blocks.{l}.attn.{gqa_uscore}b_K"] = torch.zeros( - n_kv_heads, - cfg.d_head, - dtype=cfg.dtype, - device=cfg.device, - ) - state_dict[f"blocks.{l}.attn.{gqa_uscore}b_V"] = torch.zeros( - n_kv_heads, - cfg.d_head, - dtype=cfg.dtype, - device=cfg.device, - ) - - W_O = apertus.model.layers[l].self_attn.o_proj.weight - - if not getattr(cfg, "load_in_4bit", False): - W_O = einops.rearrange(W_O, "m (n h)->n h m", n=cfg.n_heads) - - state_dict[f"blocks.{l}.attn.W_O"] = W_O.to(device=cfg.device) - - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros( - cfg.d_model, dtype=cfg.dtype, device=cfg.device - ) - - state_dict[f"blocks.{l}.ln2.w"] = apertus.model.layers[l].feedforward_layernorm.weight - if not getattr(cfg, "load_in_4bit", False): - state_dict[f"blocks.{l}.mlp.W_in"] = apertus.model.layers[l].mlp.up_proj.weight.T - state_dict[f"blocks.{l}.mlp.W_out"] = apertus.model.layers[l].mlp.down_proj.weight.T - else: - state_dict[f"blocks.{l}.mlp.W_in"] = apertus.model.layers[l].mlp.up_proj.weight - state_dict[f"blocks.{l}.mlp.W_out"] = apertus.model.layers[l].mlp.down_proj.weight - - state_dict[f"blocks.{l}.mlp.b_in"] = torch.zeros( - cfg.d_mlp, dtype=cfg.dtype, device=cfg.device - ) - state_dict[f"blocks.{l}.mlp.b_out"] = torch.zeros( - cfg.d_model, dtype=cfg.dtype, device=cfg.device - ) - - # Extract trainable XIeLU activation parameters - mlp = apertus.model.layers[l].mlp - try: - if hasattr(mlp, "act_fn"): - alpha_p = mlp.act_fn.alpha_p - alpha_n = mlp.act_fn.alpha_n - beta = mlp.act_fn.beta - elif hasattr(mlp, "act"): - alpha_p = mlp.act.alpha_p - alpha_n = mlp.act.alpha_n - beta = mlp.act.beta - else: - alpha_p = mlp.alpha_p - alpha_n = mlp.alpha_n - beta = mlp.beta - state_dict[f"blocks.{l}.mlp.act_fn.alpha_p"] = alpha_p - state_dict[f"blocks.{l}.mlp.act_fn.alpha_n"] = alpha_n - state_dict[f"blocks.{l}.mlp.act_fn.beta"] = beta - except AttributeError: - logger.warning("XIeLU activation parameters not found in layer %d, using defaults", l) - state_dict[f"blocks.{l}.mlp.act_fn.alpha_p"] = torch.tensor( - 0.8, dtype=cfg.dtype, device=cfg.device - ) - state_dict[f"blocks.{l}.mlp.act_fn.alpha_n"] = torch.tensor( - 0.8, dtype=cfg.dtype, device=cfg.device - ) - state_dict[f"blocks.{l}.mlp.act_fn.beta"] = torch.tensor( - 0.5, dtype=cfg.dtype, device=cfg.device - ) - - state_dict["ln_final.w"] = apertus.model.norm.weight - - state_dict["unembed.W_U"] = apertus.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype, device=cfg.device) - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/bert.py b/transformer_lens/pretrained/weight_conversions/bert.py deleted file mode 100644 index ef98449e14..0000000000 --- a/transformer_lens/pretrained/weight_conversions/bert.py +++ /dev/null @@ -1,75 +0,0 @@ -import einops - -from transformer_lens.config import TransformerLensConfig - - -def convert_bert_weights(bert, cfg: TransformerLensConfig): - embeddings = bert.bert.embeddings - state_dict = { - "embed.embed.W_E": embeddings.word_embeddings.weight, - "embed.pos_embed.W_pos": embeddings.position_embeddings.weight, - "embed.token_type_embed.W_token_type": embeddings.token_type_embeddings.weight, - "embed.ln.w": embeddings.LayerNorm.weight, - "embed.ln.b": embeddings.LayerNorm.bias, - } - - for l in range(cfg.n_layers): - block = bert.bert.encoder.layer[l] - state_dict[f"blocks.{l}.attn.W_Q"] = einops.rearrange( - block.attention.self.query.weight, "(i h) m -> i m h", i=cfg.n_heads - ) - state_dict[f"blocks.{l}.attn.b_Q"] = einops.rearrange( - block.attention.self.query.bias, "(i h) -> i h", i=cfg.n_heads - ) - state_dict[f"blocks.{l}.attn.W_K"] = einops.rearrange( - block.attention.self.key.weight, "(i h) m -> i m h", i=cfg.n_heads - ) - state_dict[f"blocks.{l}.attn.b_K"] = einops.rearrange( - block.attention.self.key.bias, "(i h) -> i h", i=cfg.n_heads - ) - state_dict[f"blocks.{l}.attn.W_V"] = einops.rearrange( - block.attention.self.value.weight, "(i h) m -> i m h", i=cfg.n_heads - ) - state_dict[f"blocks.{l}.attn.b_V"] = einops.rearrange( - block.attention.self.value.bias, "(i h) -> i h", i=cfg.n_heads - ) - state_dict[f"blocks.{l}.attn.W_O"] = einops.rearrange( - block.attention.output.dense.weight, - "m (i h) -> i h m", - i=cfg.n_heads, - ) - state_dict[f"blocks.{l}.attn.b_O"] = block.attention.output.dense.bias - state_dict[f"blocks.{l}.ln1.w"] = block.attention.output.LayerNorm.weight - state_dict[f"blocks.{l}.ln1.b"] = block.attention.output.LayerNorm.bias - state_dict[f"blocks.{l}.mlp.W_in"] = einops.rearrange( - block.intermediate.dense.weight, "mlp model -> model mlp" - ) - state_dict[f"blocks.{l}.mlp.b_in"] = block.intermediate.dense.bias - state_dict[f"blocks.{l}.mlp.W_out"] = einops.rearrange( - block.output.dense.weight, "model mlp -> mlp model" - ) - state_dict[f"blocks.{l}.mlp.b_out"] = block.output.dense.bias - state_dict[f"blocks.{l}.ln2.w"] = block.output.LayerNorm.weight - state_dict[f"blocks.{l}.ln2.b"] = block.output.LayerNorm.bias - - pooler = bert.bert.pooler - state_dict["pooler.W"] = pooler.dense.weight.T - state_dict["pooler.b"] = pooler.dense.bias - - mlm_head = bert.cls.predictions - state_dict["mlm_head.W"] = mlm_head.transform.dense.weight.T - state_dict["mlm_head.b"] = mlm_head.transform.dense.bias - state_dict["mlm_head.ln.w"] = mlm_head.transform.LayerNorm.weight - state_dict["mlm_head.ln.b"] = mlm_head.transform.LayerNorm.bias - - # The NSP head does not have an unembedding - # so we are only using weights from the MLM head - # Note: BERT uses tied embeddings - state_dict["unembed.W_U"] = mlm_head.decoder.weight.T - state_dict["unembed.b_U"] = mlm_head.decoder.bias - - nsp_head = bert.cls.seq_relationship - state_dict["nsp_head.W"] = nsp_head.weight.T - state_dict["nsp_head.b"] = nsp_head.bias - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/bloom.py b/transformer_lens/pretrained/weight_conversions/bloom.py deleted file mode 100644 index 85ad36ad39..0000000000 --- a/transformer_lens/pretrained/weight_conversions/bloom.py +++ /dev/null @@ -1,57 +0,0 @@ -import einops - -from transformer_lens.config import TransformerLensConfig - - -def convert_bloom_weights(bloom, cfg: TransformerLensConfig): - state_dict = {} - - state_dict["embed.W_E"] = bloom.transformer.word_embeddings.weight - - # Bloom uses post embedding layer norm - state_dict["embed.ln.w"] = bloom.transformer.word_embeddings_layernorm.weight - state_dict["embed.ln.b"] = bloom.transformer.word_embeddings_layernorm.bias - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = bloom.transformer.h[l].input_layernorm.weight - state_dict[f"blocks.{l}.ln1.b"] = bloom.transformer.h[l].input_layernorm.bias - - W = bloom.transformer.h[l].self_attention.query_key_value.weight - - W_split = W.T.reshape(cfg.d_model, cfg.n_heads, 3, cfg.d_head) - - W_Q, W_K, W_V = W_split[..., 0, :], W_split[..., 1, :], W_split[..., 2, :] - W_Q = einops.rearrange(W_Q, "m n h ->n m h", n=cfg.n_heads) - W_K = einops.rearrange(W_K, "m n h ->n m h", n=cfg.n_heads) - W_V = einops.rearrange(W_V, "m n h ->n m h", n=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.W_K"] = W_K - state_dict[f"blocks.{l}.attn.W_V"] = W_V - - qkv_bias = bloom.transformer.h[l].self_attention.query_key_value.bias - qkv_bias = qkv_bias.reshape(cfg.n_heads, 3, cfg.d_head) - - state_dict[f"blocks.{l}.attn.b_Q"] = qkv_bias[:, 0, :] - state_dict[f"blocks.{l}.attn.b_K"] = qkv_bias[:, 1, :] - state_dict[f"blocks.{l}.attn.b_V"] = qkv_bias[:, 2, :] - - W_O = bloom.transformer.h[l].self_attention.dense.weight.T # [1024, 1024] - W_O = einops.rearrange(W_O, "(n h) m->n h m", n=cfg.n_heads) # [n_heads, d_head, d_model] - state_dict[f"blocks.{l}.attn.W_O"] = W_O - state_dict[f"blocks.{l}.attn.b_O"] = bloom.transformer.h[l].self_attention.dense.bias - - state_dict[f"blocks.{l}.ln2.w"] = bloom.transformer.h[l].post_attention_layernorm.weight - state_dict[f"blocks.{l}.ln2.b"] = bloom.transformer.h[l].post_attention_layernorm.bias - - W_in = bloom.transformer.h[l].mlp.dense_h_to_4h.weight.T - state_dict[f"blocks.{l}.mlp.W_in"] = W_in - state_dict[f"blocks.{l}.mlp.b_in"] = bloom.transformer.h[l].mlp.dense_h_to_4h.bias - - W_out = bloom.transformer.h[l].mlp.dense_4h_to_h.weight.T - state_dict[f"blocks.{l}.mlp.W_out"] = W_out - state_dict[f"blocks.{l}.mlp.b_out"] = bloom.transformer.h[l].mlp.dense_4h_to_h.bias - state_dict["unembed.W_U"] = bloom.lm_head.weight.T - - state_dict["ln_final.w"] = bloom.transformer.ln_f.weight - state_dict["ln_final.b"] = bloom.transformer.ln_f.bias - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/coder.py b/transformer_lens/pretrained/weight_conversions/coder.py deleted file mode 100644 index 6218efc789..0000000000 --- a/transformer_lens/pretrained/weight_conversions/coder.py +++ /dev/null @@ -1,63 +0,0 @@ -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_coder_weights(model, cfg: TransformerLensConfig): - state_dict = {} - - state_dict["embed.W_E"] = model.transformer.wte.weight - state_dict["pos_embed.W_pos"] = model.transformer.wpe.weight - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = model.transformer.h[l].ln_1.weight - state_dict[f"blocks.{l}.ln1.b"] = model.transformer.h[l].ln_1.bias - - # In GPT-2, q,k,v are produced by one big linear map, whose output is - # concat([q, k, v]) - W_KV = model.transformer.h[l].attn.kv_attn.weight # [d_model, 2 * d_head] - W_K, W_V = torch.tensor_split(W_KV, 2, dim=1) - W_Q = model.transformer.h[l].attn.q_attn.weight # [d_model, d_model] - W_Q = einops.rearrange(W_Q, "m (i h)->i m h", i=cfg.n_heads) - W_K = einops.repeat(W_K, "m h -> i m h", i=cfg.n_heads) - W_V = einops.repeat(W_V, "m h -> i m h", i=cfg.n_heads) - - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.W_K"] = W_K - state_dict[f"blocks.{l}.attn.W_V"] = W_V - - b_Q = einops.rearrange( - model.transformer.h[l].attn.q_attn.bias, - "(index head)-> index head", - index=cfg.n_heads, - head=cfg.d_head, - ) - b_KV = model.transformer.h[l].attn.kv_attn.bias # [2 * d_head] - b_K, b_V = torch.tensor_split(b_KV, 2, dim=0) - b_K = einops.repeat(b_K, "head -> index head", index=cfg.n_heads) - b_V = einops.repeat(b_V, "head -> index head", index=cfg.n_heads) - state_dict[f"blocks.{l}.attn.b_Q"] = b_Q - state_dict[f"blocks.{l}.attn.b_K"] = b_K - state_dict[f"blocks.{l}.attn.b_V"] = b_V - - W_O = model.transformer.h[l].attn.c_proj.weight - W_O = einops.rearrange(W_O, "(i h) m->i h m", i=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - state_dict[f"blocks.{l}.attn.b_O"] = model.transformer.h[l].attn.c_proj.bias - - state_dict[f"blocks.{l}.ln2.w"] = model.transformer.h[l].ln_2.weight - state_dict[f"blocks.{l}.ln2.b"] = model.transformer.h[l].ln_2.bias - - W_in = model.transformer.h[l].mlp.c_fc.weight - state_dict[f"blocks.{l}.mlp.W_in"] = W_in - state_dict[f"blocks.{l}.mlp.b_in"] = model.transformer.h[l].mlp.c_fc.bias - - W_out = model.transformer.h[l].mlp.c_proj.weight - state_dict[f"blocks.{l}.mlp.W_out"] = W_out - state_dict[f"blocks.{l}.mlp.b_out"] = model.transformer.h[l].mlp.c_proj.bias - state_dict["unembed.W_U"] = model.lm_head.weight.T - - state_dict["ln_final.w"] = model.transformer.ln_f.weight - state_dict["ln_final.b"] = model.transformer.ln_f.bias - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/gemma.py b/transformer_lens/pretrained/weight_conversions/gemma.py deleted file mode 100644 index ce84edb922..0000000000 --- a/transformer_lens/pretrained/weight_conversions/gemma.py +++ /dev/null @@ -1,139 +0,0 @@ -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_gemma_weights(gemma, cfg: TransformerLensConfig): - state_dict = {} - - assert cfg.n_key_value_heads is not None # keep mypy happy - assert cfg.d_mlp is not None # keep mypy happy - - # Multimodal detection must survive transformers 5.x, which moved - # Gemma3ForConditionalGeneration's language_model under .model — the old - # top-level probe silently reported text-only there and converted the - # multimodal model down the wrong path. - language_model = getattr(gemma, "language_model", None) - if language_model is None: - language_model = getattr(getattr(gemma, "model", None), "language_model", None) - - if language_model is not None: - # Vision tower is skipped entirely; only the text transformer converts. - base_model = getattr(language_model, "model", language_model) - else: - # Text-only Gemma3ForCausalLM has .model wrapper - base_model = gemma.model - - # Gemma Models scale embeddings by multiplying by sqrt(d_model), use hidden state type to match - # HF implementation - state_dict["embed.W_E"] = base_model.embed_tokens.weight * torch.tensor( - cfg.d_model**0.5, dtype=cfg.dtype - ) - - # Gemma has no biases anywhere - for l in range(cfg.n_layers): - # GemmaRMSNorm adds 1 to weights before multiplying by input, keep RMS calcs in float32 - state_dict[f"blocks.{l}.ln1.w"] = base_model.layers[ - l - ].input_layernorm.weight.float() + torch.ones_like( - base_model.layers[l].input_layernorm.weight, dtype=torch.float32 - ) - if getattr(cfg, "use_normalization_before_and_after", False): - # Only applies for Gemma 2 - state_dict[f"blocks.{l}.ln1_post.w"] = base_model.layers[ - l - ].post_attention_layernorm.weight.float() + torch.ones_like( - base_model.layers[l].input_layernorm.weight, dtype=torch.float32 - ) - - W_Q = base_model.layers[l].self_attn.q_proj.weight - W_K = base_model.layers[l].self_attn.k_proj.weight - W_V = base_model.layers[l].self_attn.v_proj.weight - W_Q = einops.rearrange(W_Q, "(n h) m->n m h", n=cfg.n_heads) - W_K = einops.rearrange(W_K, "(n h) m->n m h", n=cfg.n_key_value_heads) - W_V = einops.rearrange(W_V, "(n h) m->n m h", n=cfg.n_key_value_heads) - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn._W_K"] = W_K - state_dict[f"blocks.{l}.attn._W_V"] = W_V - - # Load q_norm and k_norm if they exist (Gemma 3) - # Gemma3RMSNorm adds 1 to weights in forward(), so we pre-add it here - if getattr(cfg, "use_qk_norm", False): - state_dict[f"blocks.{l}.attn.q_norm.w"] = base_model.layers[ - l - ].self_attn.q_norm.weight.float() + torch.ones_like( - base_model.layers[l].self_attn.q_norm.weight, dtype=torch.float32 - ) - state_dict[f"blocks.{l}.attn.k_norm.w"] = base_model.layers[ - l - ].self_attn.k_norm.weight.float() + torch.ones_like( - base_model.layers[l].self_attn.k_norm.weight, dtype=torch.float32 - ) - - state_dict[f"blocks.{l}.attn.b_Q"] = torch.zeros( - cfg.n_heads, cfg.d_head, dtype=cfg.dtype, device=W_Q.device - ) - state_dict[f"blocks.{l}.attn._b_K"] = torch.zeros( - cfg.n_key_value_heads, cfg.d_head, dtype=cfg.dtype, device=W_K.device - ) - state_dict[f"blocks.{l}.attn._b_V"] = torch.zeros( - cfg.n_key_value_heads, cfg.d_head, dtype=cfg.dtype, device=W_V.device - ) - - W_O = base_model.layers[l].self_attn.o_proj.weight - W_O = einops.rearrange(W_O, "m (n h)->n h m", n=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros( - cfg.d_model, dtype=cfg.dtype, device=W_O.device - ) - - # GemmaRMSNorm adds 1 to weights before multiplying by input, keep RMS calcs in float32 - if not getattr(cfg, "use_normalization_before_and_after", False): - # Only applies for Gemma 1. Confusingly post_attention_layernorm is applied to mlp_input in Gemma 1 and attn_out in Gemma 2 - state_dict[f"blocks.{l}.ln2.w"] = base_model.layers[ - l - ].post_attention_layernorm.weight.float() + torch.ones_like( - base_model.norm.weight, dtype=torch.float32 - ) - else: - # Only applies for Gemma 2 - state_dict[f"blocks.{l}.ln2.w"] = base_model.layers[ - l - ].pre_feedforward_layernorm.weight.float() + torch.ones_like( - base_model.layers[l].pre_feedforward_layernorm.weight, dtype=torch.float32 - ) - state_dict[f"blocks.{l}.ln2_post.w"] = base_model.layers[ - l - ].post_feedforward_layernorm.weight.float() + torch.ones_like( - base_model.layers[l].post_feedforward_layernorm.weight, dtype=torch.float32 - ) - - state_dict[f"blocks.{l}.mlp.W_in"] = base_model.layers[l].mlp.up_proj.weight.T - state_dict[f"blocks.{l}.mlp.W_gate"] = base_model.layers[l].mlp.gate_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_in"] = torch.zeros( - cfg.d_mlp, dtype=cfg.dtype, device=base_model.layers[l].mlp.up_proj.weight.device - ) - - state_dict[f"blocks.{l}.mlp.W_out"] = base_model.layers[l].mlp.down_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_out"] = torch.zeros( - cfg.d_model, dtype=cfg.dtype, device=base_model.layers[l].mlp.down_proj.weight.device - ) - - # GemmaRMSNorm adds 1 to weights before multiplying by input, keep RMS calcs in float32 - state_dict["ln_final.w"] = base_model.norm.weight.float() + torch.ones_like( - base_model.norm.weight, dtype=torch.float32 - ) - - # For multimodal models, lm_head might not exist or be tied to embeddings - if hasattr(gemma, "lm_head"): - state_dict["unembed.W_U"] = gemma.lm_head.weight.T - unembed_device = gemma.lm_head.weight.device - else: - # Multimodal models might use tied embeddings - state_dict["unembed.W_U"] = base_model.embed_tokens.weight.T - unembed_device = base_model.embed_tokens.weight.device - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype, device=unembed_device) - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/gpt2.py b/transformer_lens/pretrained/weight_conversions/gpt2.py deleted file mode 100644 index 7dcd8dba17..0000000000 --- a/transformer_lens/pretrained/weight_conversions/gpt2.py +++ /dev/null @@ -1,61 +0,0 @@ -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_gpt2_weights(gpt2, cfg: TransformerLensConfig): - state_dict = {} - - state_dict["embed.W_E"] = gpt2.transformer.wte.weight - - # Trim positional embeddings to n_ctx if the pretrained weights have more - # positions than the model expects. - pos_embed = gpt2.transformer.wpe.weight - if pos_embed.shape[0] > cfg.n_ctx: - pos_embed = pos_embed[: cfg.n_ctx, :] - state_dict["pos_embed.W_pos"] = pos_embed - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = gpt2.transformer.h[l].ln_1.weight - state_dict[f"blocks.{l}.ln1.b"] = gpt2.transformer.h[l].ln_1.bias - - # In GPT-2, q,k,v are produced by one big linear map, whose output is - # concat([q, k, v]) - W = gpt2.transformer.h[l].attn.c_attn.weight - W_Q_flat, W_K_flat, W_V_flat = torch.split(W, cfg.d_model, dim=1) - W_Q = einops.rearrange(W_Q_flat, "m (i h)->i m h", i=cfg.n_heads) - W_K = einops.rearrange(W_K_flat, "m (i h)->i m h", i=cfg.n_heads) - W_V = einops.rearrange(W_V_flat, "m (i h)->i m h", i=cfg.n_heads) - - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.W_K"] = W_K - state_dict[f"blocks.{l}.attn.W_V"] = W_V - - qkv_bias = gpt2.transformer.h[l].attn.c_attn.bias - qkv_bias = qkv_bias.view(3, cfg.d_model) - qkv_bias = qkv_bias.view(3, cfg.n_heads, cfg.d_head) - state_dict[f"blocks.{l}.attn.b_Q"] = qkv_bias[0] - state_dict[f"blocks.{l}.attn.b_K"] = qkv_bias[1] - state_dict[f"blocks.{l}.attn.b_V"] = qkv_bias[2] - - W_O = gpt2.transformer.h[l].attn.c_proj.weight - W_O = einops.rearrange(W_O, "(i h) m->i h m", i=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - state_dict[f"blocks.{l}.attn.b_O"] = gpt2.transformer.h[l].attn.c_proj.bias - - state_dict[f"blocks.{l}.ln2.w"] = gpt2.transformer.h[l].ln_2.weight - state_dict[f"blocks.{l}.ln2.b"] = gpt2.transformer.h[l].ln_2.bias - - W_in = gpt2.transformer.h[l].mlp.c_fc.weight - state_dict[f"blocks.{l}.mlp.W_in"] = W_in - state_dict[f"blocks.{l}.mlp.b_in"] = gpt2.transformer.h[l].mlp.c_fc.bias - - W_out = gpt2.transformer.h[l].mlp.c_proj.weight - state_dict[f"blocks.{l}.mlp.W_out"] = W_out - state_dict[f"blocks.{l}.mlp.b_out"] = gpt2.transformer.h[l].mlp.c_proj.bias - state_dict["unembed.W_U"] = gpt2.lm_head.weight.T - - state_dict["ln_final.w"] = gpt2.transformer.ln_f.weight - state_dict["ln_final.b"] = gpt2.transformer.ln_f.bias - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/gptj.py b/transformer_lens/pretrained/weight_conversions/gptj.py deleted file mode 100644 index 3056e0ba2d..0000000000 --- a/transformer_lens/pretrained/weight_conversions/gptj.py +++ /dev/null @@ -1,50 +0,0 @@ -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_gptj_weights(gptj, cfg: TransformerLensConfig): - state_dict = {} - - state_dict["embed.W_E"] = gptj.transformer.wte.weight - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = gptj.transformer.h[l].ln_1.weight - state_dict[f"blocks.{l}.ln1.b"] = gptj.transformer.h[l].ln_1.bias - - W_Q = gptj.transformer.h[l].attn.q_proj.weight - W_K = gptj.transformer.h[l].attn.k_proj.weight - W_V = gptj.transformer.h[l].attn.v_proj.weight - W_Q = einops.rearrange(W_Q, "(i h) m->i m h", i=cfg.n_heads) - W_K = einops.rearrange(W_K, "(i h) m->i m h", i=cfg.n_heads) - W_V = einops.rearrange(W_V, "(i h) m->i m h", i=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.W_K"] = W_K - state_dict[f"blocks.{l}.attn.W_V"] = W_V - - state_dict[f"blocks.{l}.attn.b_Q"] = torch.zeros(cfg.n_heads, cfg.d_head, dtype=cfg.dtype) - state_dict[f"blocks.{l}.attn.b_K"] = torch.zeros(cfg.n_heads, cfg.d_head, dtype=cfg.dtype) - state_dict[f"blocks.{l}.attn.b_V"] = torch.zeros(cfg.n_heads, cfg.d_head, dtype=cfg.dtype) - - W_O = gptj.transformer.h[l].attn.out_proj.weight - W_O = einops.rearrange(W_O, "m (i h)->i h m", i=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - # Layer Norm 1 and 2 are tied. - state_dict[f"blocks.{l}.ln2.w"] = state_dict[f"blocks.{l}.ln1.w"] - state_dict[f"blocks.{l}.ln2.b"] = state_dict[f"blocks.{l}.ln1.b"] - - state_dict[f"blocks.{l}.mlp.W_in"] = gptj.transformer.h[l].mlp.fc_in.weight.T - state_dict[f"blocks.{l}.mlp.b_in"] = gptj.transformer.h[l].mlp.fc_in.bias - - state_dict[f"blocks.{l}.mlp.W_out"] = gptj.transformer.h[l].mlp.fc_out.weight.T - state_dict[f"blocks.{l}.mlp.b_out"] = gptj.transformer.h[l].mlp.fc_out.bias - state_dict["ln_final.w"] = gptj.transformer.ln_f.weight - state_dict["ln_final.b"] = gptj.transformer.ln_f.bias - - state_dict["unembed.W_U"] = gptj.lm_head.weight.T - # Contains a bias, for some reason? - state_dict["unembed.b_U"] = gptj.lm_head.bias - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/hubert.py b/transformer_lens/pretrained/weight_conversions/hubert.py deleted file mode 100644 index b67961d520..0000000000 --- a/transformer_lens/pretrained/weight_conversions/hubert.py +++ /dev/null @@ -1,145 +0,0 @@ -import einops - -from transformer_lens.config import TransformerLensConfig - - -def convert_hubert_weights(hf_model, cfg: TransformerLensConfig): - """Convert transformer encoder weights from a HuggingFace HuBERT model - into the state_dict expected by Transformer-Lens' HookedEncoder. - - Intentionally skips the convolutional frontend and feature_projection; - those are used directly from the HF model. Use - ``model.load_state_dict(state_dict, strict=False)`` to load these. - """ - state_dict = {} - - # Try to find the encoder layer list (different HF variants use .layers or .layer) - encoder = getattr(hf_model, "encoder", None) - if encoder is None: - raise ValueError("hf_model has no .encoder attribute") - - encoder_layers = getattr(encoder, "layers", None) or getattr(encoder, "layer", None) - if encoder_layers is None: - # maybe hf_model itself is the encoder (unlikely), or a wrapped attribute - raise ValueError("Couldn't find encoder.layers or encoder.layer on hf_model.encoder") - - # Use cfg dims for reshaping - d_model = cfg.d_model - n_heads = cfg.n_heads - - for l, layer in enumerate(encoder_layers): - # --- Attention module --- - # Some HF variants might call it `attention`, others `self_attn` etc. - att = getattr(layer, "attention", None) or getattr(layer, "self_attn", None) - if att is None: - raise AttributeError(f"Encoder layer {l} has no 'attention' or 'self_attn' attribute") - - # q/k/v/out proj names in HuBERT's HubertAttention: q_proj, k_proj, v_proj, out_proj - # fall back to common alternatives if present - q_w = getattr(att, "q_proj", None) - k_w = getattr(att, "k_proj", None) - v_w = getattr(att, "v_proj", None) - o_w = getattr(att, "out_proj", None) or getattr(att, "proj", None) - - if any(x is None for x in (q_w, k_w, v_w, o_w)): - # Try alternate nested attributes like att.q, att.k, att.v, att.o - q_w = q_w or getattr(att, "q", None) - k_w = k_w or getattr(att, "k", None) - v_w = v_w or getattr(att, "v", None) - o_w = o_w or getattr(att, "o", None) - - if any(x is None for x in (q_w, k_w, v_w, o_w)): - raise AttributeError(f"Could not find q/k/v/out projections in layer {l}. Found: {att}") - - assert q_w is not None and k_w is not None and v_w is not None and o_w is not None - - # weights are Linear modules: weight shape (out, in) => same convention as Bert conversion - # reshape to Transformer-Lens expected shapes using einops - state_dict[f"blocks.{l}.attn.W_Q"] = einops.rearrange( - q_w.weight, "(i h) m -> i m h", i=n_heads - ) - if q_w.bias is not None: - state_dict[f"blocks.{l}.attn.b_Q"] = einops.rearrange( - q_w.bias, "(i h) -> i h", i=n_heads - ) - - state_dict[f"blocks.{l}.attn.W_K"] = einops.rearrange( - k_w.weight, "(i h) m -> i m h", i=n_heads - ) - if k_w.bias is not None: - state_dict[f"blocks.{l}.attn.b_K"] = einops.rearrange( - k_w.bias, "(i h) -> i h", i=n_heads - ) - - state_dict[f"blocks.{l}.attn.W_V"] = einops.rearrange( - v_w.weight, "(i h) m -> i m h", i=n_heads - ) - if v_w.bias is not None: - state_dict[f"blocks.{l}.attn.b_V"] = einops.rearrange( - v_w.bias, "(i h) -> i h", i=n_heads - ) - - state_dict[f"blocks.{l}.attn.W_O"] = einops.rearrange( - o_w.weight, "m (i h) -> i h m", i=n_heads - ) - if o_w.bias is not None: - state_dict[f"blocks.{l}.attn.b_O"] = o_w.bias - - # --- Layer norms inside the layer --- - # HuBERT layer has `layer.layer_norm` and `layer.final_layer_norm` - ln1 = getattr(layer, "layer_norm", None) - ln2 = getattr(layer, "final_layer_norm", None) - if ln1 is None or ln2 is None: - # try alternative names - ln1 = ln1 or getattr(layer, "attention_norm", None) - ln2 = ln2 or getattr(layer, "output_layer_norm", None) - - if ln1 is not None: - state_dict[f"blocks.{l}.ln1.w"] = ln1.weight - state_dict[f"blocks.{l}.ln1.b"] = ln1.bias - if ln2 is not None: - state_dict[f"blocks.{l}.ln2.w"] = ln2.weight - state_dict[f"blocks.{l}.ln2.b"] = ln2.bias - - # --- Feed-forward / MLP --- - # HuBERT uses `feed_forward` which contains intermediate_dense and output_dense - ff = ( - getattr(layer, "feed_forward", None) - or getattr(layer, "feedforward", None) - or getattr(layer, "ff", None) - ) - if ff is None: - raise AttributeError(f"Layer {l} has no feed_forward/ff attribute") - - # Many implementations name them intermediate_dense and output_dense - fc1 = ( - getattr(ff, "intermediate_dense", None) - or getattr(ff, "fc1", None) - or getattr(ff, "linear1", None) - ) - fc2 = ( - getattr(ff, "output_dense", None) - or getattr(ff, "fc2", None) - or getattr(ff, "linear2", None) - ) - - if fc1 is None or fc2 is None: - raise AttributeError(f"Could not find FFN dense layers in layer {l}: {ff}") - - # fc1.weight shape: (d_mlp, d_model) -> Transformer-Lens expects (d_model, d_mlp) - state_dict[f"blocks.{l}.mlp.W_in"] = einops.rearrange(fc1.weight, "mlp model -> model mlp") - if fc1.bias is not None: - state_dict[f"blocks.{l}.mlp.b_in"] = fc1.bias - - # fc2.weight shape: (d_model, d_mlp) -> Transformer-Lens expects (d_mlp, d_model) - state_dict[f"blocks.{l}.mlp.W_out"] = einops.rearrange(fc2.weight, "model mlp -> mlp model") - if fc2.bias is not None: - state_dict[f"blocks.{l}.mlp.b_out"] = fc2.bias - - # --- Optional: encoder-level layer_norm (HubertModel.encoder.layer_norm) --- - if hasattr(hf_model.encoder, "layer_norm"): - ln_final = hf_model.encoder.layer_norm - state_dict["ln_final.w"] = ln_final.weight - state_dict["ln_final.b"] = ln_final.bias - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/llama.py b/transformer_lens/pretrained/weight_conversions/llama.py deleted file mode 100644 index d8f2c5f551..0000000000 --- a/transformer_lens/pretrained/weight_conversions/llama.py +++ /dev/null @@ -1,96 +0,0 @@ -from typing import cast - -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_llama_weights(llama, cfg: TransformerLensConfig): - state_dict = {} - - state_dict["embed.W_E"] = llama.model.embed_tokens.weight - - # Some models with the Llama architecture use Grouped Query Attention, and so for these we need to modify - # the state dict keys for the K/V attention weight/biases, prepending "_" to the key names. - using_gqa = cfg.n_key_value_heads is not None - gqa_uscore = "_" if using_gqa else "" - # need a cast since MyPy isn't smart enough to realize that using_gqa implies n_key_value_heads is not None - n_kv_heads = cast(int, cfg.n_key_value_heads if using_gqa else cfg.n_heads) - - # llama has no biases anywhere and deals with everything else roughly like - # GPTNeoX with different names - - assert cfg.d_mlp is not None # keep mypy happy - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = llama.model.layers[l].input_layernorm.weight - - W_Q = llama.model.layers[l].self_attn.q_proj.weight - W_K = llama.model.layers[l].self_attn.k_proj.weight - W_V = llama.model.layers[l].self_attn.v_proj.weight - - # in case of quantization, - # parameters should stay as bitsandbytes.nn.modules.Params4bit - if not getattr(cfg, "load_in_4bit", False): - W_Q = einops.rearrange(W_Q, "(n h) m->n m h", n=cfg.n_heads) - W_K = einops.rearrange(W_K, "(n h) m->n m h", n=n_kv_heads) - W_V = einops.rearrange(W_V, "(n h) m->n m h", n=n_kv_heads) - - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.{gqa_uscore}W_K"] = W_K - state_dict[f"blocks.{l}.attn.{gqa_uscore}W_V"] = W_V - - state_dict[f"blocks.{l}.attn.b_Q"] = torch.zeros( - cfg.n_heads, cfg.d_head, dtype=cfg.dtype, device=cfg.device - ) - state_dict[f"blocks.{l}.attn.{gqa_uscore}b_K"] = torch.zeros( - n_kv_heads, - cfg.d_head, - dtype=cfg.dtype, - device=cfg.device, - ) - state_dict[f"blocks.{l}.attn.{gqa_uscore}b_V"] = torch.zeros( - n_kv_heads, - cfg.d_head, - dtype=cfg.dtype, - device=cfg.device, - ) - - W_O = llama.model.layers[l].self_attn.o_proj.weight - - if not getattr(cfg, "load_in_4bit", False): - W_O = einops.rearrange(W_O, "m (n h)->n h m", n=cfg.n_heads) - - state_dict[f"blocks.{l}.attn.W_O"] = W_O.to(device=cfg.device) - - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros( - cfg.d_model, dtype=cfg.dtype, device=cfg.device - ) - - state_dict[f"blocks.{l}.ln2.w"] = llama.model.layers[l].post_attention_layernorm.weight - - # in case of quantization, - # parameters should stay as bitsandbytes.nn.modules.Params4bit - if not getattr(cfg, "load_in_4bit", False): - state_dict[f"blocks.{l}.mlp.W_in"] = llama.model.layers[l].mlp.up_proj.weight.T - state_dict[f"blocks.{l}.mlp.W_gate"] = llama.model.layers[l].mlp.gate_proj.weight.T - state_dict[f"blocks.{l}.mlp.W_out"] = llama.model.layers[l].mlp.down_proj.weight.T - else: - state_dict[f"blocks.{l}.mlp.W_in"] = llama.model.layers[l].mlp.up_proj.weight - state_dict[f"blocks.{l}.mlp.W_gate"] = llama.model.layers[l].mlp.gate_proj.weight - state_dict[f"blocks.{l}.mlp.W_out"] = llama.model.layers[l].mlp.down_proj.weight - - state_dict[f"blocks.{l}.mlp.b_in"] = torch.zeros( - cfg.d_mlp, dtype=cfg.dtype, device=cfg.device - ) - state_dict[f"blocks.{l}.mlp.b_out"] = torch.zeros( - cfg.d_model, dtype=cfg.dtype, device=cfg.device - ) - - state_dict["ln_final.w"] = llama.model.norm.weight - - state_dict["unembed.W_U"] = llama.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype, device=cfg.device) - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/mingpt.py b/transformer_lens/pretrained/weight_conversions/mingpt.py deleted file mode 100644 index e6eb2e8af8..0000000000 --- a/transformer_lens/pretrained/weight_conversions/mingpt.py +++ /dev/null @@ -1,63 +0,0 @@ -import einops - -from transformer_lens.config import TransformerLensConfig - - -def convert_mingpt_weights(old_state_dict, cfg: TransformerLensConfig): - # mingpt (https://github.com/karpathy/minGPT) is mostly similar to GPT-2, - # but doesn't concat the QKV matrices. - state_dict = {} - - state_dict["embed.W_E"] = old_state_dict["tok_emb.weight"] - state_dict["pos_embed.W_pos"] = old_state_dict["pos_emb"].squeeze() - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = old_state_dict[f"blocks.{l}.ln1.weight"] - state_dict[f"blocks.{l}.ln1.b"] = old_state_dict[f"blocks.{l}.ln1.bias"] - - W_Q = old_state_dict[f"blocks.{l}.attn.query.weight"] - W_K = old_state_dict[f"blocks.{l}.attn.key.weight"] - W_V = old_state_dict[f"blocks.{l}.attn.value.weight"] - W_Q = einops.rearrange(W_Q, "(i h) m->i m h", i=cfg.n_heads) - W_K = einops.rearrange(W_K, "(i h) m->i m h", i=cfg.n_heads) - W_V = einops.rearrange(W_V, "(i h) m->i m h", i=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.W_K"] = W_K - state_dict[f"blocks.{l}.attn.W_V"] = W_V - - q_bias = einops.rearrange( - old_state_dict[f"blocks.{l}.attn.query.bias"], "(i h)->i h", i=cfg.n_heads - ) - k_bias = einops.rearrange( - old_state_dict[f"blocks.{l}.attn.key.bias"], "(i h)->i h", i=cfg.n_heads - ) - v_bias = einops.rearrange( - old_state_dict[f"blocks.{l}.attn.value.bias"], "(i h)->i h", i=cfg.n_heads - ) - - state_dict[f"blocks.{l}.attn.b_Q"] = q_bias - state_dict[f"blocks.{l}.attn.b_K"] = k_bias - state_dict[f"blocks.{l}.attn.b_V"] = v_bias - - W_O = old_state_dict[f"blocks.{l}.attn.proj.weight"] - W_O = einops.rearrange(W_O, "m (i h)->i h m", i=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - state_dict[f"blocks.{l}.attn.b_O"] = old_state_dict[f"blocks.{l}.attn.proj.bias"] - - state_dict[f"blocks.{l}.ln2.w"] = old_state_dict[f"blocks.{l}.ln2.weight"] - state_dict[f"blocks.{l}.ln2.b"] = old_state_dict[f"blocks.{l}.ln2.bias"] - - W_in = old_state_dict[f"blocks.{l}.mlp.0.weight"] - state_dict[f"blocks.{l}.mlp.W_in"] = W_in.T - state_dict[f"blocks.{l}.mlp.b_in"] = old_state_dict[f"blocks.{l}.mlp.0.bias"] - - W_out = old_state_dict[f"blocks.{l}.mlp.2.weight"] - state_dict[f"blocks.{l}.mlp.W_out"] = W_out.T - state_dict[f"blocks.{l}.mlp.b_out"] = old_state_dict[f"blocks.{l}.mlp.2.bias"] - - state_dict["unembed.W_U"] = old_state_dict["head.weight"].T - - state_dict["ln_final.w"] = old_state_dict["ln_f.weight"] - state_dict["ln_final.b"] = old_state_dict["ln_f.bias"] - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/mistral.py b/transformer_lens/pretrained/weight_conversions/mistral.py deleted file mode 100644 index 4438bcf40c..0000000000 --- a/transformer_lens/pretrained/weight_conversions/mistral.py +++ /dev/null @@ -1,57 +0,0 @@ -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_mistral_weights(mistral, cfg: TransformerLensConfig): - state_dict = {} - - state_dict["embed.W_E"] = mistral.model.embed_tokens.weight - - assert cfg.n_key_value_heads is not None # keep mypy happy - assert cfg.d_mlp is not None # keep mypy happy - - # Mistral has no biases anywhere - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = mistral.model.layers[l].input_layernorm.weight - - W_Q = mistral.model.layers[l].self_attn.q_proj.weight - W_K = mistral.model.layers[l].self_attn.k_proj.weight - W_V = mistral.model.layers[l].self_attn.v_proj.weight - W_Q = einops.rearrange(W_Q, "(n h) m->n m h", n=cfg.n_heads) - W_K = einops.rearrange(W_K, "(n h) m->n m h", n=cfg.n_key_value_heads) - W_V = einops.rearrange(W_V, "(n h) m->n m h", n=cfg.n_key_value_heads) - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn._W_K"] = W_K - state_dict[f"blocks.{l}.attn._W_V"] = W_V - - state_dict[f"blocks.{l}.attn.b_Q"] = torch.zeros(cfg.n_heads, cfg.d_head, dtype=cfg.dtype) - state_dict[f"blocks.{l}.attn._b_K"] = torch.zeros( - cfg.n_key_value_heads, cfg.d_head, dtype=cfg.dtype - ) - state_dict[f"blocks.{l}.attn._b_V"] = torch.zeros( - cfg.n_key_value_heads, cfg.d_head, dtype=cfg.dtype - ) - - W_O = mistral.model.layers[l].self_attn.o_proj.weight - W_O = einops.rearrange(W_O, "m (n h)->n h m", n=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.ln2.w"] = mistral.model.layers[l].post_attention_layernorm.weight - - state_dict[f"blocks.{l}.mlp.W_in"] = mistral.model.layers[l].mlp.up_proj.weight.T - state_dict[f"blocks.{l}.mlp.W_gate"] = mistral.model.layers[l].mlp.gate_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_in"] = torch.zeros(cfg.d_mlp, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.mlp.W_out"] = mistral.model.layers[l].mlp.down_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_out"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict["ln_final.w"] = mistral.model.norm.weight - - state_dict["unembed.W_U"] = mistral.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/mixtral.py b/transformer_lens/pretrained/weight_conversions/mixtral.py deleted file mode 100644 index 9451add061..0000000000 --- a/transformer_lens/pretrained/weight_conversions/mixtral.py +++ /dev/null @@ -1,87 +0,0 @@ -import einops -import torch - -from transformer_lens.config import TransformerLensConfig -from transformer_lens.utilities.quantization import require_readable_weight - - -def convert_mixtral_weights(mixtral, cfg: TransformerLensConfig): - # The same as Mistral, but with the MLP replaced with MoE - # As with Mistral, Mixtral has no biases - - state_dict = {} - - assert cfg.n_key_value_heads is not None # keep mypy happy - assert cfg.d_mlp is not None - assert cfg.num_experts is not None - - state_dict["embed.W_E"] = mixtral.model.embed_tokens.weight - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = mixtral.model.layers[l].input_layernorm.weight - - W_Q = mixtral.model.layers[l].self_attn.q_proj.weight - W_K = mixtral.model.layers[l].self_attn.k_proj.weight - W_V = mixtral.model.layers[l].self_attn.v_proj.weight - W_Q = einops.rearrange(W_Q, "(n h) m->n m h", n=cfg.n_heads) - W_K = einops.rearrange(W_K, "(n h) m->n m h", n=cfg.n_key_value_heads) - W_V = einops.rearrange(W_V, "(n h) m->n m h", n=cfg.n_key_value_heads) - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn._W_K"] = W_K - state_dict[f"blocks.{l}.attn._W_V"] = W_V - - state_dict[f"blocks.{l}.attn.b_Q"] = torch.zeros(cfg.n_heads, cfg.d_head, dtype=cfg.dtype) - state_dict[f"blocks.{l}.attn._b_K"] = torch.zeros( - cfg.n_key_value_heads, cfg.d_head, dtype=cfg.dtype - ) - state_dict[f"blocks.{l}.attn._b_V"] = torch.zeros( - cfg.n_key_value_heads, cfg.d_head, dtype=cfg.dtype - ) - - W_O = mixtral.model.layers[l].self_attn.o_proj.weight - W_O = einops.rearrange(W_O, "m (n h)->n h m", n=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.ln2.w"] = mixtral.model.layers[l].post_attention_layernorm.weight - - # transformers 5.x renamed the MoE block (block_sparse_moe -> mlp) and - # replaced the per-expert w1/w2/w3 Linears with batched Parameters on a - # single MixtralExperts module: - # gate_up_proj: [num_experts, 2 * d_mlp, d_model] (gate fused above up) - # down_proj: [num_experts, d_model, d_mlp] - moe = mixtral.model.layers[l].mlp - # Guarded like the experts below: load_state_dict accepts a SAME-SHAPE - # int8/FP8 router and silently casts it to float32, so nothing - # downstream catches it. - state_dict[f"blocks.{l}.mlp.W_gate.weight"] = require_readable_weight( - moe.gate.weight, operation="convert the Mixtral router weight", owner=mixtral - ) - - experts = moe.experts - gate_up = require_readable_weight( - experts.gate_up_proj, - operation="convert Mixtral expert weights (gate_up_proj)", - owner=mixtral, - ) - down = require_readable_weight( - experts.down_proj, - operation="convert Mixtral expert weights (down_proj)", - owner=mixtral, - ) - - # MixtralExperts.forward does - # gate, up = F.linear(x, gate_up_proj[e]).chunk(2, dim=-1) - # so the FIRST half of dim 1 is the gate projection and the second is up. - for e in range(cfg.num_experts): - state_dict[f"blocks.{l}.mlp.experts.{e}.W_gate.weight"] = gate_up[e, : cfg.d_mlp, :] - state_dict[f"blocks.{l}.mlp.experts.{e}.W_in.weight"] = gate_up[e, cfg.d_mlp :, :] - state_dict[f"blocks.{l}.mlp.experts.{e}.W_out.weight"] = down[e] - - state_dict["ln_final.w"] = mixtral.model.norm.weight.data - - state_dict["unembed.W_U"] = mixtral.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/nanogpt.py b/transformer_lens/pretrained/weight_conversions/nanogpt.py deleted file mode 100644 index c7e20d5601..0000000000 --- a/transformer_lens/pretrained/weight_conversions/nanogpt.py +++ /dev/null @@ -1,108 +0,0 @@ -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_nanogpt_weights(old_state_dict, cfg: TransformerLensConfig): - """For https://github.com/karpathy/nanoGPT - There are two complications with converting nanogpt models: - The first is that some state dicts have an unwanted prefix on keys that needs to be removed. - The second is that the models can be saved with or without bias. By default, there - is no bias. This function can handle both cases.""" - # Nanogpt models saved after torch.compile() have this unwanted prefix - unwanted_prefix = "_orig_mod." - for k, v in list(old_state_dict.items()): - if k.startswith(unwanted_prefix): - old_state_dict[k[len(unwanted_prefix) :]] = old_state_dict.pop(k) - - new_state_dict = {} - new_state_dict["pos_embed.W_pos"] = old_state_dict["transformer.wpe.weight"] - new_state_dict["embed.W_E"] = old_state_dict["transformer.wte.weight"] - - new_state_dict["ln_final.w"] = old_state_dict["transformer.ln_f.weight"] - new_state_dict["ln_final.b"] = torch.zeros_like(old_state_dict["transformer.ln_f.weight"]) - new_state_dict["unembed.W_U"] = old_state_dict["lm_head.weight"].T - - bias = False - if "transformer.ln_f.bias" in old_state_dict: - bias = True - new_state_dict["ln_final.b"] = old_state_dict["transformer.ln_f.bias"] - else: - new_state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - - for layer in range(cfg.n_layers): - layer_key = f"transformer.h.{layer}" - - new_state_dict[f"blocks.{layer}.ln1.w"] = old_state_dict[f"{layer_key}.ln_1.weight"] - # A bias of zeros is required for folding layer norm - new_state_dict[f"blocks.{layer}.ln1.b"] = torch.zeros_like( - old_state_dict[f"{layer_key}.ln_1.weight"] - ) - new_state_dict[f"blocks.{layer}.ln2.w"] = old_state_dict[f"{layer_key}.ln_2.weight"] - new_state_dict[f"blocks.{layer}.ln2.b"] = torch.zeros_like( - old_state_dict[f"{layer_key}.ln_2.weight"] - ) - - new_state_dict[f"blocks.{layer}.attn.IGNORE"] = torch.tensor(-torch.inf) - - W = old_state_dict[f"{layer_key}.attn.c_attn.weight"] - W_Q, W_K, W_V = torch.tensor_split(W, 3, dim=0) - W_Q = einops.rearrange(W_Q, "(i h) m->i m h", i=cfg.n_heads) - W_K = einops.rearrange(W_K, "(i h) m->i m h", i=cfg.n_heads) - W_V = einops.rearrange(W_V, "(i h) m->i m h", i=cfg.n_heads) - new_state_dict[f"blocks.{layer}.attn.W_Q"] = W_Q - new_state_dict[f"blocks.{layer}.attn.W_K"] = W_K - new_state_dict[f"blocks.{layer}.attn.W_V"] = W_V - - W_O = old_state_dict[f"{layer_key}.attn.c_proj.weight"] - W_O = einops.rearrange(W_O, "m (i h)->i h m", i=cfg.n_heads) - new_state_dict[f"blocks.{layer}.attn.W_O"] = W_O - - new_state_dict[f"blocks.{layer}.mlp.W_in"] = old_state_dict[ - f"{layer_key}.mlp.c_fc.weight" - ].T - new_state_dict[f"blocks.{layer}.mlp.W_out"] = old_state_dict[ - f"{layer_key}.mlp.c_proj.weight" - ].T - - if bias: - new_state_dict[f"blocks.{layer}.ln1.b"] = old_state_dict[f"{layer_key}.ln_1.bias"] - new_state_dict[f"blocks.{layer}.ln2.b"] = old_state_dict[f"{layer_key}.ln_2.bias"] - new_state_dict[f"blocks.{layer}.mlp.b_in"] = old_state_dict[ - f"{layer_key}.mlp.c_fc.bias" - ] - new_state_dict[f"blocks.{layer}.mlp.b_out"] = old_state_dict[ - f"{layer_key}.mlp.c_proj.bias" - ] - - B = old_state_dict[f"{layer_key}.attn.c_attn.bias"] - B_Q, B_K, B_V = torch.tensor_split(B, 3, dim=0) - B_Q = einops.rearrange(B_Q, "(i h)->i h", i=cfg.n_heads) - B_K = einops.rearrange(B_K, "(i h)->i h", i=cfg.n_heads) - B_V = einops.rearrange(B_V, "(i h)->i h", i=cfg.n_heads) - new_state_dict[f"blocks.{layer}.attn.b_Q"] = B_Q - new_state_dict[f"blocks.{layer}.attn.b_K"] = B_K - new_state_dict[f"blocks.{layer}.attn.b_V"] = B_V - new_state_dict[f"blocks.{layer}.attn.b_O"] = old_state_dict[ - f"{layer_key}.attn.c_proj.bias" - ] - else: - if cfg.d_mlp is None: - raise ValueError( - "cfg.d_mlp must be set to convert nanoGPT weights for the no-bias case." - ) - new_state_dict[f"blocks.{layer}.mlp.b_out"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - new_state_dict[f"blocks.{layer}.mlp.b_in"] = torch.zeros(cfg.d_mlp, dtype=cfg.dtype) - new_state_dict[f"blocks.{layer}.attn.b_Q"] = torch.zeros( - (cfg.n_heads, cfg.d_head), dtype=cfg.dtype - ) - new_state_dict[f"blocks.{layer}.attn.b_K"] = torch.zeros( - cfg.n_heads, cfg.d_head, dtype=cfg.dtype - ) - new_state_dict[f"blocks.{layer}.attn.b_V"] = torch.zeros( - cfg.n_heads, cfg.d_head, dtype=cfg.dtype - ) - new_state_dict[f"blocks.{layer}.attn.b_O"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - return new_state_dict diff --git a/transformer_lens/pretrained/weight_conversions/neel_solu_old.py b/transformer_lens/pretrained/weight_conversions/neel_solu_old.py deleted file mode 100644 index 4455e68eae..0000000000 --- a/transformer_lens/pretrained/weight_conversions/neel_solu_old.py +++ /dev/null @@ -1,38 +0,0 @@ -from transformer_lens.config import TransformerLensConfig - - -def convert_neel_solu_old_weights(state_dict: dict, cfg: TransformerLensConfig): - """ - Converts the weights of my old SoLU models to the HookedTransformer format. - Takes as input a state dict, *not* a model object. - - There are a bunch of dumb bugs in the original code, sorry! - - Models 1L, 2L, 4L and 6L have left facing weights (ie, weights have shape - [dim_out, dim_in]) while HookedTransformer does right facing (ie [dim_in, - dim_out]). - - 8L has *just* a left facing W_pos, the rest right facing. - - And some models were trained with - """ - # Early models have left facing W_pos - reverse_pos = cfg.n_layers <= 8 - - # Models prior to 8L have left facing everything (8L has JUST left facing W_pos - sorry! Stupid bug) - reverse_weights = cfg.n_layers <= 6 - - new_state_dict = {} - for k, v in state_dict.items(): - k = k.replace("norm", "ln") - if k.startswith("ln."): - k = k.replace("ln.", "ln_final.") - new_state_dict[k] = v - - if reverse_pos: - new_state_dict["pos_embed.W_pos"] = new_state_dict["pos_embed.W_pos"].T - if reverse_weights: - for k, v in new_state_dict.items(): - if "W_" in k and "W_pos" not in k: - new_state_dict[k] = v.transpose(-2, -1) - return new_state_dict diff --git a/transformer_lens/pretrained/weight_conversions/neo.py b/transformer_lens/pretrained/weight_conversions/neo.py deleted file mode 100644 index a36abe86b3..0000000000 --- a/transformer_lens/pretrained/weight_conversions/neo.py +++ /dev/null @@ -1,56 +0,0 @@ -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_neo_weights(neo, cfg: TransformerLensConfig): - state_dict = {} - - state_dict["embed.W_E"] = neo.transformer.wte.weight - - # Trim positional embeddings to n_ctx if the pretrained weights have more - # positions than the model expects (e.g. TinyStories models were trained with - # seq_len=512 but the HuggingFace config reports max_position_embeddings=2048). - pos_embed = neo.transformer.wpe.weight - if pos_embed.shape[0] > cfg.n_ctx: - pos_embed = pos_embed[: cfg.n_ctx, :] - state_dict["pos_embed.W_pos"] = pos_embed - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = neo.transformer.h[l].ln_1.weight - state_dict[f"blocks.{l}.ln1.b"] = neo.transformer.h[l].ln_1.bias - - W_Q = neo.transformer.h[l].attn.attention.q_proj.weight - W_K = neo.transformer.h[l].attn.attention.k_proj.weight - W_V = neo.transformer.h[l].attn.attention.v_proj.weight - W_Q = einops.rearrange(W_Q, "(i h) m->i m h", i=cfg.n_heads) - W_K = einops.rearrange(W_K, "(i h) m->i m h", i=cfg.n_heads) - W_V = einops.rearrange(W_V, "(i h) m->i m h", i=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.W_K"] = W_K - state_dict[f"blocks.{l}.attn.W_V"] = W_V - - state_dict[f"blocks.{l}.attn.b_Q"] = torch.zeros(cfg.n_heads, cfg.d_head, dtype=cfg.dtype) - state_dict[f"blocks.{l}.attn.b_K"] = torch.zeros(cfg.n_heads, cfg.d_head, dtype=cfg.dtype) - state_dict[f"blocks.{l}.attn.b_V"] = torch.zeros(cfg.n_heads, cfg.d_head, dtype=cfg.dtype) - - W_O = neo.transformer.h[l].attn.attention.out_proj.weight - W_O = einops.rearrange(W_O, "m (i h)->i h m", i=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - state_dict[f"blocks.{l}.attn.b_O"] = neo.transformer.h[l].attn.attention.out_proj.bias - - state_dict[f"blocks.{l}.ln2.w"] = neo.transformer.h[l].ln_2.weight - state_dict[f"blocks.{l}.ln2.b"] = neo.transformer.h[l].ln_2.bias - - state_dict[f"blocks.{l}.mlp.W_in"] = neo.transformer.h[l].mlp.c_fc.weight.T - state_dict[f"blocks.{l}.mlp.b_in"] = neo.transformer.h[l].mlp.c_fc.bias - - state_dict[f"blocks.{l}.mlp.W_out"] = neo.transformer.h[l].mlp.c_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_out"] = neo.transformer.h[l].mlp.c_proj.bias - state_dict["ln_final.w"] = neo.transformer.ln_f.weight - state_dict["ln_final.b"] = neo.transformer.ln_f.bias - - state_dict["unembed.W_U"] = neo.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/neox.py b/transformer_lens/pretrained/weight_conversions/neox.py deleted file mode 100644 index d2b8d00cc4..0000000000 --- a/transformer_lens/pretrained/weight_conversions/neox.py +++ /dev/null @@ -1,59 +0,0 @@ -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_neox_weights(neox, cfg: TransformerLensConfig): - state_dict = {} - - state_dict["embed.W_E"] = neox.gpt_neox.embed_in.weight - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = neox.gpt_neox.layers[l].input_layernorm.weight - state_dict[f"blocks.{l}.ln1.b"] = neox.gpt_neox.layers[l].input_layernorm.bias - - # For some inexplicable reason, NeoX both uses the concatenated QKV - # matmul of GPT-2 (afaict this has a negligible performance impact) AND - # has the flattened axis in the DIFFERENT order of (head_index qkv - # d_head) - this took me an hour to debug... - W = neox.gpt_neox.layers[l].attention.query_key_value.weight - W = einops.rearrange(W, "(i qkv h) m->qkv i m h", i=cfg.n_heads, qkv=3) - - # Fold in layer norm weights - state_dict[f"blocks.{l}.attn.W_Q"] = W[0] - state_dict[f"blocks.{l}.attn.W_K"] = W[1] - state_dict[f"blocks.{l}.attn.W_V"] = W[2] - - qkv_bias = neox.gpt_neox.layers[l].attention.query_key_value.bias - qkv_bias = einops.rearrange( - qkv_bias, - "(index qkv head)->qkv index head", - qkv=3, - index=cfg.n_heads, - head=cfg.d_head, - ) - # Fold in layer norm biases - state_dict[f"blocks.{l}.attn.b_Q"] = qkv_bias[0] - state_dict[f"blocks.{l}.attn.b_K"] = qkv_bias[1] - state_dict[f"blocks.{l}.attn.b_V"] = qkv_bias[2] - - W_O = neox.gpt_neox.layers[l].attention.dense.weight - W_O = einops.rearrange(W_O, "m (i h)->i h m", i=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - state_dict[f"blocks.{l}.attn.b_O"] = neox.gpt_neox.layers[l].attention.dense.bias - - state_dict[f"blocks.{l}.ln2.w"] = neox.gpt_neox.layers[l].post_attention_layernorm.weight - state_dict[f"blocks.{l}.ln2.b"] = neox.gpt_neox.layers[l].post_attention_layernorm.bias - - state_dict[f"blocks.{l}.mlp.W_in"] = neox.gpt_neox.layers[l].mlp.dense_h_to_4h.weight.T - state_dict[f"blocks.{l}.mlp.b_in"] = neox.gpt_neox.layers[l].mlp.dense_h_to_4h.bias - - state_dict[f"blocks.{l}.mlp.W_out"] = neox.gpt_neox.layers[l].mlp.dense_4h_to_h.weight.T - state_dict[f"blocks.{l}.mlp.b_out"] = neox.gpt_neox.layers[l].mlp.dense_4h_to_h.bias - state_dict["ln_final.w"] = neox.gpt_neox.final_layer_norm.weight - state_dict["ln_final.b"] = neox.gpt_neox.final_layer_norm.bias - - state_dict["unembed.W_U"] = neox.embed_out.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/olmo.py b/transformer_lens/pretrained/weight_conversions/olmo.py deleted file mode 100644 index daa8bc8b53..0000000000 --- a/transformer_lens/pretrained/weight_conversions/olmo.py +++ /dev/null @@ -1,50 +0,0 @@ -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_olmo_weights(olmo, cfg: TransformerLensConfig): - state_dict = {} - - assert cfg.d_mlp is not None - - state_dict["embed.W_E"] = olmo.model.embed_tokens.weight - for l in range(cfg.n_layers): - olmo_layer = olmo.model.layers[l] - - W_Q = olmo_layer.self_attn.q_proj.weight - W_K = olmo_layer.self_attn.k_proj.weight - W_V = olmo_layer.self_attn.v_proj.weight - W_Q = einops.rearrange(W_Q, "(i h) m->i m h", i=cfg.n_heads) - W_K = einops.rearrange(W_K, "(i h) m->i m h", i=cfg.n_heads) - W_V = einops.rearrange(W_V, "(i h) m->i m h", i=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.W_K"] = W_K - state_dict[f"blocks.{l}.attn.W_V"] = W_V - - W_O = olmo_layer.self_attn.o_proj.weight - W_O = einops.rearrange(W_O, "m (n h)->n h m", n=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.mlp.W_in"] = olmo_layer.mlp.up_proj.weight.T - state_dict[f"blocks.{l}.mlp.W_gate"] = olmo_layer.mlp.gate_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_in"] = torch.zeros(cfg.d_mlp, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.mlp.W_out"] = olmo_layer.mlp.down_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_out"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.ln1.w"] = torch.ones(cfg.d_model, dtype=cfg.dtype) - state_dict[f"blocks.{l}.ln1.b"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - state_dict[f"blocks.{l}.ln2.w"] = torch.ones(cfg.d_model, dtype=cfg.dtype) - state_dict[f"blocks.{l}.ln2.b"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict["ln_final.w"] = torch.ones(cfg.d_model, dtype=cfg.dtype) - state_dict["ln_final.b"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict["unembed.W_U"] = olmo.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/olmo2.py b/transformer_lens/pretrained/weight_conversions/olmo2.py deleted file mode 100644 index 017c40d3b8..0000000000 --- a/transformer_lens/pretrained/weight_conversions/olmo2.py +++ /dev/null @@ -1,57 +0,0 @@ -import einops -import torch -from transformers.models.olmo2.modeling_olmo2 import Olmo2DecoderLayer - -from transformer_lens.config import TransformerLensConfig - - -def convert_olmo2_weights(olmo2, cfg: TransformerLensConfig): - state_dict = {} - - assert cfg.d_mlp is not None - - state_dict["embed.W_E"] = olmo2.model.embed_tokens.weight - - for l in range(cfg.n_layers): - olmo2_layer = olmo2.model.layers[l] - assert isinstance(olmo2_layer, Olmo2DecoderLayer) - - W_Q = olmo2_layer.self_attn.q_proj.weight - W_K = olmo2_layer.self_attn.k_proj.weight - W_V = olmo2_layer.self_attn.v_proj.weight - W_Q = einops.rearrange(W_Q, "(n h) m->n m h", n=cfg.n_heads) - W_K = einops.rearrange(W_K, "(n h) m->n m h", n=cfg.n_heads) - W_V = einops.rearrange(W_V, "(n h) m->n m h", n=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.W_K"] = W_K - state_dict[f"blocks.{l}.attn.W_V"] = W_V - state_dict[f"blocks.{l}.attn.q_norm.w"] = olmo2_layer.self_attn.q_norm.weight - state_dict[f"blocks.{l}.attn.k_norm.w"] = olmo2_layer.self_attn.k_norm.weight - - state_dict[f"blocks.{l}.attn.b_Q"] = torch.zeros(cfg.n_heads, cfg.d_head, dtype=cfg.dtype) - state_dict[f"blocks.{l}.attn.b_K"] = torch.zeros(cfg.n_heads, cfg.d_head, dtype=cfg.dtype) - state_dict[f"blocks.{l}.attn.b_V"] = torch.zeros(cfg.n_heads, cfg.d_head, dtype=cfg.dtype) - - W_O = olmo2_layer.self_attn.o_proj.weight - W_O = einops.rearrange(W_O, "m (n h)->n h m", n=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.ln1.w"] = olmo2_layer.post_attention_layernorm.weight - - state_dict[f"blocks.{l}.mlp.W_in"] = olmo2_layer.mlp.up_proj.weight.T - state_dict[f"blocks.{l}.mlp.W_gate"] = olmo2_layer.mlp.gate_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_in"] = torch.zeros(cfg.d_mlp, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.mlp.W_out"] = olmo2_layer.mlp.down_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_out"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.ln2.w"] = olmo2_layer.post_feedforward_layernorm.weight - - state_dict["ln_final.w"] = olmo2.model.norm.weight - - state_dict["unembed.W_U"] = olmo2.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/olmo3.py b/transformer_lens/pretrained/weight_conversions/olmo3.py deleted file mode 100644 index 547bc58a12..0000000000 --- a/transformer_lens/pretrained/weight_conversions/olmo3.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Weight conversion functions for OLMo 3/3.1 models. - -OLMo 3/3.1 architecture features: -- Q/K normalization (RMSNorm on queries/keys before attention) -- Grouped Query Attention (GQA) with n_key_value_heads < n_heads -- Sliding window attention + full attention layers (mixed via layer_types) -- RMSNorm throughout (no +1 modification unlike Gemma) -- Rotary Position Embeddings (RoPE) with YARN scaling -- Gated MLP (SwiGLU-style) -- Post-normalization pattern (RMSNorm after attention and MLP) -""" - -from typing import cast - -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_olmo3_weights(olmo3, cfg: TransformerLensConfig): - state_dict = {} - - # Must match TransformerBlock's attention choice, which selects - # GroupedQueryAttention (underscore-prefixed params) whenever - # n_key_value_heads is set — including n_key_value_heads == n_heads. - using_gqa = cfg.n_key_value_heads is not None - gqa_uscore = "_" if using_gqa else "" - n_kv_heads = cast(int, cfg.n_key_value_heads if using_gqa else cfg.n_heads) - - assert cfg.d_mlp is not None # keep mypy happy - - base_model = olmo3.model - state_dict["embed.W_E"] = base_model.embed_tokens.weight - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = base_model.layers[l].post_attention_layernorm.weight - - W_Q = base_model.layers[l].self_attn.q_proj.weight - W_K = base_model.layers[l].self_attn.k_proj.weight - W_V = base_model.layers[l].self_attn.v_proj.weight - - W_Q = einops.rearrange(W_Q, "(n h) m->n m h", n=cfg.n_heads) - W_K = einops.rearrange(W_K, "(n h) m->n m h", n=n_kv_heads) - W_V = einops.rearrange(W_V, "(n h) m->n m h", n=n_kv_heads) - - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.{gqa_uscore}W_K"] = W_K - state_dict[f"blocks.{l}.attn.{gqa_uscore}W_V"] = W_V - - # OLMo 3 always has Q/K norms (applied on full projected vectors) - state_dict[f"blocks.{l}.attn.q_norm.w"] = base_model.layers[l].self_attn.q_norm.weight - state_dict[f"blocks.{l}.attn.k_norm.w"] = base_model.layers[l].self_attn.k_norm.weight - - state_dict[f"blocks.{l}.attn.b_Q"] = torch.zeros( - cfg.n_heads, cfg.d_head, dtype=cfg.dtype, device=W_Q.device - ) - state_dict[f"blocks.{l}.attn.{gqa_uscore}b_K"] = torch.zeros( - n_kv_heads, cfg.d_head, dtype=cfg.dtype, device=W_K.device - ) - state_dict[f"blocks.{l}.attn.{gqa_uscore}b_V"] = torch.zeros( - n_kv_heads, cfg.d_head, dtype=cfg.dtype, device=W_V.device - ) - - W_O = base_model.layers[l].self_attn.o_proj.weight - W_O = einops.rearrange(W_O, "m (n h)->n h m", n=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros( - cfg.d_model, dtype=cfg.dtype, device=W_O.device - ) - - state_dict[f"blocks.{l}.ln2.w"] = base_model.layers[l].post_feedforward_layernorm.weight - state_dict[f"blocks.{l}.mlp.W_in"] = base_model.layers[l].mlp.up_proj.weight.T - state_dict[f"blocks.{l}.mlp.W_gate"] = base_model.layers[l].mlp.gate_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_in"] = torch.zeros( - cfg.d_mlp, dtype=cfg.dtype, device=base_model.layers[l].mlp.up_proj.weight.device - ) - state_dict[f"blocks.{l}.mlp.W_out"] = base_model.layers[l].mlp.down_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_out"] = torch.zeros( - cfg.d_model, dtype=cfg.dtype, device=base_model.layers[l].mlp.down_proj.weight.device - ) - - state_dict["ln_final.w"] = base_model.norm.weight - - state_dict["unembed.W_U"] = olmo3.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros( - cfg.d_vocab, dtype=cfg.dtype, device=olmo3.lm_head.weight.device - ) - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/olmoe.py b/transformer_lens/pretrained/weight_conversions/olmoe.py deleted file mode 100644 index 8852980a11..0000000000 --- a/transformer_lens/pretrained/weight_conversions/olmoe.py +++ /dev/null @@ -1,81 +0,0 @@ -import einops -import torch - -from transformer_lens.config import TransformerLensConfig -from transformer_lens.utilities.quantization import require_readable_weight - - -def convert_olmoe_weights(olmoe, cfg: TransformerLensConfig): - state_dict = {} - - assert cfg.n_key_value_heads is not None - assert cfg.d_mlp is not None - assert cfg.num_experts is not None - - state_dict["embed.W_E"] = olmoe.model.embed_tokens.weight - - for l in range(cfg.n_layers): - olmoe_layer = olmoe.model.layers[l] - state_dict[f"blocks.{l}.ln1.w"] = olmoe_layer.input_layernorm.weight - - W_Q = olmoe_layer.self_attn.q_proj.weight - W_K = olmoe_layer.self_attn.k_proj.weight - W_V = olmoe_layer.self_attn.v_proj.weight - W_Q = einops.rearrange(W_Q, "(n h) m->n m h", n=cfg.n_heads) - W_K = einops.rearrange(W_K, "(n h) m->n m h", n=cfg.n_key_value_heads) - W_V = einops.rearrange(W_V, "(n h) m->n m h", n=cfg.n_key_value_heads) - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn._W_K"] = W_K - state_dict[f"blocks.{l}.attn._W_V"] = W_V - state_dict[f"blocks.{l}.attn.q_norm.w"] = olmoe_layer.self_attn.q_norm.weight - state_dict[f"blocks.{l}.attn.k_norm.w"] = olmoe_layer.self_attn.k_norm.weight - - state_dict[f"blocks.{l}.attn.b_Q"] = torch.zeros(cfg.n_heads, cfg.d_head, dtype=cfg.dtype) - state_dict[f"blocks.{l}.attn._b_K"] = torch.zeros( - cfg.n_key_value_heads, cfg.d_head, dtype=cfg.dtype - ) - state_dict[f"blocks.{l}.attn._b_V"] = torch.zeros( - cfg.n_key_value_heads, cfg.d_head, dtype=cfg.dtype - ) - - W_O = olmoe_layer.self_attn.o_proj.weight - W_O = einops.rearrange(W_O, "m (n h)->n h m", n=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.ln2.w"] = olmoe_layer.post_attention_layernorm.weight - - state_dict[f"blocks.{l}.mlp.W_gate.weight"] = require_readable_weight( - olmoe_layer.mlp.gate.weight, - operation="convert the OLMoE router weight", - owner=olmoe, - ) - - # HF OLMoE uses batched expert weights: - # gate_up_proj: [num_experts, 2 * intermediate_size, hidden_size] - # down_proj: [num_experts, hidden_size, intermediate_size] - # The gate_up_proj fuses gate and up projections along dim 1. - experts = olmoe_layer.mlp.experts - gate_up = require_readable_weight( - experts.gate_up_proj, - operation="convert OLMoE expert weights (gate_up_proj)", - owner=olmoe, - ) # [num_experts, 2*d_mlp, d_model] - down = require_readable_weight( - experts.down_proj, - operation="convert OLMoE expert weights (down_proj)", - owner=olmoe, - ) # [num_experts, d_model, d_mlp] - - for e in range(cfg.num_experts): - state_dict[f"blocks.{l}.mlp.experts.{e}.W_gate.weight"] = gate_up[e, : cfg.d_mlp, :] - state_dict[f"blocks.{l}.mlp.experts.{e}.W_in.weight"] = gate_up[e, cfg.d_mlp :, :] - state_dict[f"blocks.{l}.mlp.experts.{e}.W_out.weight"] = down[e] - - state_dict["ln_final.w"] = olmoe.model.norm.weight - - state_dict["unembed.W_U"] = olmoe.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/openai.py b/transformer_lens/pretrained/weight_conversions/openai.py deleted file mode 100644 index a3341df1e6..0000000000 --- a/transformer_lens/pretrained/weight_conversions/openai.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Weight conversion for OpenAI GPT-OSS models. - -GPT-OSS has a unique MoE architecture: -- GptOssExperts stores all expert weights in merged tensors (not individual modules) -- gate_up_proj: (num_experts, hidden_size, 2*expert_dim) with interleaved gate/up columns -- down_proj: (num_experts, expert_dim, hidden_size) -- Router (GptOssTopKRouter) uses weight + bias -""" - -import einops -import torch - -from transformer_lens.config import TransformerLensConfig -from transformer_lens.utilities.quantization import require_readable_weight - -# Phrased to hold for any quantization: the guard catches int8 and FP8 too, and -# cannot know which one it caught, so it must not assert this *is* MXFP4. -_GPT_OSS_REMEDY = ( - "If this is a packed-MXFP4 checkpoint, load it dequantized so the converter " - "sees plain tensors: pass hf_model=AutoModelForCausalLM.from_pretrained(name, " - "quantization_config=Mxfp4Config(dequantize=True)), or load by model name and " - "TransformerLens dequantizes automatically. Otherwise reload without a " - "quantization_config. Quantized *forward* passes remain supported." -) - - -def convert_gpt_oss_weights(gpt_oss, cfg: TransformerLensConfig): - state_dict = {} - - assert cfg.n_key_value_heads is not None - assert cfg.d_mlp is not None - assert cfg.num_experts is not None - - state_dict["embed.W_E"] = gpt_oss.model.embed_tokens.weight - - for l in range(cfg.n_layers): - layer = gpt_oss.model.layers[l] - - # LayerNorms - state_dict[f"blocks.{l}.ln1.w"] = layer.input_layernorm.weight - state_dict[f"blocks.{l}.ln2.w"] = layer.post_attention_layernorm.weight - - # Attention - W_Q = einops.rearrange(layer.self_attn.q_proj.weight, "(n h) m -> n m h", n=cfg.n_heads) - W_K = einops.rearrange( - layer.self_attn.k_proj.weight, "(n h) m -> n m h", n=cfg.n_key_value_heads - ) - W_V = einops.rearrange( - layer.self_attn.v_proj.weight, "(n h) m -> n m h", n=cfg.n_key_value_heads - ) - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn._W_K"] = W_K - state_dict[f"blocks.{l}.attn._W_V"] = W_V - - if layer.self_attn.q_proj.bias is not None: - state_dict[f"blocks.{l}.attn.b_Q"] = einops.rearrange( - layer.self_attn.q_proj.bias, "(n h) -> n h", n=cfg.n_heads - ) - state_dict[f"blocks.{l}.attn._b_K"] = einops.rearrange( - layer.self_attn.k_proj.bias, "(n h) -> n h", n=cfg.n_key_value_heads - ) - state_dict[f"blocks.{l}.attn._b_V"] = einops.rearrange( - layer.self_attn.v_proj.bias, "(n h) -> n h", n=cfg.n_key_value_heads - ) - else: - state_dict[f"blocks.{l}.attn.b_Q"] = torch.zeros( - cfg.n_heads, cfg.d_head, dtype=cfg.dtype, device=cfg.device - ) - state_dict[f"blocks.{l}.attn._b_K"] = torch.zeros( - cfg.n_key_value_heads, cfg.d_head, dtype=cfg.dtype, device=cfg.device - ) - state_dict[f"blocks.{l}.attn._b_V"] = torch.zeros( - cfg.n_key_value_heads, cfg.d_head, dtype=cfg.dtype, device=cfg.device - ) - - state_dict[f"blocks.{l}.attn.sinks"] = layer.self_attn.sinks - - W_O = einops.rearrange(layer.self_attn.o_proj.weight, "m (n h) -> n h m", n=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - - if hasattr(layer.self_attn.o_proj, "bias") and layer.self_attn.o_proj.bias is not None: - state_dict[f"blocks.{l}.attn.b_O"] = layer.self_attn.o_proj.bias - else: - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros( - cfg.d_model, dtype=cfg.dtype, device=cfg.device - ) - - # MoE - Router (GPT-OSS uses 'router' with bias) - state_dict[f"blocks.{l}.mlp.W_gate.weight"] = require_readable_weight( - layer.mlp.router.weight, - operation="convert the gpt-oss router weight", - owner=gpt_oss, - remedy=_GPT_OSS_REMEDY, - ) - state_dict[f"blocks.{l}.mlp.W_gate.bias"] = layer.mlp.router.bias - - # MoE - Experts - # GPT-OSS stores all experts in merged tensors: - # gate_up_proj: (num_experts, hidden_size, 2*expert_dim) - interleaved gate/up - # down_proj: (num_experts, expert_dim, hidden_size) - experts = layer.mlp.experts - gate_up_proj = experts.gate_up_proj # (num_experts, hidden_size, 2*expert_dim) - gate_up_bias = require_readable_weight( - experts.gate_up_proj_bias, - operation=f"convert gpt-oss expert biases (blocks.{l}.mlp.experts.gate_up_proj_bias)", - owner=gpt_oss, - remedy=_GPT_OSS_REMEDY, - ) # (num_experts, 2*expert_dim) - down_proj = experts.down_proj # (num_experts, expert_dim, hidden_size) - down_bias = require_readable_weight( - experts.down_proj_bias, - operation=f"convert gpt-oss expert biases (blocks.{l}.mlp.experts.down_proj_bias)", - owner=gpt_oss, - remedy=_GPT_OSS_REMEDY, - ) # (num_experts, hidden_size) - - # Packed MXFP4 wraps these in a triton-kernels object (confusingly also - # named "Tensor"), but int8 and FP8 gpt-oss finetunes slice without - # complaint and would emit plausible garbage, so check the dtype too. - gate_up_proj = require_readable_weight( - gate_up_proj, - operation=f"convert gpt-oss expert weights (blocks.{l}.mlp.experts.gate_up_proj)", - owner=gpt_oss, - remedy=_GPT_OSS_REMEDY, - ) - down_proj = require_readable_weight( - down_proj, - operation=f"convert gpt-oss expert weights (blocks.{l}.mlp.experts.down_proj)", - owner=gpt_oss, - remedy=_GPT_OSS_REMEDY, - ) - - for e in range(cfg.num_experts): - # Split interleaved gate_up_proj into separate gate and up (in) projections - # Even columns → gate path, Odd columns → up/in path - state_dict[f"blocks.{l}.mlp.experts.{e}.W_gate.weight"] = gate_up_proj[ - e, :, ::2 - ].T.contiguous() - state_dict[f"blocks.{l}.mlp.experts.{e}.W_gate.bias"] = gate_up_bias[ - e, ::2 - ].contiguous() - - state_dict[f"blocks.{l}.mlp.experts.{e}.W_in.weight"] = gate_up_proj[ - e, :, 1::2 - ].T.contiguous() - state_dict[f"blocks.{l}.mlp.experts.{e}.W_in.bias"] = gate_up_bias[e, 1::2].contiguous() - - state_dict[f"blocks.{l}.mlp.experts.{e}.W_out.weight"] = down_proj[e].T.contiguous() - state_dict[f"blocks.{l}.mlp.experts.{e}.W_out.bias"] = down_bias[e].contiguous() - - state_dict["ln_final.w"] = gpt_oss.model.norm.weight - state_dict["unembed.W_U"] = gpt_oss.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype, device=cfg.device) - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/opt.py b/transformer_lens/pretrained/weight_conversions/opt.py deleted file mode 100644 index 277a8138f8..0000000000 --- a/transformer_lens/pretrained/weight_conversions/opt.py +++ /dev/null @@ -1,84 +0,0 @@ -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_opt_weights(opt, cfg: TransformerLensConfig): - state_dict = {} - - state_dict["embed.W_E"] = opt.model.decoder.embed_tokens.weight - state_dict["pos_embed.W_pos"] = opt.model.decoder.embed_positions.weight[2:, :] - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = opt.model.decoder.layers[l].self_attn_layer_norm.weight - state_dict[f"blocks.{l}.ln1.b"] = opt.model.decoder.layers[l].self_attn_layer_norm.bias - - W_Q = opt.model.decoder.layers[l].self_attn.q_proj.weight - W_K = opt.model.decoder.layers[l].self_attn.k_proj.weight - W_V = opt.model.decoder.layers[l].self_attn.v_proj.weight - W_Q = einops.rearrange( - W_Q, - "(index d_head) d_model->index d_model d_head", - index=cfg.n_heads, - ) - W_K = einops.rearrange( - W_K, - "(index d_head) d_model->index d_model d_head", - index=cfg.n_heads, - ) - W_V = einops.rearrange( - W_V, - "(index d_head) d_model->index d_model d_head", - index=cfg.n_heads, - ) - - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.W_K"] = W_K - state_dict[f"blocks.{l}.attn.W_V"] = W_V - - q_bias = einops.rearrange( - opt.model.decoder.layers[l].self_attn.q_proj.bias, - "(head_index d_head)->head_index d_head", - head_index=cfg.n_heads, - d_head=cfg.d_head, - ) - k_bias = einops.rearrange( - opt.model.decoder.layers[l].self_attn.k_proj.bias, - "(head_index d_head)->head_index d_head", - head_index=cfg.n_heads, - d_head=cfg.d_head, - ) - v_bias = einops.rearrange( - opt.model.decoder.layers[l].self_attn.v_proj.bias, - "(head_index d_head)->head_index d_head", - head_index=cfg.n_heads, - d_head=cfg.d_head, - ) - - state_dict[f"blocks.{l}.attn.b_Q"] = q_bias - state_dict[f"blocks.{l}.attn.b_K"] = k_bias - state_dict[f"blocks.{l}.attn.b_V"] = v_bias - - W_O = opt.model.decoder.layers[l].self_attn.out_proj.weight - W_O = einops.rearrange( - W_O, - "d_model (index d_head)->index d_head d_model", - index=cfg.n_heads, - ) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - state_dict[f"blocks.{l}.attn.b_O"] = opt.model.decoder.layers[l].self_attn.out_proj.bias - - state_dict[f"blocks.{l}.ln2.w"] = opt.model.decoder.layers[l].final_layer_norm.weight - state_dict[f"blocks.{l}.ln2.b"] = opt.model.decoder.layers[l].final_layer_norm.bias - - state_dict[f"blocks.{l}.mlp.W_in"] = opt.model.decoder.layers[l].fc1.weight.T - state_dict[f"blocks.{l}.mlp.W_out"] = opt.model.decoder.layers[l].fc2.weight.T - - state_dict[f"blocks.{l}.mlp.b_in"] = opt.model.decoder.layers[l].fc1.bias - state_dict[f"blocks.{l}.mlp.b_out"] = opt.model.decoder.layers[l].fc2.bias - state_dict["ln_final.w"] = opt.model.decoder.final_layer_norm.weight - state_dict["ln_final.b"] = opt.model.decoder.final_layer_norm.bias - state_dict["unembed.W_U"] = opt.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/phi.py b/transformer_lens/pretrained/weight_conversions/phi.py deleted file mode 100644 index d8ad5de19e..0000000000 --- a/transformer_lens/pretrained/weight_conversions/phi.py +++ /dev/null @@ -1,64 +0,0 @@ -import einops - -from transformer_lens.config import TransformerLensConfig - - -def convert_phi_weights(phi, cfg: TransformerLensConfig): - state_dict = {} - - state_dict["embed.W_E"] = phi.model.embed_tokens.weight - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = phi.model.layers[l].input_layernorm.weight - state_dict[f"blocks.{l}.ln1.b"] = phi.model.layers[l].input_layernorm.bias - - W_Q = phi.model.layers[l].self_attn.q_proj.weight - W_K = phi.model.layers[l].self_attn.k_proj.weight - W_V = phi.model.layers[l].self_attn.v_proj.weight - W_Q = einops.rearrange( - W_Q, "(n_head d_head) d_model -> n_head d_model d_head", n_head=cfg.n_heads - ) - W_K = einops.rearrange( - W_K, "(n_head d_head) d_model -> n_head d_model d_head", n_head=cfg.n_heads - ) - W_V = einops.rearrange( - W_V, "(n_head d_head) d_model -> n_head d_model d_head", n_head=cfg.n_heads - ) - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.W_K"] = W_K - state_dict[f"blocks.{l}.attn.W_V"] = W_V - - b_Q = phi.model.layers[l].self_attn.q_proj.bias - b_K = phi.model.layers[l].self_attn.k_proj.bias - b_V = phi.model.layers[l].self_attn.v_proj.bias - b_Q = einops.rearrange(b_Q, "(n_head d_head) -> n_head d_head", n_head=cfg.n_heads) - b_K = einops.rearrange(b_K, "(n_head d_head) -> n_head d_head", n_head=cfg.n_heads) - b_V = einops.rearrange(b_V, "(n_head d_head) -> n_head d_head", n_head=cfg.n_heads) - state_dict[f"blocks.{l}.attn.b_Q"] = b_Q - state_dict[f"blocks.{l}.attn.b_K"] = b_K - state_dict[f"blocks.{l}.attn.b_V"] = b_V - - W_O = phi.model.layers[l].self_attn.dense.weight - W_O = einops.rearrange( - W_O, "d_model (n_head d_head) -> n_head d_head d_model", n_head=cfg.n_heads - ) - - state_dict[f"blocks.{l}.attn.W_O"] = W_O - state_dict[f"blocks.{l}.attn.b_O"] = phi.model.layers[l].self_attn.dense.bias - - # Layer Norm 1 and 2 are tied. - state_dict[f"blocks.{l}.ln2.w"] = state_dict[f"blocks.{l}.ln1.w"] - state_dict[f"blocks.{l}.ln2.b"] = state_dict[f"blocks.{l}.ln1.b"] - - state_dict[f"blocks.{l}.mlp.W_in"] = phi.model.layers[l].mlp.fc1.weight.T - state_dict[f"blocks.{l}.mlp.b_in"] = phi.model.layers[l].mlp.fc1.bias - state_dict[f"blocks.{l}.mlp.W_out"] = phi.model.layers[l].mlp.fc2.weight.T - state_dict[f"blocks.{l}.mlp.b_out"] = phi.model.layers[l].mlp.fc2.bias - - state_dict["ln_final.w"] = phi.model.final_layernorm.weight - state_dict["ln_final.b"] = phi.model.final_layernorm.bias - - state_dict["unembed.W_U"] = phi.lm_head.weight.T - state_dict["unembed.b_U"] = phi.lm_head.bias - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/phi3.py b/transformer_lens/pretrained/weight_conversions/phi3.py deleted file mode 100644 index 79226a67e9..0000000000 --- a/transformer_lens/pretrained/weight_conversions/phi3.py +++ /dev/null @@ -1,78 +0,0 @@ -from typing import cast - -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_phi3_weights(phi, cfg: TransformerLensConfig): - state_dict = {} - state_dict["embed.W_E"] = phi.model.embed_tokens.weight - - # Some models with this architecture use Grouped Query Attention, and so for these we need to modify - # the state dict keys for the K/V attention weight/biases, prepending "_" to the key names. - using_gqa = cfg.n_key_value_heads is not None - gqa_uscore = "_" if using_gqa else "" - # need a cast since MyPy isn't smart enough to realize that using_gqa implies n_key_value_heads is not None - n_kv_heads = cast(int, cfg.n_key_value_heads if using_gqa else cfg.n_heads) - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = phi.model.layers[l].input_layernorm.weight - state_dict[f"blocks.{l}.ln1.b"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - - W = phi.model.layers[l].self_attn.qkv_proj.weight - q_dim = cfg.n_heads * cfg.d_head - kv_dim = n_kv_heads * cfg.d_head - W_Q, W_K, W_V = W.split([q_dim, kv_dim, kv_dim], dim=0) - - W_Q = einops.rearrange( - W_Q, "(n_head d_head) d_model -> n_head d_model d_head", n_head=cfg.n_heads - ) - W_K = einops.rearrange( - W_K, "(n_kv_head d_head) d_model -> n_kv_head d_model d_head", n_kv_head=n_kv_heads - ) - W_V = einops.rearrange( - W_V, "(n_kv_head d_head) d_model -> n_kv_head d_model d_head", n_kv_head=n_kv_heads - ) - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.{gqa_uscore}W_K"] = W_K - state_dict[f"blocks.{l}.attn.{gqa_uscore}W_V"] = W_V - - state_dict[f"blocks.{l}.attn.b_Q"] = torch.zeros( - cfg.n_heads, cfg.d_head, dtype=cfg.dtype, device=cfg.device - ) - state_dict[f"blocks.{l}.attn.{gqa_uscore}b_K"] = torch.zeros( - n_kv_heads, - cfg.d_head, - dtype=cfg.dtype, - ) - state_dict[f"blocks.{l}.attn.{gqa_uscore}b_V"] = torch.zeros( - n_kv_heads, - cfg.d_head, - dtype=cfg.dtype, - ) - - W_O = phi.model.layers[l].self_attn.o_proj.weight - W_O = einops.rearrange( - W_O, "d_model (n_head d_head) -> n_head d_head d_model", n_head=cfg.n_heads - ) - - state_dict[f"blocks.{l}.attn.W_O"] = W_O - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.ln2.w"] = phi.model.layers[l].post_attention_layernorm.weight - state_dict[f"blocks.{l}.ln2.b"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - - W = phi.model.layers[l].mlp.gate_up_proj.weight.T - W_gate, W_in = torch.tensor_split(W, 2, dim=1) - state_dict[f"blocks.{l}.mlp.W_in"] = W_in - state_dict[f"blocks.{l}.mlp.W_gate"] = W_gate - state_dict[f"blocks.{l}.mlp.W_out"] = phi.model.layers[l].mlp.down_proj.weight.T - - state_dict["ln_final.w"] = phi.model.norm.weight - - state_dict["unembed.W_U"] = phi.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/qwen.py b/transformer_lens/pretrained/weight_conversions/qwen.py deleted file mode 100644 index ebb8599c48..0000000000 --- a/transformer_lens/pretrained/weight_conversions/qwen.py +++ /dev/null @@ -1,65 +0,0 @@ -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_qwen_weights(qwen, cfg: TransformerLensConfig): - state_dict = {} - model = qwen.transformer - state_dict["embed.W_E"] = model.wte.weight - - assert cfg.d_mlp is not None # keep mypy happy - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = model.h[l].ln_1.weight - - W_Q, W_K, W_V = model.h[l].attn.c_attn.weight.split(split_size=cfg.d_model, dim=0) - W_Q = einops.rearrange(W_Q, "(n h) m->n m h", n=cfg.n_heads) - W_K = einops.rearrange(W_K, "(n h) m->n m h", n=cfg.n_heads) - W_V = einops.rearrange(W_V, "(n h) m->n m h", n=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.W_K"] = W_K - state_dict[f"blocks.{l}.attn.W_V"] = W_V - - b_Q, b_K, b_V = model.h[l].attn.c_attn.bias.split(split_size=cfg.d_model, dim=0) - b_Q = einops.rearrange( - b_Q, - "(n_head d_head) -> n_head d_head", - n_head=cfg.n_heads, - ) - b_K = einops.rearrange( - b_K, - "(n_head d_head) -> n_head d_head", - n_head=cfg.n_heads, - ) - b_V = einops.rearrange( - b_V, - "(n_head d_head) -> n_head d_head", - n_head=cfg.n_heads, - ) - state_dict[f"blocks.{l}.attn.b_Q"] = b_Q - state_dict[f"blocks.{l}.attn.b_K"] = b_K - state_dict[f"blocks.{l}.attn.b_V"] = b_V - - W_O = model.h[l].attn.c_proj.weight - W_O = einops.rearrange(W_O, "m (n h)->n h m", n=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.ln2.w"] = model.h[l].ln_2.weight - - state_dict[f"blocks.{l}.mlp.W_in"] = model.h[l].mlp.w1.weight.T - state_dict[f"blocks.{l}.mlp.W_gate"] = model.h[l].mlp.w2.weight.T - state_dict[f"blocks.{l}.mlp.b_in"] = torch.zeros(cfg.d_mlp, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.mlp.W_out"] = model.h[l].mlp.c_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_out"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict["ln_final.w"] = model.ln_f.weight - - state_dict["unembed.W_U"] = qwen.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/qwen2.py b/transformer_lens/pretrained/weight_conversions/qwen2.py deleted file mode 100644 index f7941208d6..0000000000 --- a/transformer_lens/pretrained/weight_conversions/qwen2.py +++ /dev/null @@ -1,76 +0,0 @@ -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_qwen2_weights(qwen, cfg: TransformerLensConfig): - # Note that this method is also applied for Qwen1.5 models, since they - # have architecture type Qwen2ForCausalLM. - - state_dict = {} - - state_dict["embed.W_E"] = qwen.model.embed_tokens.weight - - assert cfg.d_mlp is not None # keep mypy happy - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = qwen.model.layers[l].input_layernorm.weight - - W_Q = qwen.model.layers[l].self_attn.q_proj.weight - W_K = qwen.model.layers[l].self_attn.k_proj.weight - W_V = qwen.model.layers[l].self_attn.v_proj.weight - W_Q = einops.rearrange(W_Q, "(n h) m->n m h", n=cfg.n_heads) - W_K = einops.rearrange(W_K, "(n h) m->n m h", n=cfg.n_key_value_heads) - W_V = einops.rearrange(W_V, "(n h) m->n m h", n=cfg.n_key_value_heads) - - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn._W_K"] = W_K - state_dict[f"blocks.{l}.attn._W_V"] = W_V - - b_Q = qwen.model.layers[l].self_attn.q_proj.bias - b_Q = einops.rearrange( - b_Q, - "(n_head d_head) -> n_head d_head", - n_head=cfg.n_heads, - ) - - b_K = qwen.model.layers[l].self_attn.k_proj.bias - b_K = einops.rearrange( - b_K, - "(n_head d_head) -> n_head d_head", - n_head=cfg.n_key_value_heads, - ) - - b_V = qwen.model.layers[l].self_attn.v_proj.bias - b_V = einops.rearrange( - b_V, - "(n_head d_head) -> n_head d_head", - n_head=cfg.n_key_value_heads, - ) - - state_dict[f"blocks.{l}.attn.b_Q"] = b_Q - state_dict[f"blocks.{l}.attn._b_K"] = b_K - state_dict[f"blocks.{l}.attn._b_V"] = b_V - - W_O = qwen.model.layers[l].self_attn.o_proj.weight - W_O = einops.rearrange(W_O, "m (n h)->n h m", n=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.ln2.w"] = qwen.model.layers[l].post_attention_layernorm.weight - - state_dict[f"blocks.{l}.mlp.W_in"] = qwen.model.layers[l].mlp.up_proj.weight.T - state_dict[f"blocks.{l}.mlp.W_gate"] = qwen.model.layers[l].mlp.gate_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_in"] = torch.zeros(cfg.d_mlp, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.mlp.W_out"] = qwen.model.layers[l].mlp.down_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_out"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict["ln_final.w"] = qwen.model.norm.weight - - state_dict["unembed.W_U"] = qwen.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/qwen3.py b/transformer_lens/pretrained/weight_conversions/qwen3.py deleted file mode 100644 index 702ba3cb13..0000000000 --- a/transformer_lens/pretrained/weight_conversions/qwen3.py +++ /dev/null @@ -1,69 +0,0 @@ -from typing import Any - -import einops -import torch - -from transformer_lens.config import TransformerLensConfig - - -def convert_qwen3_weights(qwen: Any, cfg: TransformerLensConfig): - """Convert Qwen3 weights to TransformerLens format.""" - state_dict = {} - - state_dict["embed.W_E"] = qwen.model.embed_tokens.weight - - if cfg.n_key_value_heads is None: - gqa_uscore = "" - n_kv_heads = cfg.n_heads - else: - gqa_uscore = "_" - n_kv_heads = cfg.n_key_value_heads - - assert cfg.d_mlp is not None # keep mypy happy - - for l in range(cfg.n_layers): - state_dict[f"blocks.{l}.ln1.w"] = qwen.model.layers[l].input_layernorm.weight - - W_Q = qwen.model.layers[l].self_attn.q_proj.weight - W_K = qwen.model.layers[l].self_attn.k_proj.weight - W_V = qwen.model.layers[l].self_attn.v_proj.weight - W_Q = einops.rearrange(W_Q, "(n h) m->n m h", n=cfg.n_heads) - W_K = einops.rearrange(W_K, "(n h) m->n m h", n=n_kv_heads) - W_V = einops.rearrange(W_V, "(n h) m->n m h", n=n_kv_heads) - - state_dict[f"blocks.{l}.attn.W_Q"] = W_Q - state_dict[f"blocks.{l}.attn.{gqa_uscore}W_K"] = W_K - state_dict[f"blocks.{l}.attn.{gqa_uscore}W_V"] = W_V - - state_dict[f"blocks.{l}.attn.q_norm.w"] = qwen.model.layers[l].self_attn.q_norm.weight - state_dict[f"blocks.{l}.attn.k_norm.w"] = qwen.model.layers[l].self_attn.k_norm.weight - - state_dict[f"blocks.{l}.attn.b_Q"] = torch.zeros(cfg.n_heads, cfg.d_head, dtype=cfg.dtype) - state_dict[f"blocks.{l}.attn.{gqa_uscore}b_K"] = torch.zeros( - n_kv_heads, cfg.d_head, dtype=cfg.dtype - ) - state_dict[f"blocks.{l}.attn.{gqa_uscore}b_V"] = torch.zeros( - n_kv_heads, cfg.d_head, dtype=cfg.dtype - ) - - W_O = qwen.model.layers[l].self_attn.o_proj.weight - W_O = einops.rearrange(W_O, "m (n h)->n h m", n=cfg.n_heads) - state_dict[f"blocks.{l}.attn.W_O"] = W_O - - state_dict[f"blocks.{l}.attn.b_O"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.ln2.w"] = qwen.model.layers[l].post_attention_layernorm.weight - - state_dict[f"blocks.{l}.mlp.W_in"] = qwen.model.layers[l].mlp.up_proj.weight.T - state_dict[f"blocks.{l}.mlp.W_gate"] = qwen.model.layers[l].mlp.gate_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_in"] = torch.zeros(cfg.d_mlp, dtype=cfg.dtype) - - state_dict[f"blocks.{l}.mlp.W_out"] = qwen.model.layers[l].mlp.down_proj.weight.T - state_dict[f"blocks.{l}.mlp.b_out"] = torch.zeros(cfg.d_model, dtype=cfg.dtype) - - state_dict["ln_final.w"] = qwen.model.norm.weight - - state_dict["unembed.W_U"] = qwen.lm_head.weight.T - state_dict["unembed.b_U"] = torch.zeros(cfg.d_vocab, dtype=cfg.dtype) - - return state_dict diff --git a/transformer_lens/pretrained/weight_conversions/t5.py b/transformer_lens/pretrained/weight_conversions/t5.py deleted file mode 100644 index 8cf4f15431..0000000000 --- a/transformer_lens/pretrained/weight_conversions/t5.py +++ /dev/null @@ -1,101 +0,0 @@ -import einops - -from transformer_lens.config import TransformerLensConfig - - -def convert_t5_weights(t5, cfg: TransformerLensConfig): - state_dict = { - "embed.W_E": t5.encoder.embed_tokens.weight, - "unembed.W_U": t5.encoder.embed_tokens.weight.T, - "encoder.0.attn.rel_pos_bias.weight": t5.encoder.block[0] - .layer[0] - .SelfAttention.relative_attention_bias.weight, - } - - for l in range(cfg.n_layers): - block = t5.encoder.block[l] - state_dict[f"encoder.{l}.attn.W_Q"] = einops.rearrange( - block.layer[0].SelfAttention.q.weight, "(i h) m -> i m h", i=cfg.n_heads - ) - state_dict[f"encoder.{l}.attn.W_K"] = einops.rearrange( - block.layer[0].SelfAttention.k.weight, "(i h) m -> i m h", i=cfg.n_heads - ) - - state_dict[f"encoder.{l}.attn.W_V"] = einops.rearrange( - block.layer[0].SelfAttention.v.weight, "(i h) m -> i m h", i=cfg.n_heads - ) - - state_dict[f"encoder.{l}.attn.W_O"] = einops.rearrange( - block.layer[0].SelfAttention.o.weight, - "m (i h) -> i h m", - i=cfg.n_heads, - ) - state_dict[f"encoder.{l}.ln1.w"] = block.layer[0].layer_norm.weight - - # fixme DenseReluDense may be T5DenseGatedActDense instead - state_dict[f"encoder.{l}.mlp.W_in"] = einops.rearrange( - block.layer[1].DenseReluDense.wi.weight, "mlp model -> model mlp" - ) - - state_dict[f"encoder.{l}.mlp.W_out"] = einops.rearrange( - block.layer[1].DenseReluDense.wo.weight, "model mlp -> mlp model" - ) - state_dict[f"encoder.{l}.ln2.w"] = block.layer[1].layer_norm.weight - - state_dict["encoder_final_ln.w"] = t5.encoder.final_layer_norm.weight - - state_dict["decoder.0.attn.rel_pos_bias.weight"] = ( - t5.decoder.block[0].layer[0].SelfAttention.relative_attention_bias.weight - ) - - for l in range(cfg.n_layers): - block = t5.decoder.block[l] - state_dict[f"decoder.{l}.attn.W_Q"] = einops.rearrange( - block.layer[0].SelfAttention.q.weight, "(i h) m -> i m h", i=cfg.n_heads - ) - - state_dict[f"decoder.{l}.attn.W_K"] = einops.rearrange( - block.layer[0].SelfAttention.k.weight, "(i h) m -> i m h", i=cfg.n_heads - ) - state_dict[f"decoder.{l}.attn.W_V"] = einops.rearrange( - block.layer[0].SelfAttention.v.weight, "(i h) m -> i m h", i=cfg.n_heads - ) - - state_dict[f"decoder.{l}.attn.W_O"] = einops.rearrange( - block.layer[0].SelfAttention.o.weight, - "m (i h) -> i h m", - i=cfg.n_heads, - ) - - state_dict[f"decoder.{l}.ln1.w"] = block.layer[0].layer_norm.weight - - state_dict[f"decoder.{l}.cross_attn.W_Q"] = einops.rearrange( - block.layer[1].EncDecAttention.q.weight, "(i h) m -> i m h", i=cfg.n_heads - ) - - state_dict[f"decoder.{l}.cross_attn.W_K"] = einops.rearrange( - block.layer[1].EncDecAttention.k.weight, "(i h) m -> i m h", i=cfg.n_heads - ) - - state_dict[f"decoder.{l}.cross_attn.W_V"] = einops.rearrange( - block.layer[1].EncDecAttention.v.weight, "(i h) m -> i m h", i=cfg.n_heads - ) - state_dict[f"decoder.{l}.cross_attn.W_O"] = einops.rearrange( - block.layer[1].EncDecAttention.o.weight, - "m (i h) -> i h m", - i=cfg.n_heads, - ) - state_dict[f"decoder.{l}.ln2.w"] = block.layer[1].layer_norm.weight - - # fixme DenseReluDense may be T5DenseGatedActDense instead - state_dict[f"decoder.{l}.mlp.W_in"] = einops.rearrange( - block.layer[2].DenseReluDense.wi.weight, "mlp model -> model mlp" - ) - state_dict[f"decoder.{l}.mlp.W_out"] = einops.rearrange( - block.layer[2].DenseReluDense.wo.weight, "model mlp -> mlp model" - ) - state_dict[f"decoder.{l}.ln3.w"] = block.layer[2].layer_norm.weight - - state_dict["decoder_final_ln.w"] = t5.decoder.final_layer_norm.weight - - return state_dict diff --git a/transformer_lens/supported_models.py b/transformer_lens/supported_models.py index bd670c20f8..6bd53ac1a5 100644 --- a/transformer_lens/supported_models.py +++ b/transformer_lens/supported_models.py @@ -268,3 +268,29 @@ DEFAULT_MODEL_ALIASES: list[str] = [ MODEL_ALIASES[name][0] if name in MODEL_ALIASES else name for name in OFFICIAL_MODEL_NAMES ] + + +def _model_alias_map() -> dict[str, str]: + """Lowercased-key map from every alias (and official name) to its official + name — mirrors the deleted loading_from_pretrained.make_model_alias_map so + resolution stays case-insensitive.""" + alias_map: dict[str, str] = {} + for official_name in OFFICIAL_MODEL_NAMES: + for alias in MODEL_ALIASES.get(official_name, []): + alias_map[alias.lower()] = official_name + alias_map[official_name.lower()] = official_name + return alias_map + + +def get_official_model_name(model_name: str) -> str: + """Resolve a HookedTransformer-era alias to its official HF name. + + Case-insensitive (as the deleted ``loading_from_pretrained`` resolver was); + identity for an already-official name; raises for an unknown one. Rehomed so + the legacy-compatibility ledger and alias-drift tooling keep a canonical + resolver. + """ + official_name = _model_alias_map().get(model_name.lower()) + if official_name is None: + raise ValueError(f"{model_name!r} is not an official model name or a known alias.") + return official_name diff --git a/transformer_lens/train.py b/transformer_lens/train.py deleted file mode 100644 index 46b1b16593..0000000000 --- a/transformer_lens/train.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Deprecated: train.py moved to tools.training. - -Use transformer_lens.tools.training instead. -""" - -import warnings - -from torch.utils.data import Dataset - -from transformer_lens.model_protocol import TrainableTransformerLensModel -from transformer_lens.tools.training import TrainConfig as _TrainConfig -from transformer_lens.tools.training import train as _train - - -class HookedTransformerTrainConfig(_TrainConfig): - """Deprecated alias for TrainConfig. Use transformer_lens.tools.training.TrainConfig instead.""" - - def __init__(self, *args: object, **kwargs: object) -> None: - warnings.warn( - "HookedTransformerTrainConfig is deprecated; use " - "transformer_lens.tools.training.TrainConfig instead.", - DeprecationWarning, - stacklevel=2, - ) - super().__init__(*args, **kwargs) # type: ignore[arg-type] - - -def train( - model: "TrainableTransformerLensModel", config: _TrainConfig, dataset: Dataset -) -> "TrainableTransformerLensModel": - """Deprecated. Use transformer_lens.tools.training.train instead.""" - warnings.warn( - "transformer_lens.train is deprecated; use " - "transformer_lens.tools.training.train instead.", - DeprecationWarning, - stacklevel=2, - ) - return _train(model, config, dataset) diff --git a/transformer_lens/utilities/multi_gpu.py b/transformer_lens/utilities/multi_gpu.py index a855cbc5a9..735ffc354c 100644 --- a/transformer_lens/utilities/multi_gpu.py +++ b/transformer_lens/utilities/multi_gpu.py @@ -11,11 +11,15 @@ from torch import nn if TYPE_CHECKING: - from transformer_lens.config.hooked_transformer_config import ( - HookedTransformerConfig as ConfigType, + from transformer_lens.config.transformer_bridge_config import ( + TransformerBridgeConfig as ConfigType, + ) + from transformer_lens.config.transformer_lens_config import ( + TransformerLensConfig as BaseConfigType, ) else: ConfigType = Any + BaseConfigType = Any AvailableDeviceMemory = list[tuple[int, int]] """ @@ -104,7 +108,7 @@ def get_best_available_device( """Gets the best available device to be used based on the passed in arguments Args: - cfg: The HookedTransformerConfig object containing device configuration + cfg: The bridge config object containing device configuration Returns: torch.device: The best available device @@ -120,7 +124,7 @@ def get_best_available_device( def get_device_for_block_index( index: int, - cfg: ConfigType, + cfg: "BaseConfigType", device: Optional[Union[torch.device, str]] = None, ): """ @@ -154,7 +158,8 @@ def get_device_for_block_index( # the divide-by-zero when n_layers < n_devices. The naive form # `index // (n_layers // n_devices)` floors the divisor and overshoots when # n_layers is not a multiple of n_devices (e.g. 62 layers / 8 devices → 8). - device_index = (device.index or 0) + (index * cfg.n_devices) // cfg.n_layers + n_devices = getattr(cfg, "n_devices", 1) + device_index = (device.index or 0) + (index * n_devices) // cfg.n_layers return torch.device(device.type, device_index) diff --git a/transformer_lens/utils.py b/transformer_lens/utils.py deleted file mode 100644 index 8cecb5b524..0000000000 --- a/transformer_lens/utils.py +++ /dev/null @@ -1,64 +0,0 @@ -"""utils. - -This module is deprecated, but imports from the new utilities to maintain backwards compatibility. -""" - -import warnings - -from transformer_lens.utilities import * # noqa: F401,F403 - -warnings.warn( - "The 'utils' module has been deprecated. Please use 'transformer_lens.utilities' instead. " - "Importing from utils.py will be removed in TransformerLens 4.0.", - DeprecationWarning, - stacklevel=2, -) - - -__all__ = [ - "download_file_from_hf", - "clear_huggingface_cache", - "keep_single_column", - "get_dataset", - "print_gpu_mem", - "get_device", - "get_corner", - "to_numpy", - "remove_batch_dim", - "transpose", - "is_square", - "is_lower_triangular", - "check_structure", - "composition_scores", - "get_offset_position_ids", - "get_cumsum_along_dim", - "repeat_along_head_dimension", - "filter_dict_by_prefix", - "lm_cross_entropy_loss", - "lm_accuracy", - "gelu_new", - "gelu_fast", - "solu", - "calc_fan_in_and_fan_out", - "init_xavier_uniform_", - "init_xavier_normal_", - "init_kaiming_uniform_", - "init_kaiming_normal_", - "is_library_available", - "tokenize_and_concatenate", - "get_tokenizer_with_bos", - "get_input_with_manually_prepended_bos", - "get_tokens_with_bos_removed", - "get_attention_mask", - "sample_logits", - "SliceInput", - "Slice", - "get_act_name", - "get_nested_attr", - "set_nested_attr", - "override_or_use_default_value", - "LocallyOverridenDefaults", - "USE_DEFAULT_VALUE", - "test_prompt", - "warn_if_mps", -] diff --git a/uv.lock b/uv.lock index 230cb66c26..df22232fd4 100644 --- a/uv.lock +++ b/uv.lock @@ -592,15 +592,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/cd/30110dc0ffcf3b131156077b90e9f60ed75711223f306da4db08eff8403b/beautifulsoup4-4.13.4-py3-none-any.whl", hash = "sha256:9bbbb14bfde9d79f38b8cd5f8c7c85f4b8f2523190ebed90e950a8dea4cb1c4b", size = 187285, upload-time = "2025-04-15T17:05:12.221Z" }, ] -[[package]] -name = "better-abc" -version = "0.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8b/72/3d630f781659015357cc08cad32aa636b252e007df0bae31184a3d872427/better-abc-0.0.3.tar.gz", hash = "sha256:a880fd6bc9675da2ec991e8712a555bffa0f12722efed78c739f78343cf989f6", size = 2852, upload-time = "2020-11-10T22:47:31.303Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/e8/7d00a23039ab74c5741736ce05d7700eb6237e83747aac4df07a5bf2d074/better_abc-0.0.3-py3-none-any.whl", hash = "sha256:3ae73b473fbeb536a548f542984976e80b821676ae6e18f14e24d8e180647187", size = 3475, upload-time = "2020-11-10T22:47:30.354Z" }, -] - [[package]] name = "bitsandbytes" version = "0.49.2" @@ -5010,15 +5001,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351, upload-time = "2024-01-28T18:52:31.981Z" }, ] -[[package]] -name = "muutils" -version = "0.8.11" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ae/80/38cfd93c6e17356cb5be1d31d06e835b1a9f603f7fe35acce98d009db744/muutils-0.8.11.tar.gz", hash = "sha256:391abd59c57c81df5a2eef2a12217d4797b735256c6b01e20ed27b49bc475505", size = 3094363, upload-time = "2025-07-08T03:20:07.511Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/00/e4872f5da08e12ee3130889a96a2074c783b35e3cd096004203e62d3d659/muutils-0.8.11-py3-none-any.whl", hash = "sha256:a98718c4b216f37637bd6c2480494a330de758dfd5f334c2c28bbd18799ee767", size = 126722, upload-time = "2025-07-08T03:20:04.876Z" }, -] - [[package]] name = "mypy" version = "1.17.0" @@ -9523,7 +9505,6 @@ source = { editable = "." } dependencies = [ { name = "accelerate" }, { name = "beartype" }, - { name = "better-abc" }, { name = "datasets" }, { name = "einops" }, { name = "fancy-einsum" }, @@ -9584,7 +9565,6 @@ dev = [ ] docs = [ { name = "furo" }, - { name = "muutils" }, { name = "myst-parser" }, { name = "nbconvert" }, { name = "nbsphinx" }, @@ -9621,7 +9601,6 @@ tokenizers = [ requires-dist = [ { name = "accelerate", specifier = ">=1.1.0" }, { name = "beartype", specifier = ">=0.14.1" }, - { name = "better-abc", specifier = ">=0.0.3" }, { name = "chardet", marker = "extra == 'evals'", specifier = "<6" }, { name = "datasets", specifier = ">=2.7.1" }, { name = "einops", specifier = ">=0.6.0" }, @@ -9671,7 +9650,6 @@ dev = [ ] docs = [ { name = "furo", specifier = ">=2023.3.27,<2024.0.0" }, - { name = "muutils", specifier = ">=0.6.13" }, { name = "myst-parser", specifier = ">=2.0.0,<3.0.0" }, { name = "nbconvert", specifier = ">=7.9.2" }, { name = "nbsphinx", specifier = ">=0.9.3" }, From 9b3275efceaeb5b8768942ed63a9a6d14c55784f Mon Sep 17 00:00:00 2001 From: ZacharyZcR <zacharyzcr1984@gmail.com> Date: Wed, 9 Sep 2026 11:26:44 +0800 Subject: [PATCH 68/87] fix(demos): port LIT integration demo model loading to TransformerBridge (#1758) In #1740 the import in LIT_Integration_Demo.ipynb was updated to TransformerBridge, but the model loading cell still called HookedTransformer.from_pretrained, which raised a NameError since HookedTransformer was no longer imported. Update the cell to load via TransformerBridge.boot_transformers with enable_compatibility_mode(). Co-authored-by: Jonah Larson <jonahalarson@comcast.net> --- demos/LIT_Integration_Demo.ipynb | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/demos/LIT_Integration_Demo.ipynb b/demos/LIT_Integration_Demo.ipynb index ce146d5b69..17da0dca8d 100644 --- a/demos/LIT_Integration_Demo.ipynb +++ b/demos/LIT_Integration_Demo.ipynb @@ -186,12 +186,15 @@ } ], "source": [ - "# Load GPT-2 Small (124M parameters)\n", - "# Other options: \"gpt2-medium\", \"gpt2-large\", \"gpt2-xl\", \"pythia-70m\", etc.\n", - "model_name = \"gpt2-small\"\n", + "# Load GPT-2 (124M parameters)\n", + "# Other options: \"gpt2-medium\", \"gpt2-large\", \"gpt2-xl\", \"EleutherAI/pythia-70m\", etc.\n", + "model_name = \"gpt2\"\n", "\n", "print(f\"Loading {model_name}...\")\n", - "model = TransformerBridge.boot_transformers(model_name, device=device)\n", + "model = TransformerBridge.boot_transformers(\n", + " model_name,\n", + " device=device,\n", + ")\n", "model.enable_compatibility_mode()\n", "print(f\"Loaded model: {model.cfg.model_name}\")\n", "print(f\" Layers: {model.cfg.n_layers}\")\n", @@ -206,7 +209,7 @@ "source": [ "# Create LIT Wrapper\n", "\n", - "Now we wrap the HookedTransformer with our LIT wrapper to enable all the visualization features." + "Now we wrap the model with our LIT wrapper to enable all the visualization features." ] }, { From be40a9b49106ffda245e1c4db335891292eca443 Mon Sep 17 00:00:00 2001 From: emerardd <113128214+emerardd@users.noreply.github.com> Date: Sun, 13 Sep 2026 07:10:30 +0800 Subject: [PATCH 69/87] Fix IOIDataset BOS handling (#1773) * Fix IOIDataset BOS handling * Document IOIDataset BOS behavior --- tests/unit/test_evals_ioi.py | 85 +++++++++++++++++++++++++++++++++--- transformer_lens/evals.py | 9 ++-- 2 files changed, 84 insertions(+), 10 deletions(-) diff --git a/tests/unit/test_evals_ioi.py b/tests/unit/test_evals_ioi.py index a58b51f3df..4dcecf739e 100644 --- a/tests/unit/test_evals_ioi.py +++ b/tests/unit/test_evals_ioi.py @@ -1,21 +1,37 @@ -"""Tests for IOIDataset in transformer_lens/evals.py. - -Regression test for https://github.com/TransformerLensOrg/TransformerLens/issues/515 -""" +"""Tests for IOIDataset and ioi_eval in transformer_lens/evals.py.""" from unittest.mock import MagicMock -from transformer_lens.evals import IOIDataset +import torch + +from transformer_lens.evals import IOIDataset, ioi_eval def _make_tokenizer(): """Minimal mock tokenizer sufficient for IOIDataset.""" tok = MagicMock() - tok.encode.side_effect = lambda text: [1, 2, 3] + tok.encode.side_effect = lambda text, add_special_tokens=True: [1, 2, 3] tok.bos_token_id = 0 return tok +def _make_automatic_bos_tokenizer(): + """Tokenizer stub whose default encode path inserts a BOS token.""" + tok = MagicMock() + tok.bos_token_id = 0 + + def encode(text, add_special_tokens=True): + if text.startswith(" "): + payload = [30 if text.strip() == "Alice" else 31] + else: + target = 30 if text.split()[-1] == "Alice" else 31 + payload = [10, 20, target] + return ([tok.bos_token_id] if add_special_tokens else []) + payload + + tok.encode.side_effect = encode + return tok + + def test_ioi_dataset_produces_diverse_samples(): """IOIDataset must generate varied samples, not all-identical ones. @@ -63,3 +79,60 @@ def test_ioi_dataset_symmetric(): tokenizer = _make_tokenizer() dataset = IOIDataset(tokenizer, num_samples=10, symmetric=True) assert len(dataset.samples) == 10 + + +def test_ioi_dataset_prepend_bos_is_the_only_source_of_special_tokens(): + tokenizer = _make_automatic_bos_tokenizer() + with_bos = IOIDataset( + tokenizer, + templates=["[A] met [B] and [A]"], + names=["Alice", "Bob"], + nouns={}, + num_samples=1, + prepend_bos=True, + seed=0, + )[0] + without_bos = IOIDataset( + tokenizer, + templates=["[A] met [B] and [A]"], + names=["Alice", "Bob"], + nouns={}, + num_samples=1, + prepend_bos=False, + seed=0, + )[0] + + assert with_bos["prompt"].tolist() == [0, 10, 20, 31] + assert without_bos["prompt"].tolist() == [10, 20, 31] + for item in (with_bos, without_bos): + assert {tuple(item["IO"].tolist()), tuple(item["S"].tolist())} == {(30,), (31,)} + + +def test_ioi_eval_reads_logits_before_the_first_answer_token(): + tokenizer = _make_automatic_bos_tokenizer() + dataset = IOIDataset( + tokenizer, + templates=["[A] met [B] and [A]"], + names=["Alice", "Bob"], + nouns={}, + num_samples=1, + prepend_bos=True, + seed=0, + ) + + class PositionSensitiveModel: + def __call__(self, tokens, return_type): + assert return_type == "logits" + logits = torch.zeros(tokens.shape[0], tokens.shape[1], 32) + for row, prompt in enumerate(tokens): + answer = int(prompt[-1]) + distractor = 31 if answer == 30 else 30 + logits[row, 2, answer] = 1 + logits[row, 2, distractor] = -1 + logits[row, 3, answer] = -1 + logits[row, 3, distractor] = 1 + return logits + + result = ioi_eval(PositionSensitiveModel(), dataset=dataset, batch_size=1, tokenizer=tokenizer) + + assert result == {"Logit Difference": 2.0, "Accuracy": 1.0} diff --git a/transformer_lens/evals.py b/transformer_lens/evals.py index 69bbe6e2ed..e1cc0d4bbb 100644 --- a/transformer_lens/evals.py +++ b/transformer_lens/evals.py @@ -367,7 +367,8 @@ def __init__( nouns: Dict mapping placeholder names to lists of nouns. Defaults to built-in nouns. num_samples: Number of samples to generate. symmetric: If True, generate both orderings of each name pair. - prepend_bos: If True, prepend the BOS token to each prompt. + prepend_bos: If True, prepend one BOS token to each prompt. Tokenizer-added special + tokens are disabled, so False leaves the prompt without a BOS. seed: Optional random seed for reproducibility. If None, the current random state is used (samples will vary across runs). """ @@ -391,14 +392,14 @@ def __len__(self): def __getitem__(self, idx): sample = self.samples[idx] - prompt = self.tokenizer.encode(sample["text"]) + prompt = self.tokenizer.encode(sample["text"], add_special_tokens=False) if self.prepend_bos: prompt = [self.tokenizer.bos_token_id] + prompt return { "prompt": torch.LongTensor(prompt), - "IO": torch.LongTensor(self.tokenizer.encode(sample["IO"])), - "S": torch.LongTensor(self.tokenizer.encode(sample["S"])), + "IO": torch.LongTensor(self.tokenizer.encode(sample["IO"], add_special_tokens=False)), + "S": torch.LongTensor(self.tokenizer.encode(sample["S"], add_special_tokens=False)), } def get_sample(self, symmetric=False) -> List[Dict[str, str]]: From 0106853c8009ffbf62e7e41cddc5da7918fc2edf Mon Sep 17 00:00:00 2001 From: emerardd <113128214+emerardd@users.noreply.github.com> Date: Mon, 14 Sep 2026 21:36:13 +0800 Subject: [PATCH 70/87] Fix flaky direct path patching correctness test (#1783) tiny_model was built from the unseeded global RNG, so its weights varied with xdist scheduling. The linear-LN approximation error is weight-dependent (~17% of random inits exceed the 1e-3 tolerance), making test_correctness_against_actual_ln_forward flaky. Seed the fixture and correct the docstrings that claimed folding LN makes the approximation exact. --- tests/unit/test_direct_path_patching.py | 16 +++++++++------- .../tools/analysis/direct_path_patching.py | 7 ++++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/unit/test_direct_path_patching.py b/tests/unit/test_direct_path_patching.py index bc5b50e6cf..2224a14b0d 100644 --- a/tests/unit/test_direct_path_patching.py +++ b/tests/unit/test_direct_path_patching.py @@ -39,6 +39,8 @@ def tiny_model(): normalization_type="LN", attn_only=False, ) + # Seed locally: under xdist the global RNG state here depends on which tests ran earlier. + torch.manual_seed(0) model = HookedTransformer(cfg) model.process_weights_() model.eval() @@ -275,17 +277,17 @@ class TestCorrectness: def test_correctness_against_actual_ln_forward(self, tiny_model, tokens_and_caches): """Logit-diff metric: linear-LN approximation should match actual LN within 1e-3. - process_weights_() folds LN into the weight matrices, so the linear - approximation is exact and the tolerance can be tight. Using logit diff - (correct_tok - incorrect_tok) cancels the centering offset introduced by - process_weights_() and gives a numerically clean comparison. + Folding LN does not make the approximation exact: LayerNormPre still divides + by an input-dependent norm, while the approximation reuses the corrupted + run's scale. The error depends on the weights, so the tolerance holds for + the seeded tiny_model but is not guaranteed for arbitrary initialisations. + Using logit diff (correct_tok - incorrect_tok) cancels the centering offset + introduced by process_weights_(). """ _, corrupted_tokens, clean_cache, corrupted_cache = tokens_and_caches src_layer, src_head = 0, 0 dst_layer, dst_head = 2, 1 - # Pick stable token indices for the logit-diff metric - torch.manual_seed(0) correct_tok = 17 incorrect_tok = 42 @@ -339,7 +341,7 @@ def true_hook(value, hook): assert abs(our_metric - ref_metric) < 1e-3, ( f"Linear-LN approx {our_metric:.6f} disagrees with actual-LN ref {ref_metric:.6f} " - f"(diff={abs(our_metric - ref_metric):.2e}). process_weights_() should make these exact." + f"(diff={abs(our_metric - ref_metric):.2e})." ) def test_all_sources_consistent_with_single(self, tiny_model, tokens_and_caches): diff --git a/transformer_lens/tools/analysis/direct_path_patching.py b/transformer_lens/tools/analysis/direct_path_patching.py index 9714b2e490..b2855d71de 100644 --- a/transformer_lens/tools/analysis/direct_path_patching.py +++ b/transformer_lens/tools/analysis/direct_path_patching.py @@ -21,9 +21,10 @@ delta_q = (delta_resid / ln1_scale) @ W_Q[hb] # [batch, pos, d_head] patched_q = corrupted_q + delta_q -This is exact under linear layer norm (no learned offset changes the scale -in a way that matters for the perturbation), and matches the gradient-based -approximation used in attribution patching. +This is an approximation, not exact even with LayerNorm folded: it freezes the +corrupted run's ln1 scale, whereas the true scale shifts with the patched +residual. The error grows with how much ``delta_resid`` changes the residual +norm at the destination layer. Usage ----- From 3441b6bf556b38c78adb0995320ecb140a48a101 Mon Sep 17 00:00:00 2001 From: Janmenjaya Panda <83154020+janmenjayap@users.noreply.github.com> Date: Mon, 14 Sep 2026 19:09:03 +0530 Subject: [PATCH 71/87] feat(backward_lens): generalize Backward Lens to dense-MLP decoder-only Bridges (GPT-2, Pythia, GPT-NeoX) (#1778) * refactor(backward_lens): derive MLP weight layout from Bridge oracle Replace the GPT2ArchitectureAdapter isinstance check and the Conv1D-only projection guard with the Bridge's own weight_layout_in_out oracle, so support is decided per MLP projection instead of by model class. Dense MLPBridge input/output projections now resolve to "in_out" (Conv1D, e.g. GPT-2) or "out_in" (torch.nn.Linear, e.g. Pythia/GPT-NeoX), and the resolved layout threads through gradient-factor capture instead of the previous hardcoded "in_out". A projection whose wrapped module the oracle cannot orient is rejected with a clear error. Rename _require_raw_gpt2_bridge, _get_gpt2_mlp_projections, _capture_gpt2_mlp_gradient_factors, _GPT2GradientCapture, and _GPT2LayerGradientFactors to their model-agnostic names, and update call sites and docstrings accordingly. Repurpose the non-Conv1D rejection test to assert rejection of an unorientable component, since torch.nn.Linear projections are now accepted. Behavior-preserving for GPT-2: unit and integration suites for this module pass unchanged. * test(backward_lens): cover nn.Linear layout + Pythia-70m reconstruction Add model-free unit tests for dense-MLP projection discovery that exercise the Bridge weight-layout oracle directly, using real MLPBridge/LinearBridge instances rather than a booted model: a torch.nn.Linear MLP resolves to the "out_in" layout and accepts its transposed weight shape, a Conv1D MLP resolves to "in_out" with GPT-2-parity shapes, an unorientable wrapped module is rejected with a clear error, and a gated MLP is rejected regardless of orientation. Parametrize the core gradient-reconstruction integration test over a raw GPT-2 Bridge and a raw Pythia-70m Bridge sharing the same prompt and target token, asserting the same reconstruction tolerance bands for both and that each model's projections resolve to its own weight layout without any model-class conditional. Extend the weight/hook state-preservation and cleanup-on-failure integration tests to both models as well. * docs(backward_lens): state dense-MLP contract; generalize demo note Update the Backward Lens doc and demo notebook to describe the generalized dense-MLP decoder-only contract (GPT-2 and Pythia/GPT-NeoX) instead of the prior GPT-2-only framing. Explain both weight layouts (Conv1D [in, out] and torch.nn.Linear [out, in]) and that BackwardLens reads the layout from the Bridge projection component rather than the model class. Add a minimal Pythia-70m reconstruction example to the doc and a corresponding single-layer demo cell in the notebook. --- demos/Backward_Lens_Demo.ipynb | 51 ++++- docs/source/content/backward_lens.md | 52 +++-- tests/integration/test_backward_lens.py | 177 +++++++++++------- tests/unit/tools/test_backward_lens.py | 87 +++++++++ .../tools/analysis/backward_lens.py | 151 +++++++++------ 5 files changed, 372 insertions(+), 146 deletions(-) diff --git a/demos/Backward_Lens_Demo.ipynb b/demos/Backward_Lens_Demo.ipynb index 6e15b1238f..483b69dba1 100644 --- a/demos/Backward_Lens_Demo.ipynb +++ b/demos/Backward_Lens_Demo.ipynb @@ -5,9 +5,9 @@ "id": "97d2c037", "metadata": {}, "source": [ - "# Backward Lens: GPT-2 gradient factors in vocabulary space\n", + "# Backward Lens: gradient factors in vocabulary space\n", "\n", - "This notebook demonstrates `BackwardLens` on a small, fixed set of GPT-2 next-token targets. It reproduces the *method* of projecting forward inputs and backward signals into vocabulary space; it is not a paper-scale replication and makes no causal claim from token rankings alone. All examples and layers are declared before analysis, and all outcomes are shown without filtering.\n", + "This notebook demonstrates `BackwardLens` on a small, fixed set of GPT-2 next-token targets, then repeats one reconstruction against a raw Pythia-70m Bridge to show the same dense-MLP contract generalizes beyond `Conv1D` weights. It reproduces the *method* of projecting forward inputs and backward signals into vocabulary space; it is not a paper-scale replication and makes no causal claim from token rankings alone. All examples and layers are declared before analysis, and all outcomes are shown without filtering.\n", "\n", "The implementation follows [Katz et al. (2024)](https://aclanthology.org/2024.emnlp-main.142/). Gradient signs below are raw `d(loss) / d(tensor)` signs; gradient descent subtracts them." ] @@ -19,7 +19,7 @@ "source": [ "## Geometry\n", "\n", - "For GPT-2's `[in, out]` `Conv1D` weight layout, token-position factors reconstruct the gradient as\n", + "Each MLP linear projection stores its weight as `[in, out]` (`Conv1D`, e.g. GPT-2) or `[out, in]` (`torch.nn.Linear`, e.g. Pythia/GPT-NeoX). `BackwardLens` reads which layout applies from the Bridge projection component, not the model class, and token-position factors reconstruct the gradient as\n", "\n", "$$\\nabla_W L = \\sum_i x_i \\delta_i^\\mathsf{T} = X^\\mathsf{T}\\Delta.$$\n", "\n", @@ -1721,6 +1721,51 @@ ")" ] }, + { + "cell_type": "markdown", + "id": "390b96bc", + "metadata": {}, + "source": [ + "## Beyond GPT-2: a dense-MLP Bridge contract\n", + "\n", + "`BackwardLens` accepts any raw, non-gated dense-MLP `TransformerBridge` and reads each MLP projection's weight layout from the Bridge component instead of the model class. GPT-2's `Conv1D` projections store `[in, out]`; Pythia/GPT-NeoX's `torch.nn.Linear` projections store `[out, in]`. The single-layer reconstruction below repeats the first example above against a raw Pythia-70m Bridge and reports its resolved weight layout alongside the reconstruction error." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "1020c8d5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Pythia-70m input_projection weight layout: out_in\n", + "Pythia-70m reconstruction error within tolerance: True\n" + ] + } + ], + "source": [ + "with warnings.catch_warnings():\n", + " warnings.simplefilter(\"ignore\")\n", + " pythia_model = TransformerBridge.boot_transformers(\n", + " \"EleutherAI/pythia-70m\", device=DEVICE, dtype=torch.float32\n", + " )\n", + "pythia_model.eval()\n", + "\n", + "pythia_result = BackwardLens(pythia_model).analyze(prompt, target, [0])\n", + "pythia_factors = pythia_result.layer(0).input_projection.factors\n", + "print(f\"Pythia-70m input_projection weight layout: {pythia_factors.weight_layout}\")\n", + "print(\n", + " \"Pythia-70m reconstruction error within tolerance: \"\n", + " f\"{pythia_factors.absolute_reconstruction_error < 1e-4}\"\n", + ")\n", + "\n", + "del pythia_model\n", + "_ = gc.collect()" + ] + }, { "cell_type": "markdown", "id": "e3df0a77", diff --git a/docs/source/content/backward_lens.md b/docs/source/content/backward_lens.md index cc9e6fe6dc..bb69dfe2db 100644 --- a/docs/source/content/backward_lens.md +++ b/docs/source/content/backward_lens.md @@ -6,28 +6,36 @@ with the forward inputs and backward signals that compose a gradient. A readable vocabulary projection is a diagnostic, not by itself evidence that a token or neuron causes a model behavior. -TransformerLens currently provides a focused GPT-2 implementation through -`TransformerBridge`. It follows the method introduced by -[Katz et al. (2024)](https://aclanthology.org/2024.emnlp-main.142/). +TransformerLens supports dense-MLP decoder-only implementations through +`TransformerBridge` (for example GPT-2 and Pythia/GPT-NeoX). It follows the method +introduced by [Katz et al. (2024)](https://aclanthology.org/2024.emnlp-main.142/). ## Gradient factorization For one linear projection and one prompt, let $x_i \in \mathbb{R}^{d_{in}}$ be the input at token position $i$, and let $\delta_i = \partial L / \partial y_i \in \mathbb{R}^{d_{out}}$ be the loss -gradient at its output. GPT-2 stores `Conv1D` weights in `[in, out]` order, so +gradient at its output. A dense MLP projection stores its weight either as +`[in, out]` (`Conv1D`, e.g. GPT-2) or as `[out, in]` (`torch.nn.Linear`, e.g. +Pythia/GPT-NeoX). `BackwardLens` reads which layout applies from the Bridge +projection component itself, not from the model class, then computes $$ -\nabla_W L = \sum_i x_i \delta_i^\mathsf{T} = X^\mathsf{T}\Delta. +\nabla_W L = \sum_i x_i \delta_i^\mathsf{T} = X^\mathsf{T}\Delta, $$ +reporting the result in that projection's own storage layout: directly for +`[in, out]` storage, transposed for `[out, in]` storage. + `BackwardLens` captures both factors and independently computes the weight gradient. Each matrix result includes the reconstructed gradient and maximum absolute and scale-aware relative reconstruction errors. -### The two GPT-2 MLP matrices +### The two dense MLP matrices -The two projections expose different residual-width factors: +The two projections expose different residual-width factors. The weight shapes +below use `Conv1D` (`[in, out]`) storage, as GPT-2 uses; `torch.nn.Linear` storage +(e.g. Pythia/GPT-NeoX) reports each weight transposed. | Result | Weight shape | Projected factor | Shape before vocabulary projection | |---|---:|---|---:| @@ -121,6 +129,23 @@ according to the model and tokenizer configuration, so it includes a prepended B only when that configuration requests one. Treat `result.prompt_token_ids` as the source of truth for aligning all position-indexed factors and readouts. +The same call works unchanged against a Pythia/GPT-NeoX Bridge, whose `torch.nn.Linear` +MLP projections resolve to the `"out_in"` weight layout instead of GPT-2's `"in_out"`: + +```python +pythia = TransformerBridge.boot_transformers( + "EleutherAI/pythia-70m", device="cpu", dtype=torch.float32 +) + +pythia_result = BackwardLens(pythia).analyze( + prompt="The capital of France is", + target_token=" Paris", + layers=[0], +) + +pythia_result.layer(0).input_projection.factors.weight_layout # "out_in" +``` + ## Result structure `BackwardLens.analyze(...)` returns a detached `BackwardLensResult`: @@ -154,14 +179,17 @@ ranks use int64. The bounded default avoids retaining a The current implementation requires: -- A freshly booted, raw `TransformerBridge` using `GPT2ArchitectureAdapter`. -- Original, trainable GPT-2 `Conv1D` weights and a dense, non-gated MLP. +- A freshly booted, raw `TransformerBridge` whose dense MLP projections have a + Bridge-orientable weight layout (`Conv1D`, e.g. GPT-2, or `torch.nn.Linear`, + e.g. Pythia/GPT-NeoX). +- Original, trainable weights and a dense, non-gated MLP. - Compatibility mode and weight processing to remain disabled. - One non-empty prompt, one single-token target, and unique valid layer indices. It does not currently support batched prompts, multi-token target losses, gated MLPs, -other architecture families, compatibility-mode weights, model editing, or causal -claims about the displayed vocabulary rankings. +architecture families whose MLP projections have an unknown weight layout, +compatibility-mode weights, model editing, or causal claims about the displayed +vocabulary rankings. ## Model-state safety @@ -176,7 +204,7 @@ activation-editing hooks still affect the analyzed computation. | Symptom | Cause and resolution | |---|---| | Raw-Bridge or processed-weight error | Reboot with `TransformerBridge.boot_transformers(...)`; do not enable compatibility mode or process weights. | -| Target encodes to zero or multiple tokens | Choose text that maps to one GPT-2 token without BOS; check leading whitespace. | +| Target encodes to zero or multiple tokens | Choose text that maps to one token under the model's tokenizer without BOS; check leading whitespace. | | Duplicate or out-of-range layer error | Pass a non-empty sequence of unique indices in `[0, model.cfg.n_layers)`. | | Normalized logits were not requested | Call `analyze(..., normalized=True)` before using `logits(normalized=True)` or normalized ranks. | | Full logits were not retained | Call `analyze(..., return_full_logits=True)` before using `logits(...)` or ranking a token other than the analyzed target. | diff --git a/tests/integration/test_backward_lens.py b/tests/integration/test_backward_lens.py index 918fe9d26e..58a1f29242 100644 --- a/tests/integration/test_backward_lens.py +++ b/tests/integration/test_backward_lens.py @@ -12,6 +12,7 @@ PROMPT = "The capital of France is" TARGET = " Paris" LAYERS = (0, 11) +PYTHIA_LAYERS = (0, 3, 5) OVERLONG_PROMPT = " token" * 1024 DEVICE_DTYPE_CASES = [pytest.param("cpu", torch.bfloat16, id="cpu-bfloat16")] @@ -28,13 +29,22 @@ def gpt2_bridge(): return TransformerBridge.boot_transformers("gpt2", device="cpu", dtype=torch.float32) +@pytest.fixture(scope="module") +def pythia_bridge(): + from transformer_lens.model_bridge import TransformerBridge + + return TransformerBridge.boot_transformers( + "EleutherAI/pythia-70m", device="cpu", dtype=torch.float32 + ) + + @pytest.fixture(scope="module") def gradient_capture(gpt2_bridge): from transformer_lens.tools.analysis.backward_lens import ( - _capture_gpt2_mlp_gradient_factors, + _capture_dense_mlp_gradient_factors, ) - return _capture_gpt2_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, LAYERS) + return _capture_dense_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, LAYERS) @pytest.fixture(scope="module") @@ -61,36 +71,60 @@ def _projection_hook_snapshots(model, layer: int) -> list[tuple[int, ...]]: ] -def test_real_gpt2_factors_reconstruct_both_mlp_weight_gradients( - gradient_capture, gpt2_bridge +@pytest.mark.parametrize( + ("model_fixture", "layers", "d_model", "d_mlp", "expected_weight_layout", "asserts_bos"), + [ + pytest.param("gpt2_bridge", LAYERS, 768, 3072, "in_out", True, id="gpt2"), + pytest.param("pythia_bridge", PYTHIA_LAYERS, 512, 2048, "out_in", False, id="pythia-70m"), + ], +) +def test_real_dense_mlp_factors_reconstruct_both_mlp_weight_gradients( + request, + model_fixture: str, + layers: tuple[int, ...], + d_model: int, + d_mlp: int, + expected_weight_layout: str, + asserts_bos: bool, ) -> None: - assert gradient_capture.target_token_id == 6342 + from transformer_lens.tools.analysis.backward_lens import ( + _capture_dense_mlp_gradient_factors, + ) + + model = request.getfixturevalue(model_fixture) + gradient_capture = _capture_dense_mlp_gradient_factors(model, PROMPT, TARGET, layers) + expected_target_tokens = model.to_tokens(TARGET, prepend_bos=False) + assert gradient_capture.target_token_id == int(expected_target_tokens.item()) assert gradient_capture.loss > 0 - assert gradient_capture.prompt_token_ids.shape == (1, 6) assert gradient_capture.prompt_token_ids.device.type == "cpu" - expected_tokens = gpt2_bridge.to_tokens(PROMPT) + expected_tokens = model.to_tokens(PROMPT) assert torch.equal(gradient_capture.prompt_token_ids, expected_tokens) - assert gpt2_bridge.tokenizer is not None - assert gradient_capture.prompt_token_ids[0, 0].item() == gpt2_bridge.tokenizer.bos_token_id + prompt_length = int(expected_tokens.shape[1]) + assert gradient_capture.prompt_token_ids.shape == (1, prompt_length) + if asserts_bos: + assert model.tokenizer is not None + assert gradient_capture.prompt_token_ids[0, 0].item() == model.tokenizer.bos_token_id with torch.no_grad(): - logits = gpt2_bridge(expected_tokens) + logits = model(expected_tokens) expected_loss = F.cross_entropy( logits[:, -1, :], torch.tensor([gradient_capture.target_token_id]) ) assert gradient_capture.loss == pytest.approx(float(expected_loss), abs=1e-6, rel=1e-6) - assert [result.layer for result in gradient_capture.layers] == list(LAYERS) + assert [result.layer for result in gradient_capture.layers] == list(layers) + input_shape = (d_model, d_mlp) if expected_weight_layout == "in_out" else (d_mlp, d_model) + output_shape = (d_mlp, d_model) if expected_weight_layout == "in_out" else (d_model, d_mlp) for result in gradient_capture.layers: first = result.input_projection second = result.output_projection - assert first.forward_inputs.shape == (6, 768) - assert first.output_gradients.shape == (6, 3072) - assert first.weight_gradient.shape == (768, 3072) - assert second.forward_inputs.shape == (6, 3072) - assert second.output_gradients.shape == (6, 768) - assert second.weight_gradient.shape == (3072, 768) + assert first.forward_inputs.shape == (prompt_length, d_model) + assert first.output_gradients.shape == (prompt_length, d_mlp) + assert first.weight_gradient.shape == input_shape + assert second.forward_inputs.shape == (prompt_length, d_mlp) + assert second.output_gradients.shape == (prompt_length, d_model) + assert second.weight_gradient.shape == output_shape for factors in (first, second): - assert factors.weight_layout == "in_out" + assert factors.weight_layout == expected_weight_layout assert factors.reconstructed_gradient.shape == factors.weight_gradient.shape for tensor in ( factors.forward_inputs, @@ -313,50 +347,45 @@ def test_capture_rejects_invalid_analysis_inputs( match: str, ) -> None: from transformer_lens.tools.analysis.backward_lens import ( - _capture_gpt2_mlp_gradient_factors, + _capture_dense_mlp_gradient_factors, ) with pytest.raises(error, match=match): - _capture_gpt2_mlp_gradient_factors(gpt2_bridge, prompt, target, layers) + _capture_dense_mlp_gradient_factors(gpt2_bridge, prompt, target, layers) def test_capture_rejects_non_bridge_and_non_raw_states(gpt2_bridge, monkeypatch) -> None: from transformer_lens.tools.analysis.backward_lens import ( - _capture_gpt2_mlp_gradient_factors, + _capture_dense_mlp_gradient_factors, ) with pytest.raises(TypeError, match="TransformerBridge only"): - _capture_gpt2_mlp_gradient_factors(object(), PROMPT, TARGET, [0]) + _capture_dense_mlp_gradient_factors(object(), PROMPT, TARGET, [0]) monkeypatch.setattr(gpt2_bridge, "compatibility_mode", True) with pytest.raises(ValueError, match="compatibility mode"): - _capture_gpt2_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) + _capture_dense_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) monkeypatch.setattr(gpt2_bridge, "compatibility_mode", False) monkeypatch.setattr(gpt2_bridge, "_weights_processed", True) with pytest.raises(ValueError, match="processed"): - _capture_gpt2_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) + _capture_dense_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) monkeypatch.setattr(gpt2_bridge, "_weights_processed", False) - adapter = gpt2_bridge.adapter - monkeypatch.setattr(gpt2_bridge, "adapter", object()) - with pytest.raises(NotImplementedError, match="GPT2ArchitectureAdapter"): - _capture_gpt2_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) - monkeypatch.setattr(gpt2_bridge, "adapter", adapter) monkeypatch.setattr(gpt2_bridge.cfg, "gated_mlp", True) with pytest.raises(NotImplementedError, match="non-gated"): - _capture_gpt2_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) + _capture_dense_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) monkeypatch.setattr(gpt2_bridge.cfg, "gated_mlp", False) monkeypatch.setattr(gpt2_bridge, "tokenizer", None) with pytest.raises(ValueError, match="tokenizer"): - _capture_gpt2_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) + _capture_dense_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) def test_capture_rejects_inference_mode(gpt2_bridge) -> None: from transformer_lens.tools.analysis.backward_lens import ( - _capture_gpt2_mlp_gradient_factors, + _capture_dense_mlp_gradient_factors, ) with torch.inference_mode(): with pytest.raises(ValueError, match="inference_mode"): - _capture_gpt2_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) + _capture_dense_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) def test_capture_rejects_multi_device_before_projection_validation( @@ -370,29 +399,29 @@ def projection_validation_must_not_run(*_args, **_kwargs) -> None: monkeypatch.setattr(gpt2_bridge.cfg, "n_devices", 2) monkeypatch.setattr( backward_lens, - "_get_gpt2_mlp_projections", + "_get_dense_mlp_projections", projection_validation_must_not_run, ) with pytest.raises(ValueError, match=r"single-device.*co-located.*n_devices=2"): - backward_lens._capture_gpt2_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) + backward_lens._capture_dense_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) def test_capture_rejects_a_per_layer_gate(gpt2_bridge, monkeypatch) -> None: - from transformer_lens.tools.analysis.backward_lens import _get_gpt2_mlp_projections + from transformer_lens.tools.analysis.backward_lens import _get_dense_mlp_projections mlp = gpt2_bridge.blocks[0].mlp monkeypatch.setattr(mlp, "gate", torch.nn.Identity(), raising=False) with pytest.raises(ValueError, match="dense, non-gated MLPBridge"): - _get_gpt2_mlp_projections(gpt2_bridge, (0,)) + _get_dense_mlp_projections(gpt2_bridge, (0,)) -def test_capture_rejects_a_non_conv1d_component(gpt2_bridge, monkeypatch) -> None: - from transformer_lens.tools.analysis.backward_lens import _get_gpt2_mlp_projections +def test_capture_rejects_an_unorientable_component(gpt2_bridge, monkeypatch) -> None: + from transformer_lens.tools.analysis.backward_lens import _get_dense_mlp_projections projection = getattr(gpt2_bridge.blocks[0].mlp, "in") - monkeypatch.setitem(projection._modules, "_original_component", torch.nn.Linear(1, 1)) - with pytest.raises(ValueError, match="input projection must wrap GPT-2 Conv1D"): - _get_gpt2_mlp_projections(gpt2_bridge, (0,)) + monkeypatch.setitem(projection._modules, "_original_component", torch.nn.Identity()) + with pytest.raises(ValueError, match="unknown weight layout"): + _get_dense_mlp_projections(gpt2_bridge, (0,)) @pytest.mark.parametrize( @@ -409,7 +438,7 @@ def test_capture_rejects_an_invalid_weight( invalidity: str, match: str, ) -> None: - from transformer_lens.tools.analysis.backward_lens import _get_gpt2_mlp_projections + from transformer_lens.tools.analysis.backward_lens import _get_dense_mlp_projections component = getattr(gpt2_bridge.blocks[0].mlp, "in").original_component if invalidity == "shape": @@ -420,12 +449,12 @@ def test_capture_rejects_an_invalid_weight( ) monkeypatch.setattr(component, "weight", replacement_weight) with pytest.raises(ValueError, match=match): - _get_gpt2_mlp_projections(gpt2_bridge, (0,)) + _get_dense_mlp_projections(gpt2_bridge, (0,)) def test_capture_rejects_a_frozen_original_weight(gpt2_bridge) -> None: from transformer_lens.tools.analysis.backward_lens import ( - _capture_gpt2_mlp_gradient_factors, + _capture_dense_mlp_gradient_factors, ) weight = getattr(gpt2_bridge.blocks[0].mlp, "in").original_component.weight @@ -433,28 +462,32 @@ def test_capture_rejects_a_frozen_original_weight(gpt2_bridge) -> None: weight.requires_grad_(False) try: with pytest.raises(ValueError, match="trainable Parameter"): - _capture_gpt2_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) + _capture_dense_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) finally: weight.requires_grad_(original_requires_grad) +@pytest.mark.parametrize( + "model_fixture", ["gpt2_bridge", "pythia_bridge"], ids=["gpt2", "pythia-70m"] +) def test_capture_preserves_model_state_hooks_and_uses_one_autograd_call( - gpt2_bridge, monkeypatch + request, model_fixture: str, monkeypatch ) -> None: from transformer_lens.model_bridge.generalized_components.normalization import ( NATIVE_PATH_BWD_FALLBACK_WARNING, NATIVE_PATH_EDIT_FALLBACK_WARNING, ) from transformer_lens.tools.analysis.backward_lens import ( - _capture_gpt2_mlp_gradient_factors, + _capture_dense_mlp_gradient_factors, ) - mlp = gpt2_bridge.blocks[0].mlp + model = request.getfixturevalue(model_fixture) + mlp = model.blocks[0].mlp projections = (getattr(mlp, "in"), mlp.out) weights = [projection.original_component.weight for projection in projections] saved_grads = [weight.grad for weight in weights] weight_copies = [weight.detach().clone() for weight in weights] - training = gpt2_bridge.training + training = model.training outer_rng = torch.random.get_rng_state() hook_calls = 0 autograd_calls = 0 @@ -473,19 +506,19 @@ def counting_grad(*args, **kwargs): hook_point.add_hook(existing_hook) existing_handle = hook_point.fwd_hooks[-1] try: - gpt2_bridge.train(True) + model.train(True) torch.manual_seed(1234) rng_before = torch.random.get_rng_state() for index, weight in enumerate(weights): weight.grad = torch.full_like(weight, index + 1.0) grad_copies = [weight.grad.clone() for weight in weights] requires_grad = [weight.requires_grad for weight in weights] - hooks_before = _projection_hook_snapshots(gpt2_bridge, 0) + hooks_before = _projection_hook_snapshots(model, 0) monkeypatch.setattr(torch.autograd, "grad", counting_grad) with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - result = _capture_gpt2_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) + result = _capture_dense_mlp_gradient_factors(model, PROMPT, TARGET, [0]) assert len(result.layers) == 1 assert autograd_calls == 1 @@ -496,8 +529,8 @@ def counting_grad(*args, **kwargs): for warning in caught ) assert torch.equal(torch.random.get_rng_state(), rng_before) - assert gpt2_bridge.training is True - assert _projection_hook_snapshots(gpt2_bridge, 0) == hooks_before + assert model.training is True + assert _projection_hook_snapshots(model, 0) == hooks_before for weight, saved_weight, saved_grad, expected_requires_grad in zip( weights, weight_copies, grad_copies, requires_grad, strict=True ): @@ -510,13 +543,13 @@ def counting_grad(*args, **kwargs): for weight, saved_grad in zip(weights, saved_grads, strict=True): weight.grad = saved_grad - gpt2_bridge.train(training) + model.train(training) torch.random.set_rng_state(outer_rng) def test_capture_reconstructs_with_existing_activation_edits(gpt2_bridge) -> None: from transformer_lens.tools.analysis.backward_lens import ( - _capture_gpt2_mlp_gradient_factors, + _capture_dense_mlp_gradient_factors, ) mlp = gpt2_bridge.blocks[0].mlp @@ -534,7 +567,7 @@ def scale_output(tensor, hook=None): output_hook_point.add_hook(scale_output) output_handle = output_hook_point.fwd_hooks[-1] try: - result = _capture_gpt2_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) + result = _capture_dense_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) assert input_handle in input_hook_point.fwd_hooks assert output_handle in output_hook_point.fwd_hooks finally: @@ -557,7 +590,7 @@ def scale_output(tensor, hook=None): def test_capture_cleans_owned_hooks_when_autograd_raises(gpt2_bridge, monkeypatch) -> None: from transformer_lens.tools.analysis.backward_lens import ( - _capture_gpt2_mlp_gradient_factors, + _capture_dense_mlp_gradient_factors, ) hook_point = gpt2_bridge.blocks[0].mlp.out.hook_out @@ -576,7 +609,7 @@ def fail_autograd(*args, **kwargs): monkeypatch.setattr(torch.autograd, "grad", fail_autograd) try: with pytest.raises(RuntimeError, match="forced autograd failure"): - _capture_gpt2_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) + _capture_dense_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) assert hook_calls == 1 assert existing_handle in hook_point.fwd_hooks assert _projection_hook_snapshots(gpt2_bridge, 0) == hooks_before @@ -585,17 +618,21 @@ def fail_autograd(*args, **kwargs): hook_point.fwd_hooks.remove(existing_handle) -def test_capture_removes_only_owned_hooks_when_forward_fails(gpt2_bridge) -> None: +@pytest.mark.parametrize( + "model_fixture", ["gpt2_bridge", "pythia_bridge"], ids=["gpt2", "pythia-70m"] +) +def test_capture_removes_only_owned_hooks_when_forward_fails(request, model_fixture: str) -> None: from transformer_lens.tools.analysis.backward_lens import ( - _capture_gpt2_mlp_gradient_factors, + _capture_dense_mlp_gradient_factors, ) - mlp = gpt2_bridge.blocks[0].mlp + model = request.getfixturevalue(model_fixture) + mlp = model.blocks[0].mlp projections = (getattr(mlp, "in"), mlp.out) weights = [projection.original_component.weight for projection in projections] saved_grads = [weight.grad for weight in weights] weight_copies = [weight.detach().clone() for weight in weights] - training = gpt2_bridge.training + training = model.training outer_rng = torch.random.get_rng_state() hook_point = mlp.out.hook_out @@ -604,19 +641,19 @@ def fail(_module, _inputs, _output) -> None: existing_handle = hook_point.register_forward_hook(fail) try: - gpt2_bridge.train(True) + model.train(True) torch.manual_seed(5678) rng_before = torch.random.get_rng_state() for index, weight in enumerate(weights): weight.grad = torch.full_like(weight, index + 3.0) grad_copies = [weight.grad.clone() for weight in weights] requires_grad = [weight.requires_grad for weight in weights] - hooks_before = _projection_hook_snapshots(gpt2_bridge, 0) + hooks_before = _projection_hook_snapshots(model, 0) with pytest.raises(RuntimeError, match="forced existing-hook failure"): - _capture_gpt2_mlp_gradient_factors(gpt2_bridge, PROMPT, TARGET, [0]) - assert _projection_hook_snapshots(gpt2_bridge, 0) == hooks_before + _capture_dense_mlp_gradient_factors(model, PROMPT, TARGET, [0]) + assert _projection_hook_snapshots(model, 0) == hooks_before assert torch.equal(torch.random.get_rng_state(), rng_before) - assert gpt2_bridge.training is True + assert model.training is True for weight, saved_weight, saved_grad, expected_requires_grad in zip( weights, weight_copies, grad_copies, requires_grad, strict=True ): @@ -627,7 +664,7 @@ def fail(_module, _inputs, _output) -> None: existing_handle.remove() for weight, saved_grad in zip(weights, saved_grads, strict=True): weight.grad = saved_grad - gpt2_bridge.train(training) + model.train(training) torch.random.set_rng_state(outer_rng) diff --git a/tests/unit/tools/test_backward_lens.py b/tests/unit/tools/test_backward_lens.py index 2095b91c11..b346ad22ee 100644 --- a/tests/unit/tools/test_backward_lens.py +++ b/tests/unit/tools/test_backward_lens.py @@ -2,23 +2,51 @@ from dataclasses import FrozenInstanceError from types import SimpleNamespace +from typing import Any import pytest import torch import torch.nn.functional as F +from transformers.pytorch_utils import Conv1D +from transformer_lens.model_bridge.generalized_components.linear import LinearBridge +from transformer_lens.model_bridge.generalized_components.mlp import MLPBridge from transformer_lens.tools.analysis.backward_lens import ( BackwardLensMatrixResult, LinearGradientFactors, _build_linear_gradient_factors, _build_matrix_result, _factor_norms_and_normalized_rows, + _get_dense_mlp_projections, _project_residual_factors, _rank_vocabulary_logits, _single_batch_matrix, ) +def _dense_mlp_bridge( + *, input_component: torch.nn.Module, output_component: torch.nn.Module, gate: Any = None +) -> MLPBridge: + """Build a real MLPBridge wrapping the given input/output projection modules.""" + mlp = MLPBridge(name="mlp") + input_projection = LinearBridge(name="mlp.in") + input_projection.set_original_component(input_component) + output_projection = LinearBridge(name="mlp.out") + output_projection.set_original_component(output_component) + setattr(mlp, "in", input_projection) + setattr(mlp, "out", output_projection) + if gate is not None: + setattr(mlp, "gate", gate) + return mlp + + +def _dense_mlp_model(mlp: MLPBridge, *, d_model: int, d_mlp: int) -> SimpleNamespace: + return SimpleNamespace( + cfg=SimpleNamespace(d_model=d_model, d_mlp=d_mlp), + blocks=[SimpleNamespace(mlp=mlp)], + ) + + class _ReadoutModel: def __init__(self, *, affine: bool = True): self.cfg = SimpleNamespace(d_model=3) @@ -502,6 +530,65 @@ def test_ranking_accessors_return_owned_retained_prefixes() -> None: assert torch.equal(result.normalized_bottom_ranking.indices, saved_bottom_indices) +def test_get_dense_mlp_projections_resolves_nn_linear_as_out_in() -> None: + d_model, d_mlp = 4, 6 + mlp = _dense_mlp_bridge( + input_component=torch.nn.Linear(d_model, d_mlp), + output_component=torch.nn.Linear(d_mlp, d_model), + ) + model = _dense_mlp_model(mlp, d_model=d_model, d_mlp=d_mlp) + + projections = _get_dense_mlp_projections(model, (0,)) + + input_record, output_record = projections[0] + assert input_record.weight_layout == "out_in" + assert output_record.weight_layout == "out_in" + assert tuple(input_record.projection.original_component.weight.shape) == (d_mlp, d_model) + assert tuple(output_record.projection.original_component.weight.shape) == (d_model, d_mlp) + + +def test_get_dense_mlp_projections_resolves_conv1d_as_in_out() -> None: + d_model, d_mlp = 4, 6 + mlp = _dense_mlp_bridge( + input_component=Conv1D(d_mlp, d_model), + output_component=Conv1D(d_model, d_mlp), + ) + model = _dense_mlp_model(mlp, d_model=d_model, d_mlp=d_mlp) + + projections = _get_dense_mlp_projections(model, (0,)) + + input_record, output_record = projections[0] + assert input_record.weight_layout == "in_out" + assert output_record.weight_layout == "in_out" + assert tuple(input_record.projection.original_component.weight.shape) == (d_model, d_mlp) + assert tuple(output_record.projection.original_component.weight.shape) == (d_mlp, d_model) + + +def test_get_dense_mlp_projections_rejects_an_unorientable_component() -> None: + d_model, d_mlp = 4, 6 + mlp = _dense_mlp_bridge( + input_component=torch.nn.Identity(), + output_component=torch.nn.Linear(d_mlp, d_model), + ) + model = _dense_mlp_model(mlp, d_model=d_model, d_mlp=d_mlp) + + with pytest.raises(ValueError, match="unknown weight layout"): + _get_dense_mlp_projections(model, (0,)) + + +def test_get_dense_mlp_projections_rejects_a_gated_mlp() -> None: + d_model, d_mlp = 4, 6 + mlp = _dense_mlp_bridge( + input_component=torch.nn.Linear(d_model, d_mlp), + output_component=torch.nn.Linear(d_mlp, d_model), + gate=LinearBridge(name="mlp.gate"), + ) + model = _dense_mlp_model(mlp, d_model=d_model, d_mlp=d_mlp) + + with pytest.raises(ValueError, match="dense, non-gated"): + _get_dense_mlp_projections(model, (0,)) + + def test_public_backward_lens_symbols_are_exported() -> None: from transformer_lens.tools.analysis import ( BackwardLens, diff --git a/transformer_lens/tools/analysis/backward_lens.py b/transformer_lens/tools/analysis/backward_lens.py index 419fb2fdef..3743623fba 100644 --- a/transformer_lens/tools/analysis/backward_lens.py +++ b/transformer_lens/tools/analysis/backward_lens.py @@ -2,7 +2,9 @@ The Backward Lens represents a linear weight gradient as a sum of token-position outer products and projects residual-width factors into the model vocabulary. -The public API currently supports raw GPT-2 ``TransformerBridge`` models. +The public API supports raw dense-MLP decoder-only ``TransformerBridge`` models +(for example GPT-2 and Pythia/GPT-NeoX), reading each MLP projection's weight +layout from the Bridge component rather than the model class. """ from __future__ import annotations @@ -55,7 +57,7 @@ class VocabularyRanking: @dataclass(frozen=True) class BackwardLensMatrixResult: - """Factors and vocabulary readouts for one GPT-2 MLP weight matrix. + """Factors and vocabulary readouts for one dense MLP weight matrix. ``factors`` contains the full linear factorization. ``projected_factor`` says whether its residual-width ``forward_inputs`` or raw-gradient @@ -228,8 +230,8 @@ def layer(self, layer: int) -> BackwardLensLayerResult: @dataclass(frozen=True) -class _GPT2LayerGradientFactors: - """Detached gradient factors for both MLP projections in one GPT-2 layer.""" +class _MLPLayerGradientFactors: + """Detached gradient factors for both MLP projections in one layer.""" layer: int input_projection: LinearGradientFactors @@ -237,17 +239,16 @@ class _GPT2LayerGradientFactors: @dataclass(frozen=True) -class _GPT2GradientCapture: - """Private Commit-2 result for one GPT-2 next-token loss. +class _DenseMLPGradientCapture: + """Private capture result for one dense-MLP next-token loss. - Tensor fields are detached, owned CPU copies. Public vocabulary-facing result - contracts are introduced with the projection API. + Tensor fields are detached, owned CPU copies. """ prompt_token_ids: Int[torch.Tensor, "1 position"] target_token_id: int loss: float - layers: tuple[_GPT2LayerGradientFactors, ...] + layers: tuple[_MLPLayerGradientFactors, ...] def _validate_floating_matrix(name: str, tensor: Any) -> Float[torch.Tensor, "rows columns"]: @@ -546,16 +547,17 @@ def _validate_requested_layers(model: Any, layers: Sequence[int]) -> tuple[int, return requested -def _require_raw_gpt2_bridge(model: Any) -> None: - """Require the raw GPT-2 Bridge capabilities used by gradient capture.""" +def _require_raw_dense_mlp_bridge(model: Any) -> None: + """Require the raw dense-MLP Bridge capabilities used by gradient capture. + + The architecture is not constrained by class; support is decided per + projection from the Bridge weight-layout oracle in projection discovery. + """ from transformer_lens.model_bridge import TransformerBridge - from transformer_lens.model_bridge.supported_architectures.gpt2 import ( - GPT2ArchitectureAdapter, - ) if not isinstance(model, TransformerBridge): raise TypeError( - "Backward Lens supports TransformerBridge only; load GPT-2 with " + "Backward Lens supports TransformerBridge only; load the model with " "TransformerBridge.boot_transformers(...)." ) if getattr(model, "compatibility_mode", False): @@ -564,7 +566,7 @@ def _require_raw_gpt2_bridge(model: Any) -> None: ) if getattr(model, "_weights_processed", False): raise ValueError( - "Backward Lens requires original GPT-2 weights; this Bridge processed its weights" + "Backward Lens requires original model weights; this Bridge processed its weights" ) if int(model.cfg.n_devices) > 1: raise ValueError( @@ -572,48 +574,73 @@ def _require_raw_gpt2_bridge(model: Any) -> None: "projections, final normalization, and unembed must be co-located; " f"device-map dispatch with cfg.n_devices={model.cfg.n_devices} is not supported" ) - if not isinstance(model.adapter, GPT2ArchitectureAdapter): - raise NotImplementedError( - "Backward Lens currently supports the GPT2ArchitectureAdapter only; " - f"got {type(model.adapter).__name__}" - ) if bool(getattr(model.cfg, "gated_mlp", False)): - raise NotImplementedError("Backward Lens currently requires dense, non-gated GPT-2 MLPs") + raise NotImplementedError("Backward Lens currently requires dense, non-gated MLPs") if model.tokenizer is None: - raise ValueError("Backward Lens requires a GPT-2 Bridge with a tokenizer") + raise ValueError("Backward Lens requires a TransformerBridge with a tokenizer") for component in ("blocks", "ln_final", "unembed"): if not hasattr(model, component): raise ValueError(f"Backward Lens requires the standard {component} component") -def _get_gpt2_mlp_projections(model: Any, layers: tuple[int, ...]) -> dict[int, tuple[Any, Any]]: - """Return validated live GPT-2 Conv1D input/output projection bridges.""" - from transformers.pytorch_utils import Conv1D +@dataclass(frozen=True) +class _MLPLinear: + """One validated dense-MLP linear projection with its resolved weight layout.""" + + projection: Any + weight_layout: WeightLayout + + +def _get_dense_mlp_projections( + model: Any, layers: tuple[int, ...] +) -> dict[int, tuple[_MLPLinear, _MLPLinear]]: + """Return validated live dense-MLP input/output projection bridges. + Each projection's storage orientation is resolved from the Bridge weight-layout + oracle, so Conv1D ``[in, out]`` and ``torch.nn.Linear`` ``[out, in]`` weights are + both accepted without inspecting the model class. Projections whose wrapped + module the oracle cannot orient are rejected. + """ from transformer_lens.hook_points import HookPoint from transformer_lens.model_bridge.generalized_components import ( LinearBridge, MLPBridge, ) - - expected_shapes = ( - (int(model.cfg.d_model), int(model.cfg.d_mlp)), - (int(model.cfg.d_mlp), int(model.cfg.d_model)), + from transformer_lens.model_bridge.generalized_components.mlp import ( + weight_layout_in_out, ) - projections: dict[int, tuple[Any, Any]] = {} + + d_model = int(model.cfg.d_model) + d_mlp = int(model.cfg.d_mlp) + # Feature counts are fixed by the MLP role; storage order follows the layout. + # Input maps d_model -> d_mlp, output maps d_mlp -> d_model. + feature_pairs = ((d_model, d_mlp), (d_mlp, d_model)) + projections: dict[int, tuple[_MLPLinear, _MLPLinear]] = {} for layer in layers: mlp = model.blocks[layer].mlp if not isinstance(mlp, MLPBridge) or getattr(mlp, "gate", None) is not None: raise ValueError(f"layer {layer} must have a dense, non-gated MLPBridge") pair = (getattr(mlp, "in", None), getattr(mlp, "out", None)) - for name, projection, expected_shape in zip( - ("input", "output"), pair, expected_shapes, strict=True + records: list[_MLPLinear] = [] + for name, projection, feature_pair in zip( + ("input", "output"), pair, feature_pairs, strict=True ): if not isinstance(projection, LinearBridge): raise ValueError(f"layer {layer} {name} projection must be a LinearBridge") - if not isinstance(projection.original_component, Conv1D): - raise ValueError(f"layer {layer} {name} projection must wrap GPT-2 Conv1D") - weight = projection.original_component.weight + layout_flag = weight_layout_in_out(projection) + if layout_flag is None: + raise ValueError( + f"layer {layer} {name} projection has an unknown weight layout; " + "Backward Lens supports Conv1D or torch.nn.Linear MLP projections" + ) + weight_layout: WeightLayout = "in_out" if layout_flag else "out_in" + in_features, out_features = feature_pair + expected_shape = ( + (in_features, out_features) + if weight_layout == "in_out" + else (out_features, in_features) + ) + weight = getattr(projection.original_component, "weight", None) if not isinstance(weight, torch.nn.Parameter): raise ValueError( f"layer {layer} {name} original weight must be a trainable Parameter" @@ -631,7 +658,8 @@ def _get_gpt2_mlp_projections(model: Any, layers: tuple[int, ...]) -> dict[int, projection.hook_out, HookPoint ): raise ValueError(f"layer {layer} {name} projection is missing Bridge hook points") - projections[layer] = pair + records.append(_MLPLinear(projection=projection, weight_layout=weight_layout)) + projections[layer] = (records[0], records[1]) return projections @@ -652,14 +680,15 @@ def capture(_module: torch.nn.Module, _inputs: tuple[Any, ...], output: Any) -> @contextmanager def _capture_projection_tensors( - projections: dict[int, tuple[Any, Any]], + projections: dict[int, tuple[_MLPLinear, _MLPLinear]], ) -> Iterator[dict[tuple[int, str, str], torch.Tensor]]: """Capture exact linear boundaries while preserving every pre-existing hook.""" captured: dict[tuple[int, str, str], torch.Tensor] = {} handles: list[Any] = [] try: for layer, pair in projections.items(): - for name, projection in zip(("input", "output"), pair, strict=True): + for name, record in zip(("input", "output"), pair, strict=True): + projection = record.projection input_key = (layer, name, "forward_input") output_key = (layer, name, "output") # Existing hook_in edits must run first so this is the actual linear input. @@ -712,13 +741,13 @@ def _preserve_model_rng(model: Any) -> Iterator[None]: torch.mps.set_rng_state(mps_state) -def _capture_gpt2_mlp_gradient_factors( +def _capture_dense_mlp_gradient_factors( model: Any, prompt: str, target_token: str, layers: Sequence[int], -) -> _GPT2GradientCapture: - """Capture exact GPT-2 MLP weight-gradient factors for one next-token loss. +) -> _DenseMLPGradientCapture: + """Capture exact dense-MLP weight-gradient factors for one next-token loss. The analysis performs one grad-enabled forward and exactly one :func:`torch.autograd.grad` call. It does not call ``backward``, touch @@ -729,9 +758,9 @@ def _capture_gpt2_mlp_gradient_factors( "Backward Lens cannot capture gradients inside torch.inference_mode(); " "exit inference_mode before running the analysis" ) - _require_raw_gpt2_bridge(model) + _require_raw_dense_mlp_bridge(model) requested_layers = _validate_requested_layers(model, layers) - projections = _get_gpt2_mlp_projections(model, requested_layers) + projections = _get_dense_mlp_projections(model, requested_layers) if not isinstance(prompt, str): raise TypeError("prompt must be a string") if prompt == "": @@ -759,17 +788,15 @@ def _capture_gpt2_mlp_gradient_factors( input_device = next(model.original_model.parameters()).device prompt_tokens = prompt_tokens.to(input_device) weights = [ - projection.original_component.weight + record.projection.original_component.weight for layer in requested_layers - for projection in projections[layer] + for record in projections[layer] ] with _preserve_model_rng(model), torch.enable_grad(): with _capture_projection_tensors(projections) as captured: logits = model(prompt_tokens) if not isinstance(logits, torch.Tensor) or logits.ndim != 3 or logits.shape[0] != 1: - raise RuntimeError( - "GPT-2 Bridge must return logits with shape [1, position, vocab]" - ) + raise RuntimeError("the Bridge must return logits with shape [1, position, vocab]") target = torch.tensor([target_token_id], device=logits.device) loss = F.cross_entropy(logits[:, -1, :], target) if not bool(torch.isfinite(loss)): @@ -786,6 +813,7 @@ def _capture_gpt2_mlp_gradient_factors( layer_results = [] for index, layer in enumerate(requested_layers): input_offset = 2 * index + input_record, output_record = projections[layer] input_factors = _build_linear_gradient_factors( _single_batch_matrix( f"layer {layer} input projection input", @@ -795,7 +823,7 @@ def _capture_gpt2_mlp_gradient_factors( f"layer {layer} input projection gradient", output_gradients[input_offset] ), weight_gradients[input_offset], - weight_layout="in_out", + weight_layout=input_record.weight_layout, ) output_factors = _build_linear_gradient_factors( _single_batch_matrix( @@ -807,16 +835,16 @@ def _capture_gpt2_mlp_gradient_factors( output_gradients[input_offset + 1], ), weight_gradients[input_offset + 1], - weight_layout="in_out", + weight_layout=output_record.weight_layout, ) layer_results.append( - _GPT2LayerGradientFactors( + _MLPLayerGradientFactors( layer=layer, input_projection=input_factors, output_projection=output_factors, ) ) - return _GPT2GradientCapture( + return _DenseMLPGradientCapture( prompt_token_ids=prompt_tokens.detach().cpu().clone(), target_token_id=target_token_id, loss=float(loss.detach()), @@ -825,16 +853,17 @@ def _capture_gpt2_mlp_gradient_factors( class BackwardLens: - """Analyze GPT-2 MLP weight gradients in the output vocabulary basis. + """Analyze dense MLP weight gradients in the output vocabulary basis. - The analyzer accepts a fresh, raw GPT-2 :class:`TransformerBridge`. Results - retain no model or tokenizer reference and contain detached CPU-owned tensors. - Raw backward signals are loss gradients; gradient descent subtracts them. + The analyzer accepts a fresh, raw dense-MLP :class:`TransformerBridge` such as + GPT-2 or Pythia/GPT-NeoX. Results retain no model or tokenizer reference and + contain detached CPU-owned tensors. Raw backward signals are loss gradients; + gradient descent subtracts them. """ def __init__(self, model: Any): - """Validate and retain the raw GPT-2 Bridge used for analyses.""" - _require_raw_gpt2_bridge(model) + """Validate and retain the raw dense-MLP Bridge used for analyses.""" + _require_raw_dense_mlp_bridge(model) self._model = model def analyze( @@ -852,7 +881,7 @@ def analyze( Args: prompt: Non-empty unbatched prompt text. target_token: Text encoding to exactly one token without BOS. - layers: Unique GPT-2 layer indices in desired result order. + layers: Unique layer indices in desired result order. normalized: Also project unit-normalized nonzero factors using the Normalized Logit Lens. Raw projections are always returned. top_k: Number of largest and smallest values and token ids retained @@ -873,7 +902,7 @@ def analyze( raise ValueError(f"top_k must be in [1, {vocabulary_size}]; got {top_k!r}") if not isinstance(return_full_logits, bool): raise TypeError("return_full_logits must be a bool") - capture = _capture_gpt2_mlp_gradient_factors(self._model, prompt, target_token, layers) + capture = _capture_dense_mlp_gradient_factors(self._model, prompt, target_token, layers) layer_results: list[BackwardLensLayerResult] = [] absolute_errors: list[float] = [] relative_errors: list[float] = [] From 058804311e99d3d8bbb2aa8ffb10ab418863a3b1 Mon Sep 17 00:00:00 2001 From: emerardd <113128214+emerardd@users.noreply.github.com> Date: Mon, 14 Sep 2026 21:40:19 +0800 Subject: [PATCH 72/87] Reject fabricated MoE weights in SVDInterpreter (#1780) * fix: reject fabricated MoE weights in SVDInterpreter * test: sparse MoE layers are omitted from get_params, not zero-filled --- .../model_bridge/test_mixtral_adapter.py | 19 ++++++++++++ .../model_bridge/test_moe_dense_dispatch.py | 7 +++-- .../unit/model_bridge/test_get_params_util.py | 23 ++++++++++++++ transformer_lens/SVDInterpreter.py | 23 ++++++++++++-- .../model_bridge/get_params_util.py | 31 +++++++++---------- 5 files changed, 81 insertions(+), 22 deletions(-) diff --git a/tests/integration/model_bridge/test_mixtral_adapter.py b/tests/integration/model_bridge/test_mixtral_adapter.py index e4d260f801..2186389294 100644 --- a/tests/integration/model_bridge/test_mixtral_adapter.py +++ b/tests/integration/model_bridge/test_mixtral_adapter.py @@ -8,6 +8,7 @@ import copy +import pytest import torch MODEL_VOCAB = 128 @@ -79,3 +80,21 @@ def grab(tensor, hook): bridge.run_with_hooks(ids, fwd_hooks=[(name, grab) for name in hooks]) for name in hooks: assert captured.get(name) == (1, 8, 64), f"{name}: {captured.get(name)}" + + def test_svd_interpreter_rejects_sparse_moe_mlp(self, monkeypatch) -> None: + """Sparse expert weights must not be replaced by plausible dense SVD results.""" + from transformer_lens import SVDInterpreter + + bridge, _ = _tiny_mixtral_pair() + params = bridge.tl_parameters() + assert "blocks.0.mlp.W_in" not in params + assert "blocks.0.mlp.W_out" not in params + + def fail_if_called(*args, **kwargs): + pytest.fail("torch.linalg.svd must not run for a sparse-MoE MLP") + + monkeypatch.setattr(torch.linalg, "svd", fail_if_called) + interpreter = SVDInterpreter(bridge) + for vector_type in ("w_in", "w_out"): + with pytest.raises(NotImplementedError, match="(?i)sparse MoE"): + interpreter.get_singular_vectors(vector_type, layer_index=0, num_vectors=4) diff --git a/tests/integration/model_bridge/test_moe_dense_dispatch.py b/tests/integration/model_bridge/test_moe_dense_dispatch.py index f6568b2326..9eb733085d 100644 --- a/tests/integration/model_bridge/test_moe_dense_dispatch.py +++ b/tests/integration/model_bridge/test_moe_dense_dispatch.py @@ -90,9 +90,12 @@ def test_dense_layer_weights_reach_get_params() -> None: # them apart only if their weights actually differ. assert not torch.equal(hf_mlp.gate_proj.weight, hf_mlp.up_proj.weight) - # Sparse layers legitimately have no single W_* and keep the placeholder. + # Sparse layers legitimately have no single W_*, so they are omitted rather + # than zero-filled: a zero placeholder reads as a real dense weight to + # weight-space analyses, an absent key does not. assert bridge.blocks[1].mlp.bound_dense is False - assert (params["blocks.1.mlp.W_in"] == 0).all() + assert "blocks.1.mlp.W_in" not in params + assert "blocks.1.mlp.W_out" not in params def test_hook_dict_follows_a_dense_to_sparse_rebind() -> None: diff --git a/tests/unit/model_bridge/test_get_params_util.py b/tests/unit/model_bridge/test_get_params_util.py index 5d384c66ff..a4ede9455d 100644 --- a/tests/unit/model_bridge/test_get_params_util.py +++ b/tests/unit/model_bridge/test_get_params_util.py @@ -152,6 +152,29 @@ def test_get_bridge_params_gate_weights(self): assert isinstance(params[gate_key], torch.Tensor) assert isinstance(params[gate_bias_key], torch.Tensor) + def test_real_mlp_without_dense_weights_is_not_zero_filled(self): + """A structurally unsupported real MLP must not masquerade as dense weights.""" + mock_bridge = self._create_mock_bridge() + mock_bridge.blocks[0].mlp = Mock(spec=[]) + + params = get_bridge_params(mock_bridge) + + assert not any(key.startswith("blocks.0.mlp.") for key in params) + assert "blocks.1.mlp.W_in" in params + assert "blocks.1.mlp.W_out" in params + + def test_missing_real_mlp_output_weight_is_not_zero_filled(self): + """A missing dense output projection must remain unavailable to consumers.""" + mock_bridge = self._create_mock_bridge() + mock_bridge.blocks[0].mlp.W_out = None + + params = get_bridge_params(mock_bridge) + + assert "blocks.0.mlp.W_in" in params + assert "blocks.0.mlp.W_out" not in params + assert "blocks.0.mlp.b_in" in params + assert "blocks.0.mlp.b_out" in params + def _create_mock_bridge(self): """Create a mock bridge with all standard components.""" mock_bridge = Mock() diff --git a/transformer_lens/SVDInterpreter.py b/transformer_lens/SVDInterpreter.py index 4d0f3e397e..844db11513 100644 --- a/transformer_lens/SVDInterpreter.py +++ b/transformer_lens/SVDInterpreter.py @@ -4,7 +4,7 @@ :class:`transformer_lens.HookedTransformer`. """ -from typing import Any, Optional, Union +from typing import Any, NoReturn, Optional, Union import torch from typing_extensions import Literal @@ -74,6 +74,10 @@ def plot_matrix(matrix, tokens, k=10, filter="topk"): layer_index: The index of the layer. num_vectors: Number of vectors. head_index: Index of the head. + + Raises: + NotImplementedError: If the requested layer does not expose a single dense MLP weight, + such as a sparse-MoE layer that requires an expert-aware interpretation. """ if head_index is None: @@ -145,7 +149,10 @@ def _get_w_in_matrix(self, layer_index: int) -> torch.Tensor: 0 <= layer_index < self.cfg.n_layers ), f"Layer index must be between 0 and {self.cfg.n_layers-1} but got {layer_index}" - w_in = self.params[f"blocks.{layer_index}.mlp.W_in"].T + key = f"blocks.{layer_index}.mlp.W_in" + if key not in self.params: + self._raise_unsupported_mlp_weight("w_in", layer_index) + w_in = self.params[key].T if f"blocks.{layer_index}.ln2.w" in self.params: # If fold_ln == False ln_2 = self.params[f"blocks.{layer_index}.ln2.w"] @@ -160,4 +167,14 @@ def _get_w_out_matrix(self, layer_index: int) -> torch.Tensor: 0 <= layer_index < self.cfg.n_layers ), f"Layer index must be between 0 and {self.cfg.n_layers-1} but got {layer_index}" - return self.params[f"blocks.{layer_index}.mlp.W_out"] + key = f"blocks.{layer_index}.mlp.W_out" + if key not in self.params: + self._raise_unsupported_mlp_weight("w_out", layer_index) + return self.params[key] + + def _raise_unsupported_mlp_weight(self, weight_name: str, layer_index: int) -> NoReturn: + raise NotImplementedError( + f"SVDInterpreter cannot analyze {weight_name} for layer {layer_index}: " + "the layer does not expose a single dense MLP weight. Sparse MoE layers " + "require an explicit expert-aware interpretation." + ) diff --git a/transformer_lens/model_bridge/get_params_util.py b/transformer_lens/model_bridge/get_params_util.py index 1b6eadf0a5..03d89bf9b3 100644 --- a/transformer_lens/model_bridge/get_params_util.py +++ b/transformer_lens/model_bridge/get_params_util.py @@ -28,8 +28,10 @@ def get_bridge_params(bridge) -> Dict[str, torch.Tensor]: Reads the bridge components' TL-layout weight properties (``W_Q``, ``W_in``, ...), which already account for layout conversion and weight - processing. For missing weights, returns zero tensors of appropriate shape - instead of raising exceptions. Skips attn keys for non-attention layers, and + processing. For absent optional weights, returns zero tensors of appropriate + shape instead of raising exceptions. Real components whose weights cannot be + represented in the dense TL layout are omitted. Skips attn keys for + non-attention layers, and omits ``pos_embed.W_pos`` for rotary models (which have no learned position table), matching HookedTransformer's parameter set. LayerNorm params (``blocks.{i}.ln1.w`` etc.) are included when the modules @@ -132,27 +134,22 @@ def _zeros(*shape) -> torch.Tensor: mlp = getattr(block, "mlp", None) w_in = _tensor_attr(mlp, "W_in") if w_in is None: - if mlp is not None: - # Zero-filling a real MLP silently yields wrong numbers downstream - # (SVD/weight analyses decompose zeros). Say so — the fill stays for - # architectures that genuinely have no MLP under this name. + if mlp is None: + params_dict[f"blocks.{layer_idx}.mlp.W_in"] = _zeros(cfg.d_model, d_mlp) + params_dict[f"blocks.{layer_idx}.mlp.W_out"] = _zeros(d_mlp, cfg.d_model) + params_dict[f"blocks.{layer_idx}.mlp.b_in"] = _zeros(d_mlp) + params_dict[f"blocks.{layer_idx}.mlp.b_out"] = _zeros(cfg.d_model) + else: logger.warning( - "Block %d MLP weights could not be extracted — emitting ZEROS " - "for blocks.%d.mlp.W_in/W_out/b_in/b_out. Any weight-space " - "analysis of this layer will be meaningless.", - layer_idx, + "Block %d MLP does not expose a single dense W_in/W_out; " + "omitting its MLP parameters from the TL-style parameter dictionary.", layer_idx, ) - params_dict[f"blocks.{layer_idx}.mlp.W_in"] = _zeros(cfg.d_model, d_mlp) - params_dict[f"blocks.{layer_idx}.mlp.W_out"] = _zeros(d_mlp, cfg.d_model) - params_dict[f"blocks.{layer_idx}.mlp.b_in"] = _zeros(d_mlp) - params_dict[f"blocks.{layer_idx}.mlp.b_out"] = _zeros(cfg.d_model) else: params_dict[f"blocks.{layer_idx}.mlp.W_in"] = w_in w_out = _tensor_attr(mlp, "W_out") - params_dict[f"blocks.{layer_idx}.mlp.W_out"] = ( - w_out if w_out is not None else _zeros(d_mlp, cfg.d_model) - ) + if w_out is not None: + params_dict[f"blocks.{layer_idx}.mlp.W_out"] = w_out b_in = _tensor_attr(mlp, "b_in") params_dict[f"blocks.{layer_idx}.mlp.b_in"] = ( b_in if b_in is not None else _zeros(d_mlp) From 133add42ad35e63836cfc317b92aaba86372a763 Mon Sep 17 00:00:00 2001 From: emerardd <113128214+emerardd@users.noreply.github.com> Date: Mon, 14 Sep 2026 22:56:11 +0800 Subject: [PATCH 73/87] docs: add analysis tool selection guide (#1782) As a newcomer to these analysis methods, I found there was no single guide explaining which tool fits which research question. Add a question-led overview of the available analysis tools, their requirements, costs, and interpretation limits, and link it from the documentation index. --- docs/source/content/analysis_tools.md | 183 ++++++++++++++++++++++++++ docs/source/index.md | 1 + 2 files changed, 184 insertions(+) create mode 100644 docs/source/content/analysis_tools.md diff --git a/docs/source/content/analysis_tools.md b/docs/source/content/analysis_tools.md new file mode 100644 index 0000000000..370f0eae1c --- /dev/null +++ b/docs/source/content/analysis_tools.md @@ -0,0 +1,183 @@ +# Choosing an analysis tool + +Start with the question you want to answer, then choose the measurement. A vocabulary +readout, a geometric similarity score, and an intervention answer different questions, +even when they highlight the same attention head or token. + +This guide covers the high-level tools in `transformer_lens.tools.analysis`. For +capturing activations or installing your own interventions, start with the +[hook system](hook_system.md). New experiments should use `TransformerBridge`; see +the [migration guide](migrating_to_v3.md) for existing `HookedTransformer` code. + +## Choose by research question + +| I want to know… | Start with | Inputs and outputs | What the result establishes | +|---|---|---|---| +| Which components contribute to an answer's logit? | **Direct Logit Attribution (DLA)** | A prompt or activation cache, answer token, and optional comparison token → labeled logit contributions. | A decomposition of the residual-stream readout for that run. | +| Which activations should I investigate with patching? | **Attribution Patching** | Aligned clean/corrupt token pairs and a differentiable scalar metric → signed node scores. | A first-order estimate of the effect of replacing corrupt activations with clean ones. | +| Does a particular head-to-head route affect my metric? | **Direct Path Patching** | Clean/corrupt caches, a source head, and a metric → destination-head patch scores. | The measured effect of the implemented path intervention, subject to its LayerNorm approximation. | +| What vocabulary directions appear in an MLP's gradient factors? | **Backward Lens** | A prompt, one target token, and selected layers → gradient factors and vocabulary rankings. | A diagnostic of the forward inputs and backward signals composing a weight gradient. | +| How can I read or edit residuals through a fitted transport map? | **Jacobian Lens** | A matching lens artifact and model, plus prompts or activations → vocabulary readouts, decompositions, or interventions. | Readouts under the fitted map; causal effects require running and measuring an intervention. | +| How much do two subspaces overlap? | **Projection Kernel** | Two subspace bases, or Bridge attention-head weight spaces → overlap scores and principal-angle information. | Shared geometric support, independent of the choice of basis within each subspace. | +| What are a head's QK/OV singular directions? | **SVD Circuits: head decomposition** | Model weights and a layer/head index → singular values, vectors, and degeneracy reports. | The linear structure of the head's weight maps. | + +These are starting points, not a required pipeline. For example, a weight-space +comparison can motivate a patching experiment, but a large overlap score alone does +not identify the head's function or prove a causal connection. + +## Check model requirements before combining tools + +A model appearing in the [model tables](model_tables.md) does not imply that every +analysis below supports its architecture. Tools also depend on weight processing, +available hooks, and the meaning of the selected tensor axes. + +| Tool | Requirements to check | Main compute or memory cost | +|---|---|---| +| DLA | Supports HT and Bridge. Bridge requires compatibility mode and the standard attention/MLP residual decomposition; Mamba/SSM/Mixer/LinearAttention hybrid layouts are rejected. | One cached forward pass, or reuse of a suitable cache. Head decomposition can require additional per-head results. | +| Attribution Patching | Targets Bridge. Clean/corrupt inputs must be `[batch, seq]` token tensors with matching shapes and aligned positions. The required embedding, head, and MLP hook aliases must exist. Run the model and all nested stochastic modules in evaluation mode. The metric must return a differentiable scalar, and the current backward-cache path requires an active autograd graph; fully frozen models are not yet supported. | Two forwards and one backward **per prompt pair**; activation and gradient caches. Pair scores are averaged across the batch. | +| Direct Path Patching | Supports HT and Bridge exposing the expected attention weights, Q/K/V hook aliases, source `hook_z`, and destination LayerNorm scales. Folded LayerNorm parameters improve the approximation. On GQA/MQA models, direct Q-path patching remains available, but `component="k"` and `"v"` currently require `n_key_value_heads == n_heads`; per-query-head K/V semantics are not yet defined. | Clean/corrupt caching, then repeated forwards over destination heads. Sweeping all source heads adds another sweep dimension. | +| Backward Lens | The implementation documented here targets raw GPT-2 Bridge models without compatibility mode. Use one target token and check the restrictions in the [tool guide](backward_lens.md). | Gradient computation plus vocabulary projections for selected layers and positions; retaining full logits increases memory. | +| Jacobian Lens | Requires a fresh, causal decoder-only Bridge with raw HF weights, without compatibility mode or weight processing. Validate the lens against the model. Fitting additionally requires all modules in evaluation mode. | Loading an existing artifact avoids fitting. The ordinary fitting estimator uses one forward and `ceil(d_model / dim_batch)` backwards per prompt; larger batches increase memory. | +| Projection Kernel | The numerical API accepts finite, real floating-point matrices via orthonormal bases. The attention-head wrapper requires Bridge weights with compatible dimensions and ranks. | Basis extraction uses SVD. All-head comparisons allocate basis stacks and a pairwise score grid; they can be large despite requiring no forward pass. | +| SVD head decomposition | Uses Bridge per-block weight accessors; requires accessible, compatible `W_Q`, `W_K`, `W_V`, and `W_O` for the selected head. No compatibility mode or activation cache is needed. | Factored QK/OV SVD, with rank bounded by `d_head`; the dense `d_model × d_model` product is not materialized. | + +**Compatibility mode is a method-specific choice.** DLA needs it on Bridge, whereas +Jacobian Lens requires raw weights. Use separately loaded models when comparing these +workflows, and collect each cache from the model configuration that will consume it. +Do not reuse a cache after changing weights or processing mode. See +[compatibility mode](compatibility_mode.md) for the numerical conventions. + +Evaluation mode and gradient recording are separate choices: `model.eval()` disables +training behavior such as dropout; it does not disable autograd. Do not wrap a +gradient-based analysis in `torch.no_grad()` or `torch.inference_mode()`. + +## Read the output with the right interpretation + +### DLA: contributions, not ablation effects + +Use `direct_logit_attribution(..., unit="component")` for additive component +contributions, or `unit="head"` for heads plus a remainder. With `unit="layer"`, +the entries are **cumulative** residual readouts after sublayers, not independent +contributions to sum together. + +An answer-minus-comparison-token direction often makes the question more specific +than a single answer logit. String answers must encode to one token; a leading space +can change tokenization. A complete component decomposition accounts for the +residual contribution, so the unembedding bias `b_U` is excluded. Removing a component +can change downstream computation, so its DLA score is not its ablation effect. + +API: {func}`~transformer_lens.tools.analysis.direct_logit_attribution.direct_logit_attribution`. + +### Attribution Patching: screen candidates, then measure interventions + +The score uses `(clean_activation - corrupt_activation) · corrupt_gradient`. +A positive value predicts an increase in the chosen metric when moving the corrupt +activation toward the clean one. Reversing the metric reverses this interpretation. +Large activation changes and nonlinear downstream behavior can make the estimate +inaccurate; compare promising candidates with actual activation replacements using +the [hook system](hook_system.md). + +The current implementation supports node granularity with `ig_steps=1`. +`granularity="edge"` and `ig_steps>1` are declared options but raise +`NotImplementedError`; do not treat them as available EAP or EAP-IG workflows. + +API: {func}`~transformer_lens.tools.analysis.attribution_patching.attribution_patch`. + +### Direct Path Patching: state the path and its approximation + +`get_act_patch_direct_path` fixes a source head and sweeps later destination heads, +patching into their Q, K, or V inputs. It reconstructs the source head's output from +cached `hook_z` and `W_O`, so caching `hook_result` is unnecessary. The calculation +projects the source-output change using cached LayerNorm scaling; it does not +recompute the full nonlinear normalization response to that change. Folded +parameters do not remove this fixed-scale assumption. + +Interpret a score relative to the unpatched corrupt metric. Entries at or before the +source layer are zero placeholders, not measured interventions. State the source head, +destination input, prompt pair, and normalization convention when reporting a path. + +API: {func}`~transformer_lens.tools.analysis.direct_path_patching.get_act_patch_direct_path`. + +### Backward Lens: preserve the gradient sign + +Vocabulary rankings describe factors of `d(loss) / d(weight)`. Gradient descent +**subtracts** this gradient. The highest raw-gradient token is therefore not +automatically the token favored by an update. Use the documented bottom rankings +and gradient-descent target ranks where appropriate. Align positions using +`result.prompt_token_ids`, which records the actual tokenization. + +See [Backward Lens](backward_lens.md) for the example, sign conventions, and result +structure. + +### Jacobian Lens: distinguish readout, reconstruction, and intervention + +Start with a matching published artifact when possible and call +`lens.validate_model(model)`. Pin model and artifact revisions. A vocabulary readout +uses the fitted transport map; sparse decomposition describes an activation in its +dictionary. A small reconstruction error does not establish that editing a +coordinate will cause the predicted answer. + +For a causal question, install the relevant hooks and measure model outputs against +an unedited baseline and suitable controls. Check the selected intervention's layer +and position semantics: a multi-layer edit is not generally equivalent to independent +single-layer edits. + +See [fitting and provenance](jacobian_lens_fitting.md) and the +[decomposition demo](../generated/demos/Jacobian_Lens_Decomposition_Demo.ipynb). + +### Weight-space tools: use geometry to form hypotheses + +Projection Kernel compares subspaces, without measuring weight magnitude or model +behavior on a prompt. In attention-head results, use `valid_mask` when interpreting +scores: invalid pairs are stored as zeros. See [Projection Kernel](projection_kernel.md) +for rank selection, normalization, and all-head memory costs. + +For SVD head decomposition, inspect the rank report before assigning meaning to an +individual direction. Near-equal singular values define a subspace whose basis can +rotate; numerically null directions are also unsuitable for individual attribution. +The weight decomposition alone is not a causal validation of a proposed subfunction. + +API: {func}`~transformer_lens.tools.analysis.svd_circuits.decompose_head`. + +## Try a geometry question without downloading a model + +The following subspaces share exactly one axis. Their raw Projection Kernel score +is 1, and the normalized score is 0.5 because both have rank 2. + +```python +import torch + +from transformer_lens.tools.analysis import orthonormal_subspace, projection_kernel + +axes = torch.eye(3) +first = orthonormal_subspace(axes[:, [0, 1]]) +second = orthonormal_subspace(axes[:, [0, 2]]) +result = projection_kernel(first, second) + +print(result.score.item()) # 1.0 +print(result.normalized.item()) # 0.5 +``` + +Replacing these matrices with head-weight bases gives a geometric comparison of +heads. Establishing what those heads do still requires prompts and behavioral +measurements. + +## Plan a focused experiment + +For a question such as “which components help the model prefer Paris to London?”: + +1. Define the prompt set, answer token ids, final position, and signed logit-difference + metric. Check the unmodified model's answers before interpreting an intervention. +2. Use DLA for a readout decomposition, or aligned clean/corrupt pairs with + Attribution Patching to shortlist intervention candidates. These scores need not + agree because they measure different quantities. +3. Replace selected activations and record the observed metric change. Add a no-op + replacement to check the hook setup and controls matched to the experimental + claim, such as alternative sites or perturbations of comparable magnitude. +4. Inspect per-prompt results before aggregating. Report failures and exclusions, + and evaluate the hypothesis on held-out prompts when making a general claim. +5. Record model/tokenizer revisions, dtype, weight-processing mode, token ids, + hook names, positions, metric definition, and any lens artifact or random seed. + +The goal is to connect an interpretable measurement to a clearly specified +experiment, with enough information for another researcher to repeat it. diff --git a/docs/source/index.md b/docs/source/index.md index 565bb91cf8..00674a90d4 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -51,6 +51,7 @@ content/model_structure :caption: Resources content/migrating_to_v3 +content/analysis_tools content/tutorials content/citation content/contributing From a4717b41ac09b7b9f21a3bcd086543964c1815eb Mon Sep 17 00:00:00 2001 From: Jonah Larson <jonahalarson@comcast.net> Date: Mon, 14 Sep 2026 09:56:26 -0500 Subject: [PATCH 74/87] improved direct path patching test (#1786) --- tests/unit/test_direct_path_patching.py | 43 +++++++++---------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/tests/unit/test_direct_path_patching.py b/tests/unit/test_direct_path_patching.py index 2224a14b0d..8f7b739b67 100644 --- a/tests/unit/test_direct_path_patching.py +++ b/tests/unit/test_direct_path_patching.py @@ -274,16 +274,9 @@ def test_clean_equals_corrupted_gives_zero_delta(self, tiny_model): class TestCorrectness: - def test_correctness_against_actual_ln_forward(self, tiny_model, tokens_and_caches): - """Logit-diff metric: linear-LN approximation should match actual LN within 1e-3. - - Folding LN does not make the approximation exact: LayerNormPre still divides - by an input-dependent norm, while the approximation reuses the corrupted - run's scale. The error depends on the weights, so the tolerance holds for - the seeded tiny_model but is not guaranteed for arbitrary initialisations. - Using logit diff (correct_tok - incorrect_tok) cancels the centering offset - introduced by process_weights_(). - """ + @pytest.mark.parametrize("component", ["q", "k", "v"]) + def test_matches_frozen_ln_scale_reference(self, tiny_model, tokens_and_caches, component): + """Matches a hand-written hook dividing delta_resid by the corrupted run's ln1 scale.""" _, corrupted_tokens, clean_cache, corrupted_cache = tokens_and_caches src_layer, src_head = 0, 0 dst_layer, dst_head = 2, 1 @@ -300,28 +293,22 @@ def logit_diff(logits): corrupted_z = corrupted_cache[f"blocks.{src_layer}.attn.hook_z"][:, :, src_head, :] delta_resid = (clean_z @ W_O[src_head]) - (corrupted_z @ W_O[src_head]) # type: ignore[index] - # Independent reference: patch through actual LayerNorm forward - corrupted_resid = corrupted_cache[f"blocks.{dst_layer}.hook_resid_pre"] - patched_resid = corrupted_resid + delta_resid - - with torch.no_grad(): - ln1 = tiny_model.blocks[dst_layer].ln1 # type: ignore[index] - patched_normed = ln1(patched_resid) - corrupted_normed = ln1(corrupted_resid) + # Not the true LN forward: its gap to the approximation is weight-dependent and, on + # tiny_model, about as large as the patch effect, so it can't catch scale bugs. + ln_scale = corrupted_cache[f"blocks.{dst_layer}.ln1.hook_scale"] + W_comp = getattr(tiny_model.blocks[dst_layer].attn, f"W_{component.upper()}")[dst_head] + ref_delta = (delta_resid / ln_scale) @ W_comp - W_Q_dst = tiny_model.blocks[dst_layer].attn.W_Q[dst_head] # type: ignore[index,union-attr] - true_delta_q = (patched_normed - corrupted_normed) @ W_Q_dst - - def true_hook(value, hook): + def ref_hook(value, hook): if value.requires_grad: value = value.clone() - value[:, :, dst_head, :] = value[:, :, dst_head, :] + true_delta_q + value[:, :, dst_head, :] = value[:, :, dst_head, :] + ref_delta return value with torch.no_grad(): ref_logits = tiny_model.run_with_hooks( corrupted_tokens, - fwd_hooks=[(f"blocks.{dst_layer}.attn.hook_q", true_hook)], + fwd_hooks=[(f"blocks.{dst_layer}.attn.hook_{component}", ref_hook)], ) ref_metric = logit_diff(ref_logits).item() @@ -334,14 +321,14 @@ def true_hook(value, hook): patching_metric=logit_diff, src_layer=src_layer, src_head=src_head, - component="q", + component=component, verbose=False, ) our_metric = results[dst_layer, dst_head].item() - assert abs(our_metric - ref_metric) < 1e-3, ( - f"Linear-LN approx {our_metric:.6f} disagrees with actual-LN ref {ref_metric:.6f} " - f"(diff={abs(our_metric - ref_metric):.2e})." + assert abs(our_metric - ref_metric) < 1e-5, ( + f"Direct-path patch {our_metric:.6f} disagrees with frozen-scale reference " + f"{ref_metric:.6f} (diff={abs(our_metric - ref_metric):.2e})." ) def test_all_sources_consistent_with_single(self, tiny_model, tokens_and_caches): From 552217afe93827923b01064c71b058f90f0aa1ec Mon Sep 17 00:00:00 2001 From: jlarson4 <jonahalarson@comcast.net> Date: Mon, 14 Sep 2026 11:19:48 -0500 Subject: [PATCH 75/87] Fixed bug with solu folding in boot_native --- tests/integration/test_head_detector.py | 10 ++--- .../unit/model_bridge/test_native_features.py | 29 ++++++++++++++ tests/unit/test_weight_processing.py | 12 ++++++ transformer_lens/weight_processing.py | 39 +++++++++---------- 4 files changed, 64 insertions(+), 26 deletions(-) diff --git a/tests/integration/test_head_detector.py b/tests/integration/test_head_detector.py index 81d47af0a0..8a9651149f 100644 --- a/tests/integration/test_head_detector.py +++ b/tests/integration/test_head_detector.py @@ -3,7 +3,6 @@ import pytest import torch -from transformer_lens.model_bridge import TransformerBridge from transformer_lens.head_detector import ( HEAD_NAMES, ErrorMeasure, @@ -12,17 +11,16 @@ get_induction_head_detection_pattern, get_previous_token_head_detection_pattern, ) +from transformer_lens.model_bridge import TransformerBridge MODEL = "NeelNanda/SoLU_2L512W_C4_Code" # solu-2l; boot_tl_legacy takes the repo id, not the alias ATOL = 1e-4 # ATOL is set to 1e-4 because the tensors we check on are also to 4 decimal places. -# The expected tensors below were captured from HookedTransformer.from_pretrained; -# attention patterns are invariant under its weight processing (fold/center are -# function-preserving), so no_processing keeps the goldens valid — and fold_ln -# currently cannot handle this model's mid-MLP solu_ln layout on the bridge. +# The expected tensors below were captured from HookedTransformer.from_pretrained, +# whose default weight processing compatibility mode reproduces. model = TransformerBridge.boot_tl_legacy(MODEL, device="cpu") -model.enable_compatibility_mode(no_processing=True, disable_warnings=True) +model.enable_compatibility_mode(disable_warnings=True) test_regular_sequence = " four token sequence" # Four tokens including BOS test_duplicated_sequence = " seven token sequence seven token sequence" test_duplicated_sequence2 = " one two three one two three" diff --git a/tests/unit/model_bridge/test_native_features.py b/tests/unit/model_bridge/test_native_features.py index b658f2c06d..62338bc18e 100644 --- a/tests/unit/model_bridge/test_native_features.py +++ b/tests/unit/model_bridge/test_native_features.py @@ -329,6 +329,35 @@ def test_ln_default_uses_layernorm(): assert isinstance(ln1_bridge.original_component, torch.nn.LayerNorm) +def test_solu_ln_weight_processing_preserves_the_forward(): + """fold_ln reaches the mid-MLP LayerNorm through nn.Linear's [d_model, d_mlp] W_out.""" + bridge = TransformerBridge.boot_native(_cfg(n_layers=2, act_fn="solu_ln")) + bridge.eval() + generator = torch.Generator().manual_seed(1) + # Native init leaves every norm at identity, which would make the fold a no-op. + with torch.no_grad(): + for module in bridge.original_model.modules(): + if isinstance(module, torch.nn.LayerNorm): + module.weight.add_(torch.randn(module.weight.shape, generator=generator) * 0.5) + module.bias.add_(torch.randn(module.bias.shape, generator=generator) * 0.5) + tokens = torch.randint(0, bridge.cfg.d_vocab, (2, bridge.cfg.n_ctx), generator=generator) + with torch.no_grad(): + expected = torch.log_softmax(bridge(tokens).double(), dim=-1) + + bridge.process_weights() + + mid_ln = bridge.blocks[0].mlp.ln + torch.testing.assert_close(mid_ln.weight, torch.ones_like(mid_ln.weight)) + torch.testing.assert_close(mid_ln.bias, torch.zeros_like(mid_ln.bias)) + # Downstream LayerNorms hide which axis W_out was centered on, so check it directly. + for block in bridge.blocks: + row_means = block.mlp.out.weight.mean(-1) + torch.testing.assert_close(row_means, torch.zeros_like(row_means), atol=1e-6, rtol=0) + with torch.no_grad(): + actual = torch.log_softmax(bridge(tokens).double(), dim=-1) + torch.testing.assert_close(actual, expected, atol=1e-4, rtol=0) + + # -- GQA ---------------------------------------------------------------------- diff --git a/tests/unit/test_weight_processing.py b/tests/unit/test_weight_processing.py index 67bad18842..5cbcd400dd 100644 --- a/tests/unit/test_weight_processing.py +++ b/tests/unit/test_weight_processing.py @@ -374,6 +374,7 @@ def test_fold_layer_norm_solu(self, solu_config): state_dict["unembed.W_U"] = torch.randn(cfg.d_model, 100) state_dict["unembed.b_U"] = torch.randn(100) + original = deep_copy_state_dict(state_dict) processed_dict = ProcessWeights.fold_layer_norm(state_dict, cfg) # Check that SoLU ln weights are replaced with identity values @@ -389,6 +390,17 @@ def test_fold_layer_norm_solu(self, solu_config): torch.zeros_like(processed_dict[f"blocks.{l}.mlp.ln.b"]), ) + # HookedTransformer's SoLU fold on TL-layout [d_mlp, d_model] W_out. + for l in range(cfg.n_layers): + W_out = original[f"blocks.{l}.mlp.W_out"] + ln_w = original[f"blocks.{l}.mlp.ln.w"] + ln_b = original[f"blocks.{l}.mlp.ln.b"] + expected_b_out = original[f"blocks.{l}.mlp.b_out"] + (W_out * ln_b[:, None]).sum(0) + expected_W_out = W_out * ln_w[:, None] + expected_W_out = expected_W_out - expected_W_out.mean(0, keepdim=True) + torch.testing.assert_close(processed_dict[f"blocks.{l}.mlp.b_out"], expected_b_out) + torch.testing.assert_close(processed_dict[f"blocks.{l}.mlp.W_out"], expected_W_out) + def test_center_writing_weights(self, basic_config, basic_state_dict): """Test weight centering functionality.""" processed_dict = ProcessWeights.center_writing_weights(basic_state_dict, basic_config) diff --git a/transformer_lens/weight_processing.py b/transformer_lens/weight_processing.py index 083af3b3ea..485d711f9e 100644 --- a/transformer_lens/weight_processing.py +++ b/transformer_lens/weight_processing.py @@ -876,35 +876,34 @@ def _fold_mlp_layer_norm( assert mlp_ln_b is not None, f"MLP ln.b not found at key {mlp_ln_b_key}" assert mlp_ln_w is not None, f"MLP ln.w not found at key {mlp_ln_w_key}" + # TL keys hold W_out as [d_mlp, d_model]; nn.Linear bridges hold [d_model, d_mlp]. + # The fold and the centering must use the same neuron axis, so resolve it once. + d_mlp = mlp_ln_w.shape[0] + if mlp_W_out.shape[0] == d_mlp and mlp_W_out.shape[-1] != d_mlp: + neuron_dim = 0 + elif mlp_W_out.shape[-1] == d_mlp and mlp_W_out.shape[0] != d_mlp: + neuron_dim = -1 + else: + raise ValueError( + f"Cannot resolve the neuron axis of MLP W_out {tuple(mlp_W_out.shape)} " + f"against the mid-MLP LayerNorm (d_mlp={d_mlp}) at layer {layer}." + ) + ln_shape = (-1, 1) if neuron_dim == 0 else (1, -1) + if fold_biases: - new_mlp_b_out = mlp_b_out + (mlp_W_out * mlp_ln_b[:, None]).sum(-2) + new_mlp_b_out = mlp_b_out + (mlp_W_out * mlp_ln_b.reshape(ln_shape)).sum(neuron_dim) state_dict[mlp_b_out_key] = ProcessWeights.convert_tensor_to_hf_format( mlp_b_out_key, new_mlp_b_out, cfg, adapter, layer ) if mlp_ln_b_key in state_dict: state_dict[mlp_ln_b_key] = torch.zeros_like(mlp_ln_b) - new_mlp_W_out = mlp_W_out * mlp_ln_w[:, None] + new_mlp_W_out = mlp_W_out * mlp_ln_w.reshape(ln_shape) if center_weights: - # Center along d_mlp dimension. Detect format: - # TL format [d_mlp, d_model] -> center along dim=0 - # HF format [d_model, d_mlp] -> center along dim=-1 - d_model_val = cfg.d_model if cfg is not None else None - if ( - d_model_val is not None - and new_mlp_W_out.shape[-1] == d_model_val - and new_mlp_W_out.shape[0] != d_model_val - ): - new_mlp_W_out = new_mlp_W_out - new_mlp_W_out.mean(0, keepdim=True) - elif ( - d_model_val is not None - and new_mlp_W_out.shape[0] == d_model_val - and new_mlp_W_out.shape[-1] != d_model_val - ): - new_mlp_W_out = new_mlp_W_out - new_mlp_W_out.mean(-1, keepdim=True) - else: - new_mlp_W_out = new_mlp_W_out - new_mlp_W_out.mean(0, keepdim=True) + # The folded mid-MLP LayerNorm emits activations that are mean-zero across + # neurons, so a per-row constant along that axis never reaches the output. + new_mlp_W_out = new_mlp_W_out - new_mlp_W_out.mean(neuron_dim, keepdim=True) state_dict[mlp_W_out_key] = ProcessWeights.convert_tensor_to_hf_format( mlp_W_out_key, new_mlp_W_out, cfg, adapter, layer From ab42d85c3f868b24c687f706532c2df7a4369d90 Mon Sep 17 00:00:00 2001 From: emerardd <113128214+emerardd@users.noreply.github.com> Date: Wed, 16 Sep 2026 03:37:44 +0800 Subject: [PATCH 76/87] fix: require eval mode for attribution patching (#1788) --- tests/unit/tools/test_attribution_patching.py | 62 ++++++++++++++++++- .../tools/analysis/_model_state.py | 31 ++++++++++ .../tools/analysis/attribution_patching.py | 13 +++- .../tools/analysis/jacobian_lens.py | 29 +-------- 4 files changed, 105 insertions(+), 30 deletions(-) create mode 100644 transformer_lens/tools/analysis/_model_state.py diff --git a/tests/unit/tools/test_attribution_patching.py b/tests/unit/tools/test_attribution_patching.py index fe73391463..eda937a39c 100644 --- a/tests/unit/tools/test_attribution_patching.py +++ b/tests/unit/tools/test_attribution_patching.py @@ -78,6 +78,7 @@ def __init__(self, *, dtype: torch.dtype = torch.float32) -> None: ) self.compatibility_mode = False self._weights_processed = False + self.forward_calls = 0 self.embed = nn.Embedding(D_VOCAB, D_MODEL, dtype=dtype) nn.init.normal_(self.embed.weight, std=0.2) self.hook_embed = HookPoint() @@ -88,6 +89,7 @@ def __init__(self, *, dtype: torch.dtype = torch.float32) -> None: self.ln_final = nn.Identity() self.unembed = nn.Linear(D_MODEL, D_VOCAB, bias=False, dtype=dtype) nn.init.normal_(self.unembed.weight, std=0.2) + self.eval() @property def hook_dict(self) -> dict[str, HookPoint]: @@ -118,6 +120,7 @@ def to_tokens(self, prompt: str) -> torch.Tensor: def forward( self, tokens: torch.Tensor, return_type: str | None = "logits" ) -> torch.Tensor | None: + self.forward_calls += 1 residual = self.hook_embed(self.embed(tokens)) for block in self.blocks: residual = block(residual) @@ -364,6 +367,7 @@ def __init__(self, d_model: int, n_heads: int, d_head: int, layer: int, dtype: t self.w_z = nn.Linear(d_model, n_heads * d_head, bias=False, dtype=dtype) self.w_o = nn.Linear(n_heads * d_head, d_model, bias=False, dtype=dtype) self.w_mlp = nn.Linear(d_model, d_model, bias=False, dtype=dtype) + self.dropout = nn.Dropout(p=0.5) for linear in (self.w_z, self.w_o, self.w_mlp): nn.init.normal_(linear.weight, std=0.2) self.hook_z = HookPoint() @@ -374,7 +378,8 @@ def __init__(self, d_model: int, n_heads: int, d_head: int, layer: int, dtype: t def forward(self, residual: torch.Tensor) -> torch.Tensor: batch, seq, _ = residual.shape z = self.hook_z(self.w_z(residual).reshape(batch, seq, self.n_heads, self.d_head)) - residual = residual + self.w_o(z.reshape(batch, seq, self.n_heads * self.d_head)) + attn_out = self.w_o(z.reshape(batch, seq, self.n_heads * self.d_head)) + residual = residual + self.dropout(attn_out) mlp_out = self.hook_mlp_out(self.w_mlp(residual)) return residual + mlp_out @@ -403,6 +408,7 @@ def __init__(self, *, dtype: torch.dtype = torch.float32) -> None: ) self.compatibility_mode = False self._weights_processed = False + self.forward_calls = 0 self.embed = nn.Embedding(D_VOCAB, D_MODEL, dtype=dtype) nn.init.normal_(self.embed.weight, std=0.2) self.hook_embed = HookPoint() @@ -413,6 +419,7 @@ def __init__(self, *, dtype: torch.dtype = torch.float32) -> None: self.ln_final = nn.Identity() self.unembed = nn.Linear(D_MODEL, D_VOCAB, bias=False, dtype=dtype) nn.init.normal_(self.unembed.weight, std=0.2) + self.eval() @property def hook_dict(self) -> dict[str, HookPoint]: @@ -425,6 +432,7 @@ def hook_dict(self) -> dict[str, HookPoint]: def forward( self, tokens: torch.Tensor, return_type: str | None = "logits" ) -> torch.Tensor | None: + self.forward_calls += 1 residual = self.hook_embed(self.embed(tokens)) for block in self.blocks: residual = block(residual) @@ -513,6 +521,57 @@ def test_attribution_patch_raises_on_batch_size_mismatch() -> None: attribution_patch(model, clean, corrupt, _metric_fn(answer=1, wrong=2)) +def test_attribution_patch_rejects_model_in_training_mode() -> None: + model = _NodeGraphToyBridge() + model.train() + tokens = torch.tensor([[1, 2, 3]]) + + with pytest.raises(ValueError, match=r"model\.eval\(\)"): + attribution_patch(model, tokens, tokens, _metric_fn(answer=1, wrong=2)) + assert model.training is True + assert model.forward_calls == 0 + + +def test_attribution_patch_rejects_nested_submodule_in_training_mode() -> None: + model = _NodeGraphToyBridge() + model.blocks[1].dropout.train() + tokens = torch.tensor([[1, 2, 3]]) + assert model.training is False + assert model.blocks[1].training is False + + with pytest.raises(ValueError, match=r"model\.eval\(\)"): + attribution_patch(model, tokens, tokens, _metric_fn(answer=1, wrong=2)) + assert model.training is False + assert model.blocks[1].training is False + assert model.blocks[1].dropout.training is True + assert model.forward_calls == 0 + + +def test_attribution_patch_rejects_hidden_original_model_training_mode() -> None: + model = _NodeGraphToyBridge() + original_model = nn.Sequential(nn.Linear(D_MODEL, D_MODEL)) + original_model.eval() + original_model[0].train() + model.__dict__["original_model"] = original_model + tokens = torch.tensor([[1, 2, 3]]) + + with pytest.raises(ValueError, match="original_model"): + attribution_patch(model, tokens, tokens, _metric_fn(answer=1, wrong=2)) + assert original_model.training is False + assert original_model[0].training is True + assert model.forward_calls == 0 + + +def test_attribution_patch_identical_inputs_are_exactly_zero_in_eval_mode() -> None: + model = _NodeGraphToyBridge() + tokens = torch.tensor([[1, 2, 3]]) + + result = attribution_patch(model, tokens, tokens, _metric_fn(answer=1, wrong=2)) + + assert result.node_scores + assert all(score == 0.0 for score in result.node_scores.values()) + + # --------------------------------------------------------------------------- # Commit 5 — linear-model reconstruction identity # --------------------------------------------------------------------------- @@ -670,6 +729,7 @@ def __init__(self, *, dtype: torch.dtype = torch.float32) -> None: for layer in range(N_LAYERS) ] ) + self.eval() def test_nonlinear_node_scores_read_the_corrupt_run_gradient() -> None: diff --git a/transformer_lens/tools/analysis/_model_state.py b/transformer_lens/tools/analysis/_model_state.py new file mode 100644 index 0000000000..4ef787e756 --- /dev/null +++ b/transformer_lens/tools/analysis/_model_state.py @@ -0,0 +1,31 @@ +"""Shared model-state validation for analysis tools.""" + +from typing import Any + +import torch + + +def require_eval_mode(model: Any, *, operation: str) -> None: + """Reject training state anywhere in a wrapped model without mutating it.""" + training_modules: dict[int, str] = {} + roots = (("", model), ("original_model", getattr(model, "original_model", None))) + for prefix, root in roots: + if not isinstance(root, torch.nn.Module): + continue + for name, module in root.named_modules(): + if not module.training: + continue + qualified_name = ".".join(part for part in (prefix, name) if part) + training_modules.setdefault(id(module), qualified_name or "<root>") + if not training_modules: + return + + names = list(training_modules.values()) + preview = ", ".join(names[:3]) + if len(names) > 3: + preview += f", and {len(names) - 3} more" + raise ValueError( + f"{operation} requires the model and all submodules to be in evaluation " + f"mode; found training mode at {preview}. Call model.eval() before running " + "the analysis." + ) diff --git a/transformer_lens/tools/analysis/attribution_patching.py b/transformer_lens/tools/analysis/attribution_patching.py index 1d5f17461a..a0041ca4ef 100644 --- a/transformer_lens/tools/analysis/attribution_patching.py +++ b/transformer_lens/tools/analysis/attribution_patching.py @@ -36,6 +36,8 @@ import torch +from transformer_lens.tools.analysis._model_state import require_eval_mode + MetricFn = Callable[[torch.Tensor], torch.Tensor] NamesFilter = Union[str, Sequence[str], Callable[[str], bool], None] @@ -460,6 +462,10 @@ def attribution_patch( reconstruction identity holds) and per-node scores are averaged across the batch before ranking. + The model and every submodule must be in evaluation mode. Separate clean and + corrupt forwards cannot produce meaningful activation differences if stochastic + training layers such as dropout remain active. + Args: model: A ``TransformerBridge`` (or compatible) exposing ``cfg.n_layers``, ``hook_dict``, and ``hooks()``. @@ -476,8 +482,9 @@ def attribution_patch( Raises: ValueError: if ``clean``/``corrupt`` are not 2D, hold a different number of - pairs, or a pair tokenizes to different lengths (activations must align - position-by-position). + pairs, a pair tokenizes to different lengths (activations must align + position-by-position), or the model or one of its submodules is in + training mode. """ del config # node granularity + ig_steps=1 only; enforced at construction. @@ -498,6 +505,8 @@ def attribution_patch( "Attribution patching aligns activations position-by-position." ) + require_eval_mode(model, operation="attribution_patch()") + node_hook_names = _required_hook_names(int(model.cfg.n_layers)) batch = int(clean.shape[0]) totals: dict[Node, float] = {} diff --git a/transformer_lens/tools/analysis/jacobian_lens.py b/transformer_lens/tools/analysis/jacobian_lens.py index a5b2d7102c..34e7b59699 100644 --- a/transformer_lens/tools/analysis/jacobian_lens.py +++ b/transformer_lens/tools/analysis/jacobian_lens.py @@ -81,6 +81,7 @@ from tqdm.auto import tqdm from transformer_lens.ActivationCache import ActivationCache +from transformer_lens.tools.analysis._model_state import require_eval_mode from transformer_lens.tools.analysis.jacobian_lens_coordinate_patch import ( CoordinatePatch, solve_coordinate_patch, @@ -1701,7 +1702,7 @@ def fit( or layer indices, or if no prompt was long enough to fit on. """ _require_raw_bridge(model) - _require_eval_mode_for_fit(model) + require_eval_mode(model, operation="JacobianLens.fit()") if not isinstance(corpus, str) or not corpus.strip(): raise ValueError("corpus must be a non-empty provenance identifier") n_layers = model.cfg.n_layers @@ -1869,32 +1870,6 @@ def _require_raw_bridge(model: Any) -> None: ) -def _require_eval_mode_for_fit(model: Any) -> None: - """Reject stochastic training state without mutating the caller's model.""" - training_modules: Dict[int, str] = {} - roots = (("", model), ("original_model", getattr(model, "original_model", None))) - for prefix, root in roots: - if not isinstance(root, torch.nn.Module): - continue - for name, module in root.named_modules(): - if not module.training: - continue - qualified_name = ".".join(part for part in (prefix, name) if part) - training_modules.setdefault(id(module), qualified_name or "<root>") - if not training_modules: - return - - names = list(training_modules.values()) - preview = ", ".join(names[:3]) - if len(names) > 3: - preview += f", and {len(names) - 3} more" - raise ValueError( - "JacobianLens.fit() requires the model and all submodules to be in " - f"evaluation mode; found training mode at {preview}. Call model.eval() " - "before fitting." - ) - - def _validate_metadata(metadata: Dict[str, Any]) -> None: """Reject values that ``torch.load(weights_only=True)`` cannot reload.""" From 5a7b3467f1fc816d9031e57832af79be170544b5 Mon Sep 17 00:00:00 2001 From: Janmenjaya Panda <83154020+janmenjayap@users.noreply.github.com> Date: Wed, 16 Sep 2026 05:42:41 +0530 Subject: [PATCH 77/87] feat(attribution_patching): edge attribution (EAP) scoring on TransformerBridge (#1781) * feat(attribution_patching): per-head hook requirements on the Bridge Edges into and out of attention heads read attn.hook_result (writer, a head's contribution before the sum into resid) and the split attn.hook_q_input/hook_k_input/hook_v_input (reader, each head's separate Q/K/V input). Both hook families exist on the Bridge unconditionally but only fire once cfg.use_attn_result and cfg.use_split_qkv_input are on. Add an edge-granularity variant of the required hook set, a helper that turns on both flags before caching, and generalize the missing-hook ValueError enumerate_nodes already raises into a shared check so any granularity's cache can be validated against it instead of duplicating the check per graph. * feat(attribution_patching): edge enumeration in the typed graph Add enumerate_edges(model, cache), producing every writer -> reader pair in the residual-stream graph. Extend NodeKind with the reader kinds q_input/k_input/v_input/mlp_in and extend Node's invariants and hook_name accordingly. At a fixed sequence position the residual stream is a running sum, so a reader is fed by every writer enumerated ahead of it; the graph is built position-by-position, tracking that "available" writer set. A writer feeding both a direct edge and a through-MLP edge yields two distinct writer-reader pairs rather than a summed one; a dedicated uniqueness guard enforces that invariant. Edges into the MLP additionally need the MLP-entry hook point (hook_mlp_in per layer), so edge enumeration validates its presence alongside the per-head hook set already required for edge granularity. * feat(attribution_patching): edge scoring in attribution_patch Wire granularity="edge" through attribution_patch: enumerate the writer -> reader edges from the cached activations and gradients, score each with (a_clean[writer] - a_corrupt[writer]) . d(metric)/d(input of reader), and populate AttributionResult.edge_scores. node_scores for an edge sweep becomes each writer's aggregate over its own outgoing edge scores rather than a direct node measurement. Remove the granularity == "edge" raise from EdgeAttributionConfig; the ig_steps > 1 raise is untouched. top_edges() ranks by absolute score and returns the ranked edge list instead of raising. Enable use_hook_mlp_in alongside the existing per-head flags so MLP-input edges have a populated reader hook. * test(attribution_patching): exact-patch parity and mutation-checked reconstruction Add the two Risk-1 guards for edge scoring: a genuine single-edge activation patch (adding one writer's clean-minus-corrupt delta into a reader's cached input, everything else held at corrupt) matches _edge_effects' sign and, on this fully linear toy Bridge, its magnitude too; and perturbing one writer's captured contribution changes only the edges that writer feeds, leaving every other edge's score (and sibling heads/positions in the same cached tensor) untouched. Also add a sum-to-total check for the edge form: a reader's incoming edge scores sum to the same value as scoring its own cached input-delta against its own gradient directly, an identity that holds unconditionally from the residual stream's additive construction. generic_activation_patch is not used for the exact-patch check: its model: HookedTransformer parameter is enforced at runtime by this repo's jaxtyping/beartype pytest configuration, which rejects any argument that is not actually a HookedTransformer instance, including a real TransformerBridge. The patch is driven directly through the Bridge's own hooks(), the same mechanism generic_activation_patch uses internally. * feat(attribution_patching): scope edge hook-flag mutation with save/restore Add _edge_hook_flags, a context manager that enables the Bridge flags an edge sweep needs (use_attn_result, use_split_qkv_input, use_hook_mlp_in) and restores the caller's prior flag state on exit, including on error. The prior one-way helper only turned flags on, so every forward after an edge sweep kept materializing the per-head tensors, and a caller who arrived with use_attn_in set hit the use_split_qkv_input exclusivity error and was still left mutated. The context manager snapshots all four flags, clears use_attn_in before enabling the split input, and restores each flag in a finally block so a raise mid-sweep cannot leave the model mutated. Extend the edge-hook toy Bridge with use_attn_in and its setter (mutually exclusive with use_split_qkv_input) and cover enable-in-scope, restore on normal exit and on raise, already-enabled flags, and the use_attn_in caller path. * feat(attribution_patching): add terminal logits reader and parallel_attn_mlp handling to edge enumeration * feat(attribution_patching): batch edge scoring and scope the sweep's hook-flag mutation * test(attribution_patching): score edge parity and mutation at the readout position and cover multi-pair averaging and head-to-logits ranking * test(attribution_patching): construct edge toy bridges in eval mode --- tests/unit/tools/test_attribution_patching.py | 1176 ++++++++++++++++- transformer_lens/tools/analysis/__init__.py | 8 +- .../tools/analysis/attribution_patching.py | 555 +++++++- 3 files changed, 1675 insertions(+), 64 deletions(-) diff --git a/tests/unit/tools/test_attribution_patching.py b/tests/unit/tools/test_attribution_patching.py index eda937a39c..f0dab2244f 100644 --- a/tests/unit/tools/test_attribution_patching.py +++ b/tests/unit/tools/test_attribution_patching.py @@ -24,9 +24,18 @@ EdgeAttributionConfig, GradientCache, Node, + _assert_edges_unique, + _check_required_hooks, + _edge_effects, + _edge_hook_flags, + _edge_hook_names, + _ensure_edge_hook_flags, _node_effects, + _required_hook_names, + _writer_hook_name, attribution_patch, cache_activation_and_gradient, + enumerate_edges, enumerate_nodes, ) @@ -317,9 +326,13 @@ def test_config_rejects_invalid_values() -> None: EdgeAttributionConfig(ig_steps=0) -def test_config_unsupported_paths_raise_not_implemented() -> None: - with pytest.raises(NotImplementedError, match="edge"): - EdgeAttributionConfig(granularity="edge") +def test_config_accepts_edge_granularity() -> None: + config = EdgeAttributionConfig(granularity="edge") + assert config.granularity == "edge" + assert config.ig_steps == 1 + + +def test_config_ig_steps_above_one_raises_not_implemented() -> None: with pytest.raises(NotImplementedError, match="integrated gradient"): EdgeAttributionConfig(ig_steps=5) @@ -339,11 +352,34 @@ def test_top_nodes_ranks_by_effect_magnitude() -> None: assert [node for node, _ in result.top_nodes(k=10)] == [big_negative, medium, small] -def test_top_edges_not_implemented() -> None: +def test_top_edges_ranks_by_effect_magnitude() -> None: + small = (Node(kind="embed", position=0), Node(kind="mlp_in", layer=0, position=0)) + big_negative = ( + Node(kind="embed", position=0), + Node(kind="q_input", layer=0, head=0, position=0), + ) + medium = ( + Node(kind="attn_head_out", layer=0, head=0, position=0), + Node(kind="mlp_in", layer=1, position=0), + ) + result = AttributionResult( + node_scores={}, + edge_scores={small: 0.1, big_negative: -5.0, medium: 2.0}, + ) + + ranked = result.top_edges(k=2) + assert [(writer, reader) for writer, reader, _ in ranked] == [big_negative, medium] + + # k beyond the edge count returns every edge, still magnitude-ordered. + full = result.top_edges(k=10) + assert [(writer, reader) for writer, reader, _ in full] == [big_negative, medium, small] + assert [score for _, _, score in full] == [-5.0, 2.0, 0.1] + + +def test_top_edges_empty_when_no_edge_sweep_has_run() -> None: result = AttributionResult(node_scores={}) assert result.edge_scores == {} - with pytest.raises(NotImplementedError, match="edge"): - result.top_edges() + assert result.top_edges() == [] # --------------------------------------------------------------------------- @@ -796,3 +832,1131 @@ def test_nonlinear_node_scores_read_the_corrupt_run_gradient() -> None: assert (delta_m > 0) == (score > 0) # denoising sign convention checked += 1 assert checked > 0 + + +# --------------------------------------------------------------------------- +# Edge-granularity hook requirements +# --------------------------------------------------------------------------- +# +# Edges into and out of an attention head read hook points a node sweep never +# needs: the per-head ``attn.hook_result`` (writer) and the split +# ``attn.hook_q_input``/``hook_k_input``/``hook_v_input`` (reader). Both +# families exist on the Bridge unconditionally but only fire once their owning +# config flag is on. + + +class _GatedEdgeHookBlock(nn.Module): + """A block exposing the per-head hook points edge granularity reads. + + ``hook_result`` and the split ``hook_q_input``/``hook_k_input``/ + ``hook_v_input`` mirror the real Bridge's fire-time gating: the HookPoints + exist unconditionally, but the block only calls them when the matching + ``cfg`` flag is on, so their activation is absent from a cache built while + the flag is off rather than present with a placeholder value. + """ + + def __init__( + self, + d_model: int, + n_heads: int, + d_head: int, + layer: int, + dtype: torch.dtype, + cfg: SimpleNamespace, + ) -> None: + super().__init__() + self.cfg = cfg + self.n_heads = n_heads + self.d_head = d_head + self.w_z = nn.Linear(d_model, n_heads * d_head, bias=False, dtype=dtype) + self.w_o = nn.Linear(n_heads * d_head, d_model, bias=False, dtype=dtype) + self.w_mlp = nn.Linear(d_model, d_model, bias=False, dtype=dtype) + for linear in (self.w_z, self.w_o, self.w_mlp): + nn.init.normal_(linear.weight, std=0.2) + self.hook_z = HookPoint() + self.hook_z.name = f"blocks.{layer}.attn.hook_z" + self.hook_mlp_out = HookPoint() + self.hook_mlp_out.name = f"blocks.{layer}.hook_mlp_out" + self.hook_result = HookPoint() + self.hook_result.name = f"blocks.{layer}.attn.hook_result" + self.hook_q_input = HookPoint() + self.hook_q_input.name = f"blocks.{layer}.attn.hook_q_input" + self.hook_k_input = HookPoint() + self.hook_k_input.name = f"blocks.{layer}.attn.hook_k_input" + self.hook_v_input = HookPoint() + self.hook_v_input.name = f"blocks.{layer}.attn.hook_v_input" + + def forward(self, residual: torch.Tensor) -> torch.Tensor: + batch, seq, d_model = residual.shape + if self.cfg.use_split_qkv_input: + per_head_residual = residual.unsqueeze(-2).expand(batch, seq, self.n_heads, d_model) + self.hook_q_input(per_head_residual) + self.hook_k_input(per_head_residual) + self.hook_v_input(per_head_residual) + + z = self.hook_z(self.w_z(residual).reshape(batch, seq, self.n_heads, self.d_head)) + if self.cfg.use_attn_result: + # Distributive over per-head weight slicing: summing this per-head + # decomposition equals self.w_o(z_flat) exactly (no bias term to split). + w_o_per_head = self.w_o.weight.reshape(d_model, self.n_heads, self.d_head).permute( + 1, 2, 0 + ) + per_head_out = self.hook_result(torch.einsum("bshd,hdm->bshm", z, w_o_per_head)) + attn_out = per_head_out.sum(dim=-2) + else: + attn_out = self.w_o(z.reshape(batch, seq, self.n_heads * self.d_head)) + + residual = residual + attn_out + mlp_out = self.hook_mlp_out(self.w_mlp(residual)) + return residual + mlp_out + + +class _EdgeHookToyBridge(_LinearToyBridge): + """A tiny ``TransformerBridge`` whose blocks carry the edge-granularity hook points. + + ``cfg.use_attn_result``/``cfg.use_split_qkv_input`` default to ``False``, + matching a real Bridge; ``set_use_attn_result``/``set_use_split_qkv_input`` + toggle them. + """ + + def __init__(self, *, dtype: torch.dtype = torch.float32) -> None: + nn.Module.__init__(self) + self._hook_registry: dict[str, HookPoint] = {} + self.context_level = 0 + torch.manual_seed(0) + self.cfg = SimpleNamespace( + n_layers=N_LAYERS, + d_model=D_MODEL, + d_vocab=D_VOCAB, + d_vocab_out=D_VOCAB, + model_name="edge-hook-toy-bridge", + dtype=dtype, + device="cpu", + use_attn_result=False, + use_split_qkv_input=False, + use_hook_mlp_in=False, + use_attn_in=False, + ) + self.compatibility_mode = False + self._weights_processed = False + self.embed = nn.Embedding(D_VOCAB, D_MODEL, dtype=dtype) + nn.init.normal_(self.embed.weight, std=0.2) + self.hook_embed = HookPoint() + self.hook_embed.name = "hook_embed" + self.blocks = nn.ModuleList( + [ + _GatedEdgeHookBlock(D_MODEL, N_HEADS, D_HEAD, layer, dtype, self.cfg) + for layer in range(N_LAYERS) + ] + ) + self.ln_final = nn.Identity() + self.unembed = nn.Linear(D_MODEL, D_VOCAB, bias=False, dtype=dtype) + nn.init.normal_(self.unembed.weight, std=0.2) + self.eval() + + @property + def hook_dict(self) -> dict[str, HookPoint]: + hooks: dict[str, HookPoint] = {"hook_embed": self.hook_embed} + for layer, block in enumerate(self.blocks): + hooks[f"blocks.{layer}.attn.hook_z"] = block.hook_z + hooks[f"blocks.{layer}.hook_mlp_out"] = block.hook_mlp_out + hooks[f"blocks.{layer}.attn.hook_result"] = block.hook_result + hooks[f"blocks.{layer}.attn.hook_q_input"] = block.hook_q_input + hooks[f"blocks.{layer}.attn.hook_k_input"] = block.hook_k_input + hooks[f"blocks.{layer}.attn.hook_v_input"] = block.hook_v_input + return hooks + + def forward( + self, tokens: torch.Tensor, return_type: str | None = "logits" + ) -> torch.Tensor | None: + residual = self.hook_embed(self.embed(tokens)) + for block in self.blocks: + residual = block(residual) + if return_type is None: + return None + return self.unembed(self.ln_final(residual)) + + def set_use_attn_result(self, use_attn_result: bool) -> None: + self.cfg.use_attn_result = use_attn_result + + def set_use_split_qkv_input(self, use_split_qkv_input: bool) -> None: + if use_split_qkv_input and self.cfg.use_attn_in: + raise ValueError("use_split_qkv_input and use_attn_in are mutually exclusive.") + self.cfg.use_split_qkv_input = use_split_qkv_input + + def set_use_hook_mlp_in(self, use_hook_mlp_in: bool) -> None: + self.cfg.use_hook_mlp_in = use_hook_mlp_in + + def set_use_attn_in(self, use_attn_in: bool) -> None: + if use_attn_in and self.cfg.use_split_qkv_input: + raise ValueError("use_attn_in and use_split_qkv_input are mutually exclusive.") + self.cfg.use_attn_in = use_attn_in + + +def test_required_hook_names_edge_granularity_adds_per_head_families() -> None: + node_names = set(_required_hook_names(N_LAYERS)) + edge_names = set(_required_hook_names(N_LAYERS, granularity="edge")) + + assert node_names < edge_names + for layer in range(N_LAYERS): + assert f"blocks.{layer}.attn.hook_result" in edge_names + assert f"blocks.{layer}.attn.hook_q_input" in edge_names + assert f"blocks.{layer}.attn.hook_k_input" in edge_names + assert f"blocks.{layer}.attn.hook_v_input" in edge_names + + +def test_ensure_edge_hook_flags_enables_required_bridge_flags() -> None: + model = _EdgeHookToyBridge() + assert model.cfg.use_attn_result is False + assert model.cfg.use_split_qkv_input is False + assert model.cfg.use_hook_mlp_in is False + + _ensure_edge_hook_flags(model) + + assert model.cfg.use_attn_result is True + assert model.cfg.use_split_qkv_input is True + assert model.cfg.use_hook_mlp_in is True + + +def test_edge_hook_flags_enables_required_flags_inside_the_scope() -> None: + model = _EdgeHookToyBridge() + assert model.cfg.use_attn_result is False + assert model.cfg.use_split_qkv_input is False + assert model.cfg.use_hook_mlp_in is False + + with _edge_hook_flags(model): + assert model.cfg.use_attn_result is True + assert model.cfg.use_split_qkv_input is True + assert model.cfg.use_hook_mlp_in is True + + +def test_edge_hook_flags_restore_prior_state_on_exit() -> None: + model = _EdgeHookToyBridge() + + with _edge_hook_flags(model): + pass + + assert model.cfg.use_attn_result is False + assert model.cfg.use_split_qkv_input is False + assert model.cfg.use_hook_mlp_in is False + + +def test_edge_hook_flags_leave_already_enabled_flags_enabled() -> None: + model = _EdgeHookToyBridge() + model.set_use_attn_result(True) + model.set_use_hook_mlp_in(True) + + with _edge_hook_flags(model): + assert model.cfg.use_attn_result is True + assert model.cfg.use_hook_mlp_in is True + + assert model.cfg.use_attn_result is True + assert model.cfg.use_hook_mlp_in is True + assert model.cfg.use_split_qkv_input is False + + +def test_edge_hook_flags_restore_flags_when_the_body_raises() -> None: + model = _EdgeHookToyBridge() + + with pytest.raises(RuntimeError, match="sweep exploded"): + with _edge_hook_flags(model): + raise RuntimeError("sweep exploded") + + assert model.cfg.use_attn_result is False + assert model.cfg.use_split_qkv_input is False + assert model.cfg.use_hook_mlp_in is False + assert model.cfg.use_attn_in is False + + +def test_edge_hook_flags_tolerate_a_caller_with_use_attn_in_set() -> None: + model = _EdgeHookToyBridge() + model.set_use_attn_in(True) + assert model.cfg.use_attn_in is True + + with _edge_hook_flags(model): + # use_attn_in is mutually exclusive with use_split_qkv_input, so the + # scope must clear it before enabling the split input. + assert model.cfg.use_attn_in is False + assert model.cfg.use_split_qkv_input is True + + assert model.cfg.use_attn_in is True + assert model.cfg.use_split_qkv_input is False + + +def test_edge_hook_flags_populate_the_per_head_cache() -> None: + model = _EdgeHookToyBridge() + _ensure_edge_hook_flags(model) + tokens = model.to_tokens("prompt") + metric = _metric_fn(answer=1, wrong=2) + edge_names = _required_hook_names(N_LAYERS, granularity="edge") + + cache = cache_activation_and_gradient(model, tokens, metric, names_filter=edge_names) + + assert set(cache.activations) == set(edge_names) + for layer in range(N_LAYERS): + assert cache.activations[f"blocks.{layer}.attn.hook_result"].shape == ( + 1, + SEQ_LEN, + N_HEADS, + D_MODEL, + ) + for input_hook in ("hook_q_input", "hook_k_input", "hook_v_input"): + assert cache.activations[f"blocks.{layer}.attn.{input_hook}"].shape == ( + 1, + SEQ_LEN, + N_HEADS, + D_MODEL, + ) + + # No missing hook points: the shared check is a no-op. + _check_required_hooks(cache, edge_names, "edge graph", "hint unused when nothing is missing") + + +def test_missing_edge_hook_raises_instead_of_silently_shrinking_the_graph() -> None: + model = _EdgeHookToyBridge() # flags left off: per-head hooks never fire + tokens = model.to_tokens("prompt") + metric = _metric_fn(answer=1, wrong=2) + edge_names = _required_hook_names(N_LAYERS, granularity="edge") + + cache = cache_activation_and_gradient(model, tokens, metric, names_filter=edge_names) + + assert "blocks.0.attn.hook_result" not in cache.activations + + with pytest.raises(ValueError, match="attn.hook_result"): + _check_required_hooks(cache, edge_names, "edge graph", "enable use_attn_result.") + + +# --------------------------------------------------------------------------- +# Edge enumeration in the typed graph +# --------------------------------------------------------------------------- +# +# An edge u -> v carries writer u's residual-stream contribution into reader +# v's input. Readers are the per-head split-QKV inputs (q_input/k_input/ +# v_input) and the MLP entry (mlp_in); writers are the existing node kinds +# (embed, attn_head_out, mlp_out). At a fixed sequence position, a reader +# connects to every writer that precedes it in the residual stream so far -- +# the graph never mixes across positions, matching the per-position Node key. + + +def _synthetic_edge_cache( + n_layers: int = N_LAYERS, + seq_len: int = SEQ_LEN, + n_heads: int = N_HEADS, + d_head: int = D_HEAD, + d_model: int = D_MODEL, +) -> GradientCache: + """A cache whose keys/shapes carry the edge-granularity hook points. + + Edge enumeration only reads hook names and tensor shapes, so the contents + can be zeros; this keeps the graph test model-free and independent of any + forward pass. + """ + cache = _synthetic_node_cache( + n_layers=n_layers, seq_len=seq_len, n_heads=n_heads, d_head=d_head, d_model=d_model + ) + for layer in range(n_layers): + cache.activations[f"blocks.{layer}.attn.hook_result"] = torch.zeros( + 1, seq_len, n_heads, d_model + ) + for input_hook in ("hook_q_input", "hook_k_input", "hook_v_input"): + cache.activations[f"blocks.{layer}.attn.{input_hook}"] = torch.zeros( + 1, seq_len, n_heads, d_model + ) + cache.activations[f"blocks.{layer}.hook_mlp_in"] = torch.zeros(1, seq_len, d_model) + cache.activations[f"blocks.{n_layers - 1}.hook_resid_post"] = torch.zeros(1, seq_len, d_model) + cache.gradients = {name: None for name in cache.activations} + return cache + + +def test_reader_node_hook_name_and_key_validation() -> None: + assert ( + Node(kind="q_input", layer=0, head=1, position=2).hook_name == "blocks.0.attn.hook_q_input" + ) + assert ( + Node(kind="k_input", layer=1, head=0, position=0).hook_name == "blocks.1.attn.hook_k_input" + ) + assert ( + Node(kind="v_input", layer=0, head=0, position=0).hook_name == "blocks.0.attn.hook_v_input" + ) + assert Node(kind="mlp_in", layer=1, position=0).hook_name == "blocks.1.hook_mlp_in" + + # The typed key rejects malformed reader nodes rather than building a wrong graph. + with pytest.raises(ValueError): + Node(kind="q_input", layer=0, position=0) # head missing + with pytest.raises(ValueError): + Node(kind="mlp_in", layer=0, head=0, position=0) # head not allowed + + +def _expected_edge_count(n_layers: int, n_heads: int, seq_len: int) -> int: + """Independent count of writer -> reader pairs for the dense per-position graph. + + At each position, a reader connects to every writer enumerated so far: the + three per-head QKV readers at a layer see everything upstream of that + layer's attention, and the MLP reader additionally sees that layer's own + attention writes. After the layers, the terminal logits reader connects to + every writer available at that position. + """ + total_per_position = 0 + available = 1 # embed + for _ in range(n_layers): + total_per_position += 3 * n_heads * available # q/k/v readers + available += n_heads # this layer's attn_head_out writers + total_per_position += available # mlp_in reader + available += 1 # this layer's mlp_out writer + total_per_position += available # logits reader sees every writer + return seq_len * total_per_position + + +def test_enumerate_edges_returns_expected_writer_reader_pairs() -> None: + cache = _synthetic_edge_cache() + edges = enumerate_edges(_cfg_stub(), cache) + + assert len(edges) == _expected_edge_count(N_LAYERS, N_HEADS, SEQ_LEN) + assert len(set(edges)) == len(edges) # no duplicate (u, v) pairs + assert all(writer.position == reader.position for writer, reader in edges) + + # embed is the first writer at its position, so every reader at that + # position has a direct edge from it. + readers_at_0 = {reader for writer, reader in edges if writer.position == 0} + for reader in readers_at_0: + assert (Node(kind="embed", position=0), reader) in edges + + # the final layer's MLP output reaches the terminal logits reader, so it is + # a writer in this graph even though no per-layer reader sees it. + terminal_writer = Node(kind="mlp_out", layer=N_LAYERS - 1, position=0) + logits_reader = Node(kind="logits", layer=N_LAYERS - 1, position=0) + assert (terminal_writer, logits_reader) in edges + + +def test_enumerate_edges_connects_every_final_writer_to_the_logits_reader() -> None: + cache = _synthetic_edge_cache() + edges = enumerate_edges(_cfg_stub(), cache) + + for position in range(SEQ_LEN): + logits_reader = Node(kind="logits", layer=N_LAYERS - 1, position=position) + writers_to_logits = {writer for writer, reader in edges if reader == logits_reader} + + expected = {Node(kind="embed", position=position)} + for layer in range(N_LAYERS): + for head in range(N_HEADS): + expected.add(Node(kind="attn_head_out", layer=layer, head=head, position=position)) + expected.add(Node(kind="mlp_out", layer=layer, position=position)) + + assert writers_to_logits == expected + # the logits reader is a reader only; it never appears as a writer. + assert all(writer != logits_reader for writer, _ in edges) + + +def test_enumerate_edges_raises_on_missing_reader_hook() -> None: + cache = _synthetic_edge_cache() + del cache.activations["blocks.0.hook_mlp_in"] + + with pytest.raises(ValueError, match="blocks.0.hook_mlp_in"): + enumerate_edges(_cfg_stub(), cache) + + +def test_assert_edges_unique_raises_on_duplicate() -> None: + reader = Node(kind="mlp_in", layer=0, position=0) + writer = Node(kind="embed", position=0) + + with pytest.raises(ValueError, match="more than once"): + _assert_edges_unique([(writer, reader), (writer, reader)]) + + +def test_enumerate_edges_drops_same_layer_heads_from_mlp_reader_on_parallel_attn_mlp() -> None: + cache = _synthetic_edge_cache() + parallel_stub = SimpleNamespace(cfg=SimpleNamespace(n_layers=N_LAYERS, parallel_attn_mlp=True)) + + edges = enumerate_edges(parallel_stub, cache) + _assert_edges_unique(edges) + + for position in range(SEQ_LEN): + # A layer's own heads do not feed that layer's MLP: it reads the layer + # input, not the post-attention residual. + for layer in range(N_LAYERS): + mlp_reader = Node(kind="mlp_in", layer=layer, position=position) + for head in range(N_HEADS): + same_layer_head = Node( + kind="attn_head_out", layer=layer, head=head, position=position + ) + assert (same_layer_head, mlp_reader) not in edges + + # Cross-layer head -> mlp_in and terminal ->logits edges are unaffected. + upstream_head = Node(kind="attn_head_out", layer=0, head=0, position=position) + assert (upstream_head, Node(kind="mlp_in", layer=1, position=position)) in edges + logits_reader = Node(kind="logits", layer=N_LAYERS - 1, position=position) + assert (upstream_head, logits_reader) in edges + + # The sequential graph does keep the same-layer head -> mlp_in edge dropped here. + sequential_edges = enumerate_edges(_cfg_stub(), cache) + head0 = Node(kind="attn_head_out", layer=0, head=0, position=0) + assert (head0, Node(kind="mlp_in", layer=0, position=0)) in sequential_edges + assert (head0, Node(kind="mlp_in", layer=0, position=0)) not in edges + + +# --------------------------------------------------------------------------- +# Commit 3 - edge scoring in attribution_patch +# --------------------------------------------------------------------------- +# +# Edge scoring needs every reader hook to actually participate in the forward +# computation, unlike _GatedEdgeHookBlock's probes above, which fire but are +# discarded: a reader with no forward-graph path to the metric would receive +# no gradient, leaving nothing genuine for _edge_effects to score. +# _EdgeScoringBlock wires every per-head QKV input and the MLP entry into the +# actual computation, so a cache built from it carries real, distinguishable +# per-edge gradients. + + +class _EdgeScoringBlock(nn.Module): + """A block whose split-QKV inputs and MLP entry are read, not just probed. + + Each head's z is a linear combination of that head's own hook_q_input / + hook_k_input / hook_v_input, through independent per-head weights, so an + edge into one input family is distinguishable from an edge into another. + Whether a cfg flag is on only changes whether the corresponding HookPoint + fires (and so whether it is visible in a cache); the per-head + decomposition of the residual is distributive, so the computed value is + identical either way, mirroring _GatedEdgeHookBlock's hook_result split. + """ + + def __init__( + self, + d_model: int, + n_heads: int, + d_head: int, + layer: int, + dtype: torch.dtype, + cfg: SimpleNamespace, + ) -> None: + super().__init__() + self.cfg = cfg + self.n_heads = n_heads + self.d_head = d_head + self.w_q = nn.Linear(d_model, n_heads * d_head, bias=False, dtype=dtype) + self.w_k = nn.Linear(d_model, n_heads * d_head, bias=False, dtype=dtype) + self.w_v = nn.Linear(d_model, n_heads * d_head, bias=False, dtype=dtype) + self.w_o = nn.Linear(n_heads * d_head, d_model, bias=False, dtype=dtype) + self.w_mlp = nn.Linear(d_model, d_model, bias=False, dtype=dtype) + for linear in (self.w_q, self.w_k, self.w_v, self.w_o, self.w_mlp): + nn.init.normal_(linear.weight, std=0.2) + self.hook_z = HookPoint() + self.hook_z.name = f"blocks.{layer}.attn.hook_z" + self.hook_mlp_out = HookPoint() + self.hook_mlp_out.name = f"blocks.{layer}.hook_mlp_out" + self.hook_result = HookPoint() + self.hook_result.name = f"blocks.{layer}.attn.hook_result" + self.hook_q_input = HookPoint() + self.hook_q_input.name = f"blocks.{layer}.attn.hook_q_input" + self.hook_k_input = HookPoint() + self.hook_k_input.name = f"blocks.{layer}.attn.hook_k_input" + self.hook_v_input = HookPoint() + self.hook_v_input.name = f"blocks.{layer}.attn.hook_v_input" + self.hook_mlp_in = HookPoint() + self.hook_mlp_in.name = f"blocks.{layer}.hook_mlp_in" + self.hook_resid_post = HookPoint() + self.hook_resid_post.name = f"blocks.{layer}.hook_resid_post" + + def forward(self, residual: torch.Tensor) -> torch.Tensor: + batch, seq, d_model = residual.shape + per_head_residual = residual.unsqueeze(-2).expand(batch, seq, self.n_heads, d_model) + if self.cfg.use_split_qkv_input: + q_input = self.hook_q_input(per_head_residual) + k_input = self.hook_k_input(per_head_residual) + v_input = self.hook_v_input(per_head_residual) + else: + q_input = k_input = v_input = per_head_residual + + w_q_per_head = self.w_q.weight.reshape(self.n_heads, self.d_head, d_model) + w_k_per_head = self.w_k.weight.reshape(self.n_heads, self.d_head, d_model) + w_v_per_head = self.w_v.weight.reshape(self.n_heads, self.d_head, d_model) + z = self.hook_z( + torch.einsum("bshm,hdm->bshd", q_input, w_q_per_head) + + torch.einsum("bshm,hdm->bshd", k_input, w_k_per_head) + + torch.einsum("bshm,hdm->bshd", v_input, w_v_per_head) + ) + + w_o_per_head = self.w_o.weight.reshape(d_model, self.n_heads, self.d_head).permute(1, 2, 0) + per_head_out_raw = torch.einsum("bshd,hdm->bshm", z, w_o_per_head) + if self.cfg.use_attn_result: + per_head_out = self.hook_result(per_head_out_raw) + else: + per_head_out = per_head_out_raw + residual = residual + per_head_out.sum(dim=-2) + + mlp_in = self.hook_mlp_in(residual) if self.cfg.use_hook_mlp_in else residual + mlp_out = self.hook_mlp_out(self.w_mlp(mlp_in)) + return self.hook_resid_post(residual + mlp_out) + + +class _EdgeScoringToyBridge(_LinearToyBridge): + """A tiny ``TransformerBridge`` whose edge-granularity hooks all sit on the data path. + + Every reader hook point (``hook_q_input``/``hook_k_input``/``hook_v_input``, + ``hook_mlp_in``) and writer hook point (``hook_result``) that + :func:`enumerate_edges` connects is read by the forward computation itself, + so ``cache_activation_and_gradient``'s backward hooks capture real, + per-edge gradients. + """ + + def __init__(self, *, dtype: torch.dtype = torch.float32) -> None: + nn.Module.__init__(self) + self._hook_registry: dict[str, HookPoint] = {} + self.context_level = 0 + torch.manual_seed(0) + self.cfg = SimpleNamespace( + n_layers=N_LAYERS, + d_model=D_MODEL, + d_vocab=D_VOCAB, + d_vocab_out=D_VOCAB, + model_name="edge-scoring-toy-bridge", + dtype=dtype, + device="cpu", + use_attn_result=False, + use_split_qkv_input=False, + use_hook_mlp_in=False, + ) + self.compatibility_mode = False + self._weights_processed = False + self.embed = nn.Embedding(D_VOCAB, D_MODEL, dtype=dtype) + nn.init.normal_(self.embed.weight, std=0.2) + self.hook_embed = HookPoint() + self.hook_embed.name = "hook_embed" + self.blocks = nn.ModuleList( + [ + _EdgeScoringBlock(D_MODEL, N_HEADS, D_HEAD, layer, dtype, self.cfg) + for layer in range(N_LAYERS) + ] + ) + self.ln_final = nn.Identity() + self.unembed = nn.Linear(D_MODEL, D_VOCAB, bias=False, dtype=dtype) + nn.init.normal_(self.unembed.weight, std=0.2) + self.eval() + + @property + def hook_dict(self) -> dict[str, HookPoint]: + hooks: dict[str, HookPoint] = {"hook_embed": self.hook_embed} + for layer, block in enumerate(self.blocks): + hooks[f"blocks.{layer}.attn.hook_z"] = block.hook_z + hooks[f"blocks.{layer}.hook_mlp_out"] = block.hook_mlp_out + hooks[f"blocks.{layer}.attn.hook_result"] = block.hook_result + hooks[f"blocks.{layer}.attn.hook_q_input"] = block.hook_q_input + hooks[f"blocks.{layer}.attn.hook_k_input"] = block.hook_k_input + hooks[f"blocks.{layer}.attn.hook_v_input"] = block.hook_v_input + hooks[f"blocks.{layer}.hook_mlp_in"] = block.hook_mlp_in + hooks[f"blocks.{layer}.hook_resid_post"] = block.hook_resid_post + return hooks + + def forward( + self, tokens: torch.Tensor, return_type: str | None = "logits" + ) -> torch.Tensor | None: + residual = self.hook_embed(self.embed(tokens)) + for block in self.blocks: + residual = block(residual) + if return_type is None: + return None + return self.unembed(self.ln_final(residual)) + + def set_use_attn_result(self, use_attn_result: bool) -> None: + self.cfg.use_attn_result = use_attn_result + + def set_use_split_qkv_input(self, use_split_qkv_input: bool) -> None: + self.cfg.use_split_qkv_input = use_split_qkv_input + + def set_use_hook_mlp_in(self, use_hook_mlp_in: bool) -> None: + self.cfg.use_hook_mlp_in = use_hook_mlp_in + + +def test_attribution_patch_edge_granularity_scores_every_edge_with_finite_values() -> None: + model = _EdgeScoringToyBridge() + clean = torch.tensor([[1, 2, 3]]) + corrupt = torch.tensor([[3, 2, 1]]) + metric = _metric_fn(answer=1, wrong=2) + + result = attribution_patch( + model, clean, corrupt, metric, config=EdgeAttributionConfig(granularity="edge") + ) + + assert isinstance(result, AttributionResult) + assert len(result.edge_scores) == _expected_edge_count(N_LAYERS, N_HEADS, SEQ_LEN) + assert all(isinstance(score, float) for score in result.edge_scores.values()) + assert all(math.isfinite(score) for score in result.edge_scores.values()) + + top = result.top_edges(k=5) + assert len(top) == 5 + magnitudes = [abs(score) for _, _, score in top] + assert magnitudes == sorted(magnitudes, reverse=True) + for writer, reader, score in top: + assert (writer, reader) in result.edge_scores + assert result.edge_scores[(writer, reader)] == score + + +def test_attribution_patch_edge_aggregation_reconstructs_direct_node_scores() -> None: + """A writer's edge-score aggregate equals its direct node score, exactly. + + A writer reaches the metric through every edge enumerate_edges gives it, + including the edge to the terminal logits reader. That final edge carries the + writer's direct skip-connection contribution -- the part of its residual write + that no per-layer reader consumes, only the final readout does -- so the + aggregate over a writer's outgoing edges is its whole first-order effect and + equals the direct node score. This holds for every writer, including the final + layer's MLP output, whose only outgoing edge is the one to the logits reader. + """ + model = _EdgeScoringToyBridge() + clean = torch.tensor([[1, 2, 3]]) + corrupt = torch.tensor([[3, 2, 1]]) + metric = _metric_fn(answer=1, wrong=2) + + edge_result = attribution_patch( + model, clean, corrupt, metric, config=EdgeAttributionConfig(granularity="edge") + ) + node_result = attribution_patch( + model, clean, corrupt, metric, config=EdgeAttributionConfig(granularity="node") + ) + + # The final layer's MLP output now has an outgoing edge (to the logits + # reader), so it appears in the edge aggregate rather than being absent. + terminal_writer = Node(kind="mlp_out", layer=N_LAYERS - 1, position=0) + assert terminal_writer in edge_result.node_scores + + assert set(edge_result.node_scores) == set(node_result.node_scores) + for node, direct_score in node_result.node_scores.items(): + assert edge_result.node_scores[node] == pytest.approx(direct_score, abs=1e-5) + + +def _naive_edge_effects( + clean_cache: GradientCache, + corrupt_cache: GradientCache, + edges: list[tuple[Node, Node]], +) -> dict[tuple[Node, Node], float]: + """Per-edge reference score, recomputed the unbatched way for every edge. + + One clean-minus-corrupt subtraction and one ``float((delta_vec * + grad_vec).sum())`` per edge, so any drift in the batched + :func:`_edge_effects` shows up as a mismatch against this. + """ + scores: dict[tuple[Node, Node], float] = {} + for writer, reader in edges: + writer_name = _writer_hook_name(writer) + delta = clean_cache.activations[writer_name] - corrupt_cache.activations[writer_name] + if writer.kind == "attn_head_out": + delta_vec = delta[0, writer.position, writer.head] + else: + delta_vec = delta[0, writer.position] + grad = corrupt_cache.gradients[reader.hook_name] + assert grad is not None + if reader.kind in ("q_input", "k_input", "v_input"): + grad_vec = grad[0, reader.position, reader.head] + else: + grad_vec = grad[0, reader.position] + scores[(writer, reader)] = float((delta_vec * grad_vec).sum()) + return scores + + +def test_edge_effects_scores_logits_reader_edges_with_finite_signed_values() -> None: + """Every writer -> logits edge is scored, finite, and at least one is nonzero. + + The logits reader takes its gradient at the final ``hook_resid_post`` and + contracts over ``d_model`` at the readout position, so a writer's direct + path to the output is a real edge with a genuine signed score rather than a + structural zero. + """ + model = _EdgeScoringToyBridge() + _ensure_edge_hook_flags(model) + clean = torch.tensor([[1, 2, 3]]) + corrupt = torch.tensor([[3, 2, 1]]) + metric = _metric_fn(answer=1, wrong=2) + edge_hook_names = _edge_hook_names(N_LAYERS) + + clean_cache = cache_activation_and_gradient( + model, clean, metric, names_filter=edge_hook_names, compute_gradient=False + ) + corrupt_cache = cache_activation_and_gradient( + model, corrupt, metric, names_filter=edge_hook_names + ) + edges = enumerate_edges(model, corrupt_cache) + scores = _edge_effects(clean_cache, corrupt_cache, edges) + reference = _naive_edge_effects(clean_cache, corrupt_cache, edges) + + logits_edges = [(writer, reader) for writer, reader in edges if reader.kind == "logits"] + assert logits_edges # the terminal reader closes the graph + for edge in logits_edges: + assert math.isfinite(scores[edge]) + assert scores[edge] == pytest.approx(reference[edge], abs=1e-6) + + # The readout position is the one _metric_fn reads, so the direct paths there + # carry a genuine, signed effect, not a structural zero. + readout = SEQ_LEN - 1 + assert any(abs(scores[edge]) > 1e-6 for edge in logits_edges if edge[1].position == readout) + + +def test_edge_effects_batched_scores_match_the_naive_per_edge_reference() -> None: + """The batched scorer reproduces the unbatched per-edge scores exactly. + + _edge_effects caches each writer's delta once and scores each reader's + incoming edges with one batched product. Pinning it to the plain per-edge + reference guards against a future change to that batching silently shifting + any edge's score. + """ + model = _EdgeScoringToyBridge() + _ensure_edge_hook_flags(model) + clean = torch.tensor([[1, 2, 3]]) + corrupt = torch.tensor([[3, 2, 1]]) + metric = _metric_fn(answer=1, wrong=2) + edge_hook_names = _edge_hook_names(N_LAYERS) + + clean_cache = cache_activation_and_gradient( + model, clean, metric, names_filter=edge_hook_names, compute_gradient=False + ) + corrupt_cache = cache_activation_and_gradient( + model, corrupt, metric, names_filter=edge_hook_names + ) + edges = enumerate_edges(model, corrupt_cache) + + batched = _edge_effects(clean_cache, corrupt_cache, edges) + reference = _naive_edge_effects(clean_cache, corrupt_cache, edges) + + assert set(batched) == set(reference) + for edge in reference: + assert batched[edge] == pytest.approx(reference[edge], abs=1e-6) + + +def test_attribution_patch_edge_sweep_restores_caller_hook_flags() -> None: + """An edge sweep leaves the caller's per-head hook flags as it found them. + + attribution_patch enables use_attn_result / use_split_qkv_input / + use_hook_mlp_in only for the duration of its caching loop, so a caller whose + later forwards should not materialize the per-head tensors gets its flags + back once the sweep returns. + """ + model = _EdgeScoringToyBridge() + assert model.cfg.use_attn_result is False + assert model.cfg.use_split_qkv_input is False + assert model.cfg.use_hook_mlp_in is False + + attribution_patch( + model, + torch.tensor([[1, 2, 3]]), + torch.tensor([[3, 2, 1]]), + _metric_fn(answer=1, wrong=2), + config=EdgeAttributionConfig(granularity="edge"), + ) + + assert model.cfg.use_attn_result is False + assert model.cfg.use_split_qkv_input is False + assert model.cfg.use_hook_mlp_in is False + + +# --------------------------------------------------------------------------- +# Commit 4 - exact-patch parity and mutation-checked reconstruction +# --------------------------------------------------------------------------- +# +# Two guards on the edge-scoring identity Risk 1 warns about: a genuine +# activation patch of a single edge must match _edge_effects' estimate (sign +# always; magnitude too on this linear toy model), and mutating one writer's +# captured contribution must change only the edges that writer feeds, never a +# different edge's score. + + +def _edge_writer_delta_vector( + clean_cache: GradientCache, corrupt_cache: GradientCache, writer: Node +) -> torch.Tensor: + """The writer's own residual-stream delta at its position (and head, if any).""" + name = _writer_hook_name(writer) + delta = clean_cache.activations[name] - corrupt_cache.activations[name] + if writer.kind == "attn_head_out": + return delta[0, writer.position, writer.head] + return delta[0, writer.position] + + +def test_edge_scores_sum_to_the_readers_direct_input_change() -> None: + """Summed incoming-edge scores equal a reader's own first-order input-change score. + + A reader's cached input is the running sum of every writer enumerate_edges + connects to it, so its clean-minus-corrupt delta is exactly the sum of + those writers' individual deltas -- an algebraic identity of the residual + stream's additive construction, independent of anything downstream of the + reader (linear or not). Dotting each side with the reader's gradient and + comparing must therefore match to floating-point precision. + """ + model = _EdgeScoringToyBridge() + _ensure_edge_hook_flags(model) + clean = torch.tensor([[1, 2, 3]]) + corrupt = torch.tensor([[3, 2, 1]]) + metric = _metric_fn(answer=1, wrong=2) + edge_hook_names = _edge_hook_names(N_LAYERS) + + clean_cache = cache_activation_and_gradient( + model, clean, metric, names_filter=edge_hook_names, compute_gradient=False + ) + corrupt_cache = cache_activation_and_gradient( + model, corrupt, metric, names_filter=edge_hook_names + ) + edges = enumerate_edges(model, corrupt_cache) + _assert_edges_unique(edges) + edge_scores = _edge_effects(clean_cache, corrupt_cache, edges) + + readers = {reader for _writer, reader in edges} + checked = 0 + for reader in readers: + name = reader.hook_name + delta = clean_cache.activations[name] - corrupt_cache.activations[name] + grad = corrupt_cache.gradients[name] + assert grad is not None + if reader.kind in ("q_input", "k_input", "v_input"): + delta_vec = delta[0, reader.position, reader.head] + grad_vec = grad[0, reader.position, reader.head] + else: + delta_vec = delta[0, reader.position] + grad_vec = grad[0, reader.position] + direct_score = float((delta_vec * grad_vec).sum()) + + edge_sum = sum( + score for (_writer, edge_reader), score in edge_scores.items() if edge_reader == reader + ) + assert edge_sum == pytest.approx(direct_score, abs=1e-6) + checked += 1 + + assert checked > 0 + + +def test_edge_effects_mutation_only_changes_the_perturbed_writers_edges() -> None: + """Perturbing one writer's captured contribution changes only that writer's edges. + + ``_edge_effects`` reads each edge's writer delta and reader gradient from + two independently indexed tensors (writer position/head, reader + position/head). A slicing bug that mixed up either index could leak a + perturbation into an edge whose writer was never touched, or fail to move + an edge whose writer was. Perturbing a single head's slice of a shared + per-head tensor also exercises the narrower case: sibling heads and + positions inside the *same* cached tensor must stay untouched. + """ + model = _EdgeScoringToyBridge() + _ensure_edge_hook_flags(model) + clean = torch.tensor([[1, 2, 3]]) + corrupt = torch.tensor([[3, 2, 1]]) + metric = _metric_fn(answer=1, wrong=2) + edge_hook_names = _edge_hook_names(N_LAYERS) + + clean_cache = cache_activation_and_gradient( + model, clean, metric, names_filter=edge_hook_names, compute_gradient=False + ) + corrupt_cache = cache_activation_and_gradient( + model, corrupt, metric, names_filter=edge_hook_names + ) + edges = enumerate_edges(model, corrupt_cache) + _assert_edges_unique(edges) + baseline_scores = _edge_effects(clean_cache, corrupt_cache, edges) + + # The perturbed writer sits at the readout position (the one _metric_fn reads). + # This toy is position-wise, so a writer at any other position feeds only + # zero reader gradients and its edge scores stay zero under any perturbation; + # placing the writer at the readout position makes the mutation actually move + # the scores, so the test cannot pass on an all-zeros _edge_effects. + writer = Node(kind="attn_head_out", layer=0, head=0, position=SEQ_LEN - 1) + assert any(edge_writer == writer for edge_writer, _reader in edges) + + perturbation = torch.full((D_MODEL,), 0.37, dtype=clean_cache.activations["hook_embed"].dtype) + writer_name = _writer_hook_name(writer) + perturbed_activations = dict(clean_cache.activations) + perturbed_activations[writer_name] = perturbed_activations[writer_name].clone() + perturbed_activations[writer_name][0, writer.position, writer.head] += perturbation + perturbed_clean_cache = GradientCache( + activations=perturbed_activations, + gradients=clean_cache.gradients, + metric=clean_cache.metric, + ) + + mutated_scores = _edge_effects(perturbed_clean_cache, corrupt_cache, edges) + + changed = 0 + moved = 0 + for edge in edges: + edge_writer, reader = edge + if edge_writer == writer: + grad = corrupt_cache.gradients[reader.hook_name] + assert grad is not None + if reader.kind in ("q_input", "k_input", "v_input"): + grad_vec = grad[0, reader.position, reader.head] + else: + grad_vec = grad[0, reader.position] + shift = float((perturbation * grad_vec).sum()) + expected = baseline_scores[edge] + shift + assert mutated_scores[edge] == pytest.approx(expected) + changed += 1 + if abs(shift) > 1e-6: + assert mutated_scores[edge] != pytest.approx(baseline_scores[edge]) + moved += 1 + else: + assert mutated_scores[edge] == baseline_scores[edge] + + assert changed > 0 + assert moved > 0 # the perturbation genuinely shifts scores at the readout position + + +# --------------------------------------------------------------------------- +# Exact-patch parity +# --------------------------------------------------------------------------- +# +# generic_activation_patch (transformer_lens/patching.py) is not used for this +# check: its `model: HookedTransformer` parameter is enforced at runtime by +# this repo's jaxtyping/beartype pytest configuration +# (--jaxtyping-packages=transformer_lens,beartype.beartype), which rejects any +# argument that is not actually a HookedTransformer instance -- including a +# real TransformerBridge, not just this module's toy double. A `# type: +# ignore` only silences the static checker; it cannot satisfy a runtime +# isinstance check. The patch below is instead driven directly through the +# same hooks() mechanism generic_activation_patch itself uses internally. + + +def _patch_edge_toward_clean( + model: _EdgeScoringToyBridge, + corrupt: torch.Tensor, + reader: Node, + writer_delta: torch.Tensor, + metric_fn: Callable[[torch.Tensor], torch.Tensor], +) -> float: + """Run the corrupt forward with only one writer's contribution to `reader` patched toward clean. + + Adds `writer_delta` into the reader's cached input at its position, leaving + every other writer's contribution to that same reader -- and the rest of + the corrupt run -- untouched. This is the exact single-edge intervention + _edge_effects estimates to first order. + """ + + def hook(tensor: torch.Tensor, *, hook: Any) -> torch.Tensor: + del hook + tensor = tensor.clone() + tensor[0, reader.position] = tensor[0, reader.position] + writer_delta + return tensor + + with torch.no_grad(), model.hooks(fwd_hooks=[(reader.hook_name, hook)]): + return float(metric_fn(model(corrupt))) + + +def test_exact_edge_patch_matches_edge_effects_sign_and_magnitude() -> None: + """A genuine single-edge activation patch matches _edge_effects' estimate. + + On this fully linear toy Bridge the first-order estimate is exact, so both + sign and magnitude match; a nonlinear model would only be expected to + match in sign. + + The scored reader sits at the readout position (the one _metric_fn reads). + This toy is position-wise and the clean/corrupt tokens agree at the interior + positions, so an edge scored anywhere else would compare zero against zero + and pass even on an all-zeros _edge_effects; at the readout position the + tokens differ and the reader gradient is nonzero, so the scores are genuine. + """ + model = _EdgeScoringToyBridge() + _ensure_edge_hook_flags(model) + clean = torch.tensor([[1, 2, 3]]) + corrupt = torch.tensor([[3, 2, 1]]) + metric = _metric_fn(answer=1, wrong=2) + edge_hook_names = _edge_hook_names(N_LAYERS) + + clean_cache = cache_activation_and_gradient( + model, clean, metric, names_filter=edge_hook_names, compute_gradient=False + ) + corrupt_cache = cache_activation_and_gradient( + model, corrupt, metric, names_filter=edge_hook_names + ) + edges = enumerate_edges(model, corrupt_cache) + edge_scores = _edge_effects(clean_cache, corrupt_cache, edges) + + reader = Node(kind="mlp_in", layer=1, position=SEQ_LEN - 1) + writers = [writer for writer, edge_reader in edges if edge_reader == reader] + assert {writer.kind for writer in writers} == {"embed", "attn_head_out", "mlp_out"} + + with torch.no_grad(): + m_corrupt = float(metric(model(corrupt))) + + nonzero = 0 + for writer in writers: + writer_delta = _edge_writer_delta_vector(clean_cache, corrupt_cache, writer) + patched_metric = _patch_edge_toward_clean(model, corrupt, reader, writer_delta, metric) + + exact_delta_m = patched_metric - m_corrupt + score = edge_scores[(writer, reader)] + assert exact_delta_m == pytest.approx(score, abs=1e-5) + if abs(score) > 1e-6: + assert (exact_delta_m > 0) == (score > 0) + nonzero += 1 + + assert nonzero > 0 # the readout position mixes tokens, so the edges are not zeros + + +# --------------------------------------------------------------------------- +# Multi-pair edge averaging and the head -> logits ranking +# --------------------------------------------------------------------------- +# +# The edge branch slices, accumulates, and divides per-pair edge scores by the +# batch exactly as the node branch does. A two-pair sweep whose pairs score +# differently guards that arithmetic: scoring only the first pair, or dropping +# the / batch divide, no longer averages to the right value. The terminal logits +# reader also makes a head's direct-to-output path a rankable edge, which +# top_edges could never return before the reader existed. + + +def test_attribution_patch_edge_granularity_averages_scores_across_the_batch() -> None: + """Edge scores are the per-pair mean, mirroring the node-granularity averaging. + + The two pairs score differently, so the batched result equals neither pair + alone; a loop that scored only the first pair, or skipped the / batch divide, + would fail both the mean check and the not-equal-to-either check. + """ + model = _EdgeScoringToyBridge() + clean = torch.tensor([[1, 2, 3], [0, 4, 5]]) + corrupt = torch.tensor([[3, 2, 1], [5, 4, 0]]) + metric = _metric_fn(answer=1, wrong=2) + config = EdgeAttributionConfig(granularity="edge") + + batched = attribution_patch(model, clean, corrupt, metric, config=config) + per_example = [ + attribution_patch(model, clean[i : i + 1], corrupt[i : i + 1], metric, config=config) + for i in range(2) + ] + + assert set(batched.edge_scores) == set(per_example[0].edge_scores) + + differs = 0 + for edge in batched.edge_scores: + first = per_example[0].edge_scores[edge] + second = per_example[1].edge_scores[edge] + assert batched.edge_scores[edge] == pytest.approx((first + second) / 2) + if first != pytest.approx(second): + differs += 1 + assert differs > 0 # the pairs genuinely differ, so the / batch divide is exercised + + # The batched result matches neither single pair: exact float equality would + # hold only if the loop scored one pair or skipped the divide. + assert batched.edge_scores != per_example[0].edge_scores + assert batched.edge_scores != per_example[1].edge_scores + + +def test_top_edges_can_surface_a_head_to_logits_edge() -> None: + """top_edges can now return a head -> logits edge, previously impossible. + + Before the terminal logits reader existed, no writer had an edge to the + output, so a head's direct-to-output path was never an edge and top_edges + could never rank it. With the reader in the graph every head gains a + ->logits edge that competes for the ranking like any other, with a genuine + signed score at the readout position rather than a structural zero. + """ + model = _EdgeScoringToyBridge() + clean = torch.tensor([[1, 2, 3]]) + corrupt = torch.tensor([[3, 2, 1]]) + metric = _metric_fn(answer=1, wrong=2) + + result = attribution_patch( + model, clean, corrupt, metric, config=EdgeAttributionConfig(granularity="edge") + ) + + ranked = result.top_edges(k=len(result.edge_scores)) + head_to_logits = [ + (writer, reader, score) + for writer, reader, score in ranked + if writer.kind == "attn_head_out" and reader.kind == "logits" + ] + assert head_to_logits # the terminal reader gives every head a ->logits edge + + # At least one carries a real signed effect, so the ranking is not over zeros. + assert any(abs(score) > 1e-6 for _writer, _reader, score in head_to_logits) diff --git a/transformer_lens/tools/analysis/__init__.py b/transformer_lens/tools/analysis/__init__.py index 5d93a557ff..de11f44994 100644 --- a/transformer_lens/tools/analysis/__init__.py +++ b/transformer_lens/tools/analysis/__init__.py @@ -6,10 +6,10 @@ Tools: - attribution_patching: Attribution patching (gradient-linearized activation - patching) over residual-stream nodes — typed computational graph, a - names-filtered manual-backward gradient cache, and signed node scores. Edge - scoring (EAP), integrated gradients (EAP-IG), and faithfulness land in - follow-on PRs. + patching) over residual-stream nodes and edges: typed computational graph, + a names-filtered manual-backward gradient cache, and signed node and edge + scores (EAP). Integrated gradients (EAP-IG) and ablate-outside faithfulness + are not implemented yet. - backward_lens: GPT-2 MLP weight-gradient factors projected into vocabulary space with explicit raw-gradient sign semantics. - direct_logit_attribution: Direct Logit Attribution (DLA) over components, diff --git a/transformer_lens/tools/analysis/attribution_patching.py b/transformer_lens/tools/analysis/attribution_patching.py index a0041ca4ef..6cf392d34c 100644 --- a/transformer_lens/tools/analysis/attribution_patching.py +++ b/transformer_lens/tools/analysis/attribution_patching.py @@ -22,17 +22,18 @@ Retaining gradients at every hook point roughly doubles cache memory, so callers should filter to the hook families their analysis actually reads. -Scope: this build ships node granularity with plain attribution (``ig_steps=1``). -Edge scoring (EAP), the integrated-gradient path (EAP-IG, ``ig_steps>1``), and -ablate-outside faithfulness are not implemented yet; their API is declared here — -``granularity="edge"`` and ``ig_steps>1`` raise :class:`NotImplementedError` — so -downstream code can pin against a stable surface now. +Scope: this build ships node and edge granularity with plain attribution +(``ig_steps=1``). The integrated-gradient path (EAP-IG, ``ig_steps>1``) and +ablate-outside faithfulness are not implemented yet; their API is declared here, +and ``ig_steps>1`` raises :class:`NotImplementedError`, so downstream code can pin +against a stable surface now. """ from __future__ import annotations +from contextlib import contextmanager from dataclasses import dataclass, field -from typing import Any, Callable, Literal, Optional, Sequence, Union +from typing import Any, Callable, Iterator, Literal, Optional, Sequence, Union import torch @@ -41,7 +42,9 @@ MetricFn = Callable[[torch.Tensor], torch.Tensor] NamesFilter = Union[str, Sequence[str], Callable[[str], bool], None] -NodeKind = Literal["embed", "attn_head_out", "mlp_out"] +NodeKind = Literal[ + "embed", "attn_head_out", "mlp_out", "q_input", "k_input", "v_input", "mlp_in", "logits" +] Granularity = Literal["node", "edge"] @@ -62,16 +65,28 @@ class GradientCache: @dataclass(frozen=True) class Node: - """A node in the residual-stream computational graph at node granularity. + """A node in the residual-stream computational graph. Nodes are the typed, hashable keys the attribution sweep scores. Each node is identified by ``(kind, layer, position, head)``; ``kind`` selects the node - family and constrains which of ``layer``/``head`` apply: + family and constrains which of ``layer``/``head`` apply. Three kinds are + *writers* -- they contribute a value into the residual stream: - ``"embed"``: the token embedding write. ``layer`` and ``head`` are ``None``. - ``"attn_head_out"``: one attention head's output. ``layer`` and ``head`` set. - ``"mlp_out"``: one layer's MLP output. ``layer`` set, ``head`` is ``None``. + Five kinds are *readers* -- they consume the residual stream as an edge's + destination (see :func:`enumerate_edges`): + + - ``"q_input"`` / ``"k_input"`` / ``"v_input"``: one attention head's split + Q/K/V input. ``layer`` and ``head`` set. + - ``"mlp_in"``: one layer's MLP entry. ``layer`` set, ``head`` is ``None``. + - ``"logits"``: the terminal readout of the final residual, read at + ``blocks.{n_layers-1}.hook_resid_post``. ``layer`` is that final layer and + ``head`` is ``None``. Every writer feeds this reader, so a writer's + aggregate over its outgoing edges equals its direct node score. + ``position`` is the sequence index the node is read at. The invariants above are enforced in ``__post_init__`` so a malformed key raises rather than silently producing a wrong graph. @@ -94,6 +109,19 @@ def __post_init__(self) -> None: raise ValueError("mlp_out nodes need a layer") if self.head is not None: raise ValueError("mlp_out nodes take no head") + elif self.kind in ("q_input", "k_input", "v_input"): + if self.layer is None or self.head is None: + raise ValueError(f"{self.kind} nodes need both layer and head") + elif self.kind == "mlp_in": + if self.layer is None: + raise ValueError("mlp_in nodes need a layer") + if self.head is not None: + raise ValueError("mlp_in nodes take no head") + elif self.kind == "logits": + if self.layer is None: + raise ValueError("logits nodes need a layer") + if self.head is not None: + raise ValueError("logits nodes take no head") else: raise ValueError(f"unknown node kind {self.kind!r}") @@ -102,15 +130,23 @@ def hook_name(self) -> str: """The cache hook point this node reads from. Uses the standard ``TransformerBridge`` alias names (``hook_embed``, - ``blocks.{l}.attn.hook_z``, ``blocks.{l}.hook_mlp_out``); the per-head - ``attn_head_out`` node slices head ``self.head`` out of the shared - ``hook_z`` tensor. + ``blocks.{l}.attn.hook_z``, ``blocks.{l}.hook_mlp_out``, + ``blocks.{l}.attn.hook_q_input``/``hook_k_input``/``hook_v_input``, + ``blocks.{l}.hook_mlp_in``, ``blocks.{l}.hook_resid_post``); the per-head + nodes (``attn_head_out``, ``q_input``, ``k_input``, ``v_input``) slice + head ``self.head`` out of the shared per-head tensor. """ if self.kind == "embed": return "hook_embed" if self.kind == "attn_head_out": return f"blocks.{self.layer}.attn.hook_z" - return f"blocks.{self.layer}.hook_mlp_out" + if self.kind == "mlp_out": + return f"blocks.{self.layer}.hook_mlp_out" + if self.kind in ("q_input", "k_input", "v_input"): + return f"blocks.{self.layer}.attn.hook_{self.kind}" + if self.kind == "logits": + return f"blocks.{self.layer}.hook_resid_post" + return f"blocks.{self.layer}.hook_mlp_in" @dataclass(frozen=True) @@ -133,13 +169,12 @@ class EdgeAttributionConfig: ``ig_steps`` exceeds 1. Collapsing them removes the invalid states (e.g. ``method="attribution", ig_steps=5``). - This build implements node granularity with plain attribution only. - ``granularity="edge"`` and ``ig_steps>1`` are accepted by the type but raise - :class:`NotImplementedError` at construction, so downstream code can import and - reference this API now while edge scoring and the integrated-gradient path are - not implemented yet. Once EAP-IG lands, the default flips to ``ig_steps=5`` - (EAP-IG is the faithful default); until then the default is the only executable - value, ``ig_steps=1``. + This build implements node and edge granularity with plain attribution. + ``ig_steps>1`` is accepted by the type but raises :class:`NotImplementedError` + at construction, so downstream code can import and reference this API now + while the integrated-gradient path is not implemented yet. Once EAP-IG lands, + the default flips to ``ig_steps=5`` (EAP-IG is the faithful default); until + then the default is the only executable value, ``ig_steps=1``. Attributes: granularity: ``"node"`` or ``"edge"``. Defaults to ``"node"``. @@ -152,11 +187,6 @@ class EdgeAttributionConfig: def __post_init__(self) -> None: if self.ig_steps < 1: raise ValueError(f"ig_steps must be >= 1, got {self.ig_steps}") - if self.granularity == "edge": - raise NotImplementedError( - "granularity='edge' (EAP edge scoring) is not implemented yet; this " - "build supports granularity='node' only." - ) if self.ig_steps > 1: raise NotImplementedError( "ig_steps>1 (EAP-IG integrated gradients) is not implemented yet; this " @@ -173,10 +203,13 @@ class AttributionResult: ``(a_clean - a_corrupt) . d(metric)/d(a)``. A positive score means patching that node from corrupt toward clean moves the metric in the positive direction (the denoising convention pinned in the module - docstring). + docstring). For an edge-granularity sweep this is instead each + writer's aggregate over its own outgoing edge scores. Because the + graph has a terminal logits reader that every writer feeds, that + aggregate equals the writer's direct node score (see + :func:`attribution_patch`). edge_scores: Per-edge effect estimate keyed by ``(source, destination)``. - Declared here so the result API is stable across the PR series; it is - populated only once edge scoring lands and is empty for a node sweep. + Populated for an edge-granularity sweep; empty for a node sweep. """ node_scores: dict[Node, float] @@ -194,22 +227,147 @@ def top_nodes(self, k: int = 10) -> list[tuple[Node, float]]: return ranked[:k] def top_edges(self, k: int = 10) -> list[tuple[Node, Node, float]]: - """The ``k`` highest-magnitude edges — populated once edge scoring lands.""" - raise NotImplementedError( - "edge scoring is not implemented yet; run a node-granularity sweep and " - "use top_nodes()." - ) + """The ``k`` edges with the largest effect magnitude, strongest first. + Ranking is by absolute score, matching ``top_nodes``: a large negative + edge effect is as causally important as a large positive one. Ties keep + enumeration order (stable sort). Requesting more than the available + edges returns all of them. + """ + ranked = sorted(self.edge_scores.items(), key=lambda item: abs(item[1]), reverse=True) + return [(writer, reader, score) for (writer, reader), score in ranked[:k]] -def _required_hook_names(n_layers: int) -> list[str]: - """Hook points the node graph reads: embed plus per-layer attn-z and mlp-out.""" + +def _required_hook_names(n_layers: int, granularity: Granularity = "node") -> list[str]: + """Hook points a sweep at ``granularity`` reads. + + Node granularity needs the embed write plus each layer's attn-z and + mlp-out. Edge granularity additionally needs the per-head hook points on + both sides of an edge into or out of an attention head: ``attn.hook_result`` + (writer -- a head's own contribution before the sum into the residual + stream) and the split ``attn.hook_q_input``/``hook_k_input``/``hook_v_input`` + (reader -- the residual each head's Q/K/V projection reads separately). + """ names = ["hook_embed"] for layer in range(n_layers): names.append(f"blocks.{layer}.attn.hook_z") names.append(f"blocks.{layer}.hook_mlp_out") + if granularity == "edge": + names.append(f"blocks.{layer}.attn.hook_result") + names.append(f"blocks.{layer}.attn.hook_q_input") + names.append(f"blocks.{layer}.attn.hook_k_input") + names.append(f"blocks.{layer}.attn.hook_v_input") return names +def _ensure_edge_hook_flags(model: Any) -> None: + """Turn on the Bridge flags edge granularity's hook points require. + + ``attn.hook_result``, the split ``attn.hook_q_input``/``hook_k_input``/ + ``hook_v_input``, and ``hook_mlp_in`` all exist on the Bridge + unconditionally but only fire when their owning flag + (``cfg.use_attn_result`` / ``cfg.use_split_qkv_input`` / + ``cfg.use_hook_mlp_in``) is on, so an edge sweep must enable all three + before caching or the writer- and reader-side hook points it needs never + populate. + + Memory caveat: enabling ``use_attn_result``/``use_split_qkv_input`` makes + every cached per-head tensor ``[batch, seq, n_heads, d_model]`` instead of + the summed ``[batch, seq, d_model]`` residual. That is fine on a model the + size of gpt2-small; it does not scale to models with many heads or layers. + + Args: + model: A ``TransformerBridge`` (or compatible) exposing ``cfg`` and + ``set_use_attn_result``/``set_use_split_qkv_input``/ + ``set_use_hook_mlp_in``. + """ + if not model.cfg.use_attn_result: + model.set_use_attn_result(True) + if not model.cfg.use_split_qkv_input: + model.set_use_split_qkv_input(True) + if not model.cfg.use_hook_mlp_in: + model.set_use_hook_mlp_in(True) + + +@contextmanager +def _edge_hook_flags(model: Any) -> Iterator[None]: + """Enable the Bridge flags an edge sweep needs, then restore the caller's state. + + ``attn.hook_result``, the split ``attn.hook_q_input``/``hook_k_input``/ + ``hook_v_input``, and ``hook_mlp_in`` all exist on the Bridge unconditionally + but only fire when their owning flag (``cfg.use_attn_result`` / + ``cfg.use_split_qkv_input`` / ``cfg.use_hook_mlp_in``) is on, so an edge sweep + must enable all three before caching or the writer- and reader-side hook + points it needs never populate. + + ``use_split_qkv_input`` is mutually exclusive with ``use_attn_in``, so a + caller who arrives with ``use_attn_in`` on would otherwise trip the + exclusivity error. This turns ``use_attn_in`` off before enabling the split + input, and restores it after ``use_split_qkv_input`` has been turned back off. + + Invariant: the caller's flag state (``use_attn_result``, + ``use_split_qkv_input``, ``use_hook_mlp_in``, ``use_attn_in``) is unchanged on + return, including when the body raises. Restoration runs in a ``finally`` + block so a raise mid-sweep -- or in the caller's own later code -- cannot + leave the per-head tensors materialized on the model. + + Memory caveat: enabling ``use_attn_result``/``use_split_qkv_input`` makes + every cached per-head tensor ``[batch, seq, n_heads, d_model]`` instead of + the summed ``[batch, seq, d_model]`` residual. That is fine on a model the + size of gpt2-small; it does not scale to models with many heads or layers. + + Args: + model: A ``TransformerBridge`` (or compatible) exposing ``cfg`` and + ``set_use_attn_result``/``set_use_split_qkv_input``/ + ``set_use_hook_mlp_in``/``set_use_attn_in``. + """ + cfg = model.cfg + saved_attn_result = cfg.use_attn_result + saved_split_qkv_input = cfg.use_split_qkv_input + saved_hook_mlp_in = cfg.use_hook_mlp_in + saved_attn_in = bool(getattr(cfg, "use_attn_in", False)) + try: + # use_attn_in and use_split_qkv_input are mutually exclusive; clear + # use_attn_in first so enabling the split input cannot raise. + if saved_attn_in: + model.set_use_attn_in(False) + if not cfg.use_attn_result: + model.set_use_attn_result(True) + if not cfg.use_split_qkv_input: + model.set_use_split_qkv_input(True) + if not cfg.use_hook_mlp_in: + model.set_use_hook_mlp_in(True) + yield + finally: + model.set_use_attn_result(saved_attn_result) + model.set_use_split_qkv_input(saved_split_qkv_input) + model.set_use_hook_mlp_in(saved_hook_mlp_in) + # Re-enabling use_attn_in requires use_split_qkv_input already off; the + # line above restored it, and a caller with use_attn_in on cannot also + # have had use_split_qkv_input on, so this cannot trip the exclusivity. + if saved_attn_in: + model.set_use_attn_in(True) + + +def _check_required_hooks( + cache: GradientCache, required_names: Sequence[str], graph: str, hint: str +) -> None: + """Raise if any of ``required_names`` is absent from ``cache.activations``. + + Shared by every granularity's graph-construction step, so a cache built + with too narrow a ``names_filter`` -- or one produced while a required + Bridge flag was off -- fails loudly instead of silently producing a + truncated graph. + """ + missing = [name for name in required_names if name not in cache.activations] + if missing: + raise ValueError( + f"{graph} requires hook points missing from the cache: " + + ", ".join(missing) + + f". {hint}" + ) + + def enumerate_nodes(model: Any, cache: GradientCache) -> list[Node]: """Enumerate the full node-granularity graph from the Bridge hook graph. @@ -230,14 +388,13 @@ def enumerate_nodes(model: Any, cache: GradientCache) -> list[Node]: graph is never silently truncated. """ n_layers = int(model.cfg.n_layers) - missing = [name for name in _required_hook_names(n_layers) if name not in cache.activations] - if missing: - raise ValueError( - "node graph requires hook points missing from the cache: " - + ", ".join(missing) - + ". Cache with a names_filter that keeps hook_embed, " - "blocks.*.attn.hook_z, and blocks.*.hook_mlp_out." - ) + _check_required_hooks( + cache, + _required_hook_names(n_layers), + "node graph", + "Cache with a names_filter that keeps hook_embed, blocks.*.attn.hook_z, " + "and blocks.*.hook_mlp_out.", + ) seq_len = cache.activations["hook_embed"].shape[1] @@ -251,6 +408,149 @@ def enumerate_nodes(model: Any, cache: GradientCache) -> list[Node]: return nodes +def _assert_edges_unique(edges: Sequence[tuple[Node, Node]]) -> None: + """Raise if any writer -> reader pair appears more than once in ``edges``. + + A writer feeding two distinct readers (a head's output feeding both the + next layer's attention input and this layer's MLP input, say) is two + edges; this guards the enumeration against a construction bug that + collapses or duplicates a single ``(writer, reader)`` pair instead. + """ + seen: set[tuple[Node, Node]] = set() + for edge in edges: + if edge in seen: + raise ValueError(f"edge {edge} enumerated more than once") + seen.add(edge) + + +def _required_edge_reader_hook_names(n_layers: int) -> list[str]: + """The reader hook points edge enumeration additionally requires. + + ``_required_hook_names(..., granularity="edge")`` covers the per-head + attention hooks a writer/reader pair into or out of a head needs. Edge + enumeration reads two reader points those miss: + + - each layer's MLP entry, ``attn.hook_mlp_in``'s layer-level sibling + ``hook_mlp_in``, gated on ``cfg.use_hook_mlp_in`` the same way the per-head + hooks are gated on their own flags, and + - the terminal ``blocks.{n_layers-1}.hook_resid_post``, where the logits + reader takes its gradient. This final residual hook fires unconditionally, + so no Bridge flag gates it. + """ + names = [f"blocks.{layer}.hook_mlp_in" for layer in range(n_layers)] + names.append(f"blocks.{n_layers - 1}.hook_resid_post") + return names + + +def _edge_hook_names(n_layers: int) -> list[str]: + """Every hook point an edge-granularity sweep must cache. + + The per-head attention hooks from ``_required_hook_names(..., granularity="edge")`` + plus each layer's MLP-entry reader hook and the terminal logits reader hook. + """ + return _required_hook_names(n_layers, granularity="edge") + _required_edge_reader_hook_names( + n_layers + ) + + +def enumerate_edges(model: Any, cache: GradientCache) -> list[tuple[Node, Node]]: + """Enumerate every writer -> reader edge in the residual-stream graph. + + At a fixed sequence position, the residual stream is a running sum: a + reader (a head's split Q/K/V input, a layer's MLP entry, or the terminal + logits readout) is fed by every writer (the embed write, every attention + head's output, every layer's MLP output) that precedes it. Building the + graph position-by-position tracks which writers are "available" so far and + connects each new reader to all of them, then adds that layer's writers to + the available set before moving on -- so a writer never edges to a reader + upstream of it, and a writer feeding both a direct edge and a through-MLP + edge produces two distinct ``(u, v)`` pairs rather than one summed together. + + A terminal ``logits`` reader (read at the final ``hook_resid_post``) closes + the graph: after the per-layer loop every remaining writer -- including the + final layer's MLP output, which no per-layer reader sees -- edges to it. That + edge carries the writer's direct skip-connection contribution to the metric, + so a writer's aggregate over its outgoing edges equals its direct node score. + + On ``cfg.parallel_attn_mlp`` models (Pythia, GPT-J, Falcon, Phi) the MLP + reads the layer input, not the post-attention residual, so a layer's own + heads are not writers into that layer's MLP; those same-layer head->mlp_in + edges are dropped while the heads still feed later readers and the logits + reader. + + Args: + model: A ``TransformerBridge`` (or compatible) exposing ``cfg.n_layers`` + and, optionally, ``cfg.parallel_attn_mlp``. + cache: A :class:`GradientCache` holding at least the required hook points + for edge granularity. + + Returns: + The edge list as ``(writer, reader)`` node pairs; no pair repeats. + + Raises: + ValueError: if any required hook point is absent from ``cache`` -- the + graph is never silently truncated. + """ + n_layers = int(model.cfg.n_layers) + _check_required_hooks( + cache, + _edge_hook_names(n_layers), + "edge graph", + "Cache with a names_filter that keeps the edge-granularity hook set: " + "hook_embed, blocks.*.attn.hook_z, blocks.*.hook_mlp_out, " + "blocks.*.attn.hook_result, blocks.*.attn.hook_q_input, " + "blocks.*.attn.hook_k_input, blocks.*.attn.hook_v_input, " + "blocks.*.hook_mlp_in, and blocks.{n_layers-1}.hook_resid_post.", + ) + + parallel_attn_mlp = bool(getattr(model.cfg, "parallel_attn_mlp", False)) + + seq_len = cache.activations["hook_embed"].shape[1] + edges: list[tuple[Node, Node]] = [] + + for position in range(seq_len): + available: list[Node] = [Node(kind="embed", position=position)] + for layer in range(n_layers): + n_heads = cache.activations[f"blocks.{layer}.attn.hook_z"].shape[2] + + attn_reader_kinds: tuple[NodeKind, NodeKind, NodeKind] = ( + "q_input", + "k_input", + "v_input", + ) + attn_readers = [ + Node(kind=kind, layer=layer, head=head, position=position) + for kind in attn_reader_kinds + for head in range(n_heads) + ] + for reader in attn_readers: + edges.extend((writer, reader) for writer in available) + + layer_heads = [ + Node(kind="attn_head_out", layer=layer, head=head, position=position) + for head in range(n_heads) + ] + + mlp_reader = Node(kind="mlp_in", layer=layer, position=position) + if parallel_attn_mlp: + # The MLP reads the layer input, not the post-attention residual, + # so this layer's heads are not writers into its MLP. They still + # become available to later readers and the logits reader. + edges.extend((writer, mlp_reader) for writer in available) + available = available + layer_heads + else: + available = available + layer_heads + edges.extend((writer, mlp_reader) for writer in available) + + available = available + [Node(kind="mlp_out", layer=layer, position=position)] + + logits_reader = Node(kind="logits", layer=n_layers - 1, position=position) + edges.extend((writer, logits_reader) for writer in available) + + _assert_edges_unique(edges) + return edges + + def _as_predicate(names_filter: NamesFilter) -> Callable[[str], bool]: if names_filter is None: return lambda name: True @@ -437,6 +737,115 @@ def _node_effects( return scores +def _writer_hook_name(node: Node) -> str: + """The residual-stream hook point holding a writer node's own contribution. + + Distinct from ``Node.hook_name``: an ``attn_head_out`` node's ``hook_name`` + resolves to ``attn.hook_z``, the pre-``hook_result`` value node granularity + scores. An edge's writer contribution must instead be measured in the same + ``d_model`` space a reader's gradient lives in, which is ``attn.hook_result`` + -- the per-head decomposition of the head's contribution after it is + projected into the residual stream. + """ + if node.kind == "embed": + return "hook_embed" + if node.kind == "attn_head_out": + return f"blocks.{node.layer}.attn.hook_result" + if node.kind == "mlp_out": + return f"blocks.{node.layer}.hook_mlp_out" + raise ValueError(f"{node.kind} is a reader kind and has no writer contribution") + + +def _edge_effects( + clean_cache: GradientCache, + corrupt_cache: GradientCache, + edges: Sequence[tuple[Node, Node]], +) -> dict[tuple[Node, Node], float]: + """Score every edge with ``(a_clean[u] - a_corrupt[u]) . d(metric)/d(input of v)``. + + Mirrors ``_node_effects``: the delta is the writer's own residual + contribution (clean minus corrupt cache), dotted with the reader's + corrupt-run gradient -- the same denoising convention ``_node_effects`` + uses. Unlike a node score, the delta and the gradient are read from two + different hook points (the writer's and the reader's), since an edge + measures how much of one component's output reaches another component's + input. A ``logits`` reader takes its gradient at the final + ``hook_resid_post`` and contracts over ``d_model`` at its position, the same + shape path as an ``mlp_in`` reader. + + Two structural facts keep this off a per-edge recompute: a writer hook's + clean-minus-corrupt delta is the same tensor for every edge that writer + feeds, and every edge into one reader shares that reader's gradient vector. + Each writer delta is therefore computed once, and each reader's incoming + edges are scored with a single batched matrix-vector product over the + stacked writer deltas -- reducing to the same per-edge scalars a + ``float((delta_vec * grad_vec).sum())`` per edge would, reader gradient + reused across all of that reader's edges. + """ + scores: dict[tuple[Node, Node], float] = {} + writer_deltas: dict[str, torch.Tensor] = {} + + def writer_delta_vec(writer: Node) -> torch.Tensor: + name = _writer_hook_name(writer) + delta = writer_deltas.get(name) + if delta is None: + delta = clean_cache.activations[name] - corrupt_cache.activations[name] + writer_deltas[name] = delta + if writer.kind == "attn_head_out": + return delta[0, writer.position, writer.head] + return delta[0, writer.position] + + writers_by_reader: dict[Node, list[Node]] = {} + reader_order: list[Node] = [] + for writer, reader in edges: + bucket = writers_by_reader.get(reader) + if bucket is None: + writers_by_reader[reader] = bucket = [] + reader_order.append(reader) + bucket.append(writer) + + for reader in reader_order: + reader_name = reader.hook_name + grad = corrupt_cache.gradients.get(reader_name) + writers = writers_by_reader[reader] + if grad is None: + raise ValueError( + f"edge {(writers[0], reader)} reads its gradient at {reader_name!r}, but " + "the corrupt cache holds none there; cache with a names_filter that " + "retains this hook point." + ) + if reader.kind in ("q_input", "k_input", "v_input"): + grad_vec = grad[0, reader.position, reader.head] + else: + grad_vec = grad[0, reader.position] + delta_matrix = torch.stack([writer_delta_vec(writer) for writer in writers]) + edge_values = delta_matrix @ grad_vec + for writer, value in zip(writers, edge_values.tolist()): + scores[(writer, reader)] = value + return scores + + +def _aggregate_edge_scores_to_writer_nodes( + edge_scores: dict[tuple[Node, Node], float], +) -> dict[Node, float]: + """Sum each writer's outgoing edge scores into that writer's aggregate node score. + + A writer's aggregate is the sum of its effects along every edge it feeds. + enumerate_edges gives every writer an edge to the terminal logits reader, so + the aggregate includes the writer's direct skip-connection contribution to + the metric -- the part of its residual-stream write that no intermediate + component reads, only the final readout does -- and therefore equals the + quantity a node-granularity sweep measures directly at the writer's own hook + point. Every writer has at least that one outgoing edge, including the final + layer's MLP output, whose only reader is the logits terminal, so no writer is + missing from the aggregate. + """ + totals: dict[Node, float] = {} + for (writer, _reader), score in edge_scores.items(): + totals[writer] = totals.get(writer, 0.0) + score + return totals + + def attribution_patch( model: Any, clean: torch.Tensor, @@ -444,12 +853,17 @@ def attribution_patch( metric_fn: MetricFn, config: EdgeAttributionConfig = EdgeAttributionConfig(), ) -> AttributionResult: - """Estimate every node's causal effect on ``metric_fn`` in two forwards + one backward. + """Estimate every component's causal effect on ``metric_fn`` in two forwards + one backward. For each clean/corrupt pair this runs a clean forward (for ``a_clean``) and a corrupt forward whose backward hooks capture ``g = d(metric)/d(a)`` (for - ``a_corrupt`` and its gradient), then scores each node with the - first-order Taylor estimate ``effect(node) = (a_clean - a_corrupt) . g``. + ``a_corrupt`` and its gradient). At node granularity (``config.granularity == + "node"``) each node is scored with the first-order Taylor estimate + ``effect(node) = (a_clean - a_corrupt) . g``. At edge granularity + (``config.granularity == "edge"``) each writer -> reader edge is scored with + ``effect(edge) = (a_clean[writer] - a_corrupt[writer]) . d(metric)/d(input of + reader)``, and ``node_scores`` holds each writer's aggregate effect (the sum + of its outgoing edge scores). Sign/direction convention (denoising form): gradients are taken on the *corrupt* run and the estimate points *toward* the clean activation, so a positive score @@ -459,8 +873,8 @@ def attribution_patch( Dataset averaging: ``clean``/``corrupt`` may hold a batch of prompt pairs. Each pair is scored independently (per-example forward/backward, so its own - reconstruction identity holds) and per-node scores are averaged across the batch - before ranking. + reconstruction identity holds) and per-node (or per-edge) scores are averaged + across the batch before ranking. The model and every submodule must be in evaluation mode. Separate clean and corrupt forwards cannot produce meaningful activation differences if stochastic @@ -473,12 +887,19 @@ def attribution_patch( corrupt: Corrupt token ids, shape ``[batch, seq]``, paired row-by-row with ``clean``. metric_fn: Maps single-example logits to a scalar to differentiate. - config: Sweep configuration. This PR supports node granularity with plain - attribution (``ig_steps=1``) only; other values raise at construction. + config: Sweep configuration. Node and edge granularity are both + supported with plain attribution (``ig_steps=1``); ``ig_steps>1`` + raises at construction. Returns: An :class:`AttributionResult` whose ``node_scores`` are averaged over the - batch. ``edge_scores`` stays empty until edge scoring lands. + batch. For an edge-granularity sweep, ``edge_scores`` is populated too and + ``node_scores`` is the per-writer aggregate of those edge scores. The + graph's terminal logits reader gives every writer an edge carrying its + direct skip-connection contribution to the metric, so this aggregate + equals the quantity a node-granularity sweep on the same model returns + (including for the final layer's MLP output, whose only outgoing edge is + the one to the logits reader). Raises: ValueError: if ``clean``/``corrupt`` are not 2D, hold a different number of @@ -486,8 +907,6 @@ def attribution_patch( position-by-position), or the model or one of its submodules is in training mode. """ - del config # node granularity + ig_steps=1 only; enforced at construction. - if clean.ndim != 2 or corrupt.ndim != 2: raise ValueError( "attribution_patch expects 2D [batch, seq] token tensors, got clean " @@ -507,8 +926,36 @@ def attribution_patch( require_eval_mode(model, operation="attribution_patch()") - node_hook_names = _required_hook_names(int(model.cfg.n_layers)) batch = int(clean.shape[0]) + n_layers = int(model.cfg.n_layers) + + if config.granularity == "edge": + hook_names = _edge_hook_names(n_layers) + edge_totals: dict[tuple[Node, Node], float] = {} + + # Scope the per-head hook-flag mutation to the caching loop so the + # caller's flag state is restored on both the normal and the error path. + with _edge_hook_flags(model): + for index in range(batch): + clean_cache = cache_activation_and_gradient( + model, + clean[index : index + 1], + metric_fn, + names_filter=hook_names, + compute_gradient=False, + ) + corrupt_cache = cache_activation_and_gradient( + model, corrupt[index : index + 1], metric_fn, names_filter=hook_names + ) + edges = enumerate_edges(model, corrupt_cache) + for edge, score in _edge_effects(clean_cache, corrupt_cache, edges).items(): + edge_totals[edge] = edge_totals.get(edge, 0.0) + score + + edge_scores = {edge: total / batch for edge, total in edge_totals.items()} + node_scores = _aggregate_edge_scores_to_writer_nodes(edge_scores) + return AttributionResult(node_scores=node_scores, edge_scores=edge_scores) + + node_hook_names = _required_hook_names(n_layers) totals: dict[Node, float] = {} for index in range(batch): From 9a942424a4719b08e1265d438c2f9001c7a78d5f Mon Sep 17 00:00:00 2001 From: Janmenjaya Panda <83154020+janmenjayap@users.noreply.github.com> Date: Wed, 16 Sep 2026 05:49:11 +0530 Subject: [PATCH 78/87] feat(model_bridge): shared LN/Identity/Half relevance-rule backend (#1785) * test(relevance_rules): closed-form VJP + forward-identity primitives Add analytic tests for the LN-, Identity-, and Half-relevance-rule primitives against their closed forms, asserting both forward-identity and gradient correctness: - LN-rule: forward equals numerator/denom exactly; the VJP treats denom as constant, verified by contrast against plain autodiff (which would also differentiate through denom's dependency on the input). - Identity-rule: forward matches SiLU, exact GELU, and tanh-approximate GELU; the VJP is checked against the sigmoid closed form, the Gaussian CDF closed form, and the direct f(x)/x ratio with its 1/2 limit at x == 0 for the approximate case. - Half-rule: forward equals u * v; the VJP halves each ordinary product-rule term, verified by contrast against plain autodiff. Cases cover zero, negative, mixed-sign, non-contiguous, and batched inputs at both float32 and float64. No model fixtures, backward hooks, or .data access. The primitive module does not exist yet, so this fails collection with a single ModuleNotFoundError -- the expected red state before the primitives are implemented. * feat(relevance_rules): forward-equivalent rule primitives Implement LN-, Identity-, and Half-rule as torch.autograd.Function primitives. Each reproduces its native forward value exactly and swaps in the rule's closed-form VJP on backward: LN-rule treats the denominator as constant, Identity-rule uses f(x)/x with the 0.5 limit at zero, Half-rule splits the product-rule gradient evenly between operands. * test(model_bridge): scoped rule context forward-identity + cleanup Add tests for a use_relevance_rules(model, rules) scoped context: - torch.equal forward-identity between native and rule-enabled output on a tiny fixture block, both standalone and nested. - ordinary gradients are restored once the context exits, contrasted against the rule's modified gradient inside the context. - nested contexts are reference-counted: the inner context's exit does not disable a rule the outer context still needs active. - an exception raised inside the context (top-level or nested) leaves no installed rule and no changed forward behavior. - targeting is positional (ln1/ln2), not class-based: a same-class component mounted at q_norm is left untouched. - a component occupying a targeted mount but not implementing the rule protocol is reported as skipped rather than installed or raising. - RelevanceRules defaults every rule kind to False and is frozen. The scoped context does not exist yet, so this fails collection with a single ImportError -- the expected red state before the context is implemented. * feat(model_bridge): scoped relevance-rule context + coverage report Add RelevanceRules (frozen: normalization, activation, multiplicative_gate, attention), RelevanceRuleCoverage (installed, skipped), and use_relevance_rules(model, rules): a scoped context manager that installs each requested rule only on components sitting at that rule kind's canonical mount name, never by isinstance, and reports which mounts were installed versus skipped. Components at a canonical mount that do not implement the relevance-rule protocol are reported skipped rather than raising. Nested scopes over the same model are reference-counted, so an inner scope's exit never disables a rule an outer scope still needs, and state is restored on normal exit and on exception. No model configuration is mutated; the only state that changes lives on the participating components, only for the scope's duration. * feat(model_bridge): native-forward LN-rule on NormalizationBridge Wrap the existing native-autograd no-edit branch's own original_component(x) call in a custom autograd.Function so the rule-active forward stays bit-identical to today's native forward by construction, while backward applies the LN-rule (denominator treated as constant) for the x-path; weight and bias keep their ordinary gradient since the rule only redefines how relevance reaches the input, not parameter training gradients. NormalizationBridge now implements the relevance-rule protocol (_relevance_rule_kind / _enable_relevance_rule / _disable_relevance_rule) so use_relevance_rules can target ln1/ln2 mounts positionally. The reported kind is dynamic: it is empty for any instance that never reaches the native-autograd branch (LayerNormPreBridge/RMSNormPreBridge, or a config without layer_norm_folding), so such a mount is reported skipped instead of silently leaving ordinary gradients in place under a claimed installed rule. A backward hook on hook_scale/hook_normalized, or a forward hook that edits either, now raises RelevanceRuleConflictError while the rule is active instead of the existing warn-and-fallback: falling back would compose the rule with the hook edit and break the bit-identical-forward guarantee. Behavior is unchanged when the rule is inactive. Factor the LN-rule's core VJP (grad_output / denom) into a shared ln_rule_grad helper reused by both the primitive and this integration. * feat(model_bridge): gated-MLP rule via recompute-from-weights VJP Add the Identity-/Half-rule integration for gated MLPs, the second real consumer of the relevance-rule backend after NormalizationBridge. For the raw (non-fused) GatedMLPBridge used by the Qwen2/Llama/Gemma family, wrap the existing opaque original_component(x) call in a new _GatedMLPRecomputeRule custom autograd.Function. Forward returns that native call's own output unchanged, so torch.equal holds by construction. Backward has no access to the opaque call's internal gate/up/down intermediates, so it recomputes them checkpointing-style from the TL-oriented W_gate/W_in/W_out (the unconditional MLPBridge property accessors, never the compatibility-mode-only _processed_* attributes) and reapplies the Identity-rule to the activation and/or the Half-rule to the gate*up product, per whichever is independently active. Weight and bias gradients are read back off the same recomputed graph via torch.autograd.grad, so they keep their ordinary form -- the rules only redefine how relevance reaches the input, not parameter training gradients. The recompute is allowlisted by the underlying HF module class backing W_gate/W_in (nn.Linear vs Conv1D, via the existing weight_layout_in_out helper), not per adapter, since several adapters share the same backing class and orientation; an unrecognized backing module reports as skipped rather than silently installed. JointGateUpMLPBridge (Phi-3/GLM) already reconstructs its forward in Python as act_fn(gate_output) * up_output through separate gate/up LinearBridge submodules, so its rules attach directly at that multiplication -- no weights-recompute is needed there. A gated-MLP node answers to both "activation" (Identity-rule) and "multiplicative_gate" (Half-rule) independently at the same mlp mount, which the previous single-kind protocol could not express: _RelevanceRuleCapable now reports a tuple of supported kinds via _relevance_rule_kinds, and _enable_relevance_rule/_disable_relevance_rule take the specific kind being toggled. Refcounting in use_relevance_rules is now keyed per (module, kind) rather than per module, so nesting one kind's scope inside the other's never disables the outer kind early. NormalizationBridge is updated to the same (still single-kind) shape. * feat(model_bridge): fail-closed coverage on unsupported paths use_relevance_rules previously reported any canonical mount whose component could not honor a requested kind as "skipped", whether the component simply did not implement the relevance-rule protocol at all or implemented it but could not currently honor that specific kind. The two cases need different treatment: a component with no protocol (or a mount whose current dispatch path a rule genuinely does not wrap, such as NormalizationBridge on its python-norm path) is benign non-applicability, but a gated-MLP node at the mlp mount is exactly the kind of component a caller expects either rule to work on, so silently skipping it there would let analysis proceed as if the request had never been made. _RelevanceRuleCapable gains an optional _relevance_rule_unsupported_kinds attribute (not part of the structural protocol, so components that omit it stay isinstance-compatible) naming kinds a component is expected to honor at its mount but currently cannot. use_relevance_rules now raises the new RelevanceRuleUnsupportedError, naming the component's dotted path, for any requested kind found there, before yielding the coverage report and before any forward or backward pass runs. GatedMLPBridge implements the new attribute for two cases: an unrecognized weight-backing class disqualifies both "activation" and "multiplicative_gate" (the recompute cannot orient an opaque module's weights correctly for either rule), and a relu-family activation disqualifies only "activation" -- the Identity-rule's f(x)/x backward multiplier is the correct LRP-style rule for SiLU and both GELU variants, but relu-squared's ratio reduces to relu(x) rather than its true derivative 2*relu(x), and plain relu has no smooth two-sided derivative for the ratio to represent at the removable singularity either. The Half-rule is unaffected by activation form, so "multiplicative_gate" stays available on a relu-family activation as long as the weight backing is recognized. resolve_activation_fn's config-name lookup is factored into a shared _resolve_activation_fn_name helper reused by the new identity_rule_supports_activation check. * test(relevance_rules): tolerant parity vs pinned FarnoushRJ/RelP Add tests comparing the LN-, Identity-, and Half-relevance-rule primitives against formulas ported directly from FarnoushRJ/RelP pinned at commit 8219d6dc417c3fd7f318342cf61cd2a0c20b7250, reimplemented here since that repository targets an unrelated pre-Bridge TransformerLens fork rather than exposing an importable API: - LN-rule: matches the reference's x / scale.detach() to floating-point precision. - Half-rule: matches the reference's z / 2 + (z / 2).detach() split to floating-point precision. - Identity-rule: matches the reference's epsilon-stabilized ratio away from x == 0 within a tolerance sized to the reference's 1e-6 stabilizer constant, for SiLU, exact GELU, and tanh-approximate GELU. - Identity-rule at x == 0: asserts the known discrepancy explicitly rather than absorbing it into a tolerance -- this module's rule uses the paper-defined removable-singularity limit of 0.5, while the reference's epsilon stabilizer collapses the ratio to exactly 0. All cases pass immediately since the primitives already exist; this commit adds a second, independent oracle rather than driving new implementation. * fix(model_bridge): find relevance-rule mounts shadowed by named_modules dedup use_relevance_rules located canonical mounts (ln1, ln2, mlp) by scanning model.named_modules() for a matching leaf name. On a real assembled TransformerBridge, the same bridge component is reachable through two paths: the canonical alias (blocks.N.ln1) and the raw HF module tree the bridge wraps in place (blocks.N._original_component.input_layernorm). nn.Module.named_modules() deduplicates by object identity and keeps only the first-visited path, which is the raw HF-attribute path since it is registered before the canonical alias, so the canonical ln1/ln2 name was never seen. The LN-rule therefore never installed on any real model, and was reported neither installed nor skipped -- silently absent from both. Walk with remove_duplicate=False to recover every path, then keep the fewest-dot-separated-segments path per object so a mount name that happens to match through both the canonical alias and the raw HF attribute (mlp does, on every architecture checked) collapses to a single canonical-looking entry instead of a duplicate. * test(integration): backend on tiny Qwen2 + JointGateUp fixtures End-to-end coverage of the relevance-rule backend on tiny, fully offline HF fixtures built from a programmatic config (no network access, no checkpoint download): a tiny random Qwen2 exercising GatedMLPBridge's opaque recompute-from-weights path, and a tiny random Phi-3 exercising JointGateUpMLPBridge's already-reconstructed forward. With normalization, activation, and multiplicative_gate all active together: forward stays torch.equal to the rule-inactive baseline, every canonical mount across both blocks is reported installed with nothing skipped, and the gradient each rule-active node passes upstream matches its closed-form VJP given the gradient it actually received downstream in the real graph (captured via hook_in/hook_out, outside the LN-rule's fail-closed hook_scale/hook_normalized guard). * refactor(model_bridge): gated-MLP relevance rules without recompute * fix(model_bridge): LN-rule treats a missing norm weight as identity * fix(model_bridge): relevance-rule mount coverage --- tests/integration/test_relevance_lens.py | 222 +++++++++ .../test_gated_mlp_relevance_rule.py | 427 +++++++++++++++++ .../test_joint_gate_up_relevance_rule.py | 164 +++++++ .../test_normalization_relevance_rule.py | 415 ++++++++++++++++ .../test_relevance_rule_mount_placeholders.py | 128 +++++ .../unit/model_bridge/test_relevance_rules.py | 369 +++++++++++++++ tests/unit/tools/test_relevance_rules.py | 444 ++++++++++++++++++ .../model_bridge/_relevance_rules.py | 348 ++++++++++++++ .../generalized_components/gated_mlp.py | 260 +++++++++- .../joint_gate_up_mlp.py | 36 +- .../generalized_components/normalization.py | 163 ++++++- 11 files changed, 2965 insertions(+), 11 deletions(-) create mode 100644 tests/integration/test_relevance_lens.py create mode 100644 tests/unit/model_bridge/generalized_components/test_gated_mlp_relevance_rule.py create mode 100644 tests/unit/model_bridge/generalized_components/test_joint_gate_up_relevance_rule.py create mode 100644 tests/unit/model_bridge/generalized_components/test_normalization_relevance_rule.py create mode 100644 tests/unit/model_bridge/supported_architectures/test_relevance_rule_mount_placeholders.py create mode 100644 tests/unit/model_bridge/test_relevance_rules.py create mode 100644 tests/unit/tools/test_relevance_rules.py create mode 100644 transformer_lens/model_bridge/_relevance_rules.py diff --git a/tests/integration/test_relevance_lens.py b/tests/integration/test_relevance_lens.py new file mode 100644 index 0000000000..e790c7d17f --- /dev/null +++ b/tests/integration/test_relevance_lens.py @@ -0,0 +1,222 @@ +"""End-to-end relevance-rule backend on tiny, offline HF fixtures. + +Per-component unit tests wrap one bridge component directly in a hand-built +single-block harness, which only ever registers a canonical mount name +(``ln1``, ``mlp``) once. That harness cannot see what happens on a real, +fully assembled ``TransformerBridge``, where the same component is *also* +reachable through the raw HF module tree under its own HF attribute name +(for example ``blocks.0._original_component.input_layernorm``). These tests +build a tiny random Qwen2 (the opaque gated-MLP native-forward path) and a tiny +random Phi-3 (``JointGateUpMLPBridge``'s already-reconstructed forward) +fully offline -- random weights from a programmatic HF config, no network +access, no checkpoint download -- and exercise all three rules together on +the real block stack: forward stays bit-identical, canonical mounts are +actually found and installed, and the gradient each rule-active node passes +upstream matches its closed-form VJP given the gradient it actually +received downstream in the real graph. +""" + +import pytest +import torch +from transformers import AutoConfig, AutoModelForCausalLM + +from transformer_lens.model_bridge._relevance_rules import ( + RelevanceRules, + half_rule, + identity_rule, + ln_rule_grad, + use_relevance_rules, +) +from transformer_lens.model_bridge.bridge import TransformerBridge +from transformer_lens.model_bridge.generalized_components.gated_mlp import ( + resolve_activation_fn, +) +from transformer_lens.model_bridge.sources import build_bridge_config_from_hf +from transformer_lens.model_bridge.supported_architectures.phi3 import ( + Phi3ArchitectureAdapter, +) +from transformer_lens.model_bridge.supported_architectures.qwen2 import ( + Qwen2ArchitectureAdapter, +) + +TOKENS = torch.tensor([[1, 5, 7, 42, 9]]) +N_LAYERS = 2 +TINY_DIMS = dict( + vocab_size=97, + hidden_size=32, + intermediate_size=64, + num_hidden_layers=N_LAYERS, + num_attention_heads=4, + num_key_value_heads=2, + max_position_embeddings=64, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, +) + + +class _MockTokenizer: + """Stand-in to satisfy TransformerBridge(tokenizer=...).""" + + +def _build_qwen2_bridge() -> TransformerBridge: + hf_config = AutoConfig.for_model("qwen2", **TINY_DIMS) + torch.manual_seed(0) + hf_model = AutoModelForCausalLM.from_config(hf_config, attn_implementation="eager").eval() + bridge_config = build_bridge_config_from_hf( + hf_model.config, "Qwen2ForCausalLM", "qwen2-tiny", torch.float32 + ) + adapter = Qwen2ArchitectureAdapter(bridge_config) + return TransformerBridge(model=hf_model, adapter=adapter, tokenizer=_MockTokenizer()) + + +def _build_phi3_bridge() -> TransformerBridge: + hf_config = AutoConfig.for_model("phi3", **TINY_DIMS) + torch.manual_seed(0) + hf_model = AutoModelForCausalLM.from_config(hf_config, attn_implementation="eager").eval() + bridge_config = build_bridge_config_from_hf( + hf_model.config, "Phi3ForCausalLM", "phi3-tiny", torch.float32 + ) + adapter = Phi3ArchitectureAdapter(bridge_config) + return TransformerBridge(model=hf_model, adapter=adapter, tokenizer=_MockTokenizer()) + + +# Qwen2 exercises GatedMLPBridge's opaque native-forward path (the primary path +# per real usage), where the rules attach to the live HF activation and down +# projection; Phi-3 exercises JointGateUpMLPBridge's already-reconstructed +# forward, isolating rule bugs from Bridge-integration bugs on the +# fused-projection family. +FIXTURE_BUILDERS = { + "qwen2": _build_qwen2_bridge, + "phi3": _build_phi3_bridge, +} + + +@pytest.fixture(scope="module", params=sorted(FIXTURE_BUILDERS), ids=sorted(FIXTURE_BUILDERS)) +def bridge(request: pytest.FixtureRequest) -> TransformerBridge: + return FIXTURE_BUILDERS[request.param]() + + +def _expected_canonical_mounts() -> set[str]: + ln_mounts = {f"blocks.{i}.ln1" for i in range(N_LAYERS)} | { + f"blocks.{i}.ln2" for i in range(N_LAYERS) + } + mlp_mounts = {f"blocks.{i}.mlp" for i in range(N_LAYERS)} + return ln_mounts | mlp_mounts + + +class TestForwardIdentity: + def test_active_forward_matches_baseline_under_all_three_rules( + self, bridge: TransformerBridge + ) -> None: + with torch.no_grad(): + baseline = bridge(TOKENS) + with use_relevance_rules( + bridge, RelevanceRules(normalization=True, activation=True, multiplicative_gate=True) + ): + with torch.no_grad(): + active = bridge(TOKENS) + assert torch.equal(active, baseline) + + +class TestCoverage: + def test_every_canonical_mount_is_installed_and_none_skipped( + self, bridge: TransformerBridge + ) -> None: + with use_relevance_rules( + bridge, RelevanceRules(normalization=True, activation=True, multiplicative_gate=True) + ) as coverage: + pass + assert set(coverage.installed) == _expected_canonical_mounts() + assert coverage.skipped == () + + +class TestGradientMatchesClosedFormOracle: + """Each rule-active node's local VJP, checked against the gradient it + actually receives in the real graph -- not a hand-rederived whole-model + oracle. The rule Functions compute a purely local closed-form VJP (already + proven against analytic oracles in the primitive and single-component + tests), so the only thing a real multi-block graph can newly break is the + wiring: the wrong node's weights, a disconnected graph path, or a mount + that silently never resolves so its rule never activates. Tapping + hook_in/hook_out -- outside the LN-rule's fail-closed + hook_scale/hook_normalized guard -- exposes exactly the gradient each node + passes upstream and the gradient it receives from downstream, with no + need to reconstruct attention or the rest of the stack by hand. + """ + + def test_ln_rule_and_gated_mlp_rule_match_local_oracles( + self, bridge: TransformerBridge + ) -> None: + ln1 = bridge.blocks[0].ln1 + mlp = bridge.blocks[0].mlp + captured: dict[str, torch.Tensor] = {} + + def _capture(key: str): + def _hook(tensor: torch.Tensor, hook=None) -> None: + captured[key] = tensor.detach().clone() + + return _hook + + ln1.hook_in.add_hook(_capture("ln_x")) + ln1.hook_in.add_hook(_capture("ln_grad_in"), dir="bwd") + ln1.hook_out.add_hook(_capture("ln_grad_out"), dir="bwd") + mlp.hook_in.add_hook(_capture("mlp_x")) + mlp.hook_in.add_hook(_capture("mlp_grad_in"), dir="bwd") + mlp.hook_out.add_hook(_capture("mlp_grad_out"), dir="bwd") + ln1_weight_grad = None + try: + with use_relevance_rules( + bridge, + RelevanceRules(normalization=True, activation=True, multiplicative_gate=True), + ): + logits = bridge(TOKENS) + logits.sum().backward() + ln1_weight_grad = ln1.weight.grad.clone() + finally: + ln1.hook_in.remove_hooks(dir="both") + ln1.hook_out.remove_hooks(dir="both") + mlp.hook_in.remove_hooks(dir="both") + mlp.hook_out.remove_hooks(dir="both") + for parameter in bridge.parameters(): + parameter.grad = None + + eps = getattr(ln1.original_component, "variance_epsilon", 1e-6) + weight = ln1.weight.detach() + denom = (captured["ln_x"].pow(2).mean(-1, keepdim=True) + eps).sqrt() + expected_ln_grad_in = ln_rule_grad(captured["ln_grad_out"] * weight, denom) + torch.testing.assert_close( + captured["ln_grad_in"], expected_ln_grad_in, atol=1e-5, rtol=1e-4 + ) + # weight keeps its ordinary gradient: the rule only redefines the x-path + # VJP, not d(output)/d(weight), given the same grad_out the rule-active + # forward actually produced. + expected_weight_grad = (captured["ln_grad_out"] * (captured["ln_x"] / denom)).sum( + dim=(0, 1) + ) + torch.testing.assert_close(ln1_weight_grad, expected_weight_grad, atol=1e-5, rtol=1e-4) + + x = captured["mlp_x"].detach().requires_grad_(True) + w_gate, w_in, w_out = mlp.W_gate.detach(), mlp.W_in.detach(), mlp.W_out.detach() + b_gate = getattr(mlp.gate, "bias", None) + b_in = getattr(getattr(mlp, "in"), "bias", None) + b_out = getattr(mlp.out, "bias", None) + act_fn = resolve_activation_fn(mlp.config) + with torch.enable_grad(): + gate_output = x @ w_gate + if b_gate is not None: + gate_output = gate_output + b_gate + up_output = x @ w_in + if b_in is not None: + up_output = up_output + b_in + activated = identity_rule(gate_output, act_fn) + gated = half_rule(activated, up_output) + down = gated @ w_out + if b_out is not None: + down = down + b_out + (expected_mlp_grad_in,) = torch.autograd.grad( + down, x, grad_outputs=captured["mlp_grad_out"] + ) + torch.testing.assert_close( + captured["mlp_grad_in"], expected_mlp_grad_in, atol=1e-5, rtol=1e-4 + ) diff --git a/tests/unit/model_bridge/generalized_components/test_gated_mlp_relevance_rule.py b/tests/unit/model_bridge/generalized_components/test_gated_mlp_relevance_rule.py new file mode 100644 index 0000000000..0d2ab27d36 --- /dev/null +++ b/tests/unit/model_bridge/generalized_components/test_gated_mlp_relevance_rule.py @@ -0,0 +1,427 @@ +"""Identity-/Half-rule integration on GatedMLPBridge's opaque native-forward path. + +The raw (non-fused) gated-MLP path keeps the HF module's own forward intact and +installs the rules on its live submodules for a ``use_relevance_rules`` scope: the +Identity-rule by swapping the module's activation callable, and the Half-rule by a +forward-pre-hook that halves the gradient entering the down projection (the gate*up +product). Because the real forward runs unchanged, anything it does beyond the core +``down(act(gate) * up)`` shape -- a post-product multiplier, activation sparsity, the +module's own hooks -- survives into the backward graph, and the rule VJP is taken +through that real forward rather than a reconstruction of the core shape. The rules +attach regardless of the projection weight class, so an opaque backing that is neither +``nn.Linear`` nor ``Conv1D`` is still covered. Only the relu-family Identity-rule +exclusion and the missing-activation-callable case remain unsupported. +""" + +import pytest +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers.pytorch_utils import Conv1D + +from transformer_lens.model_bridge._relevance_rules import ( + RelevanceRules, + RelevanceRuleUnsupportedError, + half_rule, + identity_rule, + use_relevance_rules, +) +from transformer_lens.model_bridge.generalized_components.base import ( + GeneralizedComponent, +) +from transformer_lens.model_bridge.generalized_components.gated_mlp import ( + GatedMLPBridge, +) +from transformer_lens.model_bridge.generalized_components.linear import LinearBridge + + +class _Cfg: + def __init__(self, hidden_act: str = "silu"): + self.hidden_act = hidden_act + + +class _ReluSquared(nn.Module): + def forward(self, x: torch.Tensor) -> torch.Tensor: + return torch.relu(x).square() + + +_ACTIVATIONS = {"silu": nn.SiLU, "relu2": _ReluSquared} + + +class _OpaqueProj(nn.Module): + """Weight-backed projection that is neither nn.Linear nor Conv1D. + + Stands in for a backing class the retired weight-orientation allowlist would + have refused; the rules now attach without reading its weights at all. + """ + + def __init__(self, d_in: int, d_out: int, bias: bool = True): + super().__init__() + self.weight = nn.Parameter(torch.randn(d_out, d_in)) + self.bias = nn.Parameter(torch.randn(d_out)) if bias else None + + def forward(self, x: torch.Tensor) -> torch.Tensor: + out = x @ self.weight.T + return out if self.bias is None else out + self.bias + + +class _TinyGatedMLP(nn.Module): + """Mirrors the Qwen2/Llama/Gemma gated-MLP: one opaque call over its own submodules. + + The activation is an ``nn.Module`` attribute the forward calls, matching the + ``ACT2FN`` shape the Identity-rule wrap targets. + """ + + def __init__(self, gate_proj, up_proj, down_proj, hidden_act: str = "silu"): + super().__init__() + self.gate_proj = gate_proj + self.up_proj = up_proj + self.down_proj = down_proj + self.act_fn = _ACTIVATIONS[hidden_act]() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + + +class _MultiplierGatedMLP(_TinyGatedMLP): + """A gated MLP whose forward does strictly more than ``down(act(gate) * up)``. + + The constant post-product multiplier stands in for the Falcon-H1 / Gemma3n + families whose native forward applies extra scaling. A rule VJP taken through a + reconstruction of only the core gated shape would omit the multiplier and + disagree with the VJP through this real forward. + """ + + def __init__(self, gate_proj, up_proj, down_proj, multiplier: float = 1.7): + super().__init__(gate_proj, up_proj, down_proj) + self.multiplier = multiplier + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return super().forward(x) * self.multiplier + + +class _Block(nn.Module): + """Mounts a gated-MLP bridge at the canonical mlp position.""" + + def __init__(self, mlp: GeneralizedComponent): + super().__init__() + self.mlp = mlp + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.mlp(x) + + +def _make_projections(backing_class: str, d_model: int = 4, d_mlp: int = 8, bias: bool = True): + torch.manual_seed(0) + if backing_class == "nn.Linear": + return ( + nn.Linear(d_model, d_mlp, bias=bias), + nn.Linear(d_model, d_mlp, bias=bias), + nn.Linear(d_mlp, d_model, bias=bias), + ) + if backing_class == "Conv1D": + return ( + Conv1D(d_mlp, d_model), + Conv1D(d_mlp, d_model), + Conv1D(d_model, d_mlp), + ) + return ( + _OpaqueProj(d_model, d_mlp, bias=bias), + _OpaqueProj(d_model, d_mlp, bias=bias), + _OpaqueProj(d_mlp, d_model, bias=bias), + ) + + +def _wire_bridge(hf_mlp: nn.Module, config: _Cfg) -> GatedMLPBridge: + bridge = GatedMLPBridge(name="mlp", config=config) + gate_bridge = LinearBridge(name="gate_proj") + in_bridge = LinearBridge(name="up_proj") + out_bridge = LinearBridge(name="down_proj") + bridge.add_module("gate", gate_bridge) + bridge.add_module("in", in_bridge) + bridge.add_module("out", out_bridge) + bridge.set_original_component(hf_mlp) + gate_bridge.set_original_component(hf_mlp.gate_proj) + in_bridge.set_original_component(hf_mlp.up_proj) + out_bridge.set_original_component(hf_mlp.down_proj) + return bridge + + +def _make_bridge( + backing_class: str, bias: bool = True, hidden_act: str = "silu" +) -> tuple[_Block, _TinyGatedMLP]: + gate_proj, up_proj, down_proj = _make_projections(backing_class, bias=bias) + hf_mlp = _TinyGatedMLP(gate_proj, up_proj, down_proj, hidden_act=hidden_act) + bridge = _wire_bridge(hf_mlp, _Cfg(hidden_act)) + return _Block(bridge), hf_mlp + + +def _make_multiplier_bridge(multiplier: float = 1.7) -> tuple[_Block, _MultiplierGatedMLP]: + gate_proj, up_proj, down_proj = _make_projections("nn.Linear", bias=True) + hf_mlp = _MultiplierGatedMLP(gate_proj, up_proj, down_proj, multiplier=multiplier) + bridge = _wire_bridge(hf_mlp, _Cfg("silu")) + return _Block(bridge), hf_mlp + + +def _oracle_grads(hf_mlp, x, activation_active, gate_active, multiplier: float = 1.0): + # hf_mlp's parameters are shared with the bridge under test, so a prior backward + # already left gradients on them; reset first or this second backward would + # accumulate on top instead of producing an independently comparable oracle. The + # oracle runs the module's real submodules -- including the multiplier -- so it is + # the VJP through the actual forward, not through the core gated shape alone. + for p in hf_mlp.parameters(): + p.grad = None + x_oracle = x.detach().clone().requires_grad_(True) + gate_output = hf_mlp.gate_proj(x_oracle) + up_output = hf_mlp.up_proj(x_oracle) + activated = ( + identity_rule(gate_output, hf_mlp.act_fn) + if activation_active + else hf_mlp.act_fn(gate_output) + ) + gated = half_rule(activated, up_output) if gate_active else activated * up_output + down = hf_mlp.down_proj(gated) * multiplier + down.sum().backward() + grads = {n: p.grad.clone() for n, p in hf_mlp.named_parameters()} + return x_oracle.grad.clone(), grads + + +BACKING_CLASSES = ["nn.Linear", "Conv1D", "opaque"] + + +class TestGatedMLPRelevanceRuleCapability: + @pytest.mark.parametrize("backing_class", BACKING_CLASSES) + def test_capable_of_both_kinds_regardless_of_weight_backing(self, backing_class): + block, _ = _make_bridge(backing_class) + assert set(block.mlp._relevance_rule_kinds) == {"activation", "multiplicative_gate"} + + def test_identity_rule_unsupported_for_relu_squared_activation(self): + block, _ = _make_bridge("nn.Linear", hidden_act="relu2") + assert block.mlp._relevance_rule_kinds == ("multiplicative_gate",) + + with pytest.raises(RelevanceRuleUnsupportedError, match="mlp"): + with use_relevance_rules(block, RelevanceRules(activation=True)): + pass + + def test_activation_unsupported_when_no_activation_callable_is_exposed(self): + # A gated MLP whose forward uses a bare function has no callable activation + # attribute for the Identity-rule to wrap, so requesting it must raise rather + # than install a silent no-op. The Half-rule needs no activation access and + # stays available. + class _FunctionalActMLP(nn.Module): + def __init__(self): + super().__init__() + self.gate_proj = nn.Linear(4, 8) + self.up_proj = nn.Linear(4, 8) + self.down_proj = nn.Linear(8, 4) + + def forward(self, x): + return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x)) + + torch.manual_seed(0) + bridge = _wire_bridge(_FunctionalActMLP(), _Cfg("silu")) + block = _Block(bridge) + + assert bridge._relevance_rule_kinds == ("multiplicative_gate",) + with pytest.raises(RelevanceRuleUnsupportedError, match="mlp"): + with use_relevance_rules(block, RelevanceRules(activation=True)): + pass + + def test_half_rule_remains_available_under_relu_squared_activation(self): + block, _ = _make_bridge("nn.Linear", hidden_act="relu2") + x = torch.randn(2, 4) + baseline = block(x) + with use_relevance_rules(block, RelevanceRules(multiplicative_gate=True)) as coverage: + assert coverage.installed == ("mlp",) + assert torch.equal(block(x), baseline) + + +class TestGatedMLPRelevanceRuleForwardIdentity: + @pytest.mark.parametrize("backing_class", BACKING_CLASSES) + def test_forward_identical_while_rule_active(self, backing_class): + block, _ = _make_bridge(backing_class) + x = torch.randn(3, 4) + baseline = block(x) + with use_relevance_rules(block, RelevanceRules(activation=True, multiplicative_gate=True)): + active = block(x) + assert torch.equal(active, baseline) + + def test_forward_identical_while_rule_active_with_extra_forward_ops(self): + block, _ = _make_multiplier_bridge() + x = torch.randn(3, 4) + baseline = block(x) + with use_relevance_rules(block, RelevanceRules(activation=True, multiplicative_gate=True)): + active = block(x) + assert torch.equal(active, baseline) + + @pytest.mark.parametrize("backing_class", BACKING_CLASSES) + def test_forward_unchanged_when_rule_inactive(self, backing_class): + block, _ = _make_bridge(backing_class) + x = torch.randn(3, 4) + assert torch.equal(block(x), block(x)) + + def test_activation_callable_restored_after_scope(self): + block, hf_mlp = _make_bridge("nn.Linear") + original_act = hf_mlp._modules["act_fn"] + with use_relevance_rules(block, RelevanceRules(activation=True)): + assert hf_mlp._modules["act_fn"] is not original_act + assert hf_mlp._modules["act_fn"] is original_act + + +class TestGatedMLPRelevanceRuleVJP: + @pytest.mark.parametrize("backing_class", BACKING_CLASSES) + def test_both_rules_active_matches_real_forward_oracle(self, backing_class): + block, hf_mlp = _make_bridge(backing_class) + x = torch.randn(3, 4, requires_grad=True) + + with use_relevance_rules(block, RelevanceRules(activation=True, multiplicative_gate=True)): + out = block(x) + out.sum().backward() + + grad_x = x.grad.clone() + grads = {n: p.grad.clone() for n, p in hf_mlp.named_parameters()} + + expected_grad_x, expected_grads = self._expected(hf_mlp, x, True, True) + torch.testing.assert_close(grad_x, expected_grad_x, atol=1e-5, rtol=1e-5) + for name, grad in grads.items(): + # The down projection's own weight gradient stays ordinary: the Half-rule + # only halves the gradient reaching the product, not the parameter grads. + torch.testing.assert_close(grad, expected_grads[name], atol=1e-5, rtol=1e-5) + + def test_both_rules_match_vjp_through_extra_forward_ops(self): + block, hf_mlp = _make_multiplier_bridge() + x = torch.randn(3, 4, requires_grad=True) + + with use_relevance_rules(block, RelevanceRules(activation=True, multiplicative_gate=True)): + out = block(x) + out.sum().backward() + + grad_x = x.grad.clone() + grads = {n: p.grad.clone() for n, p in hf_mlp.named_parameters()} + + expected_grad_x, expected_grads = _oracle_grads( + hf_mlp, x, activation_active=True, gate_active=True, multiplier=hf_mlp.multiplier + ) + torch.testing.assert_close(grad_x, expected_grad_x, atol=1e-5, rtol=1e-5) + for name, grad in grads.items(): + torch.testing.assert_close(grad, expected_grads[name], atol=1e-5, rtol=1e-5) + + @pytest.mark.parametrize("backing_class", BACKING_CLASSES) + def test_activation_only_leaves_gate_split_ordinary(self, backing_class): + block, hf_mlp = _make_bridge(backing_class) + x = torch.randn(3, 4, requires_grad=True) + + with use_relevance_rules(block, RelevanceRules(activation=True)): + out = block(x) + out.sum().backward() + + grad_x = x.grad.clone() + expected_grad_x, _ = self._expected(hf_mlp, x, True, False) + torch.testing.assert_close(grad_x, expected_grad_x, atol=1e-5, rtol=1e-5) + + @pytest.mark.parametrize("backing_class", BACKING_CLASSES) + def test_multiplicative_gate_only_leaves_activation_ordinary(self, backing_class): + block, hf_mlp = _make_bridge(backing_class) + x = torch.randn(3, 4, requires_grad=True) + + with use_relevance_rules(block, RelevanceRules(multiplicative_gate=True)): + out = block(x) + out.sum().backward() + + grad_x = x.grad.clone() + expected_grad_x, _ = self._expected(hf_mlp, x, False, True) + torch.testing.assert_close(grad_x, expected_grad_x, atol=1e-5, rtol=1e-5) + + def test_rule_inactive_gradients_are_ordinary(self): + block, hf_mlp = _make_bridge("nn.Linear") + x = torch.randn(3, 4, requires_grad=True) + + out = block(x) + out.sum().backward() + grad_x = x.grad.clone() + grads = {n: p.grad.clone() for n, p in hf_mlp.named_parameters()} + + for p in hf_mlp.parameters(): + p.grad = None + x_plain = x.detach().clone().requires_grad_(True) + hf_mlp(x_plain).sum().backward() + + torch.testing.assert_close(grad_x, x_plain.grad) + for name, param in hf_mlp.named_parameters(): + torch.testing.assert_close(grads[name], param.grad) + + def test_bias_free_projections_are_handled(self): + block, hf_mlp = _make_bridge("nn.Linear", bias=False) + x = torch.randn(3, 4, requires_grad=True) + + with use_relevance_rules(block, RelevanceRules(activation=True, multiplicative_gate=True)): + out = block(x) + out.sum().backward() + + grad_x = x.grad.clone() + expected_grad_x, expected_grads = self._expected(hf_mlp, x, True, True) + torch.testing.assert_close(grad_x, expected_grad_x, atol=1e-5, rtol=1e-5) + for name, param in hf_mlp.named_parameters(): + torch.testing.assert_close(param.grad, expected_grads[name], atol=1e-5, rtol=1e-5) + + @staticmethod + def _expected(hf_mlp, x, activation_active, gate_active): + return _oracle_grads(hf_mlp, x, activation_active, gate_active, multiplier=1.0) + + +class TestGatedMLPRelevanceRulePreservesRealForward: + def test_modules_own_backward_hook_fires_during_rule_active_backward(self): + # The rules attach to the live module and leave its native forward in the + # graph, so a backward hook registered on an HF submodule still fires during + # a rule-active backward rather than being bypassed by a separate graph. + block, hf_mlp = _make_bridge("nn.Linear") + fired = {"count": 0} + + def _record(module, grad_input, grad_output): + fired["count"] += 1 + + handle = hf_mlp.down_proj.register_full_backward_hook(_record) + x = torch.randn(3, 4, requires_grad=True) + try: + with use_relevance_rules( + block, RelevanceRules(activation=True, multiplicative_gate=True) + ): + block(x).sum().backward() + finally: + handle.remove() + + assert fired["count"] > 0 + + +class TestGatedMLPRelevanceRuleCompatibilityMode: + """The processed-weights (compatibility) path reconstructs the forward from folded + weights, so it applies the rules inline; before this it skipped them entirely.""" + + def _make_compat_bridge(self, bias: bool = True): + block, hf_mlp = _make_bridge("nn.Linear", bias=bias) + bridge = block.mlp + bridge._use_processed_weights = True + # Plain (non-Parameter) tensors so the bridge's custom __getattr__ resolves + # them; nn.Linear stores weight as [out, in], the layout functional linear + # expects, so the folded-weight forward reproduces the native projections. + bridge._processed_W_gate = hf_mlp.gate_proj.weight.detach() + bridge._processed_b_gate = None if not bias else hf_mlp.gate_proj.bias.detach() + bridge._processed_W_in = hf_mlp.up_proj.weight.detach() + bridge._processed_b_in = None if not bias else hf_mlp.up_proj.bias.detach() + bridge._processed_W_out = hf_mlp.down_proj.weight.detach() + bridge._processed_b_out = None if not bias else hf_mlp.down_proj.bias.detach() + return block, hf_mlp + + def test_compat_forward_identical_and_rule_vjp_matches_oracle(self): + block, hf_mlp = self._make_compat_bridge() + x = torch.randn(3, 4, requires_grad=True) + + baseline = block(x.detach()) + with use_relevance_rules(block, RelevanceRules(activation=True, multiplicative_gate=True)): + active = block(x) + assert torch.equal(active.detach(), baseline) + + active.sum().backward() + grad_x = x.grad.clone() + expected_grad_x, _ = _oracle_grads(hf_mlp, x, True, True) + torch.testing.assert_close(grad_x, expected_grad_x, atol=1e-5, rtol=1e-5) diff --git a/tests/unit/model_bridge/generalized_components/test_joint_gate_up_relevance_rule.py b/tests/unit/model_bridge/generalized_components/test_joint_gate_up_relevance_rule.py new file mode 100644 index 0000000000..f8bee5e693 --- /dev/null +++ b/tests/unit/model_bridge/generalized_components/test_joint_gate_up_relevance_rule.py @@ -0,0 +1,164 @@ +"""Identity-/Half-rule integration on JointGateUpMLPBridge's reconstructed forward. + +Unlike the raw GatedMLPBridge path (opaque native forward, tested separately), +JointGateUpMLPBridge already reconstructs its forward in Python as +``act_fn(gate_output) * up_output`` through separate gate/up LinearBridge +submodules, so the rules attach directly at that multiplication inline, off the +same boolean flags -- no live-submodule hook is installed here. This covers that +the reconstructed +forward is unaffected by an inactive rule, that both rules apply correctly +together and independently, and that gradients match the oracle produced by the +already-tested Identity-/Half-rule primitives directly. +""" + +import pytest +import torch +import torch.nn as nn +import torch.nn.functional as F + +from transformer_lens.model_bridge._relevance_rules import ( + RelevanceRules, + half_rule, + identity_rule, + use_relevance_rules, +) +from transformer_lens.model_bridge.generalized_components.joint_gate_up_mlp import ( + JointGateUpMLPBridge, +) +from transformer_lens.model_bridge.generalized_components.linear import LinearBridge + + +class _Cfg: + hidden_act = "silu" + + +class _TinyPhi3MLP(nn.Module): + """Mirrors Phi-3/GLM's fused gate_up_proj structure.""" + + def __init__(self, d_model: int = 4, d_mlp: int = 8, bias: bool = False): + super().__init__() + self.gate_up_proj = nn.Linear(d_model, 2 * d_mlp, bias=bias) + self.down_proj = nn.Linear(d_mlp, d_model, bias=bias) + self.activation_fn = nn.SiLU() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + gate_up = self.gate_up_proj(x) + gate, up = gate_up.chunk(2, dim=-1) + return self.down_proj(self.activation_fn(gate) * up) + + +class _Block(nn.Module): + """Mounts a joint gate-up bridge at the canonical mlp position.""" + + def __init__(self, mlp: JointGateUpMLPBridge): + super().__init__() + self.mlp = mlp + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.mlp(x) + + +def _make_bridge(bias: bool = False) -> tuple[_Block, _TinyPhi3MLP]: + torch.manual_seed(0) + hf_mlp = _TinyPhi3MLP(bias=bias) + bridge = JointGateUpMLPBridge(name="mlp", config=_Cfg(), submodules={}) + out_bridge = LinearBridge(name="down_proj") + bridge.add_module("out", out_bridge) + bridge.set_original_component(hf_mlp) + out_bridge.set_original_component(hf_mlp.down_proj) + return _Block(bridge), hf_mlp + + +def _oracle_grads(hf_mlp, gate_proj, up_proj, x, activation_active: bool, gate_active: bool): + for p in hf_mlp.parameters(): + p.grad = None + x_oracle = x.detach().clone().requires_grad_(True) + gate_output = F.linear(x_oracle, gate_proj.weight, gate_proj.bias) + up_output = F.linear(x_oracle, up_proj.weight, up_proj.bias) + activated = identity_rule(gate_output, F.silu) if activation_active else F.silu(gate_output) + gated = half_rule(activated, up_output) if gate_active else activated * up_output + down = hf_mlp.down_proj(gated) + down.sum().backward() + return x_oracle.grad.clone() + + +class TestJointGateUpRelevanceRuleCapability: + def test_capable_of_both_kinds(self): + block, _ = _make_bridge() + assert set(block.mlp._relevance_rule_kinds) == {"activation", "multiplicative_gate"} + + +class TestJointGateUpRelevanceRuleForwardIdentity: + def test_forward_identical_while_rule_active(self): + block, _ = _make_bridge() + x = torch.randn(3, 4) + baseline = block(x) + with use_relevance_rules(block, RelevanceRules(activation=True, multiplicative_gate=True)): + active = block(x) + assert torch.equal(active, baseline) + + def test_forward_unchanged_when_rule_inactive(self): + block, _ = _make_bridge() + x = torch.randn(3, 4) + before = block(x) + after = block(x) + assert torch.equal(before, after) + + +class TestJointGateUpRelevanceRuleVJP: + @pytest.mark.parametrize( + ("activation_active", "gate_active"), + [(True, True), (True, False), (False, True)], + ) + def test_matches_manually_composed_oracle(self, activation_active, gate_active): + block, hf_mlp = _make_bridge() + x = torch.randn(3, 4, requires_grad=True) + + with use_relevance_rules( + block, + RelevanceRules( + activation=activation_active, + multiplicative_gate=gate_active, + ), + ): + out = block(x) + out.sum().backward() + + grad_x = x.grad.clone() + + gate_proj = block.mlp.gate.original_component + up_proj = getattr(block.mlp, "in").original_component + expected_grad_x = _oracle_grads( + hf_mlp, gate_proj, up_proj, x, activation_active, gate_active + ) + torch.testing.assert_close(grad_x, expected_grad_x, atol=1e-5, rtol=1e-5) + + def test_rule_inactive_gradients_are_ordinary(self): + block, hf_mlp = _make_bridge() + x = torch.randn(3, 4, requires_grad=True) + + out = block(x) + out.sum().backward() + grad_x = x.grad.clone() + + for p in hf_mlp.parameters(): + p.grad = None + x_plain = x.detach().clone().requires_grad_(True) + plain = hf_mlp(x_plain) + plain.sum().backward() + + torch.testing.assert_close(grad_x, x_plain.grad) + + def test_bias_free_and_biased_projections_are_both_handled(self): + block, hf_mlp = _make_bridge(bias=True) + x = torch.randn(3, 4, requires_grad=True) + + with use_relevance_rules(block, RelevanceRules(activation=True, multiplicative_gate=True)): + out = block(x) + out.sum().backward() + + grad_x = x.grad.clone() + gate_proj = block.mlp.gate.original_component + up_proj = getattr(block.mlp, "in").original_component + expected_grad_x = _oracle_grads(hf_mlp, gate_proj, up_proj, x, True, True) + torch.testing.assert_close(grad_x, expected_grad_x, atol=1e-5, rtol=1e-5) diff --git a/tests/unit/model_bridge/generalized_components/test_normalization_relevance_rule.py b/tests/unit/model_bridge/generalized_components/test_normalization_relevance_rule.py new file mode 100644 index 0000000000..bfd6dcd374 --- /dev/null +++ b/tests/unit/model_bridge/generalized_components/test_normalization_relevance_rule.py @@ -0,0 +1,415 @@ +"""LN-rule integration on NormalizationBridge's native-autograd path. + +Covers the commit-5 contract: the rule-wrapped native forward is bit-identical to +today's native forward by construction (the wrapping calls ``original_component(x)`` +itself rather than reproducing its numerics), the backward follows the LN-rule +(denominator treated as constant) while weight/bias keep their ordinary gradient, +targeting is positional so an ln1/ln2 mount that never reaches the native-autograd +branch reports as skipped rather than silently leaving ordinary gradients in place, +and a hook that would otherwise silently fall back instead raises while the rule is +active. +""" + + +import pytest +import torch +import torch.nn as nn +import torch.nn.functional as F + +from transformer_lens.model_bridge._relevance_rules import ( + RelevanceRuleConflictError, + RelevanceRules, + use_relevance_rules, +) +from transformer_lens.model_bridge.generalized_components.normalization import ( + LayerNormPreBridge, + NormalizationBridge, + RMSNormPreBridge, +) + + +class _Cfg: + def __init__( + self, + uses_rms_norm: bool = False, + eps: float = 1e-5, + rmsnorm_uses_offset: bool = False, + layer_norm_folding: bool = False, + ): + self.uses_rms_norm = uses_rms_norm + self.eps = eps + self.rmsnorm_uses_offset = rmsnorm_uses_offset + self.layer_norm_folding = layer_norm_folding + + +class _TinyRMSNorm(nn.Module): + """Minimal RMSNorm mirroring LlamaRMSNorm's forward.""" + + def __init__(self, d: int, eps: float = 1e-5): + super().__init__() + self.weight = nn.Parameter(torch.randn(d) * 0.1 + 1.0) + self.variance_epsilon = eps + + def forward(self, x: torch.Tensor) -> torch.Tensor: + variance = x.pow(2).mean(-1, keepdim=True) + return self.weight * x * torch.rsqrt(variance + self.variance_epsilon) + + +class _TinyGemmaRMSNorm(nn.Module): + """Minimal Gemma-style RMSNorm: weight is stored as an offset from 1.""" + + def __init__(self, d: int, eps: float = 1e-5): + super().__init__() + self.weight = nn.Parameter(torch.randn(d) * 0.1) + self.variance_epsilon = eps + + def forward(self, x: torch.Tensor) -> torch.Tensor: + variance = x.pow(2).mean(-1, keepdim=True) + x_normed = x * torch.rsqrt(variance + self.variance_epsilon) + return x_normed * (1.0 + self.weight) + + +class _TinyOlmoLayerNorm(nn.Module): + """Param-free centered LayerNorm mirroring OLMo's OlmoLayerNorm: no weight, no bias.""" + + def __init__(self, d: int, eps: float = 1e-5): + super().__init__() + self.normalized_shape = (d,) + self.eps = eps + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return F.layer_norm(x, self.normalized_shape, None, None, self.eps) + + +def _layernorm(d: int) -> nn.LayerNorm: + layer = nn.LayerNorm(d, eps=1e-5) + nn.init.normal_(layer.weight, std=0.1) + nn.init.normal_(layer.bias, std=0.1) + return layer + + +class _Block(nn.Module): + """Mounts a normalization bridge at the canonical ln1 position.""" + + def __init__(self, norm: NormalizationBridge): + super().__init__() + self.ln1 = norm + + +def _make_bridge( + native: bool, + rms: bool = False, + offset: bool = False, + d: int = 16, + layer_norm_folding: bool = False, +) -> NormalizationBridge: + layer: nn.Module + if offset: + layer = _TinyGemmaRMSNorm(d) + elif rms: + layer = _TinyRMSNorm(d) + else: + layer = _layernorm(d) + bridge = NormalizationBridge( + name="ln1", + config=_Cfg( + uses_rms_norm=rms or offset, + rmsnorm_uses_offset=offset, + layer_norm_folding=layer_norm_folding, + ), + use_native_layernorm_autograd=native, + ) + bridge.set_original_component(layer) + return bridge + + +def _denom_detached_oracle( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor | None, + uses_rms: bool, + offset: bool, + eps: float, +) -> torch.Tensor: + """Manual recompute with the denominator detached: the LN-rule's target VJP.""" + x_centered = x if uses_rms else x - x.mean(-1, keepdim=True) + denom = (x_centered.pow(2).mean(-1, keepdim=True) + eps).sqrt().detach() + w_eff = (1.0 + weight) if offset else weight + out = (x_centered / denom) * w_eff + if bias is not None: + out = out + bias + return out + + +class TestForwardIdentity: + @pytest.mark.parametrize("rms", [False, True], ids=["layernorm", "rmsnorm"]) + def test_active_forward_matches_inactive_forward(self, rms): + bridge = _make_bridge(native=True, rms=rms) + block = _Block(bridge) + x = torch.randn(2, 5, 16) + baseline = bridge(x) + with use_relevance_rules(block, RelevanceRules(normalization=True)): + active = bridge(x) + assert torch.equal(active, baseline) + + def test_active_forward_matches_original_component_directly(self): + bridge = _make_bridge(native=True) + block = _Block(bridge) + x = torch.randn(2, 5, 16) + with use_relevance_rules(block, RelevanceRules(normalization=True)): + active = bridge(x) + assert torch.equal(active, bridge.original_component(x)) + + +class TestVJPMatchesDetachedDenomOracle: + def test_layernorm(self): + bridge = _make_bridge(native=True) + block = _Block(bridge) + x = torch.randn(2, 5, 16, requires_grad=True) + with use_relevance_rules(block, RelevanceRules(normalization=True)): + y = bridge(x) + y.sum().backward() + grad_rule = x.grad.clone() + + x_oracle = x.detach().clone().requires_grad_(True) + y_oracle = _denom_detached_oracle( + x_oracle, + bridge.original_component.weight, + bridge.original_component.bias, + uses_rms=False, + offset=False, + eps=1e-5, + ) + y_oracle.sum().backward() + torch.testing.assert_close(grad_rule, x_oracle.grad) + + @pytest.mark.parametrize("offset", [False, True], ids=["plain_rms", "gemma_offset"]) + def test_rmsnorm(self, offset): + bridge = _make_bridge(native=True, rms=not offset, offset=offset) + block = _Block(bridge) + x = torch.randn(2, 5, 16, requires_grad=True) + with use_relevance_rules(block, RelevanceRules(normalization=True)): + y = bridge(x) + y.sum().backward() + grad_rule = x.grad.clone() + + x_oracle = x.detach().clone().requires_grad_(True) + y_oracle = _denom_detached_oracle( + x_oracle, + bridge.original_component.weight, + None, + uses_rms=True, + offset=offset, + eps=1e-5, + ) + y_oracle.sum().backward() + torch.testing.assert_close(grad_rule, x_oracle.grad) + + def test_rule_grad_disagrees_with_ordinary_autodiff(self): + """The whole point of the rule: it must actually change the gradient.""" + bridge = _make_bridge(native=True) + block = _Block(bridge) + x = torch.randn(2, 5, 16, requires_grad=True) + + with use_relevance_rules(block, RelevanceRules(normalization=True)): + y_rule = bridge(x) + (grad_rule,) = torch.autograd.grad(y_rule.sum(), x) + + x_plain = x.detach().clone().requires_grad_(True) + y_plain = bridge(x_plain) + (grad_plain,) = torch.autograd.grad(y_plain.sum(), x_plain) + assert not torch.allclose(grad_rule, grad_plain) + + +class TestParameterGradientsPreserved: + def test_weight_and_bias_receive_ordinary_gradient(self): + bridge = _make_bridge(native=True) + block = _Block(bridge) + x = torch.randn(2, 5, 16, requires_grad=True) + with use_relevance_rules(block, RelevanceRules(normalization=True)): + y = bridge(x) + y.sum().backward() + weight_grad = bridge.original_component.weight.grad + bias_grad = bridge.original_component.bias.grad + assert weight_grad is not None and torch.isfinite(weight_grad).all() + assert bias_grad is not None and torch.isfinite(bias_grad).all() + + # Ordinary gradient: d(output)/d(weight) = normalized value (pre-weight), + # independent of the rule's treatment of the x-path; d(output)/d(bias) = 1. + x_oracle = x.detach().clone() + normalized = _denom_detached_oracle( + x_oracle, + torch.ones_like(bridge.original_component.weight), + None, + uses_rms=False, + offset=False, + eps=1e-5, + ) + reduce_dims = tuple(range(normalized.dim() - 1)) + expected_weight_grad = normalized.sum(dim=reduce_dims) + expected_bias_grad = torch.full_like(bias_grad, x.shape[0] * x.shape[1]) + torch.testing.assert_close(weight_grad, expected_weight_grad, atol=1e-4, rtol=1e-4) + torch.testing.assert_close(bias_grad, expected_bias_grad, atol=1e-4, rtol=1e-4) + + +class TestRuleInactiveRegression: + @pytest.mark.parametrize("native", [True, False], ids=["native_autograd", "python_norm"]) + def test_forward_byte_identical_to_ordinary_call(self, native): + bridge = _make_bridge(native=native) + x = torch.randn(2, 5, 16) + expected = bridge(x) + # No use_relevance_rules context at all: today's behavior, unconditionally. + actual = bridge(x) + assert torch.equal(actual, expected) + + def test_non_native_path_rule_request_is_skipped_and_forward_unaffected(self): + """A python-norm-path bridge (no native autograd, no folding) never reaches + the branch the LN-rule wraps, so it must be reported skipped rather than + silently leaving ordinary gradients in place under a claimed rule.""" + bridge = _make_bridge(native=False) + block = _Block(bridge) + x = torch.randn(2, 5, 16) + baseline = bridge(x) + with use_relevance_rules(block, RelevanceRules(normalization=True)) as coverage: + assert coverage.installed == () + assert coverage.skipped == ("ln1",) + active = bridge(x) + assert torch.equal(active, baseline) + + def test_layer_norm_folding_config_flag_makes_the_rule_installable(self): + """layer_norm_folding also dispatches through the native-autograd branch, + so the rule must be installable there even with use_native_layernorm_autograd + left False.""" + bridge = _make_bridge(native=False, layer_norm_folding=True) + block = _Block(bridge) + x = torch.randn(2, 5, 16) + baseline = bridge(x) + with use_relevance_rules(block, RelevanceRules(normalization=True)) as coverage: + assert coverage.installed == ("ln1",) + active = bridge(x) + assert torch.equal(active, baseline) + + @pytest.mark.parametrize( + "bridge_cls", [LayerNormPreBridge, RMSNormPreBridge], ids=["ln_pre", "rms_pre"] + ) + def test_param_free_pre_norm_is_skipped_and_forward_unaffected(self, bridge_cls): + """LNPre/RMSPre always take the python-norm path regardless of the native + flag, so they must never be reported installed.""" + bridge = bridge_cls(name="ln1", config=_Cfg()) + bridge.set_original_component(nn.LayerNorm(16, eps=1e-5, elementwise_affine=False)) + block = _Block(bridge) + x = torch.randn(2, 5, 16) + baseline = bridge(x) + with use_relevance_rules(block, RelevanceRules(normalization=True)) as coverage: + assert coverage.installed == () + assert coverage.skipped == ("ln1",) + active = bridge(x) + assert torch.equal(active, baseline) + + +class TestMissingWeightTreatedAsIdentity: + """A native-autograd bridge over a parameter-free norm (OLMo's OlmoLayerNorm has + no ``weight``) reports the LN-rule installed, so its rule-active forward and + backward must treat the missing weight as a unit scale instead of dereferencing + ``self.weight`` and raising AttributeError.""" + + def _make_param_free_bridge(self, d: int = 16) -> NormalizationBridge: + bridge = NormalizationBridge( + name="ln1", + config=_Cfg(uses_rms_norm=False), + use_native_layernorm_autograd=True, + ) + bridge.set_original_component(_TinyOlmoLayerNorm(d)) + return bridge + + def test_active_forward_matches_native_forward(self): + bridge = self._make_param_free_bridge() + block = _Block(bridge) + x = torch.randn(2, 5, 16) + baseline = bridge.original_component(x) + with use_relevance_rules(block, RelevanceRules(normalization=True)) as coverage: + assert coverage.installed == ("ln1",) + active = bridge(x) + assert torch.equal(active, baseline) + + def test_vjp_matches_detached_denom_oracle_with_unit_weight(self): + bridge = self._make_param_free_bridge() + block = _Block(bridge) + d = 16 + x = torch.randn(2, 5, d, requires_grad=True) + with use_relevance_rules(block, RelevanceRules(normalization=True)): + y = bridge(x) + y.sum().backward() + grad_rule = x.grad.clone() + + x_oracle = x.detach().clone().requires_grad_(True) + y_oracle = _denom_detached_oracle( + x_oracle, + torch.ones(d), + None, + uses_rms=False, + offset=False, + eps=1e-5, + ) + y_oracle.sum().backward() + torch.testing.assert_close(grad_rule, x_oracle.grad) + + +class TestFailClosedHookPrecedence: + def test_bwd_hook_raises_while_rule_active(self): + bridge = _make_bridge(native=True) + block = _Block(bridge) + x = torch.randn(2, 5, 16, requires_grad=True) + bridge.hook_normalized.add_hook(lambda g, hook=None: g, dir="bwd") + with pytest.raises(RelevanceRuleConflictError, match="Backward hooks"): + with use_relevance_rules(block, RelevanceRules(normalization=True)): + bridge(x) + bridge.hook_normalized.remove_hooks() + + def test_forward_edit_raises_while_rule_active(self): + bridge = _make_bridge(native=True) + block = _Block(bridge) + x = torch.randn(2, 5, 16) + bridge.hook_scale.add_hook(lambda t, hook=None: t * 2.0) + with pytest.raises(RelevanceRuleConflictError, match="forward hook edited"): + with use_relevance_rules(block, RelevanceRules(normalization=True)): + bridge(x) + bridge.hook_scale.remove_hooks() + + def test_observation_only_forward_hook_does_not_raise_while_rule_active(self): + """A hook that only observes (returns None) is not an edit and must not + trip the fail-closed check.""" + bridge = _make_bridge(native=True) + block = _Block(bridge) + x = torch.randn(2, 5, 16) + baseline = bridge(x) + cache = {} + + def observe(tensor, hook=None): + cache["normalized"] = tensor.detach() + return None + + bridge.hook_normalized.add_hook(observe) + with use_relevance_rules(block, RelevanceRules(normalization=True)): + active = bridge(x) + bridge.hook_normalized.remove_hooks() + assert torch.equal(active, baseline) + assert "normalized" in cache + + def test_bwd_hook_still_falls_back_with_warning_when_rule_inactive(self): + """Control: unchanged behavior when no rule is active (regression guard).""" + bridge = _make_bridge(native=True) + x = torch.randn(2, 5, 16, requires_grad=True) + bridge.hook_normalized.add_hook(lambda g, hook=None: g, dir="bwd") + with pytest.warns(UserWarning, match="Backward hooks"): + bridge(x) + bridge.hook_normalized.remove_hooks() + + def test_edit_still_falls_back_with_warning_when_rule_inactive(self): + """Control: unchanged behavior when no rule is active (regression guard).""" + bridge = _make_bridge(native=True) + x = torch.randn(2, 5, 16) + bridge.hook_scale.add_hook(lambda t, hook=None: t * 2.0) + with pytest.warns(UserWarning, match="reconstructed from the hooked values"): + bridge(x) + bridge.hook_scale.remove_hooks() diff --git a/tests/unit/model_bridge/supported_architectures/test_relevance_rule_mount_placeholders.py b/tests/unit/model_bridge/supported_architectures/test_relevance_rule_mount_placeholders.py new file mode 100644 index 0000000000..4e9d800089 --- /dev/null +++ b/tests/unit/model_bridge/supported_architectures/test_relevance_rule_mount_placeholders.py @@ -0,0 +1,128 @@ +"""Relevance-rule coverage classification for architectures that mount q/k/v-norm +or sandwich-norm placeholders instead of a rule-capable ln1/ln2. + +``use_relevance_rules`` targets the "normalization" rule kind only at the ln1/ln2 +mount name (see ``_CANONICAL_MOUNTS`` in ``transformer_lens.model_bridge._relevance_rules``), +matched on the last segment of a live module's registered name. These tests inspect +each adapter's declared ``component_mapping`` directly (no HF weights, no live module +wiring) to lock in the structural facts that make the mechanism's "not applicable" +behavior correct for these three documented edge cases, without needing to build a +working forward-capable module tree per adapter: + +- Gemma 3n: every per-block norm (including the per-head q/k/v norms) is a plain + ``GeneralizedComponent`` placeholder, and none of them is keyed "ln1"/"ln2". +- Gemma 4: ln1/ln2 exist (sandwich norms) but are plain ``GeneralizedComponent`` + placeholders, not ``NormalizationBridge`` -- rule-incapable even though the mount + name matches. The per-head q/k/v norms are, like Gemma 3n, keyed under "self_attn", + never "ln1"/"ln2". +- StableLM: ln1/ln2 ARE real ``NormalizationBridge`` instances (a genuine rule + target), but the per-head norms are keyed "q_norm"/"k_norm" under "attn" even + though the wrapped HF module is named "q_layernorm"/"k_layernorm" -- the mount + key, not the wrapped module's own name, is what the canonical-mount check reads. +""" + +from typing import Any + +from tests.unit.model_bridge.supported_architectures.helpers import make_bridge_cfg +from transformer_lens.model_bridge._relevance_rules import _RelevanceRuleCapable +from transformer_lens.model_bridge.generalized_components import NormalizationBridge +from transformer_lens.model_bridge.generalized_components.base import ( + GeneralizedComponent, +) +from transformer_lens.model_bridge.supported_architectures.gemma3n import ( + Gemma3nArchitectureAdapter, +) +from transformer_lens.model_bridge.supported_architectures.gemma4 import ( + Gemma4ArchitectureAdapter, +) +from transformer_lens.model_bridge.supported_architectures.stablelm import ( + StableLmArchitectureAdapter, +) + + +def _block_submodules(adapter: Any) -> dict: + return dict(adapter.component_mapping["blocks"].submodules) + + +def _attn_submodules(block_submodules: dict, attn_key: str) -> dict: + return dict(block_submodules[attn_key].submodules) + + +class TestGemma3nPlaceholders: + def _adapter(self) -> Gemma3nArchitectureAdapter: + cfg = make_bridge_cfg("Gemma3nForConditionalGeneration", n_key_value_heads=2, d_head=8) + return Gemma3nArchitectureAdapter(cfg) + + def test_no_ln1_or_ln2_key_at_block_level(self): + block_submodules = _block_submodules(self._adapter()) + assert "ln1" not in block_submodules + assert "ln2" not in block_submodules + + def test_qkv_norm_placeholders_are_plain_and_rule_incapable(self): + block_submodules = _block_submodules(self._adapter()) + attn_submodules = _attn_submodules(block_submodules, "self_attn") + for key in ("q_norm", "k_norm", "v_norm"): + component = attn_submodules[key] + assert type(component) is GeneralizedComponent + assert not isinstance(component, NormalizationBridge) + assert not isinstance(component, _RelevanceRuleCapable) + + +class TestGemma4Placeholders: + def _adapter(self) -> Gemma4ArchitectureAdapter: + from types import SimpleNamespace + + cfg = make_bridge_cfg("Gemma4ForConditionalGeneration", n_key_value_heads=1, d_head=8) + cfg.vision_config = SimpleNamespace( + hidden_size=32, num_hidden_layers=2, num_attention_heads=4 + ) + cfg.vision_soft_tokens_per_image = 4 + return Gemma4ArchitectureAdapter(cfg) + + def test_ln1_ln2_are_plain_sandwich_placeholders_not_normalization_bridge(self): + block_submodules = _block_submodules(self._adapter()) + for key in ("ln1", "ln2"): + component = block_submodules[key] + assert type(component) is GeneralizedComponent + assert not isinstance(component, NormalizationBridge) + assert not isinstance(component, _RelevanceRuleCapable) + + def test_qkv_norm_placeholders_are_not_keyed_ln1_or_ln2(self): + block_submodules = _block_submodules(self._adapter()) + attn_submodules = _attn_submodules(block_submodules, "attn") + for key in ("q_norm", "k_norm", "v_norm"): + assert key not in ("ln1", "ln2") + component = attn_submodules[key] + assert type(component) is GeneralizedComponent + assert not isinstance(component, _RelevanceRuleCapable) + + +class TestStableLmPlaceholders: + def _adapter(self) -> StableLmArchitectureAdapter: + cfg = make_bridge_cfg("StableLmForCausalLM", n_key_value_heads=2, d_head=8) + return StableLmArchitectureAdapter(cfg) + + def test_ln1_and_ln2_are_genuine_rule_targets(self): + """Unlike Gemma 3n/4, StableLM's block-level norms ARE rule-capable -- + this is the genuine positive case the other two are contrasted against.""" + block_submodules = _block_submodules(self._adapter()) + for key in ("ln1", "ln2"): + component = block_submodules[key] + assert isinstance(component, NormalizationBridge) + assert isinstance(component, _RelevanceRuleCapable) + assert component._relevance_rule_kinds == ("normalization",) + + def test_per_head_norms_are_keyed_q_norm_k_norm_not_ln1_ln2(self): + """The wrapped HF module is named q_layernorm/k_layernorm, but the mount + KEY the canonical-mount check reads is q_norm/k_norm -- distinct either + way from ln1/ln2, so these stay invisible to the "normalization" rule + regardless of which naming convention the underlying HF module uses.""" + block_submodules = _block_submodules(self._adapter()) + attn_submodules = _attn_submodules(block_submodules, "attn") + assert attn_submodules["q_norm"].name == "q_layernorm" + assert attn_submodules["k_norm"].name == "k_layernorm" + for key in ("q_norm", "k_norm"): + component = attn_submodules[key] + assert key not in ("ln1", "ln2") + assert type(component) is GeneralizedComponent + assert not isinstance(component, _RelevanceRuleCapable) diff --git a/tests/unit/model_bridge/test_relevance_rules.py b/tests/unit/model_bridge/test_relevance_rules.py new file mode 100644 index 0000000000..57cca39c25 --- /dev/null +++ b/tests/unit/model_bridge/test_relevance_rules.py @@ -0,0 +1,369 @@ +"""Tests for the scoped relevance-rule context: forward-identity, cleanup, and nesting. + +The fixture component below installs the real LN-rule primitive +(``transformer_lens.model_bridge._relevance_rules.ln_rule``) through the protocol +``use_relevance_rules`` relies on to find and toggle rule-capable components. That keeps +these tests focused on the scoping mechanics -- forward identity, gradient restoration, +nested contexts, exception safety, and positional (not class-based) targeting -- rather +than on any concrete NormalizationBridge or gated-MLP integration, which land in later +commits. + +The scoped context does not exist yet, so this fails collection with a single +ImportError -- the expected red state before the context is implemented. +""" + +import dataclasses + +import pytest +import torch +import torch.nn as nn + +from transformer_lens.model_bridge._relevance_rules import ( + RelevanceRuleCoverage, + RelevanceRules, + RelevanceRuleUnsupportedError, + ln_rule, + use_relevance_rules, +) + + +class _FakeNormComponent(nn.Module): + """A minimal ln1/ln2-style target: installs the real LN-rule primitive on request.""" + + _relevance_rule_kinds = ("normalization",) + + def __init__(self, eps: float = 1e-2): + super().__init__() + # Composing this division with a plain second normalization and a linear + # readout makes the LN-rule's denom-as-constant correction shrink linearly + # with eps, so a realistic normalization epsilon (1e-6) would leave the + # correction too small for torch.allclose to detect reliably end to end. + self.eps = eps + self._rule_active = False + + def _enable_relevance_rule(self, kind: str) -> None: + self._rule_active = True + + def _disable_relevance_rule(self, kind: str) -> None: + self._rule_active = False + + def forward(self, x: torch.Tensor) -> torch.Tensor: + denom = x.abs().mean(dim=-1, keepdim=True) + self.eps + if self._rule_active: + return ln_rule(x, denom) + return x / denom + + +class _PlainMount(nn.Module): + """Occupies a targeted mount name but implements no relevance-rule protocol.""" + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x + + +class _TinyBlock(nn.Module): + """Mimics a block's ln1/ln2 mount points plus a same-class q_norm and an MLP. + + ``q_norm`` uses the identical fake-normalization class as ``ln1``/``ln2`` so tests + can assert that targeting is positional (by mount name) rather than class-based. + """ + + def __init__(self): + super().__init__() + self.ln1: nn.Module = _FakeNormComponent() + self.q_norm = _FakeNormComponent() + self.ln2: nn.Module = _FakeNormComponent() + self.mlp = nn.Linear(4, 4) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.ln1(x) + x = self.q_norm(x) + x = self.ln2(x) + return self.mlp(x) + + +def _tiny_block() -> _TinyBlock: + block = _TinyBlock() + with torch.no_grad(): + block.mlp.weight.copy_(torch.eye(4) * 0.5) + block.mlp.bias.zero_() + return block + + +class _SandwichBlock(nn.Module): + """Mimics a sandwich-norm block: pre-norms ln1/ln2 plus post-norms ln1_post/ln2_post. + + Sandwich-norm architectures mount a second normalization after attention and after + the MLP at ``ln1_post``/``ln2_post``, so the LN-rule must reach those mounts as well + as the pre-norms rather than leaving them silently out of coverage. + """ + + def __init__(self): + super().__init__() + self.ln1: nn.Module = _FakeNormComponent() + self.ln1_post: nn.Module = _FakeNormComponent() + self.ln2: nn.Module = _FakeNormComponent() + self.ln2_post: nn.Module = _FakeNormComponent() + self.mlp = nn.Linear(4, 4) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.ln1(x) + x = self.ln1_post(x) + x = self.ln2(x) + x = self.ln2_post(x) + return self.mlp(x) + + +class _FakeGatedMLPComponent(nn.Module): + """A minimal mlp-mount target that answers to two rule kinds independently. + + A real gated-MLP node answers to both "activation" (Identity-rule on its + activation function) and "multiplicative_gate" (Half-rule on its gate*up + product) at the same mount, and either can be requested without the other, + so this fixture tracks the two kinds as separate booleans rather than one. + """ + + _relevance_rule_kinds = ("activation", "multiplicative_gate") + + def __init__(self): + super().__init__() + self._activation_rule_active = False + self._gate_rule_active = False + + def _enable_relevance_rule(self, kind: str) -> None: + if kind == "activation": + self._activation_rule_active = True + elif kind == "multiplicative_gate": + self._gate_rule_active = True + + def _disable_relevance_rule(self, kind: str) -> None: + if kind == "activation": + self._activation_rule_active = False + elif kind == "multiplicative_gate": + self._gate_rule_active = False + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x + + +class _Boom(Exception): + """Marker exception raised inside a context to test cleanup on failure.""" + + +def test_relevance_rules_defaults_to_no_rules(): + rules = RelevanceRules() + assert rules.normalization is False + assert rules.activation is False + assert rules.multiplicative_gate is False + assert rules.attention is False + + +def test_relevance_rules_is_frozen(): + rules = RelevanceRules(normalization=True) + with pytest.raises(dataclasses.FrozenInstanceError): + rules.normalization = False # type: ignore[misc] + + +def test_forward_is_identical_while_rule_active(): + block = _tiny_block() + x = torch.randn(2, 3, 4) + baseline = block(x) + with use_relevance_rules(block, RelevanceRules(normalization=True)): + active = block(x) + assert torch.equal(active, baseline) + + +def test_positional_targeting_excludes_same_class_component_at_other_mount(): + block = _tiny_block() + with use_relevance_rules(block, RelevanceRules(normalization=True)) as coverage: + assert block.ln1._rule_active is True + assert block.ln2._rule_active is True + assert block.q_norm._rule_active is False + assert set(coverage.installed) == {"ln1", "ln2"} + assert block.ln1._rule_active is False + assert block.ln2._rule_active is False + + +def test_normalization_rule_installs_on_sandwich_post_norm_mounts(): + block = _SandwichBlock() + with use_relevance_rules(block, RelevanceRules(normalization=True)) as coverage: + assert block.ln1._rule_active is True + assert block.ln1_post._rule_active is True + assert block.ln2._rule_active is True + assert block.ln2_post._rule_active is True + assert set(coverage.installed) == {"ln1", "ln1_post", "ln2", "ln2_post"} + assert coverage.skipped == () + assert block.ln1_post._rule_active is False + assert block.ln2_post._rule_active is False + + +def test_sandwich_post_norm_on_python_path_is_reported_skipped_never_absent(): + block = _SandwichBlock() + # A post-norm mount whose occupant takes the python-norm path (no rule protocol) + # must surface as skipped, never vanish from coverage as it did before the mount + # names were recognized. + block.ln2_post = _PlainMount() + with use_relevance_rules(block, RelevanceRules(normalization=True)) as coverage: + assert set(coverage.installed) == {"ln1", "ln1_post", "ln2"} + assert set(coverage.skipped) == {"ln2_post"} + + +def test_requesting_a_kind_with_no_canonical_mount_raises(): + block = _tiny_block() + # "attention" is a valid RelevanceRules field but has no canonical mount, so the + # request must raise rather than install nothing and return empty coverage. + with pytest.raises(ValueError, match="attention"): + with use_relevance_rules(block, RelevanceRules(attention=True)): + pass + + +def test_unsupported_component_at_targeted_mount_is_skipped(): + block = _tiny_block() + block.ln2 = _PlainMount() + with use_relevance_rules(block, RelevanceRules(normalization=True)) as coverage: + assert isinstance(coverage, RelevanceRuleCoverage) + assert set(coverage.installed) == {"ln1"} + assert set(coverage.skipped) == {"ln2"} + + +def test_requesting_unsupported_kind_on_capable_component_raises(): + block = _tiny_block() + block.mlp = _FakeGatedMLPComponent() + # This mount implements the protocol and currently only supports "activation", + # but -- unlike a mount that never deals with "multiplicative_gate" at all -- + # names it in _relevance_rule_unsupported_kinds as one it is expected to honor + # here and currently cannot, so requesting it raises instead of being skipped. + block.mlp._relevance_rule_kinds = ("activation",) + block.mlp._relevance_rule_unsupported_kinds = ("multiplicative_gate",) + with pytest.raises(RelevanceRuleUnsupportedError, match="mlp"): + with use_relevance_rules(block, RelevanceRules(multiplicative_gate=True)): + pass + assert block.mlp._gate_rule_active is False + + +def test_no_rules_requested_installs_nothing(): + block = _tiny_block() + with use_relevance_rules(block, RelevanceRules()) as coverage: + assert coverage.installed == () + assert coverage.skipped == () + assert block.ln1._rule_active is False + assert block.ln2._rule_active is False + + +def test_ordinary_gradients_restored_after_exit(): + block = _tiny_block() + x = torch.randn(2, 3, 4, requires_grad=True) + + with use_relevance_rules(block, RelevanceRules(normalization=True)): + y_rule = block(x) + (grad_rule,) = torch.autograd.grad(y_rule.sum(), x) + + assert block.ln1._rule_active is False + assert block.ln2._rule_active is False + + x_plain = x.detach().clone().requires_grad_(True) + y_plain = block(x_plain) + (grad_plain,) = torch.autograd.grad(y_plain.sum(), x_plain) + + # The LN-rule treats the denominator as constant, so its VJP differs from + # ordinary autodiff through the same division wherever the denominator + # actually depends on the input -- true for every row of this fixture. + assert not torch.allclose(grad_rule, grad_plain) + + # A second plain pass confirms the exit left no residual rule state: it must + # reproduce grad_plain exactly rather than drifting toward grad_rule. + x_plain_again = x.detach().clone().requires_grad_(True) + y_plain_again = block(x_plain_again) + (grad_plain_again,) = torch.autograd.grad(y_plain_again.sum(), x_plain_again) + torch.testing.assert_close(grad_plain_again, grad_plain) + + +def test_nested_contexts_restore_outer_state_on_inner_exit(): + block = _tiny_block() + x = torch.randn(2, 3, 4) + baseline = block(x) + + with use_relevance_rules(block, RelevanceRules(normalization=True)) as outer_coverage: + assert block.ln1._rule_active is True + with use_relevance_rules(block, RelevanceRules(normalization=True)) as inner_coverage: + assert block.ln1._rule_active is True + assert torch.equal(block(x), baseline) + # The inner exit must not disable the rule the outer context still needs. + assert block.ln1._rule_active is True + assert block.ln2._rule_active is True + assert torch.equal(block(x), baseline) + + assert block.ln1._rule_active is False + assert block.ln2._rule_active is False + assert set(outer_coverage.installed) == {"ln1", "ln2"} + assert set(inner_coverage.installed) == {"ln1", "ln2"} + + +def test_exception_inside_context_leaves_no_rule_state(): + block = _tiny_block() + x = torch.randn(2, 3, 4) + baseline = block(x) + + with pytest.raises(_Boom): + with use_relevance_rules(block, RelevanceRules(normalization=True)): + assert block.ln1._rule_active is True + raise _Boom("failure inside the scoped context") + + assert block.ln1._rule_active is False + assert block.ln2._rule_active is False + assert torch.equal(block(x), baseline) + + +def test_exception_during_nested_context_restores_outer_state(): + block = _tiny_block() + x = torch.randn(2, 3, 4) + baseline = block(x) + + with use_relevance_rules(block, RelevanceRules(normalization=True)): + with pytest.raises(_Boom): + with use_relevance_rules(block, RelevanceRules(normalization=True)): + raise _Boom("failure inside the nested scoped context") + # The outer context is still active after the inner one unwinds. + assert block.ln1._rule_active is True + assert torch.equal(block(x), baseline) + + assert block.ln1._rule_active is False + assert block.ln2._rule_active is False + + +def test_one_kind_requested_on_a_two_kind_mount_leaves_the_other_kind_inactive(): + block = _tiny_block() + block.mlp = _FakeGatedMLPComponent() + with use_relevance_rules(block, RelevanceRules(multiplicative_gate=True)) as coverage: + assert block.mlp._gate_rule_active is True + assert block.mlp._activation_rule_active is False + assert coverage.installed == ("mlp",) + assert block.mlp._gate_rule_active is False + + +def test_both_kinds_requested_together_both_activate_on_the_same_mount(): + block = _tiny_block() + block.mlp = _FakeGatedMLPComponent() + with use_relevance_rules( + block, RelevanceRules(activation=True, multiplicative_gate=True) + ) as coverage: + assert block.mlp._activation_rule_active is True + assert block.mlp._gate_rule_active is True + assert set(coverage.installed) == {"mlp"} + assert block.mlp._activation_rule_active is False + assert block.mlp._gate_rule_active is False + + +def test_nested_scopes_over_different_kinds_on_one_mount_refcount_independently(): + block = _tiny_block() + block.mlp = _FakeGatedMLPComponent() + with use_relevance_rules(block, RelevanceRules(activation=True)): + assert block.mlp._activation_rule_active is True + with use_relevance_rules(block, RelevanceRules(multiplicative_gate=True)): + assert block.mlp._activation_rule_active is True + assert block.mlp._gate_rule_active is True + # The inner (multiplicative_gate-only) scope's exit must not disable + # the outer scope's independently refcounted activation rule. + assert block.mlp._activation_rule_active is True + assert block.mlp._gate_rule_active is False + assert block.mlp._activation_rule_active is False diff --git a/tests/unit/tools/test_relevance_rules.py b/tests/unit/tools/test_relevance_rules.py new file mode 100644 index 0000000000..c8a6e35046 --- /dev/null +++ b/tests/unit/tools/test_relevance_rules.py @@ -0,0 +1,444 @@ +"""Analytic closed-form tests for the LN-, Identity-, and Half-relevance-rule primitives. + +Model-free: every primitive is a plain ``torch.autograd.Function`` exercised on +synthetic tensors, so no model, backward hook, or ``.data`` access is involved. Each +test asserts two properties independently: the forward value is exactly the native +(ordinary-autograd) value, and the backward value matches the rule's closed-form VJP +rather than what ordinary autodiff would produce. +""" + +import math +from functools import partial + +import pytest +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers.pytorch_utils import Conv1D + +from transformer_lens.model_bridge._relevance_rules import ( + half_rule, + identity_rule, + ln_rule, +) +from transformer_lens.model_bridge.generalized_components.mlp import ( + normalize_mlp_weight, + weight_layout_in_out, +) + +DTYPES = [torch.float32, torch.float64] + + +def _leaf(values: torch.Tensor, dtype: torch.dtype) -> torch.Tensor: + return values.to(dtype).clone().requires_grad_(True) + + +def _sample_rows(dtype: torch.dtype) -> torch.Tensor: + """A batch covering an all-zero, all-negative, mixed-sign, and positive row.""" + values = torch.tensor( + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [-1.0, -2.0, -0.5, -3.0, -0.25], + [1.0, -2.0, 3.0, -4.0, 0.5], + [2.0, 4.0, 6.0, 8.0, 10.0], + ] + ) + return values.to(dtype) + + +def _normal_cdf(x: torch.Tensor) -> torch.Tensor: + return 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) + + +class TestLNRule: + """Forward equals ``numerator / denom``; the VJP treats ``denom`` as constant.""" + + @pytest.mark.parametrize("dtype", DTYPES) + def test_forward_matches_plain_division(self, dtype): + numerator = _sample_rows(dtype) + denom = numerator.abs().mean(dim=-1, keepdim=True) + 1e-6 + assert torch.equal(ln_rule(numerator, denom), numerator / denom) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_backward_treats_denom_as_constant(self, dtype): + eps = 1e-6 + base = _sample_rows(dtype) + grad_out = torch.ones_like(base) + + x_rule = _leaf(base, dtype) + denom_rule = x_rule.abs().mean(dim=-1, keepdim=True) + eps + y_rule = ln_rule(x_rule, denom_rule) + (grad_rule,) = torch.autograd.grad(y_rule, x_rule, grad_outputs=grad_out) + + # The closed-form VJP of this rule: grad_x = grad_out / denom, with no + # contribution from d(denom)/dx. + expected = grad_out / denom_rule.detach() + torch.testing.assert_close(grad_rule, expected) + + # Plain autodiff through the same expression differentiates the denom too, + # so it disagrees with the rule everywhere the denom actually depends on x + # (every row here, since eps alone would zero out that dependency). + x_plain = _leaf(base, dtype) + denom_plain = x_plain.abs().mean(dim=-1, keepdim=True) + eps + y_plain = x_plain / denom_plain + (grad_plain,) = torch.autograd.grad(y_plain, x_plain, grad_outputs=grad_out) + assert not torch.allclose(grad_plain, grad_rule) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_backward_zero_row_has_no_division_by_zero(self, dtype): + eps = 1e-6 + x = _leaf(_sample_rows(dtype)[:1], dtype) + denom = x.abs().mean(dim=-1, keepdim=True) + eps + y = ln_rule(x, denom) + (grad,) = torch.autograd.grad(y, x, grad_outputs=torch.ones_like(x)) + assert torch.isfinite(grad).all() + torch.testing.assert_close(grad, torch.full_like(x, 1.0 / eps)) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_forward_and_backward_on_non_contiguous_input(self, dtype): + base = _sample_rows(dtype).t() + assert not base.is_contiguous() + numerator = base.clone().requires_grad_(True) + denom = numerator.abs().mean(dim=-1, keepdim=True) + 1e-6 + + y = ln_rule(numerator, denom) + assert torch.equal(y, numerator.detach() / denom.detach()) + + grad_out = torch.ones_like(numerator) + (grad,) = torch.autograd.grad(y, numerator, grad_outputs=grad_out) + torch.testing.assert_close(grad, grad_out / denom.detach()) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_batched_leading_dims(self, dtype): + numerator = _leaf(torch.randn(2, 3, 4), dtype) + denom = numerator.abs().mean(dim=-1, keepdim=True) + 1e-6 + y = ln_rule(numerator, denom) + assert y.shape == numerator.shape + assert torch.equal(y, numerator.detach() / denom.detach()) + + (grad,) = torch.autograd.grad(y, numerator, grad_outputs=torch.ones_like(numerator)) + torch.testing.assert_close(grad, torch.ones_like(numerator) / denom.detach()) + + +class TestIdentityRule: + """Forward equals the native activation; the VJP is ``grad_out * phi(x)``. + + ``phi`` has a closed form for each activation tested here: ``sigmoid`` for SiLU + and the Gaussian CDF for exact GELU. Both are, by construction, the removable- + singularity limit of ``f(x) / x`` at ``x == 0``, which is the direct oracle used + for the tanh-approximate GELU, where no simpler closed form applies. + """ + + @pytest.mark.parametrize("dtype", DTYPES) + def test_forward_matches_native_silu(self, dtype): + x = _sample_rows(dtype) + assert torch.equal(identity_rule(x, F.silu), F.silu(x)) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_backward_matches_sigmoid_for_silu(self, dtype): + x = _leaf(_sample_rows(dtype), dtype) + y = identity_rule(x, F.silu) + grad_out = torch.ones_like(x) + (grad,) = torch.autograd.grad(y, x, grad_outputs=grad_out) + torch.testing.assert_close(grad, grad_out * torch.sigmoid(x)) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_forward_matches_native_gelu_exact(self, dtype): + x = _sample_rows(dtype) + act_fn = partial(F.gelu, approximate="none") + assert torch.equal(identity_rule(x, act_fn), act_fn(x)) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_backward_matches_gaussian_cdf_for_exact_gelu(self, dtype): + x = _leaf(_sample_rows(dtype), dtype) + act_fn = partial(F.gelu, approximate="none") + y = identity_rule(x, act_fn) + grad_out = torch.ones_like(x) + (grad,) = torch.autograd.grad(y, x, grad_outputs=grad_out) + torch.testing.assert_close(grad, grad_out * _normal_cdf(x)) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_forward_matches_native_gelu_approx(self, dtype): + x = _sample_rows(dtype) + act_fn = partial(F.gelu, approximate="tanh") + assert torch.equal(identity_rule(x, act_fn), act_fn(x)) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_backward_matches_ratio_with_zero_limit_for_approx_gelu(self, dtype): + x = _leaf(_sample_rows(dtype), dtype) + act_fn = partial(F.gelu, approximate="tanh") + y = identity_rule(x, act_fn) + grad_out = torch.ones_like(x) + (grad,) = torch.autograd.grad(y, x, grad_outputs=grad_out) + + with torch.no_grad(): + safe_x = torch.where(x == 0, torch.ones_like(x), x) + expected_phi = torch.where(x == 0, torch.full_like(x, 0.5), act_fn(x.detach()) / safe_x) + assert torch.isfinite(grad).all() + torch.testing.assert_close(grad, grad_out * expected_phi) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_forward_and_backward_on_non_contiguous_input(self, dtype): + base = _sample_rows(dtype).t() + assert not base.is_contiguous() + x = base.clone().requires_grad_(True) + + y = identity_rule(x, F.silu) + assert torch.equal(y, F.silu(x.detach())) + + grad_out = torch.ones_like(x) + (grad,) = torch.autograd.grad(y, x, grad_outputs=grad_out) + torch.testing.assert_close(grad, grad_out * torch.sigmoid(x.detach())) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_batched_leading_dims(self, dtype): + x = _leaf(torch.randn(2, 3, 4), dtype) + y = identity_rule(x, F.silu) + assert y.shape == x.shape + assert torch.equal(y, F.silu(x.detach())) + + (grad,) = torch.autograd.grad(y, x, grad_outputs=torch.ones_like(x)) + torch.testing.assert_close(grad, torch.sigmoid(x.detach())) + + +class TestHalfRule: + """Forward equals ``u * v``; the VJP halves each ordinary product-rule term.""" + + @pytest.mark.parametrize("dtype", DTYPES) + def test_forward_matches_plain_product(self, dtype): + u = _sample_rows(dtype) + v = _sample_rows(dtype).flip(0) + assert torch.equal(half_rule(u, v), u * v) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_backward_halves_each_operand_gradient(self, dtype): + u = _leaf(_sample_rows(dtype), dtype) + v = _leaf(_sample_rows(dtype).flip(0), dtype) + grad_out = torch.ones_like(u) + + y = half_rule(u, v) + grad_u, grad_v = torch.autograd.grad(y, (u, v), grad_outputs=grad_out) + torch.testing.assert_close(grad_u, 0.5 * grad_out * v.detach()) + torch.testing.assert_close(grad_v, 0.5 * grad_out * u.detach()) + + # Ordinary autodiff of u * v would give the full (unhalved) product-rule + # terms, so the rule must disagree with it. + u_plain = _leaf(_sample_rows(dtype), dtype) + v_plain = _leaf(_sample_rows(dtype).flip(0), dtype) + y_plain = u_plain * v_plain + grad_u_plain, grad_v_plain = torch.autograd.grad( + y_plain, (u_plain, v_plain), grad_outputs=grad_out + ) + assert not torch.allclose(grad_u_plain, grad_u) + assert not torch.allclose(grad_v_plain, grad_v) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_forward_and_backward_on_non_contiguous_input(self, dtype): + u = _sample_rows(dtype).t().clone().requires_grad_(True) + v = _sample_rows(dtype).flip(0).t().clone().requires_grad_(True) + assert not u.is_contiguous() + assert not v.is_contiguous() + + y = half_rule(u, v) + assert torch.equal(y, u.detach() * v.detach()) + + grad_out = torch.ones_like(u) + grad_u, grad_v = torch.autograd.grad(y, (u, v), grad_outputs=grad_out) + torch.testing.assert_close(grad_u, 0.5 * grad_out * v.detach()) + torch.testing.assert_close(grad_v, 0.5 * grad_out * u.detach()) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_batched_leading_dims(self, dtype): + u = _leaf(torch.randn(2, 3, 4), dtype) + v = _leaf(torch.randn(2, 3, 4), dtype) + y = half_rule(u, v) + assert y.shape == u.shape + assert torch.equal(y, u.detach() * v.detach()) + + grad_out = torch.ones_like(u) + grad_u, grad_v = torch.autograd.grad(y, (u, v), grad_outputs=grad_out) + torch.testing.assert_close(grad_u, 0.5 * grad_out * v.detach()) + torch.testing.assert_close(grad_v, 0.5 * grad_out * u.detach()) + + +class _Proj: + """Minimal stand-in for a projection bridge: carries only what + ``weight_layout_in_out``/``normalize_mlp_weight`` read (``original_component``).""" + + def __init__(self, original_component): + self.original_component = original_component + + +class TestGatedMLPWeightOrientation: + """``MLPBridge``'s ``W_gate``/``W_in``/``W_out`` read the underlying projection + through ``weight_layout_in_out``/``normalize_mlp_weight``, so an orientation bug in + those helpers would silently transpose a gate/up/down weight for one backing class. + Covers both HF module classes: ``nn.Linear`` (weight stored ``[out, in]``, + transposed to TL orientation) and ``Conv1D`` (weight stored ``[in, out]``, already + TL-oriented). + """ + + @pytest.fixture(params=["nn.Linear", "Conv1D"]) + def backing_class(self, request): + return request.param + + def _make_gate_up_down(self, backing_class: str, d_model: int = 3, d_mlp: int = 5): + torch.manual_seed(0) + if backing_class == "nn.Linear": + gate_proj = nn.Linear(d_model, d_mlp) + up_proj = nn.Linear(d_model, d_mlp) + down_proj = nn.Linear(d_mlp, d_model) + else: + gate_proj = Conv1D(d_mlp, d_model) + up_proj = Conv1D(d_mlp, d_model) + down_proj = Conv1D(d_model, d_mlp) + return gate_proj, up_proj, down_proj + + def _tl_weight(self, proj: torch.nn.Module, pattern: str) -> torch.Tensor: + wrapper = _Proj(proj) + layout = weight_layout_in_out(wrapper) + return normalize_mlp_weight(proj.weight, layout, wrapper, pattern=pattern) + + def test_tl_oriented_matmul_reproduces_native_projection(self, backing_class): + gate_proj, up_proj, down_proj = self._make_gate_up_down(backing_class) + x = torch.randn(2, 3) + + w_gate = self._tl_weight(gate_proj, pattern="in") + w_in = self._tl_weight(up_proj, pattern="in") + w_out = self._tl_weight(down_proj, pattern="out") + + assert torch.allclose(x @ w_gate + gate_proj.bias, gate_proj(x), atol=1e-6) + assert torch.allclose(x @ w_in + up_proj.bias, up_proj(x), atol=1e-6) + hidden = torch.randn(2, 5) + assert torch.allclose(hidden @ w_out + down_proj.bias, down_proj(hidden), atol=1e-6) + + +class TestPinnedReferenceParity: + """Tolerant parity against ``FarnoushRJ/RelP`` pinned at + ``8219d6dc417c3fd7f318342cf61cd2a0c20b7250``. + + That repository vendors an unrelated pre-Bridge TransformerLens fork, so its + rule formulas are reimplemented here directly from the pinned commit's + component diffs rather than imported: + + - LN-rule (``transformer_lens/components/rms_norm.py``): ``x / scale.detach()``. + - Identity-rule (``transformer_lens/utilities/activation_functions.py``, + class ``ModifiedAct``): ``zp = stabilize(x); zp * (act_fn(x) / zp).detach()``, + where ``stabilize(z) = z + ((z == 0) + sign(z)) * 1e-6`` + (``transformer_lens/lrp_utils.py``). + - Half-rule (``transformer_lens/components/mlps/gated_mlp.py``): + ``z = u * v; z / 2 + (z / 2).detach()``. + + The LN- and Half-rule reference formulas produce the same VJP as this module's + primitives to floating-point precision. The Identity-rule reference formula + does not: its epsilon stabilizer only approximates the paper-defined factor + away from ``x == 0``, and collapses to exactly zero at ``x == 0`` where the + paper-defined factor's removable-singularity limit is ``0.5``. + """ + + @staticmethod + def _reference_stabilize(z: torch.Tensor) -> torch.Tensor: + return z + ((z == 0).to(z.dtype) + torch.sign(z)) * 1e-6 + + @classmethod + def _reference_ln_rule_grad(cls, x: torch.Tensor, denom_fn) -> torch.Tensor: + x = x.clone().requires_grad_(True) + denom = denom_fn(x) + y = x / denom.detach() + (grad,) = torch.autograd.grad(y, x, grad_outputs=torch.ones_like(x)) + return grad + + @classmethod + def _reference_identity_rule_grad(cls, x: torch.Tensor, act_fn) -> torch.Tensor: + x = x.clone().requires_grad_(True) + z = act_fn(x) + zp = cls._reference_stabilize(x) + y = zp * (z / zp).detach() + (grad,) = torch.autograd.grad(y, x, grad_outputs=torch.ones_like(x)) + return grad + + @classmethod + def _reference_half_rule_grad(cls, u: torch.Tensor, v: torch.Tensor): + u = u.clone().requires_grad_(True) + v = v.clone().requires_grad_(True) + z = u * v + y = z / 2 + (z / 2).detach() + grad_u, grad_v = torch.autograd.grad(y, (u, v), grad_outputs=torch.ones_like(u)) + return grad_u, grad_v + + @pytest.mark.parametrize("dtype", DTYPES) + def test_ln_rule_matches_reference_grad(self, dtype): + def denom_fn(t: torch.Tensor) -> torch.Tensor: + return (t.pow(2).mean(-1, keepdim=True) + 1e-6).sqrt() + + x = _sample_rows(dtype) + + x_rule = _leaf(x, dtype) + denom_rule = denom_fn(x_rule) + y_rule = ln_rule(x_rule, denom_rule) + (grad_rule,) = torch.autograd.grad(y_rule, x_rule, grad_outputs=torch.ones_like(x_rule)) + + grad_reference = self._reference_ln_rule_grad(x, denom_fn) + torch.testing.assert_close(grad_rule, grad_reference) + + @pytest.mark.parametrize("dtype", DTYPES) + @pytest.mark.parametrize( + "act_fn", + [F.silu, partial(F.gelu, approximate="none"), partial(F.gelu, approximate="tanh")], + ids=["silu", "gelu_exact", "gelu_tanh"], + ) + def test_identity_rule_matches_reference_away_from_zero(self, dtype, act_fn): + x = _sample_rows(dtype) + nonzero_mask = x != 0 + + x_rule = _leaf(x, dtype) + y_rule = identity_rule(x_rule, act_fn) + (grad_rule,) = torch.autograd.grad(y_rule, x_rule, grad_outputs=torch.ones_like(x_rule)) + + grad_reference = self._reference_identity_rule_grad(x, act_fn) + + torch.testing.assert_close( + grad_rule[nonzero_mask], + grad_reference[nonzero_mask], + rtol=1e-4, + atol=1e-5, + ) + + @pytest.mark.parametrize("dtype", DTYPES) + @pytest.mark.parametrize( + "act_fn", + [F.silu, partial(F.gelu, approximate="none"), partial(F.gelu, approximate="tanh")], + ids=["silu", "gelu_exact", "gelu_tanh"], + ) + def test_identity_rule_exact_zero_discrepancy_is_documented(self, dtype, act_fn): + """At ``x == 0`` this module's Identity-rule uses the paper-defined + removable-singularity limit ``0.5``, while the pinned reference's epsilon + stabilizer yields exactly ``0``. Assert both values explicitly, rather than + letting a tolerance absorb the gap, so a change to either side's zero + handling is caught instead of silently passing. + """ + x = torch.zeros(3, dtype=dtype) + + x_rule = _leaf(x, dtype) + y_rule = identity_rule(x_rule, act_fn) + (grad_rule,) = torch.autograd.grad(y_rule, x_rule, grad_outputs=torch.ones_like(x_rule)) + torch.testing.assert_close(grad_rule, torch.full_like(x, 0.5)) + + grad_reference = self._reference_identity_rule_grad(x, act_fn) + torch.testing.assert_close(grad_reference, torch.zeros_like(x)) + + @pytest.mark.parametrize("dtype", DTYPES) + def test_half_rule_matches_reference_grad(self, dtype): + u = _sample_rows(dtype) + v = _sample_rows(dtype).flip(0) + + u_rule = _leaf(u, dtype) + v_rule = _leaf(v, dtype) + y_rule = half_rule(u_rule, v_rule) + grad_u_rule, grad_v_rule = torch.autograd.grad( + y_rule, (u_rule, v_rule), grad_outputs=torch.ones_like(u_rule) + ) + + grad_u_reference, grad_v_reference = self._reference_half_rule_grad(u, v) + torch.testing.assert_close(grad_u_rule, grad_u_reference) + torch.testing.assert_close(grad_v_rule, grad_v_reference) diff --git a/transformer_lens/model_bridge/_relevance_rules.py b/transformer_lens/model_bridge/_relevance_rules.py new file mode 100644 index 0000000000..5da1bbf80d --- /dev/null +++ b/transformer_lens/model_bridge/_relevance_rules.py @@ -0,0 +1,348 @@ +"""Forward-equivalent relevance-rule primitives, plus the scoped context that installs them. + +Each primitive is a ``torch.autograd.Function`` that reproduces its native forward +value exactly while replacing the backward pass with the rule's closed-form VJP. +``use_relevance_rules`` installs these rules on a model's canonical mount points only +for the duration of a ``with`` block, targeting components positionally (by mount +name, never by class) and reporting which mounts were installed versus skipped. +""" + +import dataclasses +from contextlib import contextmanager +from typing import ( + Any, + Callable, + Dict, + Iterator, + List, + Mapping, + Protocol, + Tuple, + runtime_checkable, +) + +import torch +import torch.nn as nn + + +def ln_rule_grad(grad_output: torch.Tensor, denom: torch.Tensor) -> torch.Tensor: + """Core LN-rule VJP: divide by ``denom`` without differentiating through it. + + Shared by the ``ln_rule`` primitive below and by any integration (such as + ``NormalizationBridge``) that wraps a component's own native forward call + instead of reproducing the division itself. + """ + return grad_output / denom + + +class _LNRule(torch.autograd.Function): + """LN-rule: forward is ``numerator / denom``; the VJP treats ``denom`` as constant.""" + + @staticmethod + def forward(ctx: Any, numerator: torch.Tensor, denom: torch.Tensor) -> torch.Tensor: + ctx.save_for_backward(denom) + return numerator / denom + + @staticmethod + def backward(ctx: Any, grad_output: torch.Tensor) -> Tuple[torch.Tensor, None]: + (denom,) = ctx.saved_tensors + return ln_rule_grad(grad_output, denom), None + + +def ln_rule(numerator: torch.Tensor, denom: torch.Tensor) -> torch.Tensor: + """Apply the LN-rule: native division forward, denom-as-constant backward.""" + result: torch.Tensor = _LNRule.apply(numerator, denom) + return result + + +class _IdentityRule(torch.autograd.Function): + """Identity-rule: forward is the native activation; the VJP is ``grad_out * phi(x)``. + + ``phi`` is ``f(x) / x``, the removable singularity at ``x == 0`` filled in with its + limit ``0.5``. This holds for any elementwise activation with ``f(0) == 0`` and a + well-defined derivative at zero, which covers SiLU and both GELU variants. + """ + + @staticmethod + def forward( + ctx: Any, x: torch.Tensor, act_fn: Callable[[torch.Tensor], torch.Tensor] + ) -> torch.Tensor: + y = act_fn(x) + ctx.save_for_backward(x, y) + return y + + @staticmethod + def backward(ctx: Any, grad_output: torch.Tensor) -> Tuple[torch.Tensor, None]: + x, y = ctx.saved_tensors + safe_x = torch.where(x == 0, torch.ones_like(x), x) + phi = torch.where(x == 0, torch.full_like(x, 0.5), y / safe_x) + return grad_output * phi, None + + +def identity_rule(x: torch.Tensor, act_fn: Callable[[torch.Tensor], torch.Tensor]) -> torch.Tensor: + """Apply the Identity-rule for an elementwise activation with ``f(0) == 0``.""" + result: torch.Tensor = _IdentityRule.apply(x, act_fn) + return result + + +class _HalfRule(torch.autograd.Function): + """Half-rule: forward is ``u * v``; the VJP halves each ordinary product-rule term.""" + + @staticmethod + def forward(ctx: Any, u: torch.Tensor, v: torch.Tensor) -> torch.Tensor: + ctx.save_for_backward(u, v) + return u * v + + @staticmethod + def backward(ctx: Any, grad_output: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + u, v = ctx.saved_tensors + return 0.5 * grad_output * v, 0.5 * grad_output * u + + +def half_rule(u: torch.Tensor, v: torch.Tensor) -> torch.Tensor: + """Apply the Half-rule: native product forward, evenly split backward.""" + result: torch.Tensor = _HalfRule.apply(u, v) + return result + + +class _ScaleGradient(torch.autograd.Function): + """Identity forward; the VJP scales the incoming gradient by a constant factor.""" + + @staticmethod + def forward(ctx: Any, x: torch.Tensor, factor: float) -> torch.Tensor: + ctx.factor = factor + return x + + @staticmethod + def backward(ctx: Any, grad_output: torch.Tensor) -> Tuple[torch.Tensor, None]: + return ctx.factor * grad_output, None + + +def scale_gradient(x: torch.Tensor, factor: float) -> torch.Tensor: + """Pass ``x`` through unchanged while scaling its gradient by ``factor``. + + The Half-rule on a product ``u * v`` halves each ordinary product-rule term, + which is the same as halving the single gradient that enters the product before + it splits. When the product is computed inside an opaque module the bridge cannot + reach term by term (its native forward is called as one unit), scaling the + gradient entering the product by ``0.5`` reproduces the Half-rule at that point + without altering the native forward value. + """ + result: torch.Tensor = _ScaleGradient.apply(x, factor) + return result + + +class RelevanceRuleConflictError(RuntimeError): + """A hook would silently break a rule-active forward/backward invariant. + + Raised instead of the ordinary warn-and-fall-back a component would use when + no rule is active, since falling back while a rule is active would compose the + rule with the hook edit and break the bit-identical-forward guarantee. + """ + + +class RelevanceRuleUnsupportedError(RuntimeError): + """A requested relevance rule cannot be installed on an otherwise-capable component. + + Raised at ``use_relevance_rules`` entry, before any forward or backward pass, when + a component reports the requested kind in its own ``_relevance_rule_unsupported_kinds`` + -- for example a gated-MLP recompute path backed by an unrecognized weight-orientation + class, or an activation form the Identity-rule does not support. Distinct from a + kind that is simply absent from ``_relevance_rule_kinds`` without being named there + (reported ``skipped``, not raised): that covers a component not implementing the + protocol at all, or one whose mount genuinely never deals with the kind (for example + normalization on a dispatch path the LN-rule does not wrap), both benign + non-applicability rather than a rule request the component was expected to honor. + """ + + +@dataclasses.dataclass(frozen=True) +class RelevanceRules: + """Which relevance rules to request for the duration of a ``use_relevance_rules`` scope. + + Each field names a rule kind. Setting it ``True`` requests that rule wherever a + component at that kind's canonical mount point implements ``_RelevanceRuleCapable``. + Unset fields (the default) leave the corresponding components untouched. + """ + + normalization: bool = False + activation: bool = False + multiplicative_gate: bool = False + attention: bool = False + + +@dataclasses.dataclass +class RelevanceRuleCoverage: + """Which canonical mounts a ``use_relevance_rules`` scope installed versus skipped. + + ``installed`` holds the dotted path of every mount where a requested rule kind was + actually enabled. ``skipped`` holds the dotted path of every mount that matched a + requested kind's canonical mount name but did not implement the relevance-rule + protocol there, so no rule could be installed. + """ + + installed: Tuple[str, ...] + skipped: Tuple[str, ...] + + +@runtime_checkable +class _RelevanceRuleCapable(Protocol): + """Structural contract a component must satisfy to accept a relevance rule. + + ``_relevance_rule_kinds`` names every ``RelevanceRules`` field the component + answers to at its current mount -- a gated-MLP node answers to both + "activation" (Identity-rule on its activation function) and + "multiplicative_gate" (Half-rule on its gate*up product) independently, since + either can be requested without the other. ``_enable_relevance_rule``/ + ``_disable_relevance_rule`` take the specific kind being toggled and touch + only that kind's state, without touching model configuration, so the + component's own state is the only thing that changes and only for the + scope's duration. + + A component may optionally also define ``_relevance_rule_unsupported_kinds`` + (a ``Tuple[str, ...]``, not part of this structural protocol so components that + omit it stay isinstance-compatible) naming kinds it is expected to honor at its + mount but currently cannot -- ``use_relevance_rules`` raises + ``RelevanceRuleUnsupportedError`` for those instead of reporting them skipped. + """ + + _relevance_rule_kinds: Tuple[str, ...] + + def _enable_relevance_rule(self, kind: str) -> None: + ... + + def _disable_relevance_rule(self, kind: str) -> None: + ... + + +# Canonical mount name per rule kind. Targeting is positional: a component is only +# considered for a kind when it sits at that kind's mount name, never by isinstance, +# so a same-class component mounted elsewhere (for example a q_norm sharing +# NormalizationBridge's class) is left untouched. The normalization kind lists both +# the pre-norm mounts (ln1, ln2) and the sandwich post-norm mounts (ln1_post, +# ln2_post) so the LN-rule reaches the post-attention/post-MLP norms that +# sandwich-norm architectures mount there, matching the pinned RelP reference. +_CANONICAL_MOUNTS: Mapping[str, Tuple[str, ...]] = { + "normalization": ("ln1", "ln2", "ln1_post", "ln2_post"), + "activation": ("mlp",), + "multiplicative_gate": ("mlp",), +} + + +def _iter_canonical_mount_candidates( + model: nn.Module, mount_names: Tuple[str, ...] +) -> Iterator[Tuple[str, _RelevanceRuleCapable]]: + """Yield each distinct module reachable at one of ``mount_names``, once. + + A bridge component reachable at a canonical mount name (for example + ``blocks.0.ln1``) is also reachable, under the same parent, through the + raw HF module tree the bridge wraps in place (for example + ``blocks.0._original_component.input_layernorm``) -- both names resolve to + the identical object. ``nn.Module.named_modules()`` deduplicates by object + identity and keeps only whichever path it visits first, which is the raw + HF-attribute path (registered before the canonical alias), so on a real + assembled model the canonical name is silently never seen. Walking with + ``remove_duplicate=False`` restores every path so the canonical name is + visible, and picking the fewest-dot-separated-segments path per object + (breaking a tie between two paths that both happen to end in a mount name, + such as ``mlp``, which HF's own attribute name also frequently matches) + reports the shallower, canonical-looking path rather than an internal one. + """ + best_by_id: Dict[int, Tuple[str, _RelevanceRuleCapable]] = {} + for name, module in model.named_modules(remove_duplicate=False): + if name.rsplit(".", 1)[-1] not in mount_names: + continue + existing = best_by_id.get(id(module)) + if existing is None or name.count(".") < existing[0].count("."): + best_by_id[id(module)] = (name, module) + yield from best_by_id.values() + + +def _acquire_rule(module: _RelevanceRuleCapable, kind: str) -> None: + """Enable ``module``'s ``kind`` rule only on the outermost scope that requests it. + + Refcounted per kind, not per module: a gated-MLP node can have its + "activation" rule and "multiplicative_gate" rule independently nested to + different depths, so one kind's inner exit must never disable the other. + """ + counts: Dict[str, int] = getattr(module, "_relevance_rule_refcounts", None) or {} + count = counts.get(kind, 0) + if count == 0: + module._enable_relevance_rule(kind) + counts[kind] = count + 1 + setattr(module, "_relevance_rule_refcounts", counts) + + +def _release_rule(module: _RelevanceRuleCapable, kind: str) -> None: + """Disable ``module``'s ``kind`` rule only once its innermost scope exits.""" + counts: Dict[str, int] = getattr(module, "_relevance_rule_refcounts", None) or {} + count = counts.get(kind, 0) - 1 + counts[kind] = max(count, 0) + setattr(module, "_relevance_rule_refcounts", counts) + if count <= 0: + module._disable_relevance_rule(kind) + + +@contextmanager +def use_relevance_rules(model: nn.Module, rules: RelevanceRules) -> Iterator[RelevanceRuleCoverage]: + """Install the requested relevance rules on ``model`` only for this scope. + + Targeting is positional: a component is considered for a rule kind only when it + sits at that kind's canonical mount name (never by class). A canonical mount + occupied by a component that does not implement ``_RelevanceRuleCapable``, or + whose ``_relevance_rule_kinds`` simply omits the requested kind, is reported as + skipped -- both are benign non-applicability, covering a structurally different + architecture or a mount whose current dispatch path the rule does not wrap. A + component that additionally names the requested kind in its own + ``_relevance_rule_unsupported_kinds`` raises ``RelevanceRuleUnsupportedError`` + instead: that names a kind the component is expected to honor at this mount but + cannot given its current configuration, so silently skipping it would let + analysis proceed as if the caller had never asked. Requesting a kind that has no + canonical mount at all (for example "attention", a valid ``RelevanceRules`` field + with no mount defined) raises ``ValueError`` before the install loop, since there + is no mount to target and the scope would otherwise return empty coverage as + though the request had succeeded. Scopes over the same model + are reference-counted, so an inner scope's exit never disables a rule an outer + scope still needs. No model configuration is mutated; the only state that + changes lives on the participating components, and only for the scope's + duration. + """ + requested_kinds = [ + field.name for field in dataclasses.fields(rules) if getattr(rules, field.name) + ] + + unmapped_kinds = [kind for kind in requested_kinds if kind not in _CANONICAL_MOUNTS] + if unmapped_kinds: + joined = ", ".join(repr(kind) for kind in unmapped_kinds) + raise ValueError( + f"No canonical mount is defined for relevance-rule kind(s) {joined}. " + "Such a kind has no mount to target, so it would install nothing and " + "silently return an empty coverage report instead of applying the rule." + ) + + installed: List[Tuple[str, _RelevanceRuleCapable, str]] = [] + skipped: List[str] = [] + for kind in requested_kinds: + mount_names = _CANONICAL_MOUNTS.get(kind, ()) + for name, module in _iter_canonical_mount_candidates(model, mount_names): + if isinstance(module, _RelevanceRuleCapable) and kind in module._relevance_rule_kinds: + installed.append((name, module, kind)) + continue + unsupported_kinds = getattr(module, "_relevance_rule_unsupported_kinds", ()) + if isinstance(module, _RelevanceRuleCapable) and kind in unsupported_kinds: + raise RelevanceRuleUnsupportedError( + f"{name!r} ({type(module).__name__}) cannot install the {kind!r} " + "relevance rule: unsupported configuration for this component." + ) + skipped.append(name) + + for _, module, kind in installed: + _acquire_rule(module, kind) + try: + yield RelevanceRuleCoverage( + installed=tuple(name for name, _, _ in installed), + skipped=tuple(skipped), + ) + finally: + for _, module, kind in installed: + _release_rule(module, kind) diff --git a/transformer_lens/model_bridge/generalized_components/gated_mlp.py b/transformer_lens/model_bridge/generalized_components/gated_mlp.py index c6d969baf4..ccc533c726 100644 --- a/transformer_lens/model_bridge/generalized_components/gated_mlp.py +++ b/transformer_lens/model_bridge/generalized_components/gated_mlp.py @@ -2,28 +2,57 @@ This module contains the bridge component for gated MLP layers (e.g., LLaMA, Gemma). """ -from typing import Any, Callable, Dict, Mapping, Optional +from typing import Any, Callable, Dict, Mapping, Optional, Tuple, cast import torch +import torch.nn as nn +from transformer_lens.model_bridge._relevance_rules import ( + half_rule, + identity_rule, + scale_gradient, +) from transformer_lens.model_bridge.generalized_components.base import ( GeneralizedComponent, ) from transformer_lens.model_bridge.generalized_components.mlp import MLPBridge +def _resolve_activation_fn_name(config: Any) -> Optional[str]: + """The raw activation-name attribute a config exposes, in adapter priority order.""" + if config is None: + return None + for attr in ("activation_function", "hidden_activation", "hidden_act", "act_fn"): + name = getattr(config, attr, None) + if name is not None: + return str(name) + return None + + +_IDENTITY_RULE_UNSUPPORTED_ACTIVATIONS = {"relu", "relu2", "relu_2", "relu_squared"} + + +def identity_rule_supports_activation(config: Any) -> bool: + """Whether the config's resolved activation form is safe for the Identity-rule. + + The Identity-rule's backward multiplier is ``f(x) / x`` (the removable-singularity + limit filled in at zero) rather than the ordinary derivative -- the correct + LRP-style rule for SiLU and both GELU variants, but not for the relu family: + relu-squared's ratio reduces to ``relu(x)``, not its true derivative + ``2 * relu(x)``, and plain relu has no smooth two-sided derivative for the ratio + to represent at the removable singularity either. Both are therefore excluded + rather than silently applying a rule that does not hold for them. + """ + return _resolve_activation_fn_name(config) not in _IDENTITY_RULE_UNSUPPORTED_ACTIVATIONS + + def resolve_activation_fn(config: Any) -> Callable: """Resolve activation function from a model config. Checks config attributes in order: activation_function, hidden_activation, hidden_act, act_fn. Maps common aliases to torch.nn.functional callables. """ - act_fn_name = None - if config is not None: - for attr in ("activation_function", "hidden_activation", "hidden_act", "act_fn"): - act_fn_name = getattr(config, attr, None) - if act_fn_name is not None: - break + act_fn_name = _resolve_activation_fn_name(config) if act_fn_name is None or act_fn_name in ("silu", "swish"): return torch.nn.functional.silu @@ -46,6 +75,25 @@ def relu_squared(x: torch.Tensor) -> torch.Tensor: return torch.nn.functional.silu +class _IdentityRuleActivation(nn.Module): + """Route a wrapped activation through the Identity-rule for a scope's duration. + + The opaque gated-MLP path keeps the HF module's own forward intact and installs + the Identity-rule by swapping the module's activation callable for this wrapper. + Its forward returns ``act_fn(x)`` unchanged, so the native forward value is + preserved, while the backward follows the Identity-rule VJP. Storing the wrapped + activation as an attribute registers it as a child module when it is itself an + ``nn.Module`` (the common ``ACT2FN`` case), so its parameters, if any, stay live. + """ + + def __init__(self, wrapped: Callable[[torch.Tensor], torch.Tensor]): + super().__init__() + self._wrapped_activation = wrapped + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return identity_rule(x, self._wrapped_activation) + + class GatedMLPBridge(MLPBridge): """Bridge component for gated MLP layers. @@ -84,6 +132,188 @@ def __init__( optional: If True, setup skips this bridge when absent (hybrid architectures). """ super().__init__(name, config, submodules=submodules or {}, optional=optional) + self._relevance_rule_activation_active = False + self._relevance_rule_gate_active = False + # Opaque-path rule installers hold their teardown state here. The activation + # wrap records (attr_name, original_value, was_child_module) so the swapped + # activation callable can be restored exactly; the gate handle is the + # forward-pre-hook that scales the gradient entering the down projection. + self._relevance_activation_wrap: Optional[Tuple[str, Any, bool]] = None + self._relevance_gate_hook_handle: Optional[Any] = None + + def _is_gated_mlp_shaped(self) -> bool: + """Whether this instance has the gate/up/down submodules a gated MLP needs. + + A container missing one of these was never wired up as a gated-MLP node at + all (a different architecture at this mount), which is benign + non-applicability rather than an unsupported configuration of a gated-MLP + node -- unlike an activation form the Identity-rule cannot honor, which + occupies exactly this node's shape but cannot be honored correctly. + """ + if self.original_component is None: + return False + gate_module = getattr(self, "gate", None) + in_module = getattr(self, "in", None) + out_module = getattr(self, "out", None) + return gate_module is not None and in_module is not None and out_module is not None + + def _find_activation_attr(self) -> Optional[str]: + """The attribute name under which the HF module holds its activation callable. + + The opaque path installs the Identity-rule by swapping this attribute, so the + activation must be reachable as a callable attribute the native forward calls + (the ``ACT2FN`` module the gated-MLP families store as ``act_fn``). Returns + ``None`` when no such attribute exists, in which case the Identity-rule cannot + be wrapped in and ``"activation"`` is reported unsupported rather than + installed as a silent no-op. + """ + component = self.original_component + if component is None: + return None + for attr in ("act_fn", "activation_fn", "act", "activation"): + if callable(getattr(component, attr, None)): + return attr + return None + + def _activation_rule_installable(self) -> bool: + """Whether the Identity-rule can be installed on this instance's activation. + + Requires both a config activation form the ratio rule is valid for (the + relu family is excluded) and, on the opaque path, an activation callable the + bridge can wrap in place. Subclasses that reconstruct the forward themselves + override this, since they call the activation directly and never wrap it. + """ + return ( + identity_rule_supports_activation(self.config) + and self._find_activation_attr() is not None + ) + + @property + def _relevance_rule_kinds(self) -> Tuple[str, ...]: + """The relevance-rule kinds this instance can currently honor. + + Empty when this is not a gated-MLP-shaped node. Otherwise always includes + ``"multiplicative_gate"`` (the Half-rule is a gradient scale at the gate*up + product and needs no weight or activation access) and includes + ``"activation"`` only when the Identity-rule can be installed on the + configured activation, so a relu-family activation, or one the bridge cannot + reach to wrap, is excluded. + """ + if not self._is_gated_mlp_shaped(): + return () + kinds: Tuple[str, ...] = ("multiplicative_gate",) + if self._activation_rule_installable(): + kinds = ("activation",) + kinds + return kinds + + @property + def _relevance_rule_unsupported_kinds(self) -> Tuple[str, ...]: + """Kinds this gated-MLP node is expected to honor but currently cannot. + + Unlike a kind simply absent from ``_relevance_rule_kinds`` because this is + not a gated-MLP-shaped node at all (benign non-applicability, reported + skipped), a gated-MLP node whose activation form or activation callable the + Identity-rule cannot honor is exactly the kind of component a caller expects + the rule to work on. Requesting ``"activation"`` there raises instead of + silently reporting the mount skipped. The Half-rule applies to every + gated-MLP node, so ``"multiplicative_gate"`` is never reported unsupported. + """ + if not self._is_gated_mlp_shaped(): + return () + if "activation" not in self._relevance_rule_kinds: + return ("activation",) + return () + + def _install_activation_rule(self) -> None: + """Swap the HF module's activation callable for the Identity-rule wrapper. + + No-op when the activation callable cannot be located; requesting the + activation rule in that case is refused earlier through + ``_relevance_rule_unsupported_kinds``. The original value and whether it was + a registered child module are recorded so teardown restores it exactly. + """ + component = self.original_component + attr = self._find_activation_attr() + if component is None or attr is None: + return + was_child_module = attr in component._modules + original = component._modules[attr] if was_child_module else getattr(component, attr, None) + # _find_activation_attr only returns an attribute whose value is callable. + wrapper = _IdentityRuleActivation(cast(Callable[[torch.Tensor], torch.Tensor], original)) + if not was_child_module: + component.__dict__.pop(attr, None) + component._modules[attr] = wrapper + self._relevance_activation_wrap = (attr, original, was_child_module) + + def _teardown_activation_rule(self) -> None: + """Restore the activation callable swapped in by ``_install_activation_rule``.""" + if self._relevance_activation_wrap is None: + return + attr, original, was_child_module = self._relevance_activation_wrap + component = self.original_component + if component is not None: + component._modules.pop(attr, None) + if was_child_module: + component._modules[attr] = original + else: + component.__dict__[attr] = original + self._relevance_activation_wrap = None + + def _install_gate_rule(self) -> None: + """Halve the gradient entering the down projection to reproduce the Half-rule. + + The gate*up product is the down projection's input, so a forward-pre-hook + that routes that input through ``scale_gradient(..., 0.5)`` halves the single + gradient feeding the product before it splits, which matches halving both + product-rule terms. The native forward value is unchanged, and the down + projection's own weight gradient stays ordinary because it is taken against + the unscaled downstream gradient. + """ + out_module = getattr(self, "out", None) + down_component = getattr(out_module, "original_component", None) + if down_component is None: + return + + def _scale_product_gradient( + module: nn.Module, args: Tuple[Any, ...] + ) -> Optional[Tuple[Any, ...]]: + if not args: + return None + return (scale_gradient(args[0], 0.5),) + tuple(args[1:]) + + self._relevance_gate_hook_handle = down_component.register_forward_pre_hook( + _scale_product_gradient + ) + + def _teardown_gate_rule(self) -> None: + """Remove the down-projection gradient-scale hook.""" + if self._relevance_gate_hook_handle is not None: + self._relevance_gate_hook_handle.remove() + self._relevance_gate_hook_handle = None + + def _enable_relevance_rule(self, kind: str) -> None: + """Activate the named rule and install its opaque-path hook. + + The boolean flag drives the reconstructed forward paths (compatibility mode + here, and the inline forward of subclasses that override it). The install + step additionally attaches the rule to the live HF submodules for the opaque + native forward, which a flag alone cannot alter. + """ + if kind == "activation": + self._relevance_rule_activation_active = True + self._install_activation_rule() + elif kind == "multiplicative_gate": + self._relevance_rule_gate_active = True + self._install_gate_rule() + + def _disable_relevance_rule(self, kind: str) -> None: + """Deactivate the named rule and tear down its opaque-path hook.""" + if kind == "activation": + self._teardown_activation_rule() + self._relevance_rule_activation_active = False + elif kind == "multiplicative_gate": + self._teardown_gate_rule() + self._relevance_rule_gate_active = False def forward(self, *args, **kwargs) -> torch.Tensor: """Forward pass through the gated MLP bridge. @@ -120,8 +350,16 @@ def forward(self, *args, **kwargs) -> torch.Tensor: if in_module is not None and hasattr(in_module, "hook_out"): linear_output = in_module.hook_out(linear_output) # type: ignore[misc] act_fn = resolve_activation_fn(self.config) - activated = act_fn(gate_output) - hidden = activated * linear_output + activated = ( + identity_rule(gate_output, act_fn) + if self._relevance_rule_activation_active + else act_fn(gate_output) + ) + hidden = ( + half_rule(activated, linear_output) + if self._relevance_rule_gate_active + else activated * linear_output + ) if hasattr(self, "out") and hasattr(self.out, "hook_in"): hidden = self.out.hook_in(hidden) output = torch.nn.functional.linear( @@ -140,6 +378,10 @@ def forward(self, *args, **kwargs) -> torch.Tensor: hidden_states = args[0] hidden_states = self.hook_in(hidden_states) new_args = (hidden_states,) + args[1:] + # The active relevance rules are attached to the live HF submodules (the + # activation callable and the down projection) by _enable_relevance_rule, + # so the native forward runs unchanged and its own internal hooks, gate + # multipliers, and activation sparsity all remain in the backward graph. output = self.original_component(*new_args, **kwargs) output = self.hook_out(output) return output diff --git a/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py b/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py index 7d40d57afd..28a1981283 100644 --- a/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py +++ b/transformer_lens/model_bridge/generalized_components/joint_gate_up_mlp.py @@ -6,11 +6,13 @@ import torch +from transformer_lens.model_bridge._relevance_rules import half_rule, identity_rule from transformer_lens.model_bridge.generalized_components.base import ( GeneralizedComponent, ) from transformer_lens.model_bridge.generalized_components.gated_mlp import ( GatedMLPBridge, + identity_rule_supports_activation, resolve_activation_fn, ) from transformer_lens.model_bridge.generalized_components.linear import LinearBridge @@ -166,6 +168,29 @@ def _resolve_activation_fn(self) -> Callable: return self._activation_fn return resolve_activation_fn(self.config) + def _activation_rule_installable(self) -> bool: + """The reconstructed forward calls the activation itself, so only the config + activation form gates the Identity-rule; the opaque-path requirement of a + wrappable activation callable does not apply here.""" + return identity_rule_supports_activation(self.config) + + # The reconstructed forward applies both rules inline off the boolean flags set + # by the base ``_enable_relevance_rule``/``_disable_relevance_rule``. The + # opaque-path installers must stay disabled: the gate hook lives on the shared + # down projection this forward also calls, so leaving it active would halve the + # gate*up gradient a second time on top of the inline ``half_rule``. + def _install_activation_rule(self) -> None: + return None + + def _teardown_activation_rule(self) -> None: + return None + + def _install_gate_rule(self) -> None: + return None + + def _teardown_gate_rule(self) -> None: + return None + def forward(self, *args: Any, **kwargs: Any) -> torch.Tensor: """Reconstructed gated MLP forward with individual hook access.""" # Delegate to GatedMLPBridge's processed-weights path only when ALL @@ -184,7 +209,16 @@ def forward(self, *args: Any, **kwargs: Any) -> torch.Tensor: up_output = getattr(self, "in")(hidden_states) act_fn = self._resolve_activation_fn() - gated = act_fn(gate_output) * up_output + activated = ( + identity_rule(gate_output, act_fn) + if self._relevance_rule_activation_active + else act_fn(gate_output) + ) + gated = ( + half_rule(activated, up_output) + if self._relevance_rule_gate_active + else activated * up_output + ) if hasattr(self, "out") and self.out is not None: output = self.out(gated) diff --git a/transformer_lens/model_bridge/generalized_components/normalization.py b/transformer_lens/model_bridge/generalized_components/normalization.py index 418cb3edb9..5aa0b59bc9 100644 --- a/transformer_lens/model_bridge/generalized_components/normalization.py +++ b/transformer_lens/model_bridge/generalized_components/normalization.py @@ -1,11 +1,15 @@ """Normalization bridge component implementation.""" import contextlib import warnings -from typing import Any, ContextManager, Dict, Optional, cast +from typing import Any, ContextManager, Dict, Optional, Tuple, cast import torch from transformer_lens.hook_points import HookPoint +from transformer_lens.model_bridge._relevance_rules import ( + RelevanceRuleConflictError, + ln_rule_grad, +) from transformer_lens.model_bridge.generalized_components.base import ( GeneralizedComponent, ) @@ -23,6 +27,83 @@ "output is reconstructed from the hooked values instead of HF's native forward. " "Output numerics may differ from the unhooked forward at float-rounding scale." ) +# While the LN-rule is active, the fallbacks above would silently compose the rule +# with the hook edit and break the bit-identical-forward guarantee, so they raise +# instead of warning. +RULE_ACTIVE_BWD_HOOK_CONFLICT = ( + "Backward hooks on hook_scale/hook_normalized are incompatible with an active " + "LN-rule on '{name}': the rule-wrapped native forward keeps these hook points " + "out of its backward graph, so a backward hook here would silently never fire." +) +RULE_ACTIVE_EDIT_HOOK_CONFLICT = ( + "A forward hook edited hook_scale/hook_normalized while the LN-rule is active on " + "'{name}': honoring the edit would require the python-norm fallback, which would " + "compose the rule with the edit and break the bit-identical-forward guarantee." +) + + +class _NativeLNRuleForward(torch.autograd.Function): + """Wrap a normalization module's own forward call in the LN-rule's VJP. + + Forward returns ``component(x)`` unchanged, so the result is bit-identical to + the native forward by construction. Backward routes the x-path gradient + through the centering op ordinarily but treats ``denom`` as a constant (the + LN-rule), while ``weight`` and ``bias`` receive their ordinary gradient since + the rule only redefines how relevance reaches the input, not parameter + training gradients. A parameter-free norm (``weight`` is ``None``, e.g. + OLMo's ``OlmoLayerNorm``) is treated as a unit scale in both directions. + """ + + @staticmethod + def forward( + ctx: Any, + x_centered: torch.Tensor, + denom: torch.Tensor, + weight: Optional[torch.Tensor], + bias: Optional[torch.Tensor], + x: torch.Tensor, + component: torch.nn.Module, + offset: bool, + input_dtype: torch.dtype, + ) -> torch.Tensor: + ctx.save_for_backward(x_centered, denom, weight) + ctx.has_bias = bias is not None + ctx.bias_requires_grad = bool(bias is not None and bias.requires_grad) + ctx.offset = offset + result = component(x) + if result.dtype != input_dtype: + result = result.to(input_dtype) + return result + + @staticmethod + def backward( + ctx: Any, grad_output: torch.Tensor + ) -> Tuple[ + torch.Tensor, + None, + Optional[torch.Tensor], + Optional[torch.Tensor], + None, + None, + None, + None, + ]: + x_centered, denom, weight = ctx.saved_tensors + if weight is None: + w_eff: torch.Tensor | float = 1.0 + else: + w_eff = (1.0 + weight) if ctx.offset else weight + reduce_dims = tuple(range(grad_output.dim() - 1)) + grad_x_centered = ln_rule_grad(grad_output * w_eff, denom) + grad_weight = ( + (grad_output * (x_centered / denom)).sum(dim=reduce_dims) + if weight is not None and weight.requires_grad + else None + ) + grad_bias = ( + grad_output.sum(dim=reduce_dims) if ctx.has_bias and ctx.bias_requires_grad else None + ) + return grad_x_centered, None, grad_weight, grad_bias, None, None, None, None class NormalizationBridge(GeneralizedComponent): @@ -60,6 +141,31 @@ def __init__( self.hook_scale = HookPoint() self.use_native_layernorm_autograd = use_native_layernorm_autograd self._uses_rms_norm_override = uses_rms_norm + self._relevance_rule_active = False + + @property + def _relevance_rule_kinds(self) -> Tuple[str, ...]: + """``("normalization",)`` only when this instance actually dispatches through + the native-autograd branch the LN-rule wraps (``_hf_autograd_forward_with_hooks``); + empty otherwise, so ``use_relevance_rules`` reports an ln1/ln2 mount that + uses the plain python-norm path (for example ``LayerNormPreBridge`` / + ``RMSNormPreBridge``, or a config without ``layer_norm_folding``) as + skipped rather than silently leaving ordinary gradients in place under a + claimed "installed" rule. + """ + if self.use_native_layernorm_autograd: + return ("normalization",) + if bool(getattr(self.config, "layer_norm_folding", False)): + return ("normalization",) + return () + + def _enable_relevance_rule(self, kind: str) -> None: + """Activate the LN-rule for this instance's native-forward branch only.""" + self._relevance_rule_active = True + + def _disable_relevance_rule(self, kind: str) -> None: + """Deactivate the LN-rule, restoring today's native-forward behavior.""" + self._relevance_rule_active = False @property def uses_rms_norm(self) -> bool: @@ -172,6 +278,10 @@ def _hf_autograd_forward_with_hooks(self, x: torch.Tensor) -> torch.Tensor: _ = self.hook_normalized(x) return x if self.hook_scale.bwd_hooks or self.hook_normalized.bwd_hooks: + if self._relevance_rule_active: + raise RelevanceRuleConflictError( + RULE_ACTIVE_BWD_HOOK_CONFLICT.format(name=self.name) + ) warnings.warn(NATIVE_PATH_BWD_FALLBACK_WARNING) return self._python_norm_forward(x) has_fwd_hooks = bool(self.hook_scale.fwd_hooks or self.hook_normalized.fwd_hooks) @@ -216,13 +326,64 @@ def _hf_autograd_forward_with_hooks(self, x: torch.Tensor) -> torch.Tensor: # identity is the edit signal. Note in-place mutation of the hook value without # returning it is NOT detected — return the tensor from the hook to edit. if hooked_scale is scale and hooked_normalized is x_normalized: + if self._relevance_rule_active: + return self._native_forward_with_ln_rule(x, input_dtype) result = self.original_component(x) if result.dtype != input_dtype: result = result.to(input_dtype) return result + if self._relevance_rule_active: + raise RelevanceRuleConflictError(RULE_ACTIVE_EDIT_HOOK_CONFLICT.format(name=self.name)) warnings.warn(NATIVE_PATH_EDIT_FALLBACK_WARNING) return self._apply_weight_and_bias(hooked_normalized, input_dtype) + def _native_forward_with_ln_rule( + self, x: torch.Tensor, input_dtype: torch.dtype + ) -> torch.Tensor: + """Call the native forward unchanged while routing its backward through the LN-rule. + + Recomputes the centered numerator and denominator independently of the + hook-observation pass above (which may run under ``torch.no_grad()``), so + the recompute here stays grad-connected to ``x`` regardless of whether + forward hooks are attached. Autograd then flows from the returned tensor + back through the centering op ordinarily and, at the LN-rule Function + boundary, treats the denominator as a constant -- the same contract as + the shared ``ln_rule`` primitive, without reproducing the native kernel's + own forward numerics. + """ + component = self.original_component + assert component is not None + x_float = x.float() if x.dtype not in (torch.float32, torch.float64) else x + if not self.uses_rms_norm: + x_centered = x_float - x_float.mean(-1, keepdim=True) + else: + x_centered = x_float + eps_tensor = getattr(component, "eps", None) + if eps_tensor is None: + eps_tensor = getattr(component, "variance_epsilon", None) + if eps_tensor is None: + eps_value: float | torch.Tensor = getattr(self.config, "eps", 1e-05) + else: + eps_value = eps_tensor + variance = x_centered.pow(2).mean(-1, keepdim=True) + denom = ( + (variance + eps_value).sqrt() + if isinstance(eps_value, torch.Tensor) + else (variance + float(eps_value)).sqrt() + ) + # A parameter-free norm (OLMo's OlmoLayerNorm) exposes no usable weight; + # pass None so the rule treats the scale as 1 in both forward and backward. + try: + weight: Optional[torch.Tensor] = cast(torch.Tensor, self.weight) + except AttributeError: + weight = None + bias = getattr(component, "bias", None) if not self.uses_rms_norm else None + offset = bool(getattr(self.config, "rmsnorm_uses_offset", False)) + result: torch.Tensor = _NativeLNRuleForward.apply( + x_centered, denom, weight, bias, x, component, offset, input_dtype + ) + return result + class LayerNormPreBridge(NormalizationBridge): """Param-free LayerNorm (LNPre): hook_scale / hook_normalized, no weight or bias.""" From 2d2d55494af60b78d0ab6cf8bb12c3d45b1b5ff0 Mon Sep 17 00:00:00 2001 From: jlarson4 <jonahalarson@comcast.net> Date: Thu, 17 Sep 2026 09:38:01 -0500 Subject: [PATCH 79/87] Release 4 announcement message --- docs/source/content/news/release-4.0.md | 223 ++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 docs/source/content/news/release-4.0.md diff --git a/docs/source/content/news/release-4.0.md b/docs/source/content/news/release-4.0.md new file mode 100644 index 0000000000..32fd6a75a5 --- /dev/null +++ b/docs/source/content/news/release-4.0.md @@ -0,0 +1,223 @@ +# TransformerLens 4.0 +**[release date TBD]** + +TransformerLens 4.0 is here. Where 3.0 changed how models are loaded via the +TransformerBridge and its architecture adapters, 4.0 expands options for **what runs the +forward pass underneath them**. The headline of this release is the new +**execution backends** (we call them Drivers, fed by model **Sources**): the +same bridge you already use with the same hook names, the same cache, the same +intervention surface can now run on **vLLM** for high-throughput capture and +steering, or inside an **`inspect_ai`** evaluation harness. + +The other significant change in this release is the removal of the legacy +`Hooked*` classes, completing the transition to TransformerBridge that began in 3.0. Continuing to support two separate implementations of the same tool led to confusion about where features lived, where changes should be implemented, and what a change needed to support. With a new, unified system, there is now a singular home for all future features. + +If you have been following the dev-4.x branch, none of this will be a surprise. +If you have been on 3.x, your bridge code carries forward unchanged. + +## What changed: the Driver system + +In 3.0, the bridge wrapped a local HuggingFace `nn.Module` and layered hook +points over it. That is still the default and the most capable path, but it +limited study to only HuggingFace model executions via `transformers`. If you wanted the throughput +of vLLM, or wanted to capture activations inside an `inspect_ai` eval, you need to write bespoke engine-specific code. + +4.0 separates the bridge system from the source of the model. A **Driver** is anything that can run a forward pass and +fire hooks, now the bridge talks to all of them through one protocol. Every backend +declares which hook points it can serve, and the *same canonical hook names* +(`blocks.0.hook_out`, `attn.hook_out`, …) work across all of them. An +analysis written against one backend transfers to another. What changes between +backends is which hooks are fireable and whether gradients exist. This comes down to the tradeoffs of each Driver. + +Three backends ship in 4.0: + +### transformers — full hooks + gradients (the reference path) + +The existing behavior, maintained as it was designed in 3.x. `TransformerBridge.boot_transformers("gpt2")` wraps +a local HF model, the full HookPoint tree fires, backward hooks and gradients +work, and weight access is available. This is the path for circuit analysis, +attribution patching, and anything that needs a real autograd surface. If you +are doing what you did in 3.x, nothing changes for you. + +### vLLM — high-throughput capture and steering + +`RemoteBridge.boot_vllm(...)` runs the forward pass on a vLLM engine +(PagedAttention, `torch.compile`, CUDA graphs) with capture hooks installed +inside the worker before compilation. Activations come back and replay through +the bridge's HookPoint tree, so `run_with_cache` works exactly as it does +locally. Unlike observation-only tooling, each hook also applies an affine +transform, so declarative interventions (`suppress` / `scale` / `add` / `set`) +propagate to downstream layers. + +```python +import torch +from transformer_lens.model_bridge import RemoteBridge + +bridge = RemoteBridge.boot_vllm("meta-llama/Llama-3.2-1B", dtype=torch.float16) +logits, cache = bridge.run_with_cache("Hello, world") + +# Declarative intervention: zero the embedding output for this forward only. +logits2, cache2 = bridge.run_with_cache( + "Hello, world", + intervene={"embed.hook_out": {"op": "suppress"}}, +) +``` + +The vLLM backend requires a CUDA GPU and is installed with the `vllm` extra +(`uv sync --extra vllm`, or `pip install "transformer-lens[vllm]"`). It supports +single-node tensor and pipeline parallelism, both GPU-validated for capture / +intervention / logit parity against the single-rank path. + +### Inspect — interpretability inside `inspect_ai` evals + +`RemoteBridge.boot_inspect(...)` wraps an +[`inspect_ai`](https://inspect.aisi.org.uk/) model provider in a bridge, so +activation capture and interventions run inside the same harness as your +behavioral evals. The default `tl_bridge` provider is HF-backed and numerically +faithful to `boot_transformers` (residual / attention / MLP capture, full affine +interventions, full-sequence logits); a vLLM-backed sibling is also available. +Install with the `inspect` extra. + +```python +from transformer_lens.model_bridge import RemoteBridge + +bridge = RemoteBridge.boot_inspect("HuggingFaceTB/SmolLM2-135M") +logits, cache = bridge.run_with_cache("Hello, world") +``` + +For capture *during* an eval, add the `capture_activations([...])` solver to a +Task's solver chain: full activations land in per-sample artifacts and a compact +summary goes to the sample store for analysis. + +### What the backends can and can't do + +The capability tiers come from the engines themselves, and the new +[Execution Backends](../drivers.md) page documents them in full. In short: +circuit-finding and anything gradient-based run on `transformers`; capture and +steering scale out on vLLM; both speak the same hook names. Serving engines are +not autograd engines, so the remote backends have no gradients, no attention +patterns or scores (the QKᵀ→softmax path is fused into the kernel), and +interventions there are declarative. + +## The Major Deprecation News: `HookedTransformer` has been removed + +3.0 introduced the bridge and kept `HookedTransformer` running to cover features that were not yet ported to the bridge system, with the stated long-term intent to remove it in the next +major version. The five legacy model classes — +`HookedTransformer`, `HookedEncoder`, `HookedEncoderDecoder`, +`HookedAudioEncoder`, and `BertNextSentencePrediction` — along with the +supporting stack (`loading_from_pretrained`, `HookedTransformerConfig`, the +`components` tree, the per-architecture weight converters, and the `train` / +`utils` shims) have been deleted. + +`TransformerBridge` is now the single model system, and it supports **15,000+ +models across 140+ architecture families**. For the vast majority of users the +migration is simple: + +```python +# Before (removed in 4.0) +from transformer_lens import HookedTransformer +model = HookedTransformer.from_pretrained("gpt2") + +# After +from transformer_lens.model_bridge import TransformerBridge +model = TransformerBridge.boot_transformers("gpt2") +model.enable_compatibility_mode() # HookedTransformer-equivalent numerics +``` + +`enable_compatibility_mode()` reproduces `HookedTransformer`'s default weight +processing (LayerNorm folding, weight centering), verified against frozen +`HookedTransformer` reference activations captured before the removal. For mappings of `Hooked*` features to their new bridge variant, there is a replacement table in +[Migrating to TransformerLens 4.0](../migrating_to_v4.md). + +**`HookedRootModule` and `HookPoint` are kept**. They are the supported way to +add TransformerLens-style hooks to an arbitrary `nn.Module` and are maintained for that purpose. + +## Breaking changes and deprecations + +This major version removes a significant amount of past public API. The specifics: + +### The `Hooked*` classes and their stack + +Every removed name now raises an `AttributeError` naming its replacement when +accessed from the top-level package (e.g. `from transformer_lens import +HookedTransformer`). See the [4.0 migration guide](../migrating_to_v4.md). + +### Known capability reductions + +Two narrow capabilities lived only in the deleted `HookedTransformer` attention +path and have no bridge equivalent: Qwen-1's `use_logn_attn` / runtime-adaptive +Dynamic-NTK long-context scaling, and `ungroup_grouped_query_attention` +(per-query-head K/V hook shapes on GQA models). + +### Prereleases + +All three 4.0 prereleases still shipped `HookedTransformer`. If you adopted a 4.0 +beta, this final release is a second breaking step for the names above. + +## Roadmap + +As with the last two major version announcements, I've broken this into three timeframes. The +mid- and long-term items are a draft and priorities can shift with user feedback. + +### Immediate - within the next month + +Smoothing rough edges on the new backends. Expect rapid 4.x patches as issues +are reported. I would especially like to hear from anyone running the vLLM +backend on their own hardware — the single-GPU path and the tensor/pipeline +parallel paths are GPU-validated, but real workloads find things a suite of tests doesn't. + +### Mid-term - within the next 3 months + +- **Broader driver coverage.** The driver protocol is deliberately small so that + new execution backends can be added without touching the bridge. vLLM and + Inspect are the first two; other serving engines are candidates. If you have sources you'd like us to add, please file an issue! +- **Documentation and recipes for the remote backends** — Usage examples & suggestions for best fit solutions. + +### Long-term - within the next year + +- **Deeper multi-node support** for the vLLM backend (Ray-based multi-node is + currently unsupported). +- **Continued adapter coverage and authoring tooling**, carrying forward the + 3.0 roadmap. The Automated adapter builder via Claude now exists under `devtools/adapter_builder`. We'd like to take this further and try to automate the creation of new adapters as new frontier architectures are released on HuggingFace. Stay tuned for any exciting new developments! + +## Contributors + +This section is only relevant to contributors; if you use TransformerLens only +as a tool, you can skip it. + +### Branch changes + +During the 4.x cycle we maintained a `dev-4.x` branch for the driver work and +the `Hooked*` removal alongside the regular `dev` branch. With 4.0 shipping, +that work has landed and `dev` is again the single active development branch. +New pull requests should target `dev`. + +### The driver contract + +New execution backends implement the `Driver` protocol in +[`driver_protocol.py`](../../../../transformer_lens/model_bridge/driver_protocol.py): +a `forward` that returns a `ForwardResult` (logits + a captured hook-name → +activation map), a `close`, a `supports` capability check, and the two declared +hook-name sets (`supported_hook_points` / `non_fireable_hook_points`). +`validate_driver` checks the contract when a bridge is constructed. Two parity +scripts (`scripts/vllm_parity_report.py` and the Inspect equivalent) diff each +remote backend against `boot_transformers` on real models. + +### Dependency changes + +`better-abc`, a runtime dependency of the deleted component tree, was dropped. +The `vllm` and `inspect` extras are optional and documented on the +[Execution Backends](../drivers.md) page; `vllm` is Linux-only and cannot +co-install with the `lit` extra. + +## Conclusion + +Thank you for keeping up with TransformerLens! 3.0 decoupled TransformerLens from any single model +*implementation*; 4.0 decouples it from any single *execution engine*. The hope +is the same as it was a year ago: that the interpretability code you write keeps +working as the field moves, whether you're tracing a circuit on a local model, +collecting a dataset on a vLLM cluster, or instrumenting a model inside an eval. +With the legacy classes now retired, the +library is smaller, more consistent, and hopefully easier to build on. + +If you hit a bug or a rough edge while migrating, please open an issue or reach out to me via Slack. I am always happy to help anyone working with TransformerLens create the best possible solution for the project they are working on. From acce84bcabf192c695bab876ea2bbad326a8ff36 Mon Sep 17 00:00:00 2001 From: jlarson4 <jonahalarson@comcast.net> Date: Thu, 17 Sep 2026 09:47:11 -0500 Subject: [PATCH 80/87] Restructuring past major release information --- docs/source/content/news/past_releases.md | 11 +++++++++++ docs/source/index.md | 3 +-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 docs/source/content/news/past_releases.md diff --git a/docs/source/content/news/past_releases.md b/docs/source/content/news/past_releases.md new file mode 100644 index 0000000000..461e0cb826 --- /dev/null +++ b/docs/source/content/news/past_releases.md @@ -0,0 +1,11 @@ +# Past Major Releases + +Announcements for earlier major versions of TransformerLens. They record what +changed at each boundary and the versioning conventions later releases build on. + +```{toctree} +:maxdepth: 1 + +release-3.0 +release-2.0 +``` diff --git a/docs/source/index.md b/docs/source/index.md index e5c36efe97..c8e9cc195a 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -34,8 +34,7 @@ content/gallery :caption: News content/news/release-4.0 -content/news/release-3.0 -content/news/release-2.0 +content/news/past_releases ``` ```{toctree} From fea3624d3e29e4e7f1776c38fffab5367a037c33 Mon Sep 17 00:00:00 2001 From: jlarson4 <jonahalarson@comcast.net> Date: Mon, 14 Sep 2026 10:35:25 -0500 Subject: [PATCH 81/87] initial v4.0 release docs --- docs/source/content/migrating_to_v3.md | 15 ++-- docs/source/content/migrating_to_v4.md | 98 ++++++++++++++++++++++++++ docs/source/content/special_cases.md | 2 +- docs/source/content/tutorials.md | 2 +- 4 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 docs/source/content/migrating_to_v4.md diff --git a/docs/source/content/migrating_to_v3.md b/docs/source/content/migrating_to_v3.md index f520b67690..237823cc8e 100644 --- a/docs/source/content/migrating_to_v3.md +++ b/docs/source/content/migrating_to_v3.md @@ -1,5 +1,12 @@ # Migrating to TransformerLens 3 +```{note} +**As of 4.0, `HookedTransformer` and the other `Hooked*` classes are removed** — +there is no longer a compatibility layer keeping the old classes running. Start +with the [4.0 migration guide](migrating_to_v4.md) for the `HookedTransformer` → bridge +mapping; the API recipes on this page still apply to `TransformerBridge`. +``` + TransformerLens 3 introduces **TransformerBridge**, a new way of loading and instrumenting models that replaces `HookedTransformer.from_pretrained` as the recommended path for new code. Existing `HookedTransformer` code continues to run through a compatibility layer, but adopting the bridge unlocks broader architecture support and puts you on the supported path going forward. This page explains the differences and gives side-by-side migration recipes for the most common patterns. @@ -264,9 +271,9 @@ Weight-matrix rows return **raw** HuggingFace weights by default. `HookedTransfo | `HookedTransformer` | `TransformerBridge` equivalent | Notes | |---|---|---| -| `model.W_pos` | `bridge.pos_embed.W_pos` | Raw weight (also `bridge.pos_embed.weight`). `center_writing_weights` centers `W_pos` in default HT loads, so it matches HT's only under matching processing (`enable_compatibility_mode()`, or HT loaded with no processing). | -| `model.W_E_pos` | `torch.cat([bridge.W_E, bridge.pos_embed.W_pos], dim=0)` | No single accessor — concatenate the token + positional matrices. Same weight-processing caveat as `W_pos` (both `W_E` and `W_pos` are centered writing-weights). | -| `HookedTransformer.from_pretrained_no_processing(name)` | `TransformerBridge.boot_transformers(name, no_processing=True)` | Both load raw weights, so these match. | +| `model.W_pos` | `bridge.W_pos` | Direct accessor (also `bridge.pos_embed.W_pos`). `center_writing_weights` centers `W_pos` in default HT loads, so it matches HT's only under matching processing (`enable_compatibility_mode()`). | +| `model.W_E_pos` | `bridge.W_E_pos` | Direct accessor for the concatenated `[W_E; W_pos]`. Same weight-processing caveat as `W_pos`. | +| `HookedTransformer.from_pretrained_no_processing(name)` | `TransformerBridge.boot_transformers(name)` | A plain boot loads raw weights (no processing); do not call `enable_compatibility_mode()`. | | `model.input_to_embed(...)`; `model(..., start_at_layer=k)` | `bridge.input_to_embed(...)`; `bridge(..., start_at_layer=k)` | The bridge accepts the residual entering block `k`. Embedding-stage hooks are excluded, but blocks `0..k-1` still execute on a discarded path before block `k` swaps in that residual. | | `model.get_caching_hooks(...)`; `model.add_caching_hooks(...)` | Same methods on `bridge` | Prefer these methods or `run_with_cache` over `cache_all` and `cache_some`, which now emit `DeprecationWarning`. | | `model.run_with_cache(..., pos_slice=..., incl_bwd=...)` | Same call on `bridge` | `pos_slice` limits cached positions. `incl_bwd=True` requires the gradients-capable transformers driver and a scalar output such as `return_type="loss"`. | @@ -278,7 +285,7 @@ Weight-matrix rows return **raw** HuggingFace weights by default. `HookedTransfo | `cfg.init_weights` | `bridge.init_weights()` | `boot_native()` honors the config flag during construction; call `init_weights()` to reinitialize a TL-native bridge in place. | | `model.all_head_labels()` | `bridge.all_head_labels` | This is a property on the bridge, so omit the call parentheses. | | `model.set_tokenizer(tokenizer)` | `TransformerBridge.boot_transformers(name, tokenizer=tokenizer)` | A bridge's tokenizer is fixed when it boots. Reboot to change it; assigning `bridge.tokenizer` directly bypasses tokenizer/config wiring. | -| `from transformer_lens.train import train, HookedTransformerTrainConfig` | `from transformer_lens.tools.training import train, TrainConfig` | The training loop moved to `tools.training` and `HookedTransformerTrainConfig` renamed to `TrainConfig`. The old imports still work but emit `DeprecationWarning`. | +| `from transformer_lens.train import train, HookedTransformerTrainConfig` | `from transformer_lens.tools.training import train, TrainConfig` | The training loop moved to `tools.training` and `HookedTransformerTrainConfig` renamed to `TrainConfig`. The old `transformer_lens.train` module was removed in 4.0. | The following example demonstrates the `W_pos` and `W_E_pos` equivalents under matching weight processing: diff --git a/docs/source/content/migrating_to_v4.md b/docs/source/content/migrating_to_v4.md new file mode 100644 index 0000000000..944876630b --- /dev/null +++ b/docs/source/content/migrating_to_v4.md @@ -0,0 +1,98 @@ +--- +title: Migrating to TransformerLens 4.0 +--- +# Migrating to TransformerLens 4.0 + +TransformerLens 4.0 **removes the legacy `Hooked*` model stack**. +`TransformerBridge` was introduced in 3.0 as the recommended path, and is now the only path. This guide will detail what was removed and how to recreate those features in the new system. For the deeper conceptual differences between the `HookedTransformer` API and the bridge (hook names, weight processing, per-API recipes), see the [3.0 migration guide](migrating_to_v3.md). The API-mapping table from `HookedTransformer` to `TransformerBridge` is accurate with the final state of all those feature. + +Our intention in making this change is to unify future research, as well as future contributions. Having two parallel systems running covering the same ground created much confusion in where features exist, how they should be accessed, and where issues needed to be fixed. Additionally, it created a dual mandate to repair both any time an issue was reported, costing additional man hours. + +We will still be accepting issues and resolving any bugs on the existing `HookedTransformer` system, but they will be silo'ed to the 3.x branch. We will not be accepting new features or models. + +## The Core Change + +```python +# Removed in 4.0 +from transformer_lens import HookedTransformer +model = HookedTransformer.from_pretrained("gpt2") + +# 4.0 +from transformer_lens.model_bridge import TransformerBridge +model = TransformerBridge.boot_transformers("gpt2") +model.enable_compatibility_mode() # HookedTransformer-equivalent numerics +``` + +For anyone migrating existing `HookedTransformer` work to the latest system, `enable_compatibility_mode()` reproduces its default weight +processing (LayerNorm folding, `center_writing_weights`, `center_unembed`), +verified against frozen `HookedTransformer` reference activations. Omit it to +work with raw HuggingFace weights (the equivalent of the old +`from_pretrained_no_processing`). + +## Removed names and their replacements + +| Removed in 4.0 | Replacement | +|---|---| +| `HookedTransformer` | `TransformerBridge.boot_transformers(name)` (+ `enable_compatibility_mode()`) | +| `HookedEncoder` | `TransformerBridge.boot_transformers(name)` on a BERT model | +| `HookedEncoderDecoder` | `TransformerBridge.boot_transformers(name)` on a T5 model | +| `HookedAudioEncoder` | `TransformerBridge.boot_transformers(name)` on a HuBERT/Wav2Vec2 model | +| `BertNextSentencePrediction` | `TransformerBridge.boot_transformers(name, model_class=BertForNextSentencePrediction).predict_next_sentence(a, b)` | +| `HookedTransformerConfig` | `TransformerBridgeConfig` (pass to `TransformerBridge.boot_native(cfg)` for toy/train-from-scratch models) | +| `transformer_lens.train` | `transformer_lens.tools.training` (`train`, `TrainConfig`) | +| `transformer_lens.utils` | `transformer_lens.utilities` — **same names** (e.g. `utils.get_act_name` → `utilities.get_act_name`) | +| `transformer_lens.loading` / `loading_from_pretrained` | Model names/aliases in `transformer_lens.supported_models`; checkpoint labels in `transformer_lens.tools.model_registry.checkpoints`; config derivation is now internal to the bridge's architecture adapters | +| `transformer_lens.components` | `transformer_lens.model_bridge.generalized_components` | +| `transformer_lens.pretrained` (weight converters) | Handled internally by the bridge's adapters; legacy TL-format repos load via `TransformerBridge.boot_tl_legacy(name)` | + +Accessing a removed name from the top-level package raises an `AttributeError` +naming its replacement (e.g. `from transformer_lens import HookedTransformer`). + +## Weight accessors: `W_pos` / `W_E_pos` + +4.0 adds direct accessors, so no manual concatenation is needed: + +```python +from transformer_lens.model_bridge import TransformerBridge + +bridge = TransformerBridge.boot_transformers("gpt2") +bridge.enable_compatibility_mode() # to match HookedTransformer's processed W_pos +bridge.W_pos # (n_ctx, d_model) +bridge.W_E_pos # concatenated [W_E; W_pos] +``` + +Without `enable_compatibility_mode()` these return raw HuggingFace weights; +`HookedTransformer`'s defaults center the writing weights, so match the +processing to match the numbers. + +## `train` / `utils` + +```python +# Removed +from transformer_lens.train import train, HookedTransformerTrainConfig +from transformer_lens import utils + +# 4.0 +from transformer_lens.tools.training import train, TrainConfig +from transformer_lens import utilities as utils # identical names +``` + +## Traps + +- **`HookedEncoder.encoder_output`** does not map to `bridge.encoder_output`. + The bridge's `encoder_output` is the **audio** frame-entry method + (HuBERT/Wav2Vec2) and refuses BERT. A name-only migration will pass a `dir()` + parity check but fail at runtime — run a BERT encoder through + `bridge(...)` / `run_with_cache` instead. + +## Kept + +`HookedRootModule` and `HookPoint` are **kept** — they are the supported way to +add TransformerLens-style hooks to an arbitrary `nn.Module`, and are unrelated +to the model-class removal. + +## Prerelease note + +All three 4.0 prereleases still shipped `HookedTransformer`. If you adopted a +4.0 beta, this final release is a second breaking step for the removed names +above. diff --git a/docs/source/content/special_cases.md b/docs/source/content/special_cases.md index f020f46587..b29edc478f 100644 --- a/docs/source/content/special_cases.md +++ b/docs/source/content/special_cases.md @@ -1,7 +1,7 @@ # Special Cases ```{warning} -`HookedTransformer` is deprecated as of TransformerLens 3.0 and will be removed in the next major version. New code should use [`TransformerBridge`](migrating_to_v3.md) instead. Existing `HookedTransformer` code continues to work through the 3.x branch via a compatibility layer. See the [migration guide](migrating_to_v3.md) for conversion recipes. +`HookedTransformer` and the other `Hooked*` classes were **removed in TransformerLens 4.0**. Use [`TransformerBridge`](migrating_to_v4.md) instead — call `enable_compatibility_mode()` after booting for HookedTransformer-equivalent numerics. See the [4.0 migration guide](migrating_to_v4.md). ``` ## Mixture of Experts error rates diff --git a/docs/source/content/tutorials.md b/docs/source/content/tutorials.md index 366b7003f1..3be77a8ad0 100644 --- a/docs/source/content/tutorials.md +++ b/docs/source/content/tutorials.md @@ -1,7 +1,7 @@ # Tutorials ```{warning} -`HookedTransformer` is deprecated as of TransformerLens 3.0 and will be removed in the next major version. New code should use [`TransformerBridge`](migrating_to_v3.md) instead. Existing `HookedTransformer` code continues to work through the 3.x branch via a compatibility layer. See the [migration guide](migrating_to_v3.md) for conversion recipes. +`HookedTransformer` and the other `Hooked*` classes were **removed in TransformerLens 4.0**. Use [`TransformerBridge`](migrating_to_v4.md) instead — call `enable_compatibility_mode()` after booting for HookedTransformer-equivalent numerics. See the [4.0 migration guide](migrating_to_v4.md). ``` - **Start with the [main demo](https://neelnanda.io/transformer-lens-demo) to learn how the library works, and the basic features**. From 5d353cb6372a22947b5a3235b0bb100da4ddc752 Mon Sep 17 00:00:00 2001 From: jlarson4 <jonahalarson@comcast.net> Date: Thu, 17 Sep 2026 11:15:36 -0500 Subject: [PATCH 82/87] bug fix for dev-4.x --- .../model_bridge/transformer_bridge.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index 0f555cab16..07ef92c095 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -234,10 +234,18 @@ def __setstate__(self, state: dict[str, Any]) -> None: def original_model(self) -> nn.Module: """The wrapped ``nn.Module``. Raises :class:`AttributeError` for non-torch drivers (vLLM, Inspect) that don't expose a local module.""" - underlying = getattr(self._driver, "underlying_model", None) + driver = getattr(self, "_driver", None) + if driver is None: + # Bridges assembled without __init__ (object.__new__ scaffolds) keep the + # module in the __dict__ mirror the setter maintains. + model = self.__dict__.get("original_model") + if model is None: + raise AttributeError(f"'{type(self).__name__}' has no driver and no original_model") + return model + underlying = getattr(driver, "underlying_model", None) if underlying is None: raise AttributeError( - f"{type(self._driver).__name__} does not expose an nn.Module — " + f"{type(driver).__name__} does not expose an nn.Module — " "non-torch drivers (vLLM, Inspect) operate without a local module." ) return underlying @@ -247,7 +255,7 @@ def original_model(self, value: nn.Module) -> None: """Used by weight-processing paths that move the model across devices.""" self.__dict__["original_model"] = value # Sync via the driver's public API; non-torch drivers don't implement it. - setter = getattr(self._driver, "set_underlying_model", None) + setter = getattr(getattr(self, "_driver", None), "set_underlying_model", None) if callable(setter): setter(value) From 42c88201057ce976c75f583446b0d7ee4e04bc34 Mon Sep 17 00:00:00 2001 From: jlarson4 <jonahalarson@comcast.net> Date: Thu, 17 Sep 2026 12:13:39 -0500 Subject: [PATCH 83/87] Clean up HT references --- .cursor/rules/transformerlens.mdc | 4 +- .github/workflows/checks.yml | 2 +- AGENTS.md | 5 +- CLAUDE.md | 2 +- README.md | 4 +- demos/ARENA_Content.ipynb | 2 +- demos/Attribution_Patching_Demo.ipynb | 2 +- demos/BERT.ipynb | 2 +- demos/Colab_Compatibility.ipynb | 2 +- demos/Config_Overhaul.ipynb | 251 ------------------ demos/Direct_Logit_Attribution_Demo.ipynb | 2 +- demos/Head_Detector_Demo.ipynb | 4 +- demos/Interactive_Neuroscope.ipynb | 2 +- demos/LLaMA.ipynb | 2 +- demos/LLaMA_GPU_Quantized.ipynb | 2 +- demos/Main_Demo.ipynb | 8 +- demos/Othello_GPT.ipynb | 2 +- demos/Patchscopes_Generation_Demo.ipynb | 2 +- demos/Qwen.ipynb | 2 +- demos/SVD_Interpreter_Demo.ipynb | 2 +- demos/Santa_Coder.ipynb | 2 +- demos/T5.ipynb | 2 +- demos/stable_lm.ipynb | 2 +- devtools/adapter_builder/CLAUDE.md | 2 - devtools/adapter_builder/README.md | 2 - .../agents/hooks/guard-hooked-transformer.sh | 70 ----- .../agents/launch-solo-pair.sh | 3 +- devtools/adapter_builder/agents/launch.sh | 5 +- devtools/adapter_builder/agents/programmer.md | 1 - devtools/adapter_builder/agents/reviewer.md | 1 - .../adapter_builder/agents/solo-programmer.md | 1 - .../adapter_builder/agents/solo-reviewer.md | 1 - .../adapter_builder/docs/hooks-reference.md | 14 - .../adapter-builder-tool.md | 4 +- docs/source/content/analysis_tools.md | 2 +- docs/source/content/compatibility_mode.md | 4 +- docs/source/content/contributing.md | 16 +- docs/source/content/hook_system.md | 3 +- docs/source/content/model_tables.md | 2 +- docs/source/content/special_cases.md | 2 +- docs/source/content/tutorials.md | 2 +- tests/AGENTS.md | 2 +- tests/QUARANTINES.md | 2 +- ...son.py => test_bridge_self_consistency.py} | 0 .../test_qwen3_next_adapter.py | 5 +- ... test_bridge_cross_run_mlp_in_patching.py} | 0 ... => test_bridge_cross_run_qkv_patching.py} | 0 .../test_vision_phase_routing.py | 2 +- tests/unit/tools/test_attribution_patching.py | 13 +- transformer_lens/ActivationCache.py | 7 +- transformer_lens/SVDInterpreter.py | 6 +- transformer_lens/benchmarks/README.md | 8 +- transformer_lens/benchmarks/main_benchmark.py | 11 +- transformer_lens/cache/key_value_cache.py | 2 +- .../config/transformer_bridge_config.py | 2 +- transformer_lens/evals.py | 4 +- .../model_bridge/sources/native/init.py | 4 +- .../supported_architectures/jamba.py | 2 +- .../supported_architectures/mamba.py | 4 +- .../supported_architectures/mamba2.py | 4 +- .../supported_architectures/nemotron_h.py | 6 +- .../supported_architectures/vit.py | 4 +- .../supported_architectures/zamba2.py | 4 +- .../model_bridge/transformer_bridge.py | 4 +- transformer_lens/model_protocol.py | 28 +- transformer_lens/supported_models.py | 4 +- .../tools/model_registry/AGENTS.md | 2 +- .../tools/model_registry/verify_models.py | 4 +- transformer_lens/utilities/defaults_utils.py | 2 +- transformer_lens/utilities/matrix.py | 8 +- transformer_lens/weight_processing.py | 2 +- 71 files changed, 117 insertions(+), 474 deletions(-) delete mode 100644 demos/Config_Overhaul.ipynb delete mode 100755 devtools/adapter_builder/agents/hooks/guard-hooked-transformer.sh rename tests/integration/model_bridge/{test_bridge_vs_hooked_comparison.py => test_bridge_self_consistency.py} (100%) rename tests/unit/model_bridge/{test_bridge_vs_hooked_transformer_mlp_in_patching.py => test_bridge_cross_run_mlp_in_patching.py} (100%) rename tests/unit/model_bridge/{test_bridge_vs_hooked_transformer_patching.py => test_bridge_cross_run_qkv_patching.py} (100%) diff --git a/.cursor/rules/transformerlens.mdc b/.cursor/rules/transformerlens.mdc index 845d076da8..d087c89305 100644 --- a/.cursor/rules/transformerlens.mdc +++ b/.cursor/rules/transformerlens.mdc @@ -3,7 +3,7 @@ description: TransformerLens project conventions for Cursor agents. alwaysApply: true --- -Read `AGENTS.md` at the repo root before doing any work. It is the single source of truth for project conventions, quickstart commands, repo layout, hook-naming rules, the HookedTransformer ↔ TransformerBridge mirroring rule, PR conventions, and hard rules. +Read `AGENTS.md` at the repo root before doing any work. It is the single source of truth for project conventions, quickstart commands, repo layout, hook-naming rules, PR conventions, and hard rules. Sub-folder `AGENTS.md` files apply when you're working in those directories — read them too: @@ -14,7 +14,7 @@ Sub-folder `AGENTS.md` files apply when you're working in those directories — Quick reminders that override common defaults: - Use `uv`, not `pip` or `poetry`. Install with `uv sync`; run commands with `uv run …` or `make` targets. -- This repo has two parallel systems (`HookedTransformer` legacy and `TransformerBridge` v3). Changes to HookedTransformer that have equivalents in TransformerBridge must be mirrored to TransformerBridge. +- `TransformerBridge` is the only model system; `HookedTransformer` and the other `Hooked*` model classes were removed in 4.0 (`HookedRootModule` / `HookPoint` remain). - Base PRs against `dev`, not `main`. Never name a branch `main` or `dev`. - No pre-commit hook is installed. Run `make format` and `uv run mypy .` manually before push. - Source `.env` (e.g. `set -a; source .env; set +a`) before any HuggingFace-Hub-hitting command. diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index c774f04903..9fc2c884f9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -191,7 +191,7 @@ jobs: --ignore=tests/unit/model_bridge/test_component_inspection.py --ignore=tests/unit/model_bridge/test_benchmark_gated_hooks_fire.py --ignore=tests/unit/model_bridge/test_bridge_generate_kv_cache.py - --ignore=tests/unit/model_bridge/test_bridge_vs_hooked_transformer_patching.py + --ignore=tests/unit/model_bridge/test_bridge_cross_run_qkv_patching.py --ignore=tests/unit/model_bridge/compatibility/ env: HF_TOKEN: ${{ secrets.HF_TOKEN }} diff --git a/AGENTS.md b/AGENTS.md index 544697ed52..6a6cf728b4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,17 +81,16 @@ Python: **>=3.10, <4.0**. CI tests 3.10, 3.11, 3.12. Format/type/docstring check | Path | What's there | |---|---| | [transformer_lens/](transformer_lens/) | Core package | -| [transformer_lens/HookedEncoder.py](transformer_lens/HookedEncoder.py), [HookedEncoderDecoder.py](transformer_lens/HookedEncoderDecoder.py), [HookedAudioEncoder.py](transformer_lens/HookedAudioEncoder.py) | Encoder-only / seq2seq / audio variants | | [transformer_lens/model_bridge/](transformer_lens/model_bridge/) | `TransformerBridge` system | | [transformer_lens/model_bridge/supported_architectures/](transformer_lens/model_bridge/supported_architectures/) | One adapter file per HF architecture | | [transformer_lens/model_bridge/generalized_components/](transformer_lens/model_bridge/generalized_components/) | Bridge-side reusable components | -| [transformer_lens/factories/](transformer_lens/factories/) | `architecture_adapter_factory.py`, `mlp_factory.py`, `activation_function_factory.py` | +| [transformer_lens/factories/](transformer_lens/factories/) | `architecture_adapter_factory.py`, `activation_function_factory.py` | | [transformer_lens/config/](transformer_lens/config/) | `TransformerBridgeConfig` / `TransformerLensConfig` | | [transformer_lens/utilities/](transformer_lens/utilities/) | Device management, weight processing, HF utilities | | [transformer_lens/hook_points.py](transformer_lens/hook_points.py) | `HookPoint` class and `LensHandle` | | [transformer_lens/supported_models.py](transformer_lens/supported_models.py) | **HT-only** registry (`OFFICIAL_MODEL_NAMES`, `MODEL_ALIASES`) | | [transformer_lens/tools/model_registry/](transformer_lens/tools/model_registry/) | Bridge-side registry + `verify_models.py` benchmark suite | -| [transformer_lens/tools/analysis/](transformer_lens/tools/analysis/) | High-level single-call analyses over the cache (e.g. `direct_logit_attribution`); works with both HT and Bridge | +| [transformer_lens/tools/analysis/](transformer_lens/tools/analysis/) | High-level single-call analyses over the cache (e.g. `direct_logit_attribution`) on TransformerBridge | | [transformer_lens/patching.py](transformer_lens/patching.py), [evals.py](transformer_lens/evals.py) | Activation patching, IOI, ROME, etc. | | [tests/unit/](tests/unit/), [tests/integration/](tests/integration/), [tests/acceptance/](tests/acceptance/), [tests/benchmarks/](tests/benchmarks/), [tests/mps/](tests/mps/) | Test tiers | | [demos/](demos/) | Jupyter notebooks; a subset runs in CI under `nbval` with sanitization from [demos/doc_sanitize.cfg](demos/doc_sanitize.cfg) | diff --git a/CLAUDE.md b/CLAUDE.md index f09bae8c31..0de170e55a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,7 +22,7 @@ ## Pointers - [AGENTS.md §10](AGENTS.md#10-hard-rules) — hard rules; load-bearing. -- [AGENTS.md §2](AGENTS.md#2-two-systems-live-in-this-repo) — HT → Bridge mirroring; most common PR-review pushback. +- [AGENTS.md §2](AGENTS.md#2-two-systems-live-in-this-repo) — the single model system (the `Hooked*` model classes were removed in 4.0). - [tests/QUARANTINES.md](tests/QUARANTINES.md) — check before debugging any failing test. The macOS-arm64 KV-cache skip is the most common time-sink. - [debugging_numerical_divergence.md](docs/source/content/debugging_numerical_divergence.md) — Bridge-vs-HF logit drift bisection. - [compatibility_mode.md](docs/source/content/compatibility_mode.md) — `bridge.enable_compatibility_mode()` contract; read before adding tests that use it. diff --git a/README.md b/README.md index 6708beec76..1bd920d0ff 100644 --- a/README.md +++ b/README.md @@ -188,8 +188,8 @@ Verification lives in the integration tests at `verify_models` benchmark suite now covers the SSM and hybrid families. Mamba-1, Mamba-2, gated-delta-net (Qwen3.5 / Qwen3-Next), NemotronH, and GraniteMoeHybrid all declare `applicable_phases = [1, 2, 3, 4]`, so their forward parity (P1, vs raw -HF), hook/cache coverage (P2/P3, which skip the HookedTransformer comparison SSMs -lack), and generation quality (P4) are benchmarked like any transformer. +HF), hook/cache self-checks and HF equivalence (P2/P3), and generation quality (P4) +are benchmarked like any transformer. ## Credits diff --git a/demos/ARENA_Content.ipynb b/demos/ARENA_Content.ipynb index 47ae2f5d86..e1a0f9e7ef 100644 --- a/demos/ARENA_Content.ipynb +++ b/demos/ARENA_Content.ipynb @@ -39,7 +39,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")\n", "\n", diff --git a/demos/Attribution_Patching_Demo.ipynb b/demos/Attribution_Patching_Demo.ipynb index e5c66511b5..7fc7e3b742 100644 --- a/demos/Attribution_Patching_Demo.ipynb +++ b/demos/Attribution_Patching_Demo.ipynb @@ -83,7 +83,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")\n", "\n", diff --git a/demos/BERT.ipynb b/demos/BERT.ipynb index 649b80f775..36e2769ba9 100644 --- a/demos/BERT.ipynb +++ b/demos/BERT.ipynb @@ -71,7 +71,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")\n", "\n", diff --git a/demos/Colab_Compatibility.ipynb b/demos/Colab_Compatibility.ipynb index 28a9edeaa8..2797664948 100644 --- a/demos/Colab_Compatibility.ipynb +++ b/demos/Colab_Compatibility.ipynb @@ -49,7 +49,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")\n", "\n", diff --git a/demos/Config_Overhaul.ipynb b/demos/Config_Overhaul.ipynb deleted file mode 100644 index 40f510dfb3..0000000000 --- a/demos/Config_Overhaul.ipynb +++ /dev/null @@ -1,251 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Overview\n", - "\n", - "The current way configuration is designed in TransformerLens has a lot of limitations. It does not\n", - "allow for outside people to pass through configurations that are not officially supported, and it\n", - "is very bug prone with something as simple as typo potentially giving you a massive headache. There\n", - "are also a number of hidden rules that are not clearly documented, which can go hidden until\n", - "different pieces of TransformerLens are activated. Allowing to pass in an optional object of configuration\n", - "with no further changes does solve a couple of these problems, but it does not solve the bigger\n", - "issues. It also introduces new problems with users potentially passing in architectures that are not\n", - "supported without having a clear way to inform the user what isn't supported.\n", - "\n", - "My proposal for how all of these problems can be resolved is to fundamentally revamp the\n", - "configuration to allow for something that I like to call configuration composition. From a technical\n", - "perspective, this involves creating a centralized class that describes all supported configurations\n", - "by TransformerLens. This class would then be used to construct specific configurations for all models\n", - "that are currently supported, and it would then allow anyone to easily see in a single place all\n", - "configuration features supported by TransformerLens while also being able to read the code to\n", - "understand how they can create their own configurations for the purpose of either submitting new\n", - "models into TransformerLens, or configuring an unofficially supported model by TransformerLens,\n", - "when TransformerLens already happens to support all of the architectural pieces separately.\n", - "\n", - "This could simple be an overhaul of the existing HookedTransformerConfig. Everything I am\n", - "describing here could be made compatible with that class to give it a more usable interface that is\n", - "then directly interacted with by the end user. At the moment, that class is not really built to be\n", - "interacted with, and is instead used as a wrapper around spreading configured anonymous objects.\n", - "Overhauling this class to do what I am about to describe is a viable path, but keeping it as it is,\n", - "and making a new class as something meant to be used by the end user would be a way to maintain\n", - "compatibility, avoid refactors, and keep model configuration only focused on putting together\n", - "configuration for models, as opposed to configuring full settings needed by HookedTransformer, which\n", - "includes checking the available environment.\n", - "\n", - "A very unscientific basic example of how this would look in code by the end user can be seen\n", - "immediately below. I will delve into details of each piece in this document." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "config = ModelConfig(\n", - " d_model=4096,\n", - " d_head=8192 // 64,\n", - " n_heads=64,\n", - " act_fn=\"silu\"\n", - " # Other universally required properties across all models go here in the constructor\n", - ")\n", - "# Enabling specific features not universal among all models\n", - "config.enabled_gated_mlp()\n", - "# Customizing optional attributes\n", - "config.set_positional_embedding_type(\"alibi\")\n", - "\n", - "# and so on, until the full configuration is set\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## The constructor\n", - "\n", - "The first piece of this I want to talk about is what will be injected into the constructor. It\n", - "should basically take everything absolutely required by all models. This keeps the code easy for\n", - "someone to understand, without adding too much clutter. All fields should be required, and if there\n", - "is ever an idea that a field should be in the constructor as an option, then that is probably an\n", - "indication that there is a good case to add a function to configure that variable in a different\n", - "point in the class. An example of what this would look like can be seen below..." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# make it easy for someone to see what activation functions are supported, this would be moved from\n", - "# HookedTransformerConfig\n", - "ActivationFunction = \"silu\" | \"gelu\"\n", - "\n", - "class ModelConfig:\n", - " def __init__(\n", - " self,\n", - " d_model: int,\n", - " eps: int,\n", - " act_fn: ActivationFunction,\n", - " remaining_required_attributes,\n", - " ):\n", - " self.d_model = d_model\n", - " self.eps = eps\n", - " self.act_fn = act_fn\n", - " # Set defaults for any remaining supported attributes that are not required here \n", - " self.gated_mlp = False\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Boolean Variables\n", - "\n", - "Within TransformerLens config, anything that is a boolean variable is essentially a feature flag.\n", - "This means that all features at the time of construction would have default values, most likely set\n", - "to false. They then get toggled on with an `enable_feature` function call on the config object.\n", - "Having these functions will make very clear for someone less familiar with TransformerLens what\n", - "features are available. It also allows us to decorate these calls, which is very important. There\n", - "are some instances where if a boolean is true, a different one cannot be true, but this requirement\n", - "is not clear anywhere without analyzing code. Decorating these functions allows us to make sure\n", - "these sort of bugs are not possible. I will use `gated_mlp` as an example here, but it is not\n", - "meant to be a real implementation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def enabled_gated_mlp(self: ModelConfig) -> ModelConfig:\n", - " self.gated_mlp = True\n", - " # Configure any side effects caused by enabling of a feature\n", - " self.another_feature = False\n", - " # Returning self allows someone to chain together config calls\n", - " return self\n", - "\n", - "ModelConfig.enabled_gated_mlp = enabled_gated_mlp" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Additional Options\n", - "\n", - "Any other options would similarly have their own functions to configure. This allows for similar\n", - "decoration as with feature flags, and it also in a way documents the architectural capabilities of\n", - "TransformerLens in a single place. If there are groups of options that are also always required\n", - "together, this then gives us a way to require all of those options as opposed to having them all be\n", - "configured at the root level. This also allows us to make changes to other attributes that may be\n", - "affected as a side affect of having some values set, which again makes it both harder for people to\n", - "introduce bugs, and also creates code that documents itself. Another off the cuff example of\n", - "something like this can be seen below." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def set_rotary_dim(self: ModelConfig, rotary_dim: int) -> ModelConfig:\n", - " self.rotary_dim = rotary_dim\n", - " # Additional settings that seem to be present whenever rotary_dim is set\n", - " self.positional_embedding_type = \"rotary\"\n", - " self.rotary_adjacent_pairs = False\n", - " return self\n", - "\n", - "ModelConfig.set_rotary_dim = set_rotary_dim" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Config Final Thoughts\n", - "\n", - "The best way to describe this idea is configuration composition. The reason being is that the user is\n", - "essentially composing a model configuration by setting the base, and then combining various options\n", - "from predefined functions. Doing it like this has a lot of advantages. One of those advantages being\n", - "that there would need to be a lot less memorization on how architectures should be combined. e.g.\n", - "maybe it's not that hard to remember that `rotary_adjacent_pairs` should be False when `rotary_dim`\n", - "is set, but these sorts of combinations accumulate. Having it interfaced out gives everyone a\n", - "place to look to see how parts of configuration work in isolation without the need to memorize a\n", - "large amount of rules.\n", - "\n", - "This would also allow us to more easily mock out fake configurations and enable specific features in\n", - "order to test that functionality in isolation. This also should make it easier for someone to at a\n", - "glance understand all model compatibilities with TransformerLens, since there would be a single file\n", - "where they would all be listed out and documented. It will also allow for people to see\n", - "compatibility limitations at a glance.\n", - "\n", - "As for compatibility, this change would be 100% compatible with the existing structure. The objects\n", - "I am suggesting are abstractions of the existing configuration dictionaries for the purpose of\n", - "communication and ease of use. This means that they can be passed around just like the current\n", - "anonymous dictionaries.\n", - "\n", - "## Further Changes\n", - "\n", - "With this, there are a number of changes that I would like to make to the actual\n", - "`loading_from_pretrained` file in order to revise it to be ready for the possibility of rapidly\n", - "supporting new models. The biggest change in this respect would be to break out what is now a\n", - "configuration dictionary for every model into having its own module where one of these configuration\n", - "objects would be constructed. That object would then be exposed, so that it can be imported into\n", - "`loading_from_pretrained`. We would then create a dictionary where the official name of the\n", - "model would have the configuration object as its value, thus completely eliminating that big giant\n", - "if else statement, and replacing it with a simple return from the dictionary. The configurations\n", - "themselves would then live in a directory structure like so...\n", - "\n", - "config/ <- where the ModelConfig file lives\n", - "config/meta-llama/ <- directory for all models from the group\n", - "config/meta-llama/Llama-2-13b.py <- name matching hugging face to make it really easy to find the\n", - " configuration\n", - "\n", - "## Impact on Testing\n", - "\n", - "This change, would allow us to directly interact with these configuration objects to allow us to\n", - "more easily assert that configurations are set properly, and to also allow us to more easily access\n", - "these configurations in tests for the purposes of writing better unit tests. \n", - "\n", - "## Summary\n", - "\n", - "This change should solve a lot of problems. It may be a big change at first from what currently\n", - "exists, but in time I think most people will find it more elegant, and easier to understand. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/demos/Direct_Logit_Attribution_Demo.ipynb b/demos/Direct_Logit_Attribution_Demo.ipynb index a64d277b30..926d705756 100644 --- a/demos/Direct_Logit_Attribution_Demo.ipynb +++ b/demos/Direct_Logit_Attribution_Demo.ipynb @@ -95,7 +95,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")\n", "\n", diff --git a/demos/Head_Detector_Demo.ipynb b/demos/Head_Detector_Demo.ipynb index 2caa4231f5..23c35f5ed6 100644 --- a/demos/Head_Detector_Demo.ipynb +++ b/demos/Head_Detector_Demo.ipynb @@ -101,7 +101,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")\n", "\n", @@ -254,7 +254,7 @@ "id": "QSVGddQDk1M6" }, "source": [ - "Utils: these will be in `transformer_lens.utils` after merging the fork to the main repo" + "Utils (also available in `transformer_lens.utilities`)" ] }, { diff --git a/demos/Interactive_Neuroscope.ipynb b/demos/Interactive_Neuroscope.ipynb index c372aa4b0a..9307eb1846 100644 --- a/demos/Interactive_Neuroscope.ipynb +++ b/demos/Interactive_Neuroscope.ipynb @@ -57,7 +57,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")\n", "\n", diff --git a/demos/LLaMA.ipynb b/demos/LLaMA.ipynb index b3ee5beeb4..ece6dd3a74 100644 --- a/demos/LLaMA.ipynb +++ b/demos/LLaMA.ipynb @@ -61,7 +61,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")\n", "\n", diff --git a/demos/LLaMA_GPU_Quantized.ipynb b/demos/LLaMA_GPU_Quantized.ipynb index 37d378cc85..5481febcc5 100644 --- a/demos/LLaMA_GPU_Quantized.ipynb +++ b/demos/LLaMA_GPU_Quantized.ipynb @@ -57,7 +57,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")\n", "\n", diff --git a/demos/Main_Demo.ipynb b/demos/Main_Demo.ipynb index 219b159c8d..db94b4af6e 100644 --- a/demos/Main_Demo.ipynb +++ b/demos/Main_Demo.ipynb @@ -873,17 +873,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## HookedTransformer architecture\n", + "## Model architecture conventions\n", "\n", - "**Note:** HookedTransformer is deprecated as of TransformerLens 3.0.\n", - "\n", - "HookedTransformer is a somewhat adapted GPT-2 architecture, but is computationally identical. The most significant changes are to the internal structure of the attention heads: \n", + "TransformerBridge presents GPT-2 in the classic TransformerLens layout, which computes the same function as the HuggingFace model. The most significant changes are to the internal structure of the attention heads: \n", "* The weights (W_K, W_Q, W_V) mapping the residual stream to queries, keys and values are 3 separate matrices, rather than big concatenated one.\n", "* The weight matrices (W_K, W_Q, W_V, W_O) and activations (keys, queries, values, z (values mixed by attention pattern)) have separate head_index and d_head axes, rather than flattening them into one big axis.\n", " * The activations all have shape `[batch, position, head_index, d_head]`\n", " * W_K, W_Q, W_V have shape `[head_index, d_model, d_head]` and W_O has shape `[head_index, d_head, d_model]`\n", "\n", - "The actual code is a bit of a mess, as there's a variety of Boolean flags to make it consistent with the various different model families in TransformerLens - to understand it and the internal structure, I instead recommend reading the code in [CleanTransformerDemo](https://colab.research.google.com/github/TransformerLensOrg/TransformerLens/blob/clean-transformer-demo/Clean_Transformer_Demo.ipynb)" + "To understand the internal structure from scratch, I recommend reading the code in [CleanTransformerDemo](https://colab.research.google.com/github/TransformerLensOrg/TransformerLens/blob/clean-transformer-demo/Clean_Transformer_Demo.ipynb)" ] }, { diff --git a/demos/Othello_GPT.ipynb b/demos/Othello_GPT.ipynb index 004304b016..7b904ceba9 100644 --- a/demos/Othello_GPT.ipynb +++ b/demos/Othello_GPT.ipynb @@ -75,7 +75,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")\n", "\n", diff --git a/demos/Patchscopes_Generation_Demo.ipynb b/demos/Patchscopes_Generation_Demo.ipynb index c10eeca609..06b370bdc1 100644 --- a/demos/Patchscopes_Generation_Demo.ipynb +++ b/demos/Patchscopes_Generation_Demo.ipynb @@ -30,7 +30,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "# NBVAL_IGNORE_OUTPUT\n# Janky code to do different setup when run in a Colab notebook vs VSCode\nimport os\n\nDEBUG_MODE = False\nIN_GITHUB = os.getenv(\"GITHUB_ACTIONS\") == \"true\"\ntry:\n import google.colab\n\n IN_COLAB = True\n print(\"Running as a Colab notebook\")\nexcept:\n IN_COLAB = False\n\nif not IN_GITHUB and not IN_COLAB:\n print(\"Running as a Jupyter notebook - intended for development only!\")\n from IPython import get_ipython\n\n ipython = get_ipython()\n # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n ipython.run_line_magic(\"load_ext\", \"autoreload\")\n ipython.run_line_magic(\"autoreload\", \"2\")\n\nif IN_COLAB or IN_GITHUB:\n %pip install transformer_lens\n %pip install torchtyping\n # Install my janky personal plotting utils\n %pip install git+https://github.com/neelnanda-io/neel-plotly.git\n # Install another version of node that makes PySvelte work way faster\n %pip install circuitsvis\n # Needed for PySvelte to work, v3 came out and broke things...\n %pip install typeguard==2.13.3\n\nimport torch\nfrom typing import List, Callable, Tuple, Union\nfrom functools import partial\nfrom jaxtyping import Float\nfrom transformer_lens.model_bridge import TransformerBridge\nfrom transformer_lens.ActivationCache import ActivationCache\nfrom transformer_lens import utilities as utils\nfrom transformer_lens.hook_points import (\n HookPoint,\n) # Hooking utilities" + "source": "# NBVAL_IGNORE_OUTPUT\n# Janky code to do different setup when run in a Colab notebook vs VSCode\nimport os\n\nDEBUG_MODE = False\nIN_GITHUB = os.getenv(\"GITHUB_ACTIONS\") == \"true\"\ntry:\n import google.colab\n\n IN_COLAB = True\n print(\"Running as a Colab notebook\")\nexcept:\n IN_COLAB = False\n\nif not IN_GITHUB and not IN_COLAB:\n print(\"Running as a Jupyter notebook - intended for development only!\")\n from IPython import get_ipython\n\n ipython = get_ipython()\n # Automatically reload edited TransformerLens code without restarting the kernel\n ipython.run_line_magic(\"load_ext\", \"autoreload\")\n ipython.run_line_magic(\"autoreload\", \"2\")\n\nif IN_COLAB or IN_GITHUB:\n %pip install transformer_lens\n %pip install torchtyping\n # Install my janky personal plotting utils\n %pip install git+https://github.com/neelnanda-io/neel-plotly.git\n # Install another version of node that makes PySvelte work way faster\n %pip install circuitsvis\n # Needed for PySvelte to work, v3 came out and broke things...\n %pip install typeguard==2.13.3\n\nimport torch\nfrom typing import List, Callable, Tuple, Union\nfrom functools import partial\nfrom jaxtyping import Float\nfrom transformer_lens.model_bridge import TransformerBridge\nfrom transformer_lens.ActivationCache import ActivationCache\nfrom transformer_lens import utilities as utils\nfrom transformer_lens.hook_points import (\n HookPoint,\n) # Hooking utilities" }, { "cell_type": "markdown", diff --git a/demos/Qwen.ipynb b/demos/Qwen.ipynb index 1f4eb2c94a..23a87cbce6 100644 --- a/demos/Qwen.ipynb +++ b/demos/Qwen.ipynb @@ -53,7 +53,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")" ] diff --git a/demos/SVD_Interpreter_Demo.ipynb b/demos/SVD_Interpreter_Demo.ipynb index 790fd43f30..86be857dcb 100644 --- a/demos/SVD_Interpreter_Demo.ipynb +++ b/demos/SVD_Interpreter_Demo.ipynb @@ -84,7 +84,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")" ] diff --git a/demos/Santa_Coder.ipynb b/demos/Santa_Coder.ipynb index ec7fe929d8..91aa0c0eed 100644 --- a/demos/Santa_Coder.ipynb +++ b/demos/Santa_Coder.ipynb @@ -44,7 +44,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")" ] diff --git a/demos/T5.ipynb b/demos/T5.ipynb index 430c9390f8..2abf3b09b5 100644 --- a/demos/T5.ipynb +++ b/demos/T5.ipynb @@ -30,7 +30,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")\n", "\n", diff --git a/demos/stable_lm.ipynb b/demos/stable_lm.ipynb index 077952edba..b5b142c271 100644 --- a/demos/stable_lm.ipynb +++ b/demos/stable_lm.ipynb @@ -56,7 +56,7 @@ " from IPython import get_ipython\n", "\n", " ipython = get_ipython()\n", - " # Code to automatically update the HookedTransformer code as its edited without restarting the kernel\n", + " # Automatically reload edited TransformerLens code without restarting the kernel\n", " ipython.run_line_magic(\"load_ext\", \"autoreload\")\n", " ipython.run_line_magic(\"autoreload\", \"2\")\n" ] diff --git a/devtools/adapter_builder/CLAUDE.md b/devtools/adapter_builder/CLAUDE.md index d47078eefa..0b74a84859 100644 --- a/devtools/adapter_builder/CLAUDE.md +++ b/devtools/adapter_builder/CLAUDE.md @@ -28,7 +28,6 @@ agents/ Agent definitions and orchestration overlord-request.sh flock-based memory lock for heavy operations hooks/ Claude Code hooks for auto-enforcement timeline-capture.sh All events — structured JSONL logging - guard-hooked-transformer.sh PreToolUse — blocks edits to deprecated files guard-git.sh PreToolUse — blocks `git commit`, `git push`, `gh pr create` guard-review-rounds.sh PreToolUse — blocks review files past round 3 guard-verify-models.sh PreToolUse — blocks verify_models on >7B or unregistered models @@ -89,7 +88,6 @@ Uses Claude Code experimental agent teams (`CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS **Auto-enforced by Claude Code hooks** (in `.claude/settings.json` per worktree): - Timeline capture: every tool call + session event → `.adapter-workspace/timeline.jsonl` -- HookedTransformer guardrail: edits to deprecated files are blocked at the framework level - Lint gate: session can't end until `mypy` + `make check-format` pass - Completion notifier: Slack fires automatically on `verification_passed: true` diff --git a/devtools/adapter_builder/README.md b/devtools/adapter_builder/README.md index 86ed252a28..bf8a1a24e9 100644 --- a/devtools/adapter_builder/README.md +++ b/devtools/adapter_builder/README.md @@ -75,7 +75,6 @@ Critical rules are enforced by Claude Code hooks, not by asking agents to rememb | Hook | Enforces | | ---- | -------- | -| **guard-hooked-transformer** | Blocks writes to deprecated `HookedTransformer.py`, `loading_from_pretrained.py`, `components/`, `pretrained/weight_conversions/` | | **guard-git** | Blocks `git commit`, `git push`, `gh pr create`, `gh release create` — agents cannot publish changes | | **guard-review-rounds** | Blocks review files past round 3 per checkpoint, forcing escalation to the user when loops stall | | **guard-verify-models** | Blocks `verify_models --model …` invocations targeting unregistered models, anything above `MAX_VERIFY_PARAMS` (default 7.5B), or `--no-hf-reference` (structural-only runs are not verification) | @@ -123,7 +122,6 @@ Use `--retry` to explicitly resume a crashed session. It's safe for planning and │ ├── overlord-request.sh # flock-based memory lock │ └── hooks/ # Runtime-enforcement hooks (see docs/hooks-reference.md) │ ├── timeline-capture.sh -│ ├── guard-hooked-transformer.sh │ ├── guard-git.sh │ ├── guard-review-rounds.sh │ ├── guard-verify-models.sh diff --git a/devtools/adapter_builder/agents/hooks/guard-hooked-transformer.sh b/devtools/adapter_builder/agents/hooks/guard-hooked-transformer.sh deleted file mode 100755 index e413705dea..0000000000 --- a/devtools/adapter_builder/agents/hooks/guard-hooked-transformer.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# ============================================================================= -# guard-hooked-transformer.sh — PreToolUse hook -# -# RUNTIME CONTEXT: -# This script lives under devtools/adapter_builder/agents/hooks/ but is executed -# by Claude Code *inside the target TransformerLens worktree* — the -# .claude/settings.json written by launch.sh references this file via an -# absolute path. `pwd` at runtime is the worktree, NOT the directory -# containing this file. The deny_patterns below match against paths -# inside that worktree. -# -# Denies any Edit/Write tool call targeting deprecated HookedTransformer files. -# Agents may read these files (for reference) but must not modify them. -# -# The hook receives the tool call payload on stdin. If the tool is Edit/Write -# and the target file is a deprecated path, we exit with code 2 and print a -# message to stderr — Claude Code interprets this as a deny decision. -# ============================================================================= - -set -euo pipefail - -# Read the hook payload from stdin -payload=$(cat) - -# Extract the tool name and file path using jq (fall back gracefully if missing) -tool_name=$(echo "$payload" | jq -r '.tool_name // empty') -file_path=$(echo "$payload" | jq -r '.tool_input.file_path // .tool_input.path // empty') - -allow_response() { - echo '{"continue": true}' - exit 0 -} - -# Only intercept Edit, Write, MultiEdit, NotebookEdit operations -case "$tool_name" in - Edit|Write|MultiEdit|NotebookEdit) ;; - *) allow_response ;; -esac - -# No file path — let it through (some tools may not have one) -[[ -z "$file_path" ]] && allow_response - -# Check against the deprecated path patterns. Paths are matched as suffixes -# (the file_path from tool_input is typically absolute, so we check whether -# it ends with the pattern). This prevents false positives on unrelated paths -# like "my_transformer_lens_backup/foo.py". -deny_patterns=( - "transformer_lens/HookedTransformer.py" - "transformer_lens/loading_from_pretrained.py" - "transformer_lens/components/" - "transformer_lens/pretrained/weight_conversions/" -) - -for pattern in "${deny_patterns[@]}"; do - # Match: path ends with the pattern, or contains /pattern (subpath match). - # The leading / ensures we match at a directory boundary, not mid-word. - if [[ "$file_path" == */"$pattern"* ]] || [[ "$file_path" == "$pattern"* ]]; then - local reason="BLOCKED: This file is part of the deprecated HookedTransformer system. Adapter work must only modify files under transformer_lens/model_bridge/ and transformer_lens/factories/architecture_adapter_factory.py. HookedTransformer files may be READ for reference but not modified." - local tl="$(pwd)/.adapter-workspace/timeline.jsonl" - if [[ -f "$tl" ]]; then - jq -n --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --arg hook "guard-hooked-transformer" --arg reason "$reason" --arg path "$file_path" \ - '{ts:$ts, event:"HookBlocked", tool:"Edit", hook:$hook, file_path:$path, reason:$reason}' >> "$tl" 2>/dev/null || true - fi - jq -n --arg reason "$reason" '{continue: false, decision: "block", reason: $reason}' - exit 2 - fi -done - -allow_response diff --git a/devtools/adapter_builder/agents/launch-solo-pair.sh b/devtools/adapter_builder/agents/launch-solo-pair.sh index 8ee53eaea9..5b365584b3 100755 --- a/devtools/adapter_builder/agents/launch-solo-pair.sh +++ b/devtools/adapter_builder/agents/launch-solo-pair.sh @@ -247,7 +247,6 @@ mkdir -p "$WORKTREE_DIR/.claude" _hook_cmd() { echo "'$SCRIPT_DIR/hooks/$1'"; } TIMELINE_HOOK=$(_hook_cmd timeline-capture.sh) -GUARD_HT_HOOK=$(_hook_cmd guard-hooked-transformer.sh) GUARD_GIT_HOOK=$(_hook_cmd guard-git.sh) GUARD_REVIEW_ROUNDS_HOOK=$(_hook_cmd guard-review-rounds.sh) GUARD_VERIFY_MODELS_HOOK=$(_hook_cmd guard-verify-models.sh) @@ -259,7 +258,7 @@ _h() { jq -n --arg cmd "$1" '{"type":"command","command":$cmd}'; } jq -n \ --argjson session_start "[{\"hooks\":[ $(_h "$TIMELINE_HOOK") ]}]" \ --argjson session_end "[{\"hooks\":[ $(_h "$TIMELINE_HOOK"), $(_h "$NOTIFY_HOOK") ]}]" \ - --argjson pre_edit "{\"matcher\":\"Edit|Write|MultiEdit|NotebookEdit\",\"hooks\":[ $(_h "$GUARD_HT_HOOK"), $(_h "$GUARD_REVIEW_ROUNDS_HOOK") ]}" \ + --argjson pre_edit "{\"matcher\":\"Edit|Write|MultiEdit|NotebookEdit\",\"hooks\":[ $(_h "$GUARD_REVIEW_ROUNDS_HOOK") ]}" \ --argjson pre_bash "{\"matcher\":\"Bash\",\"hooks\":[ $(_h "$GUARD_GIT_HOOK"), $(_h "$GUARD_VERIFY_MODELS_HOOK") ]}" \ --argjson pre_all "{\"matcher\":\"\",\"hooks\":[ $(_h "$TIMELINE_HOOK") ]}" \ --argjson post_all "[{\"matcher\":\"\",\"hooks\":[ $(_h "$TIMELINE_HOOK") ]}]" \ diff --git a/devtools/adapter_builder/agents/launch.sh b/devtools/adapter_builder/agents/launch.sh index 2743be4ea3..0c9ac6b3ff 100755 --- a/devtools/adapter_builder/agents/launch.sh +++ b/devtools/adapter_builder/agents/launch.sh @@ -382,7 +382,6 @@ mkdir -p "$WORKTREE_DIR/.claude" _hook_cmd() { echo "'$SCRIPT_DIR/hooks/$1'"; } TIMELINE_HOOK=$(_hook_cmd timeline-capture.sh) -GUARD_HT_HOOK=$(_hook_cmd guard-hooked-transformer.sh) GUARD_GIT_HOOK=$(_hook_cmd guard-git.sh) GUARD_REVIEW_ROUNDS_HOOK=$(_hook_cmd guard-review-rounds.sh) GUARD_VERIFY_MODELS_HOOK=$(_hook_cmd guard-verify-models.sh) @@ -400,7 +399,7 @@ jq -n \ --argjson session_end "[{\"hooks\":[ $(_h "$TIMELINE_HOOK"), $(_h "$NOTIFY_HOOK") ]}]" \ --argjson subagent_start "[{\"hooks\":[ $(_h "$TIMELINE_HOOK") ]}]" \ --argjson subagent_stop "[{\"hooks\":[ $(_h "$TIMELINE_HOOK"), $(_h "$GATE_REVIEWER_WRITES_HOOK") ]}]" \ - --argjson pre_edit "{\"matcher\":\"Edit|Write|MultiEdit|NotebookEdit\",\"hooks\":[ $(_h "$GUARD_HT_HOOK"), $(_h "$GUARD_REVIEW_ROUNDS_HOOK") ]}" \ + --argjson pre_edit "{\"matcher\":\"Edit|Write|MultiEdit|NotebookEdit\",\"hooks\":[ $(_h "$GUARD_REVIEW_ROUNDS_HOOK") ]}" \ --argjson pre_bash "{\"matcher\":\"Bash\",\"hooks\":[ $(_h "$GUARD_GIT_HOOK"), $(_h "$GUARD_VERIFY_MODELS_HOOK") ]}" \ --argjson pre_all "{\"matcher\":\"\",\"hooks\":[ $(_h "$TIMELINE_HOOK") ]}" \ --argjson post_all "[{\"matcher\":\"\",\"hooks\":[ $(_h "$TIMELINE_HOOK") ]}]" \ @@ -414,7 +413,7 @@ jq -n \ PostToolUse: $post_all, Stop: $stop }}' > "$SETTINGS_FILE" -ok "Hooks installed: timeline, HookedTransformer guardrail, git guardrail, review-rounds limit, verify-models gate, reviewer-writes-file gate, lint gate, completion notifier" +ok "Hooks installed: timeline, git guardrail, review-rounds limit, verify-models gate, reviewer-writes-file gate, lint gate, completion notifier" # Pre-seed workspace trust for the worktree. Without this, a first launch in # a fresh worktree hangs at Claude Code's "Do you trust this folder?" prompt diff --git a/devtools/adapter_builder/agents/programmer.md b/devtools/adapter_builder/agents/programmer.md index 61b617b376..ac369eacb3 100644 --- a/devtools/adapter_builder/agents/programmer.md +++ b/devtools/adapter_builder/agents/programmer.md @@ -24,7 +24,6 @@ accessible for inspection. - `docs/memory-lock.md` — lock protocol (read before Step 3) ## Constraints (hook-enforced) -- HookedTransformer is read-only (hook blocks edits) - No `git commit`/`push`/`gh pr` (hook blocks) - `verify_models` blocked on >7.5B or unregistered models (hook blocks) - All output → files in `.adapter-workspace/`, not terminal diff --git a/devtools/adapter_builder/agents/reviewer.md b/devtools/adapter_builder/agents/reviewer.md index e1a82bd1e2..9f94f6c3be 100644 --- a/devtools/adapter_builder/agents/reviewer.md +++ b/devtools/adapter_builder/agents/reviewer.md @@ -24,7 +24,6 @@ research even if they produce correct outputs. ## Constraints (hook-enforced) -- HookedTransformer changes → flag as CRITICAL - No git commits/pushes - Review file must exist on disk before returning decision (hook voids file-less results) diff --git a/devtools/adapter_builder/agents/solo-programmer.md b/devtools/adapter_builder/agents/solo-programmer.md index 13e473f5c2..f6676ac915 100644 --- a/devtools/adapter_builder/agents/solo-programmer.md +++ b/devtools/adapter_builder/agents/solo-programmer.md @@ -26,7 +26,6 @@ accessible for inspection. - `docs/memory-lock.md` — lock protocol (read before Step 3) ## Constraints (hook-enforced) -- HookedTransformer is read-only (hook blocks edits) - No `git commit`/`push`/`gh pr` (hook blocks) - `verify_models` blocked on >7.5B or unregistered models (hook blocks) - All output → files in `.adapter-workspace/`, not terminal diff --git a/devtools/adapter_builder/agents/solo-reviewer.md b/devtools/adapter_builder/agents/solo-reviewer.md index 465b1abf0a..a69d35a4e5 100644 --- a/devtools/adapter_builder/agents/solo-reviewer.md +++ b/devtools/adapter_builder/agents/solo-reviewer.md @@ -25,7 +25,6 @@ research even if they produce correct outputs. - `docs/artifact-templates.md` — templates for review files and completion report ## Constraints (hook-enforced) -- HookedTransformer changes → flag as CRITICAL - No git commits/pushes - Review files past round 3 are blocked (a hook enforces the iteration limit) diff --git a/devtools/adapter_builder/docs/hooks-reference.md b/devtools/adapter_builder/docs/hooks-reference.md index 554d78efb6..96392e1f92 100644 --- a/devtools/adapter_builder/docs/hooks-reference.md +++ b/devtools/adapter_builder/docs/hooks-reference.md @@ -9,7 +9,6 @@ Every hook includes a `RUNTIME CONTEXT:` header reminding the reader that the sc | Hook | Event | Matcher | Can block? | |------|-------|---------|------------| | [timeline-capture.sh](../agents/hooks/timeline-capture.sh) | `SessionStart`, `SessionEnd`, `SubagentStart`, `SubagentStop`, `PreToolUse`, `PostToolUse` | (all) | No — logging only | -| [guard-hooked-transformer.sh](../agents/hooks/guard-hooked-transformer.sh) | `PreToolUse` | `Edit\|Write\|MultiEdit\|NotebookEdit` | Yes | | [guard-review-rounds.sh](../agents/hooks/guard-review-rounds.sh) | `PreToolUse` | `Edit\|Write\|MultiEdit\|NotebookEdit` | Yes | | [guard-git.sh](../agents/hooks/guard-git.sh) | `PreToolUse` | `Bash` | Yes | | [guard-verify-models.sh](../agents/hooks/guard-verify-models.sh) | `PreToolUse` | `Bash` | Yes | @@ -25,19 +24,6 @@ Appends a structured JSON line to `.adapter-workspace/timeline.jsonl` on every t Never blocks — always emits `{"continue": true}`. If `jq` or the timeline file isn't available, the append silently fails and the tool call still proceeds. -## guard-hooked-transformer.sh - -Blocks edits to deprecated HookedTransformer files. Agents may read them for reference but must not modify them — their work must only touch the TransformerBridge system. - -**Blocks writes** whose `tool_input.file_path` contains any of: - -- `transformer_lens/HookedTransformer.py` -- `transformer_lens/loading_from_pretrained.py` -- `transformer_lens/components/` -- `transformer_lens/pretrained/weight_conversions/` - -On block, emits `{"continue": false, "decision": "block", "reason": …}` with a message explaining the adapter should only touch `transformer_lens/model_bridge/` and the factory. - ## guard-review-rounds.sh Enforces the 3-round review-iteration limit from [`agents/orchestrator.md`](../agents/orchestrator.md) § Iteration Limits. Turns a prompt-level rule into a runtime stop so runaway loops can't happen. diff --git a/docs/source/content/adapter_development/adapter-builder-tool.md b/docs/source/content/adapter_development/adapter-builder-tool.md index 6550899d9c..d5c935f087 100644 --- a/docs/source/content/adapter_development/adapter-builder-tool.md +++ b/docs/source/content/adapter_development/adapter-builder-tool.md @@ -38,8 +38,8 @@ Two coordination modes exist: **agent-teams** (an orchestrator session; requires Claude Code Max) and **solo** (two independent sessions coordinated by a signal-routing daemon; works on any tier). -Critical rules — no git writes, no edits to deprecated `HookedTransformer` -paths, no oversized or structural-only verification runs, mandatory +Critical rules — no git writes, no oversized or structural-only +verification runs, mandatory mypy/format gates — are enforced by Claude Code hooks at the framework level, not just by prompt instructions. diff --git a/docs/source/content/analysis_tools.md b/docs/source/content/analysis_tools.md index 370f0eae1c..e2a368af3b 100644 --- a/docs/source/content/analysis_tools.md +++ b/docs/source/content/analysis_tools.md @@ -7,7 +7,7 @@ even when they highlight the same attention head or token. This guide covers the high-level tools in `transformer_lens.tools.analysis`. For capturing activations or installing your own interventions, start with the [hook system](hook_system.md). New experiments should use `TransformerBridge`; see -the [migration guide](migrating_to_v3.md) for existing `HookedTransformer` code. +the [4.0 migration guide](migrating_to_v4.md) for existing `HookedTransformer` code. ## Choose by research question diff --git a/docs/source/content/compatibility_mode.md b/docs/source/content/compatibility_mode.md index f2901ff031..7a978d1ee2 100644 --- a/docs/source/content/compatibility_mode.md +++ b/docs/source/content/compatibility_mode.md @@ -83,14 +83,14 @@ An adapter author for a new post-norm or MLA-style architecture must handle thes ## The four-quadrant test matrix -The integration conftest at [`tests/integration/model_bridge/conftest.py`](../../../tests/integration/model_bridge/conftest.py) provides four bridge variants for every test model: +The shared conftest at [`tests/conftest.py`](../../../tests/conftest.py) provides three bridge variants plus frozen reference goldens: | Variant | `compatibility_mode` | `no_processing` | Tests… | |---|---|---|---| | `gpt2_bridge` | off | n/a | HF-faithful numerics | | `gpt2_bridge_compat` | on | `False` | HT-equivalent numerics | | `gpt2_bridge_compat_no_processing` | on | `True` | Hook aliases without weight processing — used to bisect numerical bugs | -| (HT side) `gpt2_hooked_processed`, `gpt2_hooked_unprocessed` | n/a | n/a | Reference HookedTransformer with/without weight processing | +| (reference) `gpt2_goldens_processed`, `gpt2_goldens_unprocessed` | n/a | n/a | Frozen HookedTransformer outputs with/without weight processing | New integration tests should use the variant that matches the property they're testing. Tests of HF parity → `gpt2_bridge`. Tests of HT-API behaviour → `gpt2_bridge_compat`. Tests of hook semantics regardless of weights → `gpt2_bridge_compat_no_processing`. diff --git a/docs/source/content/contributing.md b/docs/source/content/contributing.md index cc5ee84e84..d72b835c42 100644 --- a/docs/source/content/contributing.md +++ b/docs/source/content/contributing.md @@ -137,14 +137,14 @@ Use `uv` rather than `pip` or `poetry` — commands run via `uv run <cmd>` or th ## Two systems live in this repo -The library is mid-transition between two parallel paths: +TransformerLens 4.0 has a single model system: | System | Status | Lives in | Numerics | Registry | |---|---|---|---|---| -| `TransformerBridge` | v3 — default for new work | `transformer_lens/model_bridge/` | Raw HF weights by default; `bridge.enable_compatibility_mode()` for HT-equivalent — see [Compatibility Mode](compatibility_mode.md) | `transformer_lens/tools/model_registry/data/supported_models.json` | -| `HookedTransformer` | Legacy, maintenance mode, deprecated in 3.0 | `transformer_lens/HookedTransformer.py` + `transformer_lens/components/` | Folds LayerNorm + centres weights → does NOT match HF | `transformer_lens/supported_models.py` (**HT-only**) | +| `TransformerBridge` | The only model system in 4.0 | `transformer_lens/model_bridge/` | Raw HF weights by default; `bridge.enable_compatibility_mode()` for HookedTransformer-equivalent numerics — see [Compatibility Mode](compatibility_mode.md) | `transformer_lens/tools/model_registry/data/supported_models.json` | +| `HookedTransformer` | Removed in 4.0 — see the [4.0 migration guide](migrating_to_v4.md) | *(deleted)* | — | — | -Because the two systems are parallel implementations of the same surface, behavioural changes on one side usually need a matching change on the other. If you change a feature in `HookedTransformer` that has a counterpart in `TransformerBridge` (or vice versa), update both in the same PR — drift between them has historically been a steady source of bugs. The registries are *not* parallel, though: `supported_models.py` is HookedTransformer-only, while Bridge-only models live in the Bridge registry data file under `transformer_lens/tools/model_registry/`. +`HookedRootModule` and `HookPoint` remain the supported way to hook an arbitrary `nn.Module`. `transformer_lens/supported_models.py` is kept as the frozen legacy name/alias ledger; new models go in the Bridge registry data file under `transformer_lens/tools/model_registry/`. ## PR conventions @@ -225,13 +225,13 @@ You can reference other parts of the codebase using ```reStructuredText :mod:transformer_lens # Function or module -:const:`transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES` +:const:`transformer_lens.supported_models.OFFICIAL_MODEL_NAMES` -:class:`transformer_lens.HookedTransformer` +:class:`transformer_lens.model_bridge.TransformerBridge` -:meth:`transformer_lens.HookedTransformer.from_pretrained` +:meth:`transformer_lens.model_bridge.TransformerBridge.boot_transformers` -:attr:`transformer_lens.HookedTransformer.cfg` +:attr:`transformer_lens.model_bridge.TransformerBridge.cfg` ``` ##### Maths diff --git a/docs/source/content/hook_system.md b/docs/source/content/hook_system.md index a0c83fcca9..48982bbd50 100644 --- a/docs/source/content/hook_system.md +++ b/docs/source/content/hook_system.md @@ -48,8 +48,7 @@ logits, cache = model.run_with_cache( ``` Pass `incl_bwd=True` to also cache gradients. This runs `backward()` on the output, so the -run must return a scalar (`return_type="loss"`); gradients land under a `_grad` suffix. -Works on both `HookedTransformer` and `TransformerBridge`: +run must return a scalar (`return_type="loss"`); gradients land under a `_grad` suffix: ```python loss, cache = model.run_with_cache( diff --git a/docs/source/content/model_tables.md b/docs/source/content/model_tables.md index 64d30ab128..984f16c084 100644 --- a/docs/source/content/model_tables.md +++ b/docs/source/content/model_tables.md @@ -4,7 +4,7 @@ title: Model Tables # Model Tables ```{warning} -`HookedTransformer` was removed in TransformerLens 4.0. New code should use [`TransformerBridge`](migrating_to_v3.md), which reproduces HookedTransformer numerics via `enable_compatibility_mode()`. The HookedTransformer model table below is a frozen snapshot kept for users still on the 2.x / 3.x branches. See the [migration guide](migrating_to_v3.md) for conversion recipes. +`HookedTransformer` was removed in TransformerLens 4.0. New code should use [`TransformerBridge`](migrating_to_v4.md), which reproduces HookedTransformer numerics via `enable_compatibility_mode()`. The HookedTransformer model table below is a frozen snapshot kept for users still on the 2.x / 3.x branches. See the [migration guide](migrating_to_v3.md) for conversion recipes. ``` TransformerLens documents two model tables: diff --git a/docs/source/content/special_cases.md b/docs/source/content/special_cases.md index b29edc478f..e0c5f95186 100644 --- a/docs/source/content/special_cases.md +++ b/docs/source/content/special_cases.md @@ -12,7 +12,7 @@ rate of the logits compared to those from the default model was found to be arou There are two main ways to mitigate this: -1. **Skip weight preprocessing.** On the bridge, simply load with `TransformerBridge.boot_transformers(...)` and do not call `enable_compatibility_mode()` - the bridge preserves raw HF weights by default, so no additional flag is needed. On the legacy `HookedTransformer` path, use `HookedTransformer.from_pretrained_no_processing` instead of `HookedTransformer.from_pretrained`. +1. **Skip weight preprocessing.** On the bridge, simply load with `TransformerBridge.boot_transformers(...)` and do not call `enable_compatibility_mode()` - the bridge preserves raw HF weights by default, so no additional flag is needed. 2. **Increase the precision of the data type used in the model.** ## Qwen3.5 text-only models diff --git a/docs/source/content/tutorials.md b/docs/source/content/tutorials.md index 3be77a8ad0..ecca233d9b 100644 --- a/docs/source/content/tutorials.md +++ b/docs/source/content/tutorials.md @@ -36,4 +36,4 @@ - [**SVD Interpreter Demo**](https://colab.research.google.com/github/TransformerLensOrg/TransformerLens/blob/main/demos/SVD_Interpreter_Demo.ipynb) - Based on the [Conjecture post](https://www.lesswrong.com/posts/mkbGjzxD8d8XqKHzA/the-singular-value-decompositions-of-transformer-weight#Directly_editing_SVD_representations) about how the singular value decompositions of transformer matrices are surprisingly interpretable, this demo shows how to use TransformerLens to reproduce this and investigate further. -- [**Tracr to TransformerLens**](https://colab.research.google.com/github/TransformerLensOrg/TransformerLens/blob/main/demos/Tracr_to_Transformer_Lens_Demo.ipynb) - [Tracr](https://github.com/deepmind/tracr) is a cool new DeepMind tool that compiles a written program in [RASP](https://arxiv.org/abs/2106.06981) to transformer weights.This is a (hacky!) script to convert Tracr weights from the JAX form to a TransformerLens HookedTransformer in PyTorch. +- [**Tracr to TransformerLens**](https://colab.research.google.com/github/TransformerLensOrg/TransformerLens/blob/main/demos/Tracr_to_Transformer_Lens_Demo.ipynb) - [Tracr](https://github.com/deepmind/tracr) is a cool new DeepMind tool that compiles a written program in [RASP](https://arxiv.org/abs/2106.06981) to transformer weights.This is a (hacky!) script to convert Tracr weights from the JAX form to a TransformerLens `TransformerBridge` in PyTorch. diff --git a/tests/AGENTS.md b/tests/AGENTS.md index fbdd759f41..da59c6645c 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -29,7 +29,7 @@ Read [the root AGENTS.md](../AGENTS.md) for project-wide rules. This file covers Common combinations: `make test-pr` (unit + docstring + acceptance + integration — the PR-review surface), `make test` (everything including benchmarks + notebooks). -**Rule of thumb:** new tests that load a model should land in `integration/` by default. The `unit/` tier has a few legitimate model-loading exceptions (e.g. `test_bridge_vs_hooked_transformer_*.py` compares numerics across architectures, which is conceptually unit-scoped) — match that pattern only when the test really is testing isolated behaviour that happens to need a model. +**Rule of thumb:** new tests that load a model should land in `integration/` by default. The `unit/` tier has a few legitimate model-loading exceptions (e.g. `test_bridge_cross_run_*_patching.py` checks patching semantics across architectures, which is conceptually unit-scoped) — match that pattern only when the test really is testing isolated behaviour that happens to need a model. --- diff --git a/tests/QUARANTINES.md b/tests/QUARANTINES.md index 359eda3a2f..7a0a00d3c4 100644 --- a/tests/QUARANTINES.md +++ b/tests/QUARANTINES.md @@ -154,7 +154,7 @@ now bound solely by the surviving |---|---|---| | [`unit/factored_matrix/test_constructor.py`:54](unit/factored_matrix/test_constructor.py) | `skip` | FactoredMatrix constructor edge case | | [`unit/model_bridge/test_architecture_adapter.py`:453](unit/model_bridge/test_architecture_adapter.py) | `skip` | SoLU-style weight-processing paths (adapter under test is Gemma3, which has no `mlp.ln`) | -| [`unit/model_bridge/test_bridge_vs_hooked_transformer_patching.py`:138,142](unit/model_bridge/test_bridge_vs_hooked_transformer_patching.py) | `skipif`/`xfail` | Bridge↔HT patching parity | +| [`unit/model_bridge/test_bridge_cross_run_qkv_patching.py`:138,142](unit/model_bridge/test_bridge_cross_run_qkv_patching.py) | `skipif`/`xfail` | Bridge↔HT patching parity | | [`unit/model_bridge/test_hook_alias_resolution.py`:90](unit/model_bridge/test_hook_alias_resolution.py) | `xfail(strict=True)` per-arch | Hook-alias gaps | | [`unit/model_bridge/supported_architectures/test_qwen3_5_adapter.py`:448,464,494,514,605,700,805,947,1133](unit/model_bridge/supported_architectures/test_qwen3_5_adapter.py) | `skipif` ×9 | Qwen3_5 classes absent from installed transformers | | [`unit/model_bridge/supported_architectures/test_qwen3_next_adapter.py`:397](unit/model_bridge/supported_architectures/test_qwen3_next_adapter.py) | `skipif` | Qwen3NextForCausalLM absent from installed transformers | diff --git a/tests/integration/model_bridge/test_bridge_vs_hooked_comparison.py b/tests/integration/model_bridge/test_bridge_self_consistency.py similarity index 100% rename from tests/integration/model_bridge/test_bridge_vs_hooked_comparison.py rename to tests/integration/model_bridge/test_bridge_self_consistency.py diff --git a/tests/unit/model_bridge/supported_architectures/test_qwen3_next_adapter.py b/tests/unit/model_bridge/supported_architectures/test_qwen3_next_adapter.py index 2b69818302..00c41d418b 100644 --- a/tests/unit/model_bridge/supported_architectures/test_qwen3_next_adapter.py +++ b/tests/unit/model_bridge/supported_architectures/test_qwen3_next_adapter.py @@ -1,8 +1,7 @@ """Unit tests for the Qwen3Next architecture adapter. -Qwen3Next is supported via TransformerBridge. -The bridge reads HF config directly via the adapter and bypasses -transformer_lens.loading_from_pretrained, so no convert_hf_model_config tests here. +Qwen3Next is supported via TransformerBridge, which reads the HF config directly +through the adapter, so there are no config-conversion tests here. """ import pytest diff --git a/tests/unit/model_bridge/test_bridge_vs_hooked_transformer_mlp_in_patching.py b/tests/unit/model_bridge/test_bridge_cross_run_mlp_in_patching.py similarity index 100% rename from tests/unit/model_bridge/test_bridge_vs_hooked_transformer_mlp_in_patching.py rename to tests/unit/model_bridge/test_bridge_cross_run_mlp_in_patching.py diff --git a/tests/unit/model_bridge/test_bridge_vs_hooked_transformer_patching.py b/tests/unit/model_bridge/test_bridge_cross_run_qkv_patching.py similarity index 100% rename from tests/unit/model_bridge/test_bridge_vs_hooked_transformer_patching.py rename to tests/unit/model_bridge/test_bridge_cross_run_qkv_patching.py diff --git a/tests/unit/model_registry/test_vision_phase_routing.py b/tests/unit/model_registry/test_vision_phase_routing.py index e0bafce49f..12c8130f44 100644 --- a/tests/unit/model_registry/test_vision_phase_routing.py +++ b/tests/unit/model_registry/test_vision_phase_routing.py @@ -1,6 +1,6 @@ """Vision encoders must route to the {1, 9} verification set. -Vision architectures have no tokenizer and no HookedTransformer counterpart, so the +Vision architectures have no tokenizer or text tower, so the default text phase set ({1,2,3,4}) cannot run against them. They previously opted out of verification entirely via ``applicable_phases = []``, which made them unverifiable rather than verified — Phase 1 (HF parity on pixel input) and Phase 9 (pixel diff --git a/tests/unit/tools/test_attribution_patching.py b/tests/unit/tools/test_attribution_patching.py index f0dab2244f..c2e809a2d6 100644 --- a/tests/unit/tools/test_attribution_patching.py +++ b/tests/unit/tools/test_attribution_patching.py @@ -1797,15 +1797,10 @@ def test_edge_effects_mutation_only_changes_the_perturbed_writers_edges() -> Non # Exact-patch parity # --------------------------------------------------------------------------- # -# generic_activation_patch (transformer_lens/patching.py) is not used for this -# check: its `model: HookedTransformer` parameter is enforced at runtime by -# this repo's jaxtyping/beartype pytest configuration -# (--jaxtyping-packages=transformer_lens,beartype.beartype), which rejects any -# argument that is not actually a HookedTransformer instance -- including a -# real TransformerBridge, not just this module's toy double. A `# type: -# ignore` only silences the static checker; it cannot satisfy a runtime -# isinstance check. The patch below is instead driven directly through the -# same hooks() mechanism generic_activation_patch itself uses internally. +# generic_activation_patch (transformer_lens/patching.py) replaces indexed +# activation slices with clean-cache values; a single-edge patch instead adds +# one writer's delta to a reader's input, so it is driven directly through the +# same hooks() mechanism generic_activation_patch uses internally. def _patch_edge_toward_clean( diff --git a/transformer_lens/ActivationCache.py b/transformer_lens/ActivationCache.py index b49cbd9d85..8b456a94ed 100644 --- a/transformer_lens/ActivationCache.py +++ b/transformer_lens/ActivationCache.py @@ -102,7 +102,6 @@ class ActivationCache: Warning: :class:`ActivationCache` is designed to be used with - :class:`transformer_lens.HookedTransformer` or :class:`transformer_lens.model_bridge.TransformerBridge`. Advanced helpers expect the model to expose the TransformerLens weight-processing interface and generally expect a complete cache; some internal methods may break with other models or partial caches. @@ -147,7 +146,7 @@ def __init__( ): self.cache_dict = cache_dict # Advanced helpers (LN folding, residual-direction projection) need the - # weight-processing surface; both HookedTransformer and TransformerBridge expose it. + # weight-processing surface, which TransformerBridge exposes. self.model = cast("TransformerLensModelWithWeights", model) self.has_batch_dim = has_batch_dim self.has_embed = "hook_embed" in self.cache_dict @@ -748,8 +747,8 @@ def compute_head_results( Intended use is to enable use_attn_results when running and caching the model, but this can be useful if you forget. - Works for both HookedTransformer and TransformerBridge — bridge exposes - ``blocks[i].attn.W_O`` via its component-mapping compatibility shim. + TransformerBridge exposes ``blocks[i].attn.W_O`` via its component-mapping + compatibility shim. """ # Return if valid 4D results exist; replace stale 3D Bridge entries if needed first_key = "blocks.0.attn.hook_result" diff --git a/transformer_lens/SVDInterpreter.py b/transformer_lens/SVDInterpreter.py index 35f18ef421..593ca00655 100644 --- a/transformer_lens/SVDInterpreter.py +++ b/transformer_lens/SVDInterpreter.py @@ -25,12 +25,12 @@ class SVDInterpreter: def __init__(self, model: TransformerLensModel): self.model = model self.cfg = model.cfg - # Use tl_parameters() for TransformerBridge (returns TL-style dict) - # Fall back to named_parameters() for HookedTransformer + # Use tl_parameters() for TransformerBridge (returns TL-style dict); other + # nn.Module models with TL-style parameter names use named_parameters(). if hasattr(model, "tl_parameters"): self.params = model.tl_parameters() else: - assert isinstance(model, torch.nn.Module) # legacy fallback path + assert isinstance(model, torch.nn.Module) # named_parameters() fallback self.params = {name: param for name, param in model.named_parameters()} def get_singular_vectors( diff --git a/transformer_lens/benchmarks/README.md b/transformer_lens/benchmarks/README.md index 4224df2122..366595daaa 100644 --- a/transformer_lens/benchmarks/README.md +++ b/transformer_lens/benchmarks/README.md @@ -179,12 +179,12 @@ The benchmarks use a tiered approach for comparison: - Direct comparison with original HF implementation - Ensures bridge maintains model fidelity -2. **Second Priority**: Compare TransformerBridge → the raw HuggingFace model - - If HT version exists, compare processed outputs - - Ensures compatibility with TransformerLens ecosystem +2. **Second Priority**: HF-anchored equivalence in both weight modes + - Phases 2 and 3 compare unprocessed and compatibility-mode outputs against the saved Phase 1 HF reference + - Ensures weight processing preserves model behavior 3. **Third Priority**: TransformerBridge-only validation - - If model unavailable in HT, validate bridge independently + - Reference-free structural self-checks for hooks, cache, and gradients - Ensures basic functionality and structural correctness ## Benchmark Results diff --git a/transformer_lens/benchmarks/main_benchmark.py b/transformer_lens/benchmarks/main_benchmark.py index 29ad990cff..4e65422345 100644 --- a/transformer_lens/benchmarks/main_benchmark.py +++ b/transformer_lens/benchmarks/main_benchmark.py @@ -3,8 +3,8 @@ This module provides the main benchmark suite that compares TransformerBridge against reference implementations in an optimized multi-phase approach: Phase 1: HF + Bridge (unprocessed) - Compare against raw HuggingFace model -Phase 2: Bridge (unprocessed) + HT (unprocessed) - Compare unprocessed models -Phase 3: Bridge (processed) + HT (processed) - Full compatibility mode testing +Phase 2: Bridge (unprocessed) - Runtime self-checks + HF logits/loss equivalence +Phase 3: Bridge (processed) - Compatibility mode + HF logits/loss equivalence Phase 4: Text Quality - profile prompts scored by a pinned judge's perplexity ratio Phase 5: Granular Weight Processing Tests (optional, individual flags) Phase 6: Granular Weight Processing Tests (optional, combined flags) @@ -1102,8 +1102,7 @@ def cleanup_model(model, model_name_str: str): # ======================================================================== current_phase[0] = 2 - # OPTIMIZATION: Run generation benchmarks first (only bridge in memory) - # Then cleanup bridge before loading HT to reduce peak memory + # OPTIMIZATION: Run generation benchmarks first (only bridge in memory). if should_run_phase(2) and bridge_unprocessed: if verbose: print(f"\n{'='*80}") @@ -1613,8 +1612,8 @@ def _cleanup_bridge_unprocessed(): if verbose: print("Running Phase 3 benchmarks...\n") - # Phase 3 runs in the requested dtype end-to-end. Both bridge and HT - # operate in the same precision — no dtype restoration needed. + # Phase 3 runs in the requested dtype end-to-end, so no dtype + # restoration is needed. phase3_results = run_comparison_benchmarks( bridge_model=bridge_processed, test_text=test_text, diff --git a/transformer_lens/cache/key_value_cache.py b/transformer_lens/cache/key_value_cache.py index 807790240f..8943ff5109 100644 --- a/transformer_lens/cache/key_value_cache.py +++ b/transformer_lens/cache/key_value_cache.py @@ -39,7 +39,7 @@ def init_cache( ): # Determine device for each layer if hasattr(cfg, "n_devices"): - # HookedTransformer case: use our multi-GPU logic + # Configs that track n_devices (TransformerBridgeConfig): per-block placement device_for_layer = lambda i: get_device_for_block_index(i, cfg, device) else: # Fallback when no model is provided - use single device diff --git a/transformer_lens/config/transformer_bridge_config.py b/transformer_lens/config/transformer_bridge_config.py index 08a92a8886..0a84ec5d74 100644 --- a/transformer_lens/config/transformer_bridge_config.py +++ b/transformer_lens/config/transformer_bridge_config.py @@ -272,7 +272,7 @@ def __post_init__(self): raise ValueError(f"architecture must be a string, got {type(self.architecture)}") # Resolve the initializer_range sentinel (-1.0 means "not set by the user"). - # Mirrors HookedTransformerConfig.__post_init__ (hooked_transformer_config.py). + # Same rule the legacy HookedTransformerConfig applied. # Guarded with getattr: this method also runs once from the dataclass # parent's __init__, before self.initializer_range is assigned below. if getattr(self, "initializer_range", None) is not None: diff --git a/transformer_lens/evals.py b/transformer_lens/evals.py index 69405c9bd0..78bde05946 100644 --- a/transformer_lens/evals.py +++ b/transformer_lens/evals.py @@ -447,7 +447,7 @@ def ioi_eval(model, dataset=None, batch_size=8, num_samples=1000, tokenizer=None """Evaluate the Model on the Indirect Object Identification Task. Args: - model: A HookedTransformer or TransformerBridge model. + model: A TransformerBridge model. dataset: PyTorch Dataset that returns a dict with keys "prompt", "IO", and "S". batch_size: Batch size to use. num_samples: Number of samples to use. @@ -534,7 +534,7 @@ def mmlu_eval( Paper: https://arxiv.org/abs/2009.03300 Args: - model: A HookedTransformer or TransformerBridge model to evaluate. + model: A TransformerBridge model to evaluate. tokenizer: Tokenizer to use. If None, uses model.tokenizer. subjects: Subject(s) to evaluate on. Can be None (all 57 subjects), a single subject string, or a list of subjects. See :const:`MMLU_SUBJECTS` for valid names. diff --git a/transformer_lens/model_bridge/sources/native/init.py b/transformer_lens/model_bridge/sources/native/init.py index 4e0cc6d4c2..6f96400b90 100644 --- a/transformer_lens/model_bridge/sources/native/init.py +++ b/transformer_lens/model_bridge/sources/native/init.py @@ -102,8 +102,8 @@ def apply(t: torch.Tensor) -> torch.Tensor: std = cfg.initializer_range if cfg.initializer_range > 0 else 0.8 / math.sqrt(cfg.d_model) # NOTE: this residual output scaling (1/sqrt(2*n_layers), applied only - # to output projections below) is NOT present in HookedTransformer's - # _init_weights_gpt2 (see transformer_lens/HookedTransformer.py). + # to output projections below) was NOT present in the legacy + # HookedTransformer._init_weights_gpt2 (removed in 4.0). # Intentional delta for NativeModel: kept because it follows the # original GPT-2 paper's residual-scaling convention and improves # training stability at init for deeper models. Flagged in issue #1568 diff --git a/transformer_lens/model_bridge/supported_architectures/jamba.py b/transformer_lens/model_bridge/supported_architectures/jamba.py index cb74e61e2a..d2613230fd 100644 --- a/transformer_lens/model_bridge/supported_architectures/jamba.py +++ b/transformer_lens/model_bridge/supported_architectures/jamba.py @@ -82,7 +82,7 @@ class JambaArchitectureAdapter(ArchitectureAdapter): ablated independently. """ - # P1: exact passthrough vs raw HF; P2/P3 skip HT comparison; P4 generation. + # P1: exact passthrough vs raw HF; P2/P3 self-checks + HF equivalence; P4 generation. applicable_phases: list[int] = [1, 2, 3, 4] def __init__(self, cfg: Any) -> None: diff --git a/transformer_lens/model_bridge/supported_architectures/mamba.py b/transformer_lens/model_bridge/supported_architectures/mamba.py index 6d9bda4cd5..03b6f3d953 100644 --- a/transformer_lens/model_bridge/supported_architectures/mamba.py +++ b/transformer_lens/model_bridge/supported_architectures/mamba.py @@ -23,8 +23,8 @@ class MambaArchitectureAdapter(ArchitectureAdapter): ``_HF_PASSTHROUGH_ATTRS`` in sources/_bridge_builder.py. """ - # White-box forward: P1 is exact vs raw HF (mixer delegates to HF); P2/P3 skip - # without a HookedTransformer; P4 is generation. + # White-box forward: P1 is exact vs raw HF (mixer delegates to HF); P2/P3 run + # hook/cache self-checks and HF equivalence; P4 is generation. applicable_phases: list[int] = [1, 2, 3, 4] def __init__(self, cfg: Any) -> None: diff --git a/transformer_lens/model_bridge/supported_architectures/mamba2.py b/transformer_lens/model_bridge/supported_architectures/mamba2.py index f9d281e5c3..4716e79f02 100644 --- a/transformer_lens/model_bridge/supported_architectures/mamba2.py +++ b/transformer_lens/model_bridge/supported_architectures/mamba2.py @@ -29,8 +29,8 @@ class Mamba2ArchitectureAdapter(ArchitectureAdapter): loop with Mamba-1. """ - # White-box forward: P1 is exact vs raw HF (mixer delegates to HF); P2/P3 skip - # without a HookedTransformer; P4 is generation. + # White-box forward: P1 is exact vs raw HF (mixer delegates to HF); P2/P3 run + # hook/cache self-checks and HF equivalence; P4 is generation. applicable_phases: list[int] = [1, 2, 3, 4] def __init__(self, cfg: Any) -> None: diff --git a/transformer_lens/model_bridge/supported_architectures/nemotron_h.py b/transformer_lens/model_bridge/supported_architectures/nemotron_h.py index a80f8672fa..5a190d4e14 100644 --- a/transformer_lens/model_bridge/supported_architectures/nemotron_h.py +++ b/transformer_lens/model_bridge/supported_architectures/nemotron_h.py @@ -27,7 +27,7 @@ declared ``optional=True`` so setup skips them gracefully on non-Mamba layers. - MLP layers use ``relu2`` activation (not SwiGLU); ``gated_mlp = False``. - ``applicable_phases = [1, 2, 3, 4]``: P1 is exact vs raw HF (passthrough mixers); - P2/P3 skip without a HookedTransformer; P4 is generation. + P2/P3 run hook/cache self-checks and HF equivalence; P4 is generation. """ from typing import Any @@ -68,8 +68,8 @@ class NemotronHArchitectureAdapter(ArchitectureAdapter): is determined by ``config.layers_block_type[layer_idx]``. """ - # White-box forward: P1 is exact vs raw HF (passthrough mixers); P2/P3 skip - # without a HookedTransformer; P4 is generation. + # White-box forward: P1 is exact vs raw HF (passthrough mixers); P2/P3 run + # hook/cache self-checks and HF equivalence; P4 is generation. applicable_phases: list[int] = [1, 2, 3, 4] def __init__(self, cfg: Any) -> None: diff --git a/transformer_lens/model_bridge/supported_architectures/vit.py b/transformer_lens/model_bridge/supported_architectures/vit.py index eef75a916c..6c4bc9c426 100644 --- a/transformer_lens/model_bridge/supported_architectures/vit.py +++ b/transformer_lens/model_bridge/supported_architectures/vit.py @@ -57,8 +57,8 @@ class ViTArchitectureAdapter(ArchitectureAdapter): supports_generation: bool = False # Vision models have no tokenizer, so of the text phases only Phase 1 (HF - # parity on pixel input) applies — Phases 2/3 need a HookedTransformer - # counterpart and Phase 4 needs text generation. Phase 9 (vision hook/cache + # parity on pixel input) applies — Phases 2/3 compare text logits/loss and + # Phase 4 needs text generation. Phase 9 (vision hook/cache # tests) is gated by is_visual_model, not this list; _full_and_core_phases() # routes "vision" architectures to {1, 9}. applicable_phases: list[int] = [1] diff --git a/transformer_lens/model_bridge/supported_architectures/zamba2.py b/transformer_lens/model_bridge/supported_architectures/zamba2.py index 506ef1c864..b42c406d15 100644 --- a/transformer_lens/model_bridge/supported_architectures/zamba2.py +++ b/transformer_lens/model_bridge/supported_architectures/zamba2.py @@ -33,7 +33,7 @@ ``.input_layernorm`` or ``.mamba``). Block-level ``hook_in``/``hook_out`` still fire on every layer. - ``applicable_phases = [1, 2, 3, 4]``: P1 is exact vs raw HF (pure - passthrough); P2/P3 skip without a HookedTransformer; P4 exercises + passthrough); P2/P3 run hook/cache self-checks and HF equivalence; P4 exercises ``past_key_values`` cache threading across Mamba-2 and attention layers. """ @@ -75,7 +75,7 @@ class Zamba2ArchitectureAdapter(ArchitectureAdapter): Mamba-2 step. """ - # P1: exact passthrough vs raw HF; P2/P3: skip without HookedTransformer; + # P1: exact passthrough vs raw HF; P2/P3: hook/cache self-checks + HF equivalence; # P4: generation with past_key_values cache threading. applicable_phases: list[int] = [1, 2, 3, 4] diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index 07ef92c095..4c224b8499 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -765,7 +765,7 @@ def process_weights( adapter._fold_ln_requested = fold_ln # type: ignore[union-attr] state_dict = adapter.preprocess_weights(state_dict) - # Use unified ProcessWeights.process_weights() like HookedTransformer does. + # Use the unified ProcessWeights.process_weights() pipeline. # Float32 upcasting for precision is handled centrally in process_weights(). if verbose: print(" Processing weights (fold_ln, center_writing_weights, etc.)...") @@ -4269,8 +4269,6 @@ def set_use_attn_in(self, use_attn_in: bool): def set_use_hook_mlp_in(self, use_hook_mlp_in: bool) -> None: """Toggle the ``hook_mlp_in`` HookPoint (the MLP-branch entry: pre-ln2, or the MLP input on post-norm blocks), matching legacy semantics. - - See :py:meth:`HookedTransformer.set_use_hook_mlp_in`. """ self.cfg._set_bridge_managed_hook_flag("use_hook_mlp_in", use_hook_mlp_in) if not hasattr(self, "blocks"): diff --git a/transformer_lens/model_protocol.py b/transformer_lens/model_protocol.py index bdc66c3879..4d6833f2d8 100644 --- a/transformer_lens/model_protocol.py +++ b/transformer_lens/model_protocol.py @@ -1,14 +1,12 @@ -"""Structural type shared by :class:`HookedTransformer` and :class:`TransformerBridge`. +"""Structural model types for the model-agnostic interpretability utilities. The interpretability utilities (``patching``, ``head_detector``, ``ActivationCache``) -are model-agnostic at runtime — they only need ``cfg`` plus the ``run_with_*`` / -tokenization surface and a few weight-processing helpers. Historically their -signatures said ``HookedTransformer``, which made a ``TransformerBridge`` fail to -type-check even though it works. Typing the model parameter as this Protocol accepts -either (and any future structural match, e.g. ``RemoteBridge``). - -Members are typed loosely on purpose: this is a compatibility shim over two concrete -classes whose method signatures differ in detail but agree in use. +only need ``cfg`` plus the ``run_with_*`` / tokenization surface and a few +weight-processing helpers. Typing their model parameter as this Protocol accepts +``TransformerBridge`` and any other structural match (e.g. ``RemoteBridge``). + +Members are typed loosely on purpose: implementations differ in signature detail +but agree in use. """ from __future__ import annotations @@ -19,14 +17,14 @@ # runtime_checkable so the interp utilities' beartype-decorated signatures can -# isinstance-check the parameter at runtime (presence-only; both models qualify). +# isinstance-check the parameter at runtime (presence-only). @runtime_checkable class TransformerLensModel(Protocol): - """Minimal structural interface common to HookedTransformer and TransformerBridge.""" + """Minimal structural interface the interpretability utilities rely on.""" # Read-only property (not a bare attribute) so it is covariant: a concrete model - # whose cfg is a TransformerLensConfig *subclass* (HookedTransformerConfig / - # TransformerBridgeConfig) still conforms. A mutable attribute would be invariant. + # whose cfg is a TransformerLensConfig *subclass* (e.g. TransformerBridgeConfig) + # still conforms. A mutable attribute would be invariant. @property def cfg(self) -> "TransformerLensConfig": ... @@ -69,8 +67,8 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any: @runtime_checkable class TransformerLensModelWithWeights(TransformerLensModel, Protocol): """Adds the weight-processing surface that ``ActivationCache``'s advanced helpers - (LayerNorm folding, residual-direction projection) reach for. Both concrete models - expose these; the bridge builds them from its adapter.""" + (LayerNorm folding, residual-direction projection) reach for. The bridge builds + them from its adapter.""" @property def blocks(self) -> Any: diff --git a/transformer_lens/supported_models.py b/transformer_lens/supported_models.py index 6bd53ac1a5..0b638a21b0 100644 --- a/transformer_lens/supported_models.py +++ b/transformer_lens/supported_models.py @@ -258,8 +258,8 @@ # Model Aliases: # Canonical data lives in the bridge model registry -# (tools/model_registry/data/model_aliases.json); this re-export serves the -# legacy HookedTransformer loaders until their removal at 4.0. +# (tools/model_registry/data/model_aliases.json); re-exported here for the +# frozen legacy ledger and get_official_model_name. MODEL_ALIASES: dict[str, list[str]] = load_model_aliases() """Model aliases for models on HuggingFace.""" diff --git a/transformer_lens/tools/model_registry/AGENTS.md b/transformer_lens/tools/model_registry/AGENTS.md index ec93f26f2f..35279457dc 100644 --- a/transformer_lens/tools/model_registry/AGENTS.md +++ b/transformer_lens/tools/model_registry/AGENTS.md @@ -120,7 +120,7 @@ Never edit manually. SSM / recurrent families and the hybrids (Mamba-1/2, gated-delta-net, NemotronH, GraniteMoeHybrid, Jamba, Qwen3.5/Qwen3-Next) declare `applicable_phases = [1, 2, 3, 4]` — all four apply. P2 runs bridge self-checks (hooks, cache, gradients); P3 checks processed-weight equivalence against the Phase-1 HF reference when available. -**Non-text modalities.** `classify_architecture` routes audio architectures to `{1, 8}` and vision architectures (ViT/DeiT) to `{1, 9}` — vision has no tokenizer for P4, and neither modality has a HookedTransformer counterpart for P2/P3. Phases 1/8/9 build their input with `build_modality_input()` ([`benchmarks/utils.py`](../../benchmarks/utils.py)), which shapes it from the HF config: `[batch, max_length, num_mel_bins]` for spectrogram encoders, `[batch, samples]` for waveform encoders, `[batch, channels, image_size, image_size]` for vision. Add a new non-text architecture there rather than hardcoding a shape at the call site. +**Non-text modalities.** `classify_architecture` routes audio architectures to `{1, 8}` and vision architectures (ViT/DeiT) to `{1, 9}` — vision has no tokenizer for P4, and neither modality produces the text logits/loss that P2/P3 compare. Phases 1/8/9 build their input with `build_modality_input()` ([`benchmarks/utils.py`](../../benchmarks/utils.py)), which shapes it from the HF config: `[batch, max_length, num_mel_bins]` for spectrogram encoders, `[batch, samples]` for waveform encoders, `[batch, channels, image_size, image_size]` for vision. Add a new non-text architecture there rather than hardcoding a shape at the call site. ### Phase-score thresholds diff --git a/transformer_lens/tools/model_registry/verify_models.py b/transformer_lens/tools/model_registry/verify_models.py index d96f8e92d5..cfe4581c50 100644 --- a/transformer_lens/tools/model_registry/verify_models.py +++ b/transformer_lens/tools/model_registry/verify_models.py @@ -140,8 +140,8 @@ def _full_and_core_phases(arch: str) -> tuple[set[int], set[int]]: if kind == "audio": return {1, 8}, {1, 8} if kind == "vision": - # Vision encoders have no tokenizer and no text tower: Phases 2/3 need - # HookedTransformer, Phase 4 needs text generation, and Phase 7 covers + # Vision encoders have no tokenizer and no text tower: Phases 2/3 compare + # text logits/loss, Phase 4 needs text generation, and Phase 7 covers # vision+text multimodal models, not these. Phase 1 (HF parity) plus # Phase 9 (pixel forward/cache/stability) are the whole story. return {1, 9}, {1, 9} diff --git a/transformer_lens/utilities/defaults_utils.py b/transformer_lens/utilities/defaults_utils.py index 7745ba8acc..9590a384d3 100644 --- a/transformer_lens/utilities/defaults_utils.py +++ b/transformer_lens/utilities/defaults_utils.py @@ -41,7 +41,7 @@ def __init__(self, model, **overrides): Initializes the context manager. Args: - model (HookedTransformer): The model whose default values will be overridden. + model (TransformerBridge): The model whose default values will be overridden. overrides (dict): Key-value pairs of properties to override and their new values. """ self.model = model diff --git a/transformer_lens/utilities/matrix.py b/transformer_lens/utilities/matrix.py index 2d768fdaef..9e3896fbc3 100644 --- a/transformer_lens/utilities/matrix.py +++ b/transformer_lens/utilities/matrix.py @@ -19,8 +19,12 @@ def composition_scores( Float[torch.Tensor, "*leading_dims"], Float[torch.Tensor, "*leading_dims_left_and_right"], ]: - """ - See `HookedTransformer.all_composition_scores` for documentation. + """Composition scores between two factored matrices. + + Returns ``||left @ right||_F / (||left||_F * ||right||_F)``, computed from the factored + forms so the full products are never materialized. With ``broadcast_dims``, left and right + leading dims are broadcast against each other (left dims first), scoring every left/right + pair. See ``TransformerBridge.all_composition_scores``. """ if broadcast_dims: r_leading = right.ndim - 2 diff --git a/transformer_lens/weight_processing.py b/transformer_lens/weight_processing.py index 485d711f9e..af31135051 100644 --- a/transformer_lens/weight_processing.py +++ b/transformer_lens/weight_processing.py @@ -1636,7 +1636,7 @@ def process_weights( """Apply all weight processing transformations in the correct order. This is a convenience function that applies all the weight processing steps - in the same order as HookedTransformer.load_and_process_state_dict(). + in the same order as the legacy HookedTransformer load path. Args: state_dict (Dict[str, torch.Tensor]): State dict of the model. From 73abf7c01b2efab757a460b4fd886a791690ccb5 Mon Sep 17 00:00:00 2001 From: Arnav Bendre <arnav_b@hre.iitr.ac.in> Date: Fri, 18 Sep 2026 19:12:26 +0530 Subject: [PATCH 84/87] fix: raise on stop_at_layer when no 'blocks' stack is registered (#1789) * fix: raise on stop_at_layer when no 'blocks' stack is registered Replace the stop_at_layer reject-list with an allowlist guard and check self._modules instead of hasattr for both stop_at_layer and start_at_layer. Addresses #1769 * docs: note NotImplementedError in stop_at_layer docstring test: cover a wrapped model exposing its own .blocks Addresses #1769 * test: cover a wrapped model exposing its own .blocks Addresses #1769 --------- Co-authored-by: jlarson4 <jonahalarson@comcast.net> --- .../model_bridge/test_stop_at_layer_guard.py | 50 ++++++++++++++++ .../model_bridge/transformer_bridge.py | 59 +++++++++++-------- 2 files changed, 85 insertions(+), 24 deletions(-) create mode 100644 tests/unit/model_bridge/test_stop_at_layer_guard.py diff --git a/tests/unit/model_bridge/test_stop_at_layer_guard.py b/tests/unit/model_bridge/test_stop_at_layer_guard.py new file mode 100644 index 0000000000..e0e5db99a0 --- /dev/null +++ b/tests/unit/model_bridge/test_stop_at_layer_guard.py @@ -0,0 +1,50 @@ +from types import SimpleNamespace + +import pytest +import torch +import torch.nn as nn + +from transformer_lens.config import TransformerBridgeConfig +from transformer_lens.model_bridge import TransformerBridge + + +def _bare_bridge(**block_lists: nn.Module) -> TransformerBridge: + """A TransformerBridge with only the given block lists registered (no HF model).""" + bridge = TransformerBridge.__new__(TransformerBridge) + nn.Module.__init__(bridge) + bridge.cfg = TransformerBridgeConfig( + d_model=8, + d_head=4, + n_layers=1, + n_ctx=16, + d_vocab=32, + d_mlp=16, + n_heads=2, + architecture="RavenForCausalLM", + ) + for name, module in block_lists.items(): + bridge.add_module(name, module) + return bridge + + +def test_stop_at_layer_raises_without_blocks_stack() -> None: + """Raven-style prelude/core_block/coda lists must not silently ignore stop_at_layer.""" + bridge = _bare_bridge( + prelude=nn.ModuleList([nn.Identity()]), + core_block=nn.ModuleList([nn.Identity()]), + coda=nn.ModuleList([nn.Identity()]), + ) + with pytest.raises(NotImplementedError, match="stop_at_layer requires a 'blocks' stack"): + bridge.forward(torch.zeros(1, 3, dtype=torch.long), stop_at_layer=0) + + +def test_blocks_guard_ignores_unregistered_blocks_attribute() -> None: + """A wrapped HF model exposing `.blocks` must not satisfy the guard via __getattr__.""" + bridge = _bare_bridge() + bridge.__dict__["original_model"] = SimpleNamespace(blocks=[object()]) + assert hasattr(bridge, "blocks") # the trap: __getattr__ falls through to the HF model + assert not bridge._has_registered_blocks() + with pytest.raises(NotImplementedError, match="stop_at_layer requires a 'blocks' stack"): + bridge.forward(torch.zeros(1, 3, dtype=torch.long), stop_at_layer=0) + with pytest.raises(NotImplementedError, match="start_at_layer requires a 'blocks' stack"): + bridge.forward(torch.zeros(1, 3, 8), start_at_layer=0) diff --git a/transformer_lens/model_bridge/transformer_bridge.py b/transformer_lens/model_bridge/transformer_bridge.py index 4c224b8499..d5db979561 100644 --- a/transformer_lens/model_bridge/transformer_bridge.py +++ b/transformer_lens/model_bridge/transformer_bridge.py @@ -3,6 +3,7 @@ This module provides the bridge components that wrap remote model components and provide a consistent interface for accessing their weights and performing operations. """ + import inspect import logging import re @@ -460,6 +461,16 @@ def n_params_total(self) -> int: """ return self._n_params_total + def _has_registered_blocks(self) -> bool: + """Whether a ``blocks`` stack is registered as a submodule on this bridge. + + Checks ``_modules`` directly rather than ``hasattr``: ``__getattr__`` falls + through to the wrapped HF model, so ``hasattr(self, "blocks")`` can be True + for a model that merely exposes its own ``.blocks`` attribute. + """ + modules = self.__dict__.get("_modules") or {} + return "blocks" in modules + def __getattr__(self, name: str) -> Any: """Provide a clear error message for missing attributes.""" # Re-invoke original_model's property so its descriptive AttributeError @@ -2095,7 +2106,10 @@ def forward( output is discarded when block k swaps in the residual) but are excluded from ``run_with_cache`` output. Requires an HF model that accepts ``inputs_embeds``; only supported on the standard ``blocks`` stack. - stop_at_layer: Layer to stop forward pass at + stop_at_layer: Layer to stop forward pass at. Only supported on the + standard ``blocks`` stack; architectures that register no ``blocks`` + (e.g. Raven's ``prelude``/``core_block``/``coda``) raise + ``NotImplementedError`` rather than running to completion. pixel_values: Optional image tensor for multimodal models (e.g., LLaVA, Gemma3) and vision models (eg. ViT, DeiT). The tensor is passed directly to the underlying HuggingFace model. @@ -2135,26 +2149,18 @@ def forward( if start_at_layer is not None: input = self._setup_start_at_layer(input, start_at_layer) - # Set stop_at_layer flag on all blocks if requested if stop_at_layer is not None: - if ( - hasattr(self, "L_blocks") - or hasattr(self, "H_blocks") - or hasattr(self, "encoder_blocks") - or hasattr(self, "decoder_blocks") - ): + if not self._has_registered_blocks(): raise NotImplementedError( - "stop_at_layer is not supported on non-standard block list " - "names (L_blocks, H_blocks, encoder_blocks, decoder_blocks). " - "The bridge only supports stop_at_layer on 'blocks'." - ) - if hasattr(self, "blocks"): - effective_stop_at_layer = ( - len(self.blocks) + stop_at_layer if stop_at_layer < 0 else stop_at_layer + "stop_at_layer requires a 'blocks' stack; this architecture " + "does not register one." ) - for block in self.blocks: - block._stop_at_layer_idx = effective_stop_at_layer + effective_stop_at_layer = ( + len(self.blocks) + stop_at_layer if stop_at_layer < 0 else stop_at_layer + ) + for block in self.blocks: + block._stop_at_layer_idx = effective_stop_at_layer # Map HookedEncoderDecoder-style kwargs to HF-compatible names if "decoder_input" in kwargs: @@ -2449,8 +2455,9 @@ def _setup_start_at_layer(self, input: Any, start_at_layer: int) -> torch.Tensor "start_at_layer is only supported on the standard 'blocks' stack, " f"not {alt!r}." ) - if not hasattr(self, "blocks"): + if not self._has_registered_blocks(): raise NotImplementedError("start_at_layer requires a 'blocks' stack.") + if not (isinstance(input, torch.Tensor) and input.is_floating_point()): raise ValueError( "start_at_layer requires a residual-stream tensor [batch, pos, d_model]; " @@ -2824,18 +2831,22 @@ def _generate_tokens( temperature=temperature, freq_penalty=freq_penalty, repetition_penalty=repetition_penalty, - tokens=penalty_tokens - if _generate_from_embeds - else (decoder_tokens if is_encoder_decoder else current_tokens), + tokens=( + penalty_tokens + if _generate_from_embeds + else (decoder_tokens if is_encoder_decoder else current_tokens) + ), ).to(self.cfg.device) else: sampled_tokens = utils.sample_logits( final_logits, temperature=0.0, repetition_penalty=repetition_penalty, - tokens=penalty_tokens - if _generate_from_embeds - else (decoder_tokens if is_encoder_decoder else current_tokens), + tokens=( + penalty_tokens + if _generate_from_embeds + else (decoder_tokens if is_encoder_decoder else current_tokens) + ), ).to(self.cfg.device) # Freeze rows that finished on an earlier step so they stop emitting From 9cbcd25f187703a0483480e5f0316e8f85be8105 Mon Sep 17 00:00:00 2001 From: jlarson4 <jonahalarson@comcast.net> Date: Fri, 18 Sep 2026 14:25:59 -0500 Subject: [PATCH 85/87] merge changes --- docs/make_docs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/make_docs.py b/docs/make_docs.py index f80c32ce58..174d362661 100644 --- a/docs/make_docs.py +++ b/docs/make_docs.py @@ -759,7 +759,8 @@ def copy_demos(_app: Optional[Any] = None): if copy_to_dir.exists(): shutil.rmtree(copy_to_dir) - copy_to_dir.mkdir() + # GENERATED_DIR is gitignored, so it is absent on a clean checkout: create the parent too. + copy_to_dir.mkdir(parents=True, exist_ok=True) for filename in notebooks_to_copy: shutil.copy(DEMOS_DIR / filename, copy_to_dir) From 848d8c031e727e9a760c4141f1f7723845627fc0 Mon Sep 17 00:00:00 2001 From: jlarson4 <jonahalarson@comcast.net> Date: Fri, 18 Sep 2026 14:30:00 -0500 Subject: [PATCH 86/87] fix make docs test --- tests/unit/test_make_docs.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/unit/test_make_docs.py b/tests/unit/test_make_docs.py index 50e9cd4b72..fbfa6092b8 100644 --- a/tests/unit/test_make_docs.py +++ b/tests/unit/test_make_docs.py @@ -22,3 +22,23 @@ def test_render_bridge_models_page_interpolates_registry_constants(): # Status 2 rows render with the s0 badge, so no .bt-s2 CSS rule exists. assert "m.status===2?0" in page assert "#bt-root .bt-s2" not in page + + +def test_copy_demos_creates_generated_dir_when_absent(tmp_path, monkeypatch): + """copy_demos() is the first step of build_docs() on a clean checkout, where the + gitignored generated/ directory does not exist yet. Copies from the real demos/ so a + renamed notebook fails here rather than in the docs job.""" + from docs import make_docs + + generated = tmp_path / "generated" + monkeypatch.setattr(make_docs, "GENERATED_DIR", generated) + assert not generated.exists() + + make_docs.copy_demos() + + copied = sorted(p.name for p in (generated / "demos").iterdir()) + assert copied == [ + "Exploratory_Analysis_Demo.ipynb", + "Jacobian_Lens_Decomposition_Demo.ipynb", + "Main_Demo.ipynb", + ] From 89dcb3ca977d12ae8f5427095dd41771f1de0f2a Mon Sep 17 00:00:00 2001 From: jlarson4 <jonahalarson@comcast.net> Date: Fri, 18 Sep 2026 18:09:08 -0500 Subject: [PATCH 87/87] Fix issue with Goldens on the MPS tests --- .github/workflows/checks.yml | 2 ++ tests/goldens.py | 20 ++++++++++++++++---- tests/unit/test_goldens_infra.py | 25 +++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 9fc2c884f9..2c6c8d7520 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -135,6 +135,8 @@ jobs: mps-checks: name: MPS Checks runs-on: macos-latest + # Bounded so a stall costs 90 minutes of macOS runner time, not the 6-hour default. + timeout-minutes: 90 # Only run on PRs merging to main or pushes directly to main if: > (github.event_name == 'pull_request' && github.base_ref == 'main') || diff --git a/tests/goldens.py b/tests/goldens.py index 055738e5a0..7fea88709c 100644 --- a/tests/goldens.py +++ b/tests/goldens.py @@ -39,13 +39,24 @@ def _model_dir_name(model_name: str) -> str: return model_name.replace("/", "__") -@functools.lru_cache(maxsize=1) -def resolve_goldens_dir() -> Path | None: - """Locate the goldens root, or None if unavailable.""" +# Every cell's provenance manifest — kilobytes in total, and all the availability +# checks read. Keeps "is this cell present?" from pulling any tensor. +_MANIFEST_PATTERN = "*/*/provenance.json" + + +@functools.lru_cache(maxsize=None) +def resolve_goldens_dir(model: str | None = None, config: str | None = None) -> Path | None: + """Locate the goldens root, or None if unavailable. + + ``model``/``config`` scope the Hub fetch to that one cell. The dataset is 36 GB + across every model and config, so an unscoped fetch exceeds a CI runner's disk + to satisfy a single ~800 MB cell; unscoped calls take manifests only. + """ local = os.environ.get(_ENV_VAR) if local: path = Path(local) return path if path.is_dir() else None + patterns = [f"{_model_dir_name(model)}/{config}/*"] if model and config else [_MANIFEST_PATTERN] try: from huggingface_hub import snapshot_download @@ -54,6 +65,7 @@ def resolve_goldens_dir() -> Path | None: repo_id=GOLDENS_REPO_ID, repo_type="dataset", revision=GOLDENS_REVISION, + allow_patterns=patterns, ) ) except Exception: @@ -73,7 +85,7 @@ def goldens_available(model: str | None = None, config: str | None = None) -> bo def golden_path(model: str, config: str) -> Path: """Directory of one (model, processing-config) golden cell. Raises if absent.""" - root = resolve_goldens_dir() + root = resolve_goldens_dir(model, config) if root is None: raise FileNotFoundError( f"Goldens unavailable: set {_ENV_VAR} to a local capture directory or " diff --git a/tests/unit/test_goldens_infra.py b/tests/unit/test_goldens_infra.py index 3e1e31e209..da11dfe0bf 100644 --- a/tests/unit/test_goldens_infra.py +++ b/tests/unit/test_goldens_infra.py @@ -62,6 +62,31 @@ def test_unset_env_and_unreachable_hub_degrade_to_unavailable(self, monkeypatch) assert not goldens.goldens_available() goldens.resolve_goldens_dir.cache_clear() + def test_hub_fetch_is_scoped(self, tmp_path, monkeypatch): + """The dataset holds every model at 36 GB total, so an unscoped fetch exceeds a + runner's disk to satisfy one ~800 MB cell: loads take their own cell, and + availability checks take only manifests.""" + import huggingface_hub + + cell = tmp_path / "org__tiny" / "no_processing" + cell.mkdir(parents=True) + (cell / "provenance.json").write_text("{}") + requested: list[list[str] | None] = [] + + def fake_snapshot_download(**kwargs): + requested.append(kwargs.get("allow_patterns")) + return str(tmp_path) + + monkeypatch.delenv("TL_GOLDENS_DIR", raising=False) + monkeypatch.setattr(huggingface_hub, "snapshot_download", fake_snapshot_download) + goldens.resolve_goldens_dir.cache_clear() + + assert goldens.goldens_available("org/tiny", "no_processing") + goldens.golden_path("org/tiny", "no_processing") + goldens.resolve_goldens_dir.cache_clear() + + assert requested == [["*/*/provenance.json"], ["org__tiny/no_processing/*"]] + class TestCaptureHelpers: def test_checksum_is_deterministic_and_value_sensitive(self):